scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-15 04:19:08 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 121

Merge branch 'main' into fix/router-track-empty-model-list-live-routers

Browse source
This commit is contained in:
Quinn Xu 2026-10-10 01:13:05 +08:00 • committed by GitHub
commit 43fca7f25d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2248 changed files with 212976 additions and 104488 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

345
.circleci/config.yml
View file

@ -856,58 +856,6 @@ jobs:
- store_test_results:
path: test-results
litellm_router_unit_testing: # Runs all tests with the "router" keyword
docker:
- *python312_image
working_directory: ~/project
resource_class: large
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- install_rust
- restore_cache:
keys:
- v1-uv-cache-{{ checksum "uv.lock" }}
- run:
name: Install Dependencies
command: |
uv sync --frozen --all-groups --all-extras --python 3.12
- save_cache:
paths:
- ~/.cache/uv
key: v1-uv-cache-{{ checksum "uv.lock" }}
# Run pytest and generate JUnit XML report
- setup_litellm_enterprise_pip
- run:
name: Run tests
command: |
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/router_unit_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \
-v \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 4"
no_output_timeout: 15m
- run:
name: Rename the coverage files
command: |
mv coverage.xml router_unit_tests_coverage.xml
mv .coverage router_unit_tests_coverage
# Store test results
- store_test_results:
path: test-results
- persist_to_workspace:
root: .
paths:
- router_unit_tests_coverage.xml
- router_unit_tests_coverage
llm_translation_testing:
docker:
- *python312_image
@ -1024,16 +972,13 @@ jobs:
command: |
mkdir -p test-results
export LITELLM_LOG=WARNING
TEST_FILES=$(circleci tests glob "tests/guardrails_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \
-vv \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 2 \
--timeout=120 --timeout_method=thread"
uv run --no-sync python -m pytest --tb=short \
-vv \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
--timeout=120 --timeout_method=thread \
"tests/guardrails_tests/test_custom_guardrail.py::test_get_guardrail_dynamic_request_body_params"
no_output_timeout: 15m
- run:
name: Rename the coverage files
@ -1095,50 +1040,6 @@ jobs:
- google_generate_content_endpoint_coverage.xml
- google_generate_content_endpoint_coverage
llm_responses_api_testing:
docker:
- *python312_image
working_directory: ~/project
resource_class: large
environment:
REQUEST_TIMEOUT: "180"
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- install_rust
- restore_cache:
keys:
- v1-uv-cache-{{ checksum "uv.lock" }}
- run:
name: Install Dependencies
command: |
uv sync --frozen --all-groups --all-extras --python 3.12
- save_cache:
paths:
- ~/.cache/uv
key: v1-uv-cache-{{ checksum "uv.lock" }}
# Run pytest and generate JUnit XML report
- run:
name: Run tests
command: |
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/llm_responses_api_testing/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \
-v \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 8 \
--reruns 1 --only-rerun Timeout"
no_output_timeout: 15m
# Store test results
- store_test_results:
path: test-results
ocr_testing:
docker:
- *python312_image
@ -2074,98 +1975,6 @@ jobs:
# Store test results
- store_test_results:
path: test-results
proxy_spend_accuracy_tests:
machine:
image: ubuntu-2204:2024.04.1
resource_class: large
working_directory: ~/project
steps:
- checkout
- run:
name: Generate LiteLLM master key
command: |
key="$(openssl rand -hex 16)"
printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV"
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- install_rust
- run:
name: Install Dependencies
command: |
uv sync --frozen --all-groups --all-extras --python 3.12
- start_postgres
- start_redis
- start_fake_openai_endpoint
- attach_workspace:
at: ~/project
- run:
name: Load Docker Database Image
command: |
zstd -d litellm-docker-database.tar.zst --stdout | docker load
docker images | grep litellm-docker-database
- run:
name: Run Docker container
# Point the proxy at the job-local Redis (start_redis) instead of the
# shared remote Redis. The Redis transaction buffer uses a single
# global pod-lock key (cronjob_lock:db_spend_update_job) and a single
# global buffer list (litellm_spend_update_buffer); sharing those
# across concurrent CI pipelines causes spend flushes to stall or
# land in the wrong DB, which is what makes this test flaky.
command: |
docker run -d \
-p 4000:4000 \
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
-e REDIS_HOST=host.docker.internal \
-e REDIS_PORT=6379 \
-e LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" \
-e OPENAI_API_KEY=$OPENAI_API_KEY \
-e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \
-e LITELLM_LICENSE=$LITELLM_LICENSE \
-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
-e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
-e USE_DDTRACE=True \
-e DD_API_KEY=$DD_API_KEY \
-e DD_SITE=$DD_SITE \
-e AWS_REGION_NAME=$AWS_REGION_NAME \
-e PROXY_BATCH_WRITE_AT=2 \
-e LITELLM_LOG=ERROR \
--add-host host.docker.internal:host-gateway \
--name my-app \
-v $(pwd)/litellm/proxy/example_config_yaml/spend_tracking_config.yaml:/app/config.yaml \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000
- run:
name: Start outputting logs
command: docker logs -f my-app
background: true
- wait_for_service:
url: http://localhost:4000
timeout: "300"
- run:
name: Run tests
command: |
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/spend_tracking_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \
-vv \
--junitxml=test-results/junit.xml \
--durations=5"
no_output_timeout: 15m
- store_test_results:
path: test-results
- run:
name: Stop and remove first container
when: always
command: |
docker stop my-app
docker rm my-app
docker stop redis-cache
docker rm redis-cache
proxy_multi_instance_tests:
machine:
image: ubuntu-2204:2024.04.1
@ -2629,7 +2438,7 @@ jobs:
- run:
name: Combine Coverage
command: |
uv tool run --from 'coverage[toml]==7.10.6' coverage combine realtime_translation_coverage ocr_coverage search_coverage logging_coverage audio_coverage local_testing_part1_coverage local_testing_part2_coverage pass_through_unit_tests_coverage batches_coverage guardrails_coverage google_generate_content_endpoint_coverage litellm_utils_coverage router_unit_tests_coverage auth_ui_unit_tests_coverage
uv tool run --from 'coverage[toml]==7.10.6' coverage combine realtime_translation_coverage ocr_coverage search_coverage logging_coverage audio_coverage local_testing_part1_coverage local_testing_part2_coverage pass_through_unit_tests_coverage batches_coverage google_generate_content_endpoint_coverage litellm_utils_coverage
uv tool run --from 'coverage[toml]==7.10.6' coverage xml
- codecov/upload:
file: ./coverage.xml
@ -3145,10 +2954,10 @@ jobs:
name: Test provider capture and replay harness
command: |
mkdir -p test-results/provider-replay-harness
uv run --no-sync pytest --tb=short -q --noconftest -o addopts= -o pythonpath=tests/e2e -p no:rerunfailures \
uv run --no-sync pytest --tb=short -q --noconftest -o addopts= -o "pythonpath=tests/e2e tests/e2e_harness" -p no:rerunfailures \
--junitxml=test-results/provider-replay-harness/junit.xml \
tests/e2e/test_provider_edge.py tests/e2e/test_fixture_bundle.py \
tests/e2e/test_fixture_canonical.py tests/e2e/test_fixture_mode.py \
tests/e2e_harness/test_provider_edge.py tests/e2e_harness/test_fixture_bundle.py \
tests/e2e_harness/test_fixture_canonical.py tests/e2e_harness/test_fixture_mode.py \
tests/code_coverage_tests/test_provider_replay_harness.py \
tests/code_coverage_tests/test_provider_cache.py
- store_test_results:
@ -3569,15 +3378,38 @@ workflows:
- not:
equal: ["", << pipeline.parameters.routing_parity_base >>]
jobs:
- using_litellm_on_windows
- windows_release_wheel
- provider_replay_harness
- base_sdk_install
- local_testing_part1
- local_testing_part2
- langfuse_logging_unit_tests
- litellm_router_testing
- litellm_router_unit_testing
- using_litellm_on_windows:
filters:
branches:
only: main
- windows_release_wheel:
filters:
branches:
only: main
- provider_replay_harness:
filters:
branches:
only: main
- base_sdk_install:
filters:
branches:
only: main
- local_testing_part1:
filters:
branches:
only: main
- local_testing_part2:
filters:
branches:
only: main
- langfuse_logging_unit_tests:
filters:
branches:
only: main
- litellm_router_testing:
filters:
branches:
only: main
- auth_ui_unit_tests
- build_docker_database_image
- e2e_ui_testing
@ -3591,40 +3423,74 @@ workflows:
- proxy_logging_guardrails_model_info_tests:
requires:
- build_docker_database_image
- proxy_spend_accuracy_tests:
requires:
- build_docker_database_image
- proxy_multi_instance_tests:
requires:
- build_docker_database_image
- proxy_store_model_in_db_tests:
requires:
- build_docker_database_image
- proxy_build_from_pip_tests
- proxy_build_from_pip_tests:
filters:
branches:
only: main
- proxy_pass_through_endpoint_tests:
requires:
- build_docker_database_image
- proxy_e2e_anthropic_messages_tests:
requires:
- build_docker_database_image
- llm_translation_testing
- realtime_translation_testing
- llm_translation_testing:
filters:
branches:
only: main
- realtime_translation_testing:
filters:
branches:
only: main
- guardrails_testing
- google_generate_content_endpoint_testing
- llm_responses_api_testing
- ocr_testing
- search_testing
- batches_testing
- litellm_utils_testing
- pass_through_unit_testing
- image_gen_testing
- logging_testing
- audio_testing
- google_generate_content_endpoint_testing:
filters:
branches:
only: main
- ocr_testing:
filters:
branches:
only: main
- search_testing:
filters:
branches:
only: main
- batches_testing:
filters:
branches:
only: main
- litellm_utils_testing:
filters:
branches:
only: main
- pass_through_unit_testing:
filters:
branches:
only: main
- image_gen_testing:
filters:
branches:
only: main
- logging_testing:
filters:
branches:
only: main
- audio_testing:
filters:
branches:
only: main
- upload-coverage:
filters:
branches:
only: main
requires:
- realtime_translation_testing
- google_generate_content_endpoint_testing
- guardrails_testing
- ocr_testing
- search_testing
- batches_testing
@ -3636,17 +3502,30 @@ workflows:
- langfuse_logging_unit_tests
- local_testing_part1
- local_testing_part2
- litellm_router_unit_testing
- auth_ui_unit_tests
- db_migration_disable_update_check:
requires:
- build_docker_database_image
- installing_litellm_on_python
- installing_litellm_on_python_3_13
- installing_litellm_on_python_v2_migration_resolver
- installing_litellm_on_python:
filters:
branches:
only: main
- installing_litellm_on_python_3_13:
filters:
branches:
only: main
- installing_litellm_on_python_v2_migration_resolver:
filters:
branches:
only: main
- helm_chart_testing:
filters:
branches:
only: main
requires:
- build_docker_database_image
- test_bad_database_url:
filters:
branches:
only: main
requires:
- build_docker_database_image

4
.circleci/scripts/classify_changes.sh
View file

@ -24,8 +24,8 @@ while IFS= read -r file || [ -n "$file" ]; do
has_mcp_dependencies=true ;;
esac
case "$file" in
tests/e2e/*/*.py) : ;;
tests/e2e/*.py | tests/code_coverage_tests/test_provider_cache.py | tests/code_coverage_tests/test_provider_replay_harness.py | tests/unit/test_circleci_path_filter.py | .circleci/* | pyproject.toml | uv.lock)
tests/e2e/*/*.py | tests/e2e_harness/*/*.py) : ;;
tests/e2e/*.py | tests/e2e_harness/*.py | tests/code_coverage_tests/test_provider_cache.py | tests/code_coverage_tests/test_provider_replay_harness.py | tests/unit/test_circleci_path_filter.py | .circleci/* | pyproject.toml | uv.lock)
has_provider_harness=true ;;
esac
case "$file" in

8
.circleci/scripts/run_integration.sh
View file

@ -193,10 +193,10 @@ fi
if [ "$suite" = providers ]; then
INTEGRATION_RUN_ID="$integration_identity" .venv/bin/python -m pytest --tb=short --noconftest -o addopts= \
--strict-markers --strict-config -p no:pytest-retry -p no:rerunfailures --timeout=30 \
tests/e2e/test_provider_edge.py::TestReplayMode::test_content_drift_returns_the_miss_status_naming_both_keys \
tests/e2e/test_provider_edge.py::TestReplayMode::test_exhausted_key_returns_the_miss_status \
tests/e2e/test_provider_edge.py::TestReplayLeftover::test_partially_consumed_recording_names_the_leftover \
tests/e2e/test_provider_edge.py::TestStreamingFidelity::test_replay_of_a_stream_makes_no_provider_connection \
tests/e2e_harness/test_provider_edge.py::TestReplayMode::test_content_drift_returns_the_miss_status_naming_both_keys \
tests/e2e_harness/test_provider_edge.py::TestReplayMode::test_exhausted_key_returns_the_miss_status \
tests/e2e_harness/test_provider_edge.py::TestReplayLeftover::test_partially_consumed_recording_names_the_leftover \
tests/e2e_harness/test_provider_edge.py::TestStreamingFidelity::test_replay_of_a_stream_makes_no_provider_connection \
--junitxml="$results/replay-controls.xml"
fi

4
.github/CODEOWNERS vendored
View file

@ -1,2 +1,2 @@
/model_prices_and_context_window.json @mateo-berri @ryan-crabbe-berri @kerry-berri
/litellm/model_prices_and_context_window_backup.json @mateo-berri @ryan-crabbe-berri @kerry-berri
/model_prices_and_context_window.json @mateo-berri @ryan-crabbe-berri @kerrylu-berri
/litellm/model_prices_and_context_window_backup.json @mateo-berri @ryan-crabbe-berri @kerrylu-berri

10
.github/ci-coverage-allowlist.yml vendored
View file

@ -34,6 +34,16 @@ test_paths:
- tests/local_testing/test_caching.py
- tests/local_testing/test_disk_cache_unit_tests.py
- tests/local_testing/test_gcs_cache_unit_tests.py
- reason: >-
Mixed file that no job has ever run in full. Every CircleCI job that globs
tests/local_testing deselects it by name ("caching") or keeps only another keyword, so only
its router test ran, and that test now lives in tests/unit/router_utils/pre_call_checks.
Five of the seven left need ANTHROPIC_API_KEY or Vertex credentials. The other two,
test_litellm_anthropic_prompt_caching_tools and test_litellm_anthropic_prompt_caching_system,
are offline mocks that already fail on main against a stale expected request body; they need
that fixed before they can move to tests/unit
paths:
- tests/local_testing/test_anthropic_prompt_caching.py
- reason: >-
The end-to-end suite runs against a deployed proxy from its own in-cluster rig rather than
from a pull request; it needs a live gateway and provider credentials no PR job holds

14
.github/codeql/codeql-config.yml vendored
View file

@ -1,8 +1,5 @@
name: "LiteLLM CodeQL config"
queries:
- uses: security-and-quality
# Known OOM queries on large Python codebases:
# CodeQL builds a full data flow graph in memory. These two queries trace
# sensitive data through every log call / regex pattern, causing combinatorial
@ -14,17 +11,6 @@ query-filters:
id: py/clear-text-logging-sensitive-data # CWE-312
- exclude:
id: py/polynomial-redos # CWE-730
# Import resolution confuses stdlib types with management_endpoints/types.py.
# The generic cycle query also reports intentional deferred imports.
- exclude:
id: py/cyclic-import
- exclude:
id: py/unsafe-cyclic-import
# Known false positives on live settings and Protocol placeholders.
- exclude:
id: py/unused-global-variable
- exclude:
id: py/ineffectual-statement
paths-ignore:
- tests

15
.github/merge-smoke-tests.json vendored
View file

@ -1,15 +0,0 @@
{
"cases": {
"CHAT-JSON": "tests/unit/llms/openai/test_openai.py::test_acompletion_returns_json_reply_over_injected_transport",
"CHAT-TEXT-STREAM": "tests/unit/llms/openai/test_openai.py::test_acompletion_streams_text_deltas_over_injected_transport",
"CHAT-TOOL-STREAM": "tests/unit/llms/openai/test_openai.py::test_acompletion_streams_tool_call_arguments_over_injected_transport",
"MODEL-ALLOW": "tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py::test_can_object_call_model_allows_listed_model_for_key",
"MODEL-DENY": "tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py::test_can_object_call_model_denials_return_forbidden[key-key_model_access_denied]",
"COST-EXPLICIT": "tests/unit/test_cost_calculator.py::test_completion_cost_charges_explicit_per_token_rates_over_registered_ones",
"COST-ZERO": "tests/unit/test_cost_calculator.py::test_completion_cost_is_zero_when_explicit_rates_are_zero",
"LOG-CONTENT-ON": "tests/unit/litellm_core_utils/test_litellm_logging.py::test_standard_logging_payload_keeps_message_content_when_message_logging_is_on",
"LOG-CONTENT-OFF": "tests/unit/litellm_core_utils/test_litellm_logging.py::test_standard_logging_payload_redacts_message_content_when_message_logging_is_off",
"CALLBACK-SUCCESS": "tests/unit/litellm_core_utils/test_litellm_logging.py::test_async_success_handler_delivers_standard_logging_payload_to_custom_logger",
"CALLBACK-FAILURE": "tests/unit/litellm_core_utils/test_litellm_logging.py::test_async_failure_handler_delivers_failure_payload_to_custom_logger"
}
}

8
.github/scripts/assert_ci_coverage.py vendored
View file

@ -36,7 +36,15 @@ GLOB_CHARS = frozenset("*?")
# itself decomposed one level deeper and is checked through its own entry.
SHARDED_ROOTS: tuple[str, ...] = (
"tests/test_litellm",
"tests/unit",
"tests/unit/enterprise",
"tests/unit/enterprise/enterprise_callbacks",
"tests/unit/enterprise/proxy",
"tests/unit/llms",
"tests/unit/proxy",
"tests/unit/proxy/_experimental",
"tests/unit/responses",
"tests/unit/skills",
)

140
.github/scripts/run_merge_smoke.py vendored
View file

@ -16,29 +16,11 @@ import socket
import subprocess
import sys
import time
from collections import Counter
from collections.abc import Sequence
from dataclasses import dataclass, field
from dataclasses import dataclass
from pathlib import Path
from types import MappingProxyType
from typing import Final, NoReturn, TextIO, cast
import pytest
EXPECTED_CASES: Final = (
"CHAT-JSON",
"CHAT-TEXT-STREAM",
"CHAT-TOOL-STREAM",
"MODEL-ALLOW",
"MODEL-DENY",
"COST-EXPLICIT",
"COST-ZERO",
"LOG-CONTENT-ON",
"LOG-CONTENT-OFF",
"CALLBACK-SUCCESS",
"CALLBACK-FAILURE",
)
@dataclass(frozen=True, slots=True)
class CheckResult:
@ -57,8 +39,6 @@ class _Args:
ready_deadline: float = 120.0
shutdown_deadline: float = 20.0
poll_interval: float = 0.5
manifest: str = ""
rootdir: str | None = None
def fail(reason: str) -> NoReturn:
@ -345,120 +325,6 @@ def _terminate(proc: subprocess.Popen[bytes], log_file: TextIO) -> None:
log_file.close()
def _load_manifest(path: Path) -> MappingProxyType[str, str]:
def no_duplicates(pairs: list[tuple[object, object]]) -> dict[object, object]:
seen: dict[object, object] = {}
for key, value in pairs:
if key in seen:
raise ValueError(f"duplicate key in manifest: {key}")
seen[key] = value
return seen
raw_value: object = cast(object, json.loads(path.read_text(), object_pairs_hook=no_duplicates))
if not isinstance(raw_value, dict):
raise ValueError("manifest must be an object")
loaded: Final = cast(dict[object, object], raw_value)
cases_value: object = loaded.get("cases")
if not isinstance(cases_value, dict):
raise ValueError("manifest must be an object with a 'cases' object")
cases_any: Final = cast(dict[object, object], cases_value)
cases: Final = {k: v for k, v in cases_any.items() if isinstance(k, str) and isinstance(v, str)}
if len(cases) != len(cases_any):
raise ValueError("manifest 'cases' must map string ids to string node ids")
return MappingProxyType(cases)
@dataclass(slots=True, eq=False)
class _Recorder:
collect_failed: list[str] = field(default_factory=list)
collected: tuple[str, ...] = ()
reports: dict[str, list[tuple[str, str, bool]]] = field(default_factory=dict)
def pytest_collectreport(self, report: pytest.CollectReport) -> None:
if report.failed:
self.collect_failed.append(report.nodeid)
def pytest_collection_finish(self, session: pytest.Session) -> None:
self.collected = tuple(item.nodeid for item in session.items)
def pytest_runtest_logreport(self, report: pytest.TestReport) -> None:
self.reports.setdefault(report.nodeid, []).append((report.when, report.outcome, hasattr(report, "wasxfail")))
def cmd_pytest(args: _Args) -> int:
try:
cases: Final = _load_manifest(Path(args.manifest))
except (OSError, ValueError, json.JSONDecodeError) as exc:
fail(f"manifest invalid: {exc}")
if tuple(cases) != EXPECTED_CASES:
fail(f"manifest case ids must be exactly {list(EXPECTED_CASES)} in order, got {list(cases)}")
node_ids: Final = tuple(cases.values())
if len(set(node_ids)) != len(node_ids):
fail("manifest node ids are not unique")
argv: Final = [
*node_ids,
"-p",
"no:cacheprovider",
"-p",
"no:xdist",
"-p",
"no:rerunfailures",
"-p",
"no:randomly",
"-rA",
"-q",
*(["--rootdir", args.rootdir] if args.rootdir else []),
]
recorder: Final = _Recorder()
code: Final = pytest.main(argv, plugins=[recorder])
name_of: Final = MappingProxyType({node_id: case_id for case_id, node_id in cases.items()})
problems: Final[list[str]] = []
if code != 0:
problems.append(f"pytest exit code {code}")
for failed_id in recorder.collect_failed:
problems.append(f"collection failed: {name_of.get(failed_id, failed_id)}")
expected: Final = Counter(node_ids)
collected: Final = Counter(recorder.collected)
for node_id in expected - collected:
problems.append(f"missing case {name_of[node_id]} ({node_id})")
for node_id in collected - expected:
problems.append(f"unexpected test collected: {node_id}")
for node_id, count in collected.items():
if count > 1:
problems.append(f"duplicated test id: {node_id}")
if len(recorder.collected) != len(EXPECTED_CASES):
problems.append(f"collected {len(recorder.collected)} tests, expected {len(EXPECTED_CASES)}")
rows: Final[list[tuple[str, bool]]] = []
for case_id, node_id in cases.items():
reports = recorder.reports.get(node_id, [])
case_ok = (
bool(reports)
and all(outcome == "passed" and not wasxfail for _, outcome, wasxfail in reports)
and {when for when, _, _ in reports} >= {"setup", "call", "teardown"}
)
rows.append((case_id, case_ok))
if not reports:
problems.append(f"{case_id} ({node_id}) produced no runtest reports")
continue
for when, outcome, wasxfail in reports:
if outcome != "passed":
problems.append(f"{case_id} ({node_id}) {when} outcome={outcome}")
if wasxfail:
problems.append(f"{case_id} ({node_id}) {when} was xfail/xpass")
missing_phases = {"setup", "call", "teardown"} - {when for when, _, _ in reports}
for phase in sorted(missing_phases):
problems.append(f"{case_id} ({node_id}) missing {phase} report")
for case_id, passed in rows:
print(f"{case_id} {'PASS' if passed else 'FAIL'} {cases[case_id]}")
if problems:
for problem in problems:
print(f"merge-smoke: {problem}", file=sys.stderr)
fail("pytest verdict failed")
ok("pytest 11 cases")
return 0
def main() -> int:
parser: Final = argparse.ArgumentParser(description=__doc__)
subs: Final = parser.add_subparsers(dest="command", required=True)
@ -475,16 +341,12 @@ def main() -> int:
p_proxy.add_argument("--ready-deadline", type=float, default=120)
p_proxy.add_argument("--shutdown-deadline", type=float, default=20)
p_proxy.add_argument("--poll-interval", type=float, default=0.5)
p_test: Final = subs.add_parser("pytest")
p_test.add_argument("--manifest", required=True)
p_test.add_argument("--rootdir", default=None)
args: Final = parser.parse_args(namespace=_Args())
handlers: Final = {
"verify-isolation": cmd_verify_isolation,
"interpreter": cmd_interpreter,
"cli": cmd_cli,
"proxy-startup": cmd_proxy_startup,
"pytest": cmd_pytest,
}
return handlers[args.command](args)

283
.github/workflows/_test-unit-base.yml vendored
View file

@ -1,283 +0,0 @@
name: _Unit Test Base (Reusable)
on:
workflow_call:
inputs:
python-version:
description: "Python version used to install dependencies and run tests"
required: false
type: string
default: "3.12"
test-path:
description: >-
Space-separated pytest paths to run. A path that no longer exists is
dropped with a warning instead of being passed to pytest, because one
missing path makes pytest-xdist collect nothing and report exit 5, which
the step treats as a drained shard. Options are passed through as
written, so use the `--flag=value` form: a bare `--ignore path` would
have its path existence-checked like any other token.
required: true
type: string
workers:
description: "Number of pytest-xdist workers"
required: false
type: number
default: 2
reruns:
description: "Number of reruns for flaky tests"
required: false
type: number
default: 2
timeout-minutes:
description: >-
Timeout for the test step alone. Setup (checkout, dependency install,
Prisma client generation) gets its own allowance on top, so a slow
runner or a cold binary download can never cancel passing tests.
required: false
type: number
default: 20
job-timeout-minutes:
description: >-
Backstop for the whole job. Keep it >= `timeout-minutes` plus 40: 35 for
the per-step ceilings on the setup steps below, and 5 for the runner
overhead the job clock charges but no step owns (job init, step
transitions, post-job cleanup). That headroom is what makes the test
budget a floor rather than a hope, since setup cannot overrun into it
without failing its own step first. GitHub expressions have no
arithmetic, so the sum is passed in rather than computed.
required: false
type: number
default: 60
test-timeout-seconds:
description: >-
Per-test ceiling enforced by pytest-timeout, covering fixture setup and
teardown as well as the test body. A test that hangs fails with a
traceback of where it was stuck instead of idling the shard until
`timeout-minutes` cancels it. Timed-out tests are excluded from reruns
because pytest-timeout arms its timer once per test and
pytest-rerunfailures reruns inside that same window, so a rerun of a
timed-out test would run with no timer at all.
required: false
type: number
default: 120
max-failures:
description: "Stop after this many failures"
required: false
type: number
default: 10
dist:
description: "pytest-xdist distribution mode (loadscope|load|worksteal|loadfile|no)"
required: false
type: string
default: "loadscope"
artifact-name:
description: "Unique name for the coverage artifact (must be unique per run)"
required: true
type: string
rust-bridge-artifact:
description: "Prebuilt editable Rust bridge artifact"
required: false
type: string
default: ""
permissions:
contents: read
env:
UV_PYTHON: ${{ inputs.python-version }}
LITELLM_LOCAL_MODEL_COST_MAP: "True"
jobs:
run:
name: Run tests
runs-on: ubuntu-latest
timeout-minutes: ${{ inputs.job-timeout-minutes }}
permissions:
contents: read
pull-requests: read
outputs:
decision: ${{ steps.changes.outputs.decision }}
has-coverage: ${{ steps.tests.outputs.has-coverage }}
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
timeout-minutes: 3
with:
persist-credentials: false
- name: Detect relevant changes
id: changes
timeout-minutes: 2
uses: ./.github/actions/detect-changes
- name: Set up Python
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 3
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: ${{ env.UV_PYTHON }}
- name: Set up uv
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 3
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Cache uv dependencies
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 5
uses: ./.github/actions/cache-uv-downloads
- name: Set up the Rust build
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 5
uses: ./.github/actions/rust-bridge
with:
artifact: ${{ inputs.rust-bridge-artifact }}
- name: Install dependencies
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 8
env:
RUST_BRIDGE_ARTIFACT: ${{ inputs.rust-bridge-artifact }}
run: |
diff -u model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json
if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime
else
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --no-install-project
uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl
cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so
uv run --no-sync python -c "import importlib.metadata; import litellm.rust_bridge._native; print(importlib.metadata.version('litellm'))"
fi
uv run --no-sync python -c 'import os, sys; print(sys.version); assert f"{sys.version_info.major}.{sys.version_info.minor}" == os.environ["UV_PYTHON"]'
- name: Cache Prisma binaries
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 3
uses: ./.github/actions/cache-prisma-binaries
- name: Generate Prisma client
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 3
run: |
uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
- name: Run tests
id: tests
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: ${{ inputs.timeout-minutes }}
env:
TEST_PATH: ${{ inputs.test-path }}
MAX_FAILURES: ${{ inputs.max-failures }}
WORKERS: ${{ inputs.workers }}
RERUNS: ${{ inputs.reruns }}
TEST_TIMEOUT_SECONDS: ${{ inputs.test-timeout-seconds }}
DIST: ${{ inputs.dist }}
COVERAGE_CORE: sysmon
run: |
echo "has-coverage=false" >> "$GITHUB_OUTPUT"
selection="${TEST_PATH}"
if [ -z "${selection// /}" ]; then
echo "shard selection is empty; nothing to run"
exit 0
fi
pytest_args=()
existing_paths=0
for token in ${selection}; do
case "${token}" in
-*) pytest_args+=("${token}") ;;
*)
if [ -e "${token%%::*}" ]; then
pytest_args+=("${token}")
existing_paths=$((existing_paths + 1))
else
echo "::warning::${token} does not exist; drop it from this shard's test-path"
fi
;;
esac
done
if [ "${existing_paths}" -eq 0 ]; then
echo "No path in the selection exists (${selection}); nothing to run"
exit 0
fi
xdist_args=()
if [ "${WORKERS}" != "0" ]; then
xdist_args=(-n "${WORKERS}" --dist="${DIST}")
fi
set +e
uv run --no-sync pytest "${pytest_args[@]}" \
--tb=short -vv \
--maxfail="${MAX_FAILURES}" \
"${xdist_args[@]}" \
--reruns "${RERUNS}" \
--reruns-delay 1 \
--timeout="${TEST_TIMEOUT_SECONDS}" \
--rerun-except "from pytest-timeout" \
--durations=20 \
--cov=./litellm --cov=./enterprise/litellm_enterprise \
--cov-report=xml:coverage.xml \
--cov-config=pyproject.toml
status=$?
set -e
if [ -f coverage.xml ]; then
echo "has-coverage=true" >> "$GITHUB_OUTPUT"
fi
if [ "$status" -eq 5 ]; then
echo "pytest collected no tests from ${selection}; passing"
exit 0
fi
exit "$status"
- name: Save coverage report
if: always() && steps.changes.outputs.decision != 'skip'
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
with:
name: coverage-${{ inputs.artifact-name }}-${{ github.run_id }}-${{ github.run_attempt }}
path: coverage.xml
retention-days: 1
upload-coverage:
name: Upload coverage to Codecov
needs: run
if: always() && needs.run.outputs.decision != 'skip' && needs.run.outputs.has-coverage == 'true'
runs-on: ubuntu-latest
permissions:
contents: read
id-token: write
pull-requests: write
steps:
- name: Checkout code
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Download coverage report
uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1
with:
pattern: coverage-${{ inputs.artifact-name }}-${{ github.run_id }}-${{ github.run_attempt }}
path: coverage-reports
merge-multiple: true
- name: Upload to Codecov
id: codecov-upload
continue-on-error: true
uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5.5.5
with:
use_oidc: true
directory: coverage-reports
root_dir: ${{ github.workspace }}
flags: ${{ inputs.artifact-name }}
fail_ci_if_error: false
- name: Upload to Codecov (retry)
if: steps.codecov-upload.outcome == 'failure'
continue-on-error: true
uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5.5.5
with:
use_oidc: true
directory: coverage-reports
root_dir: ${{ github.workspace }}
flags: ${{ inputs.artifact-name }}
fail_ci_if_error: false

134
.github/workflows/check-ui-api-types.yml vendored
View file

@ -1,134 +0,0 @@
name: Check UI API Types Sync
on:
pull_request:
branches:
- main
- "litellm_**"
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
check-sync:
name: Verify schema.d.ts matches the proxy OpenAPI spec
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- name: Checkout repository
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
fetch-depth: 2
- name: Detect changes that can affect the generated types
id: changes
run: |
set -euo pipefail
if ! base="$(git rev-parse --verify --quiet HEAD^2 >/dev/null && git rev-parse HEAD^1)"; then
echo "Not a pull request merge commit, running the full check."
echo "relevant=true" >> "$GITHUB_OUTPUT"
exit 0
fi
files="$(git diff --name-only "$base" HEAD)"
if grep -Eq '^(litellm/(proxy|types)/|ui/litellm-dashboard/(src/lib/http/schema\.d\.ts|scripts/gen-api-types\.mjs|package(-lock)?\.json)$|\.github/workflows/check-ui-api-types\.yml$)' <<< "$files"; then
echo "relevant=true" >> "$GITHUB_OUTPUT"
else
echo "No proxy, types or generator changes in this pull request, nothing to verify."
echo "relevant=false" >> "$GITHUB_OUTPUT"
fi
- name: Set up Python
if: steps.changes.outputs.relevant == 'true'
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
if: steps.changes.outputs.relevant == 'true'
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Cache uv dependencies
if: steps.changes.outputs.relevant == 'true'
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: |
~/.cache/uv
.venv
key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
restore-keys: |
${{ runner.os }}-uv-
- name: Cache the Rust build
if: steps.changes.outputs.relevant == 'true'
uses: ./.github/actions/cache-cargo-build
- name: Install backend dependencies
if: steps.changes.outputs.relevant == 'true'
run: .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
- name: Cache Prisma binaries
if: steps.changes.outputs.relevant == 'true'
uses: ./.github/actions/cache-prisma-binaries
- name: Generate Prisma client
if: steps.changes.outputs.relevant == 'true'
run: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
- name: Regenerate the lazy OpenAPI snapshot
if: steps.changes.outputs.relevant == 'true'
run: uv run --no-sync python -m litellm.proxy._lazy_openapi_snapshot
- name: Fail if the lazy OpenAPI snapshot is stale
if: steps.changes.outputs.relevant == 'true'
run: |
if ! git diff --exit-code -- litellm/proxy/_lazy_openapi_snapshot.json; then
echo "::error file=litellm/proxy/_lazy_openapi_snapshot.json::The lazy OpenAPI snapshot is out of sync with the lazily loaded routes."
echo ""
echo "A lazily loaded route or model changed without regenerating the snapshot that /openapi.json serves for unloaded features."
echo "To fix, run from the repo root:"
echo " uv run python -m litellm.proxy._lazy_openapi_snapshot"
echo "then run npm run gen:api from ui/litellm-dashboard and commit both files."
exit 1
fi
echo "_lazy_openapi_snapshot.json is in sync with the lazily loaded routes."
- name: Set up Node.js
if: steps.changes.outputs.relevant == 'true'
uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
with:
node-version-file: ui/litellm-dashboard/.nvmrc
cache: "npm"
cache-dependency-path: ui/litellm-dashboard/package-lock.json
- name: Install dashboard dependencies
if: steps.changes.outputs.relevant == 'true'
working-directory: ui/litellm-dashboard
run: npm ci
- name: Regenerate types from the live spec
if: steps.changes.outputs.relevant == 'true'
working-directory: ui/litellm-dashboard
env:
LITELLM_PYTHON: "uv run --no-sync python"
run: npm run gen:api
- name: Fail if types are stale
if: steps.changes.outputs.relevant == 'true'
run: |
if ! git diff --exit-code -- ui/litellm-dashboard/src/lib/http/schema.d.ts; then
echo "::error file=ui/litellm-dashboard/src/lib/http/schema.d.ts::Generated API types are out of sync with the proxy OpenAPI spec."
echo ""
echo "A backend route or model changed without regenerating the dashboard types."
echo "To fix, run from ui/litellm-dashboard:"
echo " npm run gen:api"
echo "then commit the updated src/lib/http/schema.d.ts."
exit 1
fi
echo "schema.d.ts is in sync with the proxy OpenAPI spec."

48
.github/workflows/ci-coverage.yml vendored
View file

@ -1,48 +0,0 @@
name: "CI Coverage"
on:
pull_request:
branches:
- main
- "litellm_**"
push:
branches:
- main
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
assert-ci-coverage:
name: assert-ci-coverage
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
contents: read
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Assert every test file and Dockerfile is invoked by a job
run: |
python -m pip install "pyyaml==6.0.3"
python .github/scripts/assert_ci_coverage.py
# The census asks whether a job names a file; this asks whether that job's -k
# then throws it back out. A file both globbed and deselected everywhere runs
# nowhere while counting as covered, which is how the caching suite went unrun.
- name: Assert no -k expression deselects a file from every job that globs it
run: python .github/scripts/assert_ci_coverage.py --slices
- name: Assert .github/workflows/ holds only workflows, correctly named
run: python .github/scripts/assert_workflow_dir_hygiene.py

9
.github/workflows/codeql.yml vendored
View file

@ -1,8 +1,6 @@
name: "CodeQL"
on:
push:
branches: [main]
pull_request:
branches: [main]
schedule:
@ -43,14 +41,15 @@ jobs:
persist-credentials: false
- name: Initialize CodeQL
uses: github/codeql-action/init@ebcb5b36ded6beda4ceefea6a8bc4cc885255bb3 # v3.34.1
uses: github/codeql-action/init@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2
with:
languages: ${{ matrix.language }}
build-mode: ${{ matrix.build-mode }}
config-file: ./.github/codeql/codeql-config.yml
queries: ${{ github.event_name == 'pull_request' && '+security-extended' || '' }}
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@ebcb5b36ded6beda4ceefea6a8bc4cc885255bb3 # v3.34.1
uses: github/codeql-action/analyze@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2
with:
category: "/language:${{ matrix.language }}"
output: sarif-results
@ -83,7 +82,7 @@ jobs:
output: sarif-results/python.sarif
- name: Upload SARIF
uses: github/codeql-action/upload-sarif@ebcb5b36ded6beda4ceefea6a8bc4cc885255bb3 # v3.34.1
uses: github/codeql-action/upload-sarif@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2
with:
sarif_file: sarif-results
category: "/language:${{ matrix.language }}"

6
.github/workflows/helm_unit_test.yml vendored
View file

@ -22,6 +22,12 @@ jobs:
with:
persist-credentials: false
- name: Check Lens Compose configuration
run: |
python3 -m unittest discover -s deploy/lens -p 'test_*.py'
python3 deploy/lens/configure.py --version 1.2.3 --env-file "$RUNNER_TEMP/lens.env"
docker compose --env-file "$RUNNER_TEMP/lens.env" -f deploy/lens/stack.yaml config --quiet
- name: Set up Helm 3.11.1
uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4.3.1
with:

28
.github/workflows/image-scan.yml vendored
View file

@ -18,6 +18,7 @@ on:
- backend/Dockerfile
- backend/main.py
- deploy/lens/**
- litellm-rust/**
- litellm/proxy/lens/**
- tests/e2e/migrations/lens_compose_smoke.sh
- docker/component_entrypoint.sh
@ -52,7 +53,7 @@ jobs:
if: >-
github.event_name != 'pull_request' ||
github.event.pull_request.head.repo.full_name == github.repository
timeout-minutes: 15
timeout-minutes: 45
permissions:
contents: read
strategy:
@ -79,30 +80,7 @@ jobs:
run: |
docker run --rm --network none --read-only --cap-drop ALL \
--tmpfs /tmp:rw,noexec,nosuid,size=1g --security-opt no-new-privileges \
-e EXPECTED_RELEASE_TAG="${RELEASE_TAG}" --entrypoint python lens-worker-scan -c '
import os
import lens.worker
from lens.release import release_tag
from lens.trace_store import trace_store
assert os.getuid() == 65532
assert release_tag() == os.environ["EXPECTED_RELEASE_TAG"]
with trace_store() as store:
assert store.count() == 0
'
- name: Reject a dependency whose hash has changed
run: |
docker build --target builder -f deploy/lens/Dockerfile -t lens-worker-deps .
sed -E 's/sha256:[0-9a-f]{64}/sha256:0000000000000000000000000000000000000000000000000000000000000000/g' \
deploy/lens/requirements.lock > "$RUNNER_TEMP/tampered.lock"
if docker run --rm -v "$RUNNER_TEMP/tampered.lock:/tmp/tampered.lock:ro" \
--entrypoint uv lens-worker-deps pip sync --python /app/.venv/bin/python \
--require-hashes --only-binary :all: --reinstall --no-cache /tmp/tampered.lock \
> "$RUNNER_TEMP/hash-check.log" 2>&1; then
echo "::error::Dependency hash mismatch was accepted"
exit 1
fi
cat "$RUNNER_TEMP/hash-check.log"
grep -qi 'hash mismatch' "$RUNNER_TEMP/hash-check.log"
lens-worker-scan --version | grep -F "litellm-lens $RELEASE_TAG protocol="
- name: Download Grype v0.114.0
env:
ARCH: ${{ matrix.arch }}

94
.github/workflows/lens-install-smoke.yml vendored Normal file
View file

@ -0,0 +1,94 @@
name: Lens installation smoke
on:
workflow_dispatch:
permissions:
contents: read
concurrency:
group: lens-install-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
build-images:
runs-on: ubuntu-latest-16-cores
timeout-minutes: 45
strategy:
fail-fast: false
matrix:
include:
- component: gateway
dockerfile: gateway/Dockerfile
- component: backend
dockerfile: backend/Dockerfile
- component: ui
dockerfile: ui/Dockerfile
- component: migrations
dockerfile: migrations/Dockerfile
- component: monolith
dockerfile: Dockerfile
- component: worker
dockerfile: deploy/lens/Dockerfile
env:
COMPONENT: ${{ matrix.component }}
DOCKERFILE: ${{ matrix.dockerfile }}
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Build the matching release image
run: |
docker build --build-arg LITELLM_RELEASE_TAG=v0.0.0-lens-ci \
-f "$DOCKERFILE" -t "lens-ci-$COMPONENT:v0.0.0-lens-ci" .
- name: Save the matching release image
run: |
docker save "lens-ci-$COMPONENT:v0.0.0-lens-ci" \
| gzip -1 > "$RUNNER_TEMP/lens-install-$COMPONENT.tar.gz"
- uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
with:
name: lens-install-${{ matrix.component }}-${{ github.sha }}
path: ${{ runner.temp }}/lens-install-${{ matrix.component }}.tar.gz
compression-level: 0
retention-days: 3
if-no-files-found: error
overwrite: true
helm-install:
needs: build-images
runs-on: ubuntu-latest-16-cores
timeout-minutes: 25
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1
with:
pattern: lens-install-*-${{ github.sha }}
merge-multiple: true
path: ${{ runner.temp }}/lens-install-images
- name: Load the matching release images
run: |
for component in gateway backend ui migrations monolith worker; do
archive="$RUNNER_TEMP/lens-install-images/lens-install-$component.tar.gz"
gzip -dc "$archive" | docker load
rm "$archive"
done
- name: Install pinned Kubernetes test tools
run: |
curl --fail --location --output "$RUNNER_TEMP/kind" \
https://kind.sigs.k8s.io/dl/v0.27.0/kind-linux-amd64
echo "a6875aaea358acf0ac07786b1a6755d08fd640f4c79b7a2e46681cc13f49a04b $RUNNER_TEMP/kind" | sha256sum --check
chmod +x "$RUNNER_TEMP/kind"
curl --fail --location --output "$RUNNER_TEMP/kubectl" \
https://dl.k8s.io/release/v1.32.2/bin/linux/amd64/kubectl
echo "4f6a959dcc5b702135f8354cc7109b542a2933c46b808b248a214c1f69f817ea $RUNNER_TEMP/kubectl" | sha256sum --check
chmod +x "$RUNNER_TEMP/kubectl"
curl --fail --location --output "$RUNNER_TEMP/helm.tar.gz" \
https://get.helm.sh/helm-v3.19.0-linux-amd64.tar.gz
echo "a7f81ce08007091b86d8bd696eb4d86b8d0f2e1b9f6c714be62f82f96a594496 $RUNNER_TEMP/helm.tar.gz" | sha256sum --check
tar -xzf "$RUNNER_TEMP/helm.tar.gz" -C "$RUNNER_TEMP"
echo "$RUNNER_TEMP" >> "$GITHUB_PATH"
echo "$RUNNER_TEMP/linux-amd64" >> "$GITHUB_PATH"
- name: Install, ingest, upgrade, and restart both charts
run: bash tests/e2e/migrations/lens_helm_smoke.sh

141
.github/workflows/lens-worker.yml vendored
View file

@ -5,6 +5,7 @@ on:
branches: [main, litellm_oss_branch, "litellm_**"]
paths:
- deploy/lens/**
- litellm-rust/**
- litellm/proxy/lens/**
- tests/proxy_behavior/lens/**
- .github/workflows/lens-worker.yml
@ -12,6 +13,7 @@ on:
branches: [main]
paths:
- deploy/lens/**
- litellm-rust/**
- litellm/proxy/lens/**
- tests/proxy_behavior/lens/**
- .github/workflows/lens-worker.yml
@ -29,15 +31,35 @@ jobs:
permissions:
contents: read
packages: write
id-token: write
runs-on: ubuntu-latest
timeout-minutes: 10
runs-on: ${{ matrix.runner }}
timeout-minutes: 45
strategy:
fail-fast: false
matrix:
include:
- arch: amd64
runner: ubuntu-latest
- arch: arm64
runner: ubuntu-24.04-arm
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Build Lens worker
run: docker build --build-arg LITELLM_RELEASE_TAG=sha-${{ github.sha }} -f deploy/lens/Dockerfile -t lens-worker:${{ github.sha }} .
- name: Build native Lens service
env:
RELEASE_TAG: sha-${{ github.sha }}
run: docker build --build-arg LITELLM_RELEASE_TAG="$RELEASE_TAG" -f deploy/lens/Dockerfile -t lens-worker .
- name: Verify version and unprivileged runtime
env:
RELEASE_TAG: sha-${{ github.sha }}
run: bash deploy/lens/smoke.sh lens-worker "$RELEASE_TAG"
- name: Verify confined Python on the native architecture
env:
RELEASE_TAG: sha-${{ github.sha }}
run: |
docker build --target smoke --build-arg LITELLM_RELEASE_TAG="$RELEASE_TAG" -f deploy/lens/Dockerfile -t lens-smoke .
docker run --rm --network none --read-only --cap-drop ALL \
--tmpfs /tmp:rw,noexec,nosuid,size=1g --security-opt no-new-privileges lens-smoke
- name: Reject custom builds without a matching release tag
run: |
if docker build --progress plain -f deploy/lens/Dockerfile -t lens-worker:unversioned . > missing-tag.log 2>&1; then
@ -45,86 +67,45 @@ jobs:
exit 1
fi
grep -F 'LITELLM_RELEASE_TAG: Pass --build-arg LITELLM_RELEASE_TAG matching the gateway' missing-tag.log
- name: Verify standalone imports with a read-only filesystem
run: |
docker run --rm --network none --read-only --cap-drop ALL --tmpfs /tmp:rw,noexec,nosuid,size=1g \
--security-opt no-new-privileges --entrypoint python \
lens-worker:${{ github.sha }} -c '
import os
import lens.worker
from lens.trace_store import trace_store
assert os.getuid() == 65532
with trace_store() as store:
assert store.count() == 0
'
- name: Prepare test-only coverage tool
run: |
coverage_directory=$(mktemp -d "$RUNNER_TEMP/lens-coverage.XXXXXX")
curl --fail --silent --show-error --location \
https://files.pythonhosted.org/packages/61/e8/cb8e80d6f9f55b99588625062822bf946cf03ed06315df4bd8397f5632a1/coverage-7.14.0-py3-none-any.whl \
--output "$coverage_directory/coverage.whl"
printf '%s %s\n' 8de5b61163aee3d05c8a2beab6f47913df7981dad1baf82c414d99158c286ab1 \
"$coverage_directory/coverage.whl" | sha256sum --check
chmod 777 "$coverage_directory"
echo "LENS_COVERAGE_DIRECTORY=$coverage_directory" >> "$GITHUB_ENV"
- name: Verify confined Python execution
run: |
docker run --rm --network none --read-only --cap-drop ALL \
--tmpfs /tmp:rw,noexec,nosuid,size=1g --security-opt no-new-privileges \
-v "$PWD/tests/proxy_behavior/lens/worker_python_smoke.py:/app/python_smoke.py:ro" \
-v "$PWD/tests/proxy_behavior/lens/coverage.ini:/coverage.ini:ro" \
-v "$LENS_COVERAGE_DIRECTORY:/coverage" \
-e PYTHONPATH=/coverage/coverage.whl -e COVERAGE_RCFILE=/coverage.ini \
--entrypoint python lens-worker:${{ github.sha }} \
-m coverage run --data-file=/coverage/.coverage.python /app/python_smoke.py
- name: Verify workspace investigation and live review output
run: |
docker run --rm --network none --read-only --cap-drop ALL \
--tmpfs /tmp:rw,noexec,nosuid,size=1g --security-opt no-new-privileges \
-v "$PWD/tests/proxy_behavior/lens/worker_context_smoke.py:/app/context_smoke.py:ro" \
-v "$PWD/tests/proxy_behavior/lens/coverage.ini:/coverage.ini:ro" \
-v "$LENS_COVERAGE_DIRECTORY:/coverage" \
-e PYTHONPATH=/coverage/coverage.whl -e COVERAGE_RCFILE=/coverage.ini \
--entrypoint python lens-worker:${{ github.sha }} \
-m coverage run --data-file=/coverage/.coverage.context /app/context_smoke.py
- name: Verify default workspace recovery after Python scratch storage fills
run: |
docker run --rm --network none --read-only --cap-drop ALL \
--tmpfs /tmp:rw,noexec,nosuid,size=64k --security-opt no-new-privileges \
-v "$PWD/tests/proxy_behavior/lens/worker_storage_smoke.py:/app/storage_smoke.py:ro" \
-v "$PWD/tests/proxy_behavior/lens/coverage.ini:/coverage.ini:ro" \
-v "$LENS_COVERAGE_DIRECTORY:/coverage" \
-e PYTHONPATH=/coverage/coverage.whl -e COVERAGE_RCFILE=/coverage.ini \
--entrypoint python lens-worker:${{ github.sha }} \
-m coverage run --data-file=/coverage/.coverage.storage /app/storage_smoke.py
- name: Map native worker coverage to repository sources
if: always() && env.LENS_COVERAGE_DIRECTORY != ''
run: |
docker run --rm --network none --read-only --cap-drop ALL \
--security-opt no-new-privileges -w /workspace \
-v "$PWD/litellm/proxy/lens:/workspace/litellm/proxy/lens:ro" \
-v "$PWD/tests/proxy_behavior/lens/coverage.ini:/coverage.ini:ro" \
-v "$LENS_COVERAGE_DIRECTORY:/coverage" \
-e PYTHONPATH=/coverage/coverage.whl -e COVERAGE_RCFILE=/coverage.ini \
--entrypoint /bin/sh lens-worker:${{ github.sha }} \
-c 'python -m coverage combine && python -m coverage xml'
- name: Upload native worker coverage
if: always() && env.LENS_COVERAGE_DIRECTORY != ''
uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5.5.5
with:
use_oidc: true
files: ${{ env.LENS_COVERAGE_DIRECTORY }}/lens-worker.xml
root_dir: ${{ github.workspace }}
flags: lens-worker
fail_ci_if_error: false
- name: Publish versioned Lens worker
- name: Publish development architecture
if: github.event_name != 'pull_request' && github.repository == 'BerriAI/litellm' && github.ref == 'refs/heads/main'
env:
REGISTRY_TOKEN: ${{ secrets.GITHUB_TOKEN }}
REGISTRY_USER: ${{ github.actor }}
IMAGE: ghcr.io/berriai/litellm-lens-worker-dev:sha-${{ github.sha }}-${{ matrix.arch }}
ARCH: ${{ matrix.arch }}
run: |
printf '%s' "$REGISTRY_TOKEN" | docker login ghcr.io -u "$REGISTRY_USER" --password-stdin
docker tag lens-worker "$IMAGE"
docker push "$IMAGE"
mkdir -p digests
docker inspect --format='{{index .RepoDigests 0}}' "$IMAGE" > "digests/$ARCH"
- uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
if: github.event_name != 'pull_request' && github.repository == 'BerriAI/litellm' && github.ref == 'refs/heads/main'
with:
name: lens-digest-${{ matrix.arch }}
path: digests/
retention-days: 1
publish:
name: Publish Lens development index
needs: lens-worker-image
if: github.event_name != 'pull_request' && github.repository == 'BerriAI/litellm' && github.ref == 'refs/heads/main'
runs-on: ubuntu-latest
permissions:
packages: write
steps:
- uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1
with:
pattern: lens-digest-*
merge-multiple: true
path: digests
- name: Publish both tested architectures
env:
REGISTRY_TOKEN: ${{ secrets.GITHUB_TOKEN }}
REGISTRY_USER: ${{ github.actor }}
IMAGE: ghcr.io/berriai/litellm-lens-worker-dev:sha-${{ github.sha }}
run: |
printf '%s' "$REGISTRY_TOKEN" | docker login ghcr.io -u "$REGISTRY_USER" --password-stdin
docker tag lens-worker:${{ github.sha }} "$IMAGE"
docker push "$IMAGE"
docker buildx imagetools create --tag "$IMAGE" "$(cat digests/amd64)" "$(cat digests/arm64)"
printf 'Lens worker image: `%s`\n' "$IMAGE" >> "$GITHUB_STEP_SUMMARY"

65
.github/workflows/publish-basedpyright-base-counts.yml vendored
View file

@ -1,65 +0,0 @@
name: Publish basedpyright base counts
# Every commit on main can become a future merge-base.
# Publishing its per-rule basedpyright counts as an artifact lets
# scripts/type_check_gate.py download them in seconds instead of paying a
# 60-110s second basedpyright pass on every fresh worktree or moved merge-base.
# No concurrency group on purpose: runs must never cancel each other, because
# every sha's artifact matters (any of them can become a merge-base).
on:
push:
branches:
- main
workflow_dispatch:
inputs:
ref:
description: "Ref to compute and publish base counts for (defaults to the workflow run's commit)"
required: false
permissions:
contents: read
jobs:
publish:
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
ref: ${{ inputs.ref || github.sha }}
clean: true
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Cache the Rust build
uses: ./.github/actions/cache-cargo-build
- name: Cache Prisma binaries
uses: ./.github/actions/cache-prisma-binaries
# The gate provisions its own measurement env (.venv-typecheck: a frozen
# uv sync of its canonical dependency groups plus a generated Prisma
# client), so no install step here can drift from what local runs measure.
- name: Emit basedpyright counts for HEAD
run: |
python scripts/type_check_gate.py --emit-counts-dir "$RUNNER_TEMP/basedpyright-counts"
counts_file=$(ls "$RUNNER_TEMP"/basedpyright-counts/basedpyright-counts-*.json)
echo "COUNTS_ARTIFACT_NAME=$(basename "$counts_file" .json)" >> "$GITHUB_ENV"
- name: Upload counts artifact
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
with:
name: ${{ env.COUNTS_ARTIFACT_NAME }}
path: ${{ runner.temp }}/basedpyright-counts/
if-no-files-found: error

96
.github/workflows/publish-lint-base-counts.yml vendored Normal file
View file

@ -0,0 +1,96 @@
name: Publish lint base counts
# Every commit on main can become a future merge-base.
# Publishing its per-rule counts for each lint gate (strict ruff, type discipline,
# test quality, basedpyright) as an artifact lets the gates download them through
# scripts/lint_base_counts.py in seconds instead of scanning the merge-base tree in
# a throwaway worktree on every fresh checkout or moved merge-base.
# No concurrency group on purpose: runs must never cancel each other, because
# every sha's artifact matters (any of them can become a merge-base).
on:
push:
branches:
- main
workflow_dispatch:
inputs:
ref:
description: "Ref to compute and publish base counts for (defaults to the workflow run's commit)"
required: false
permissions:
contents: read
jobs:
publish:
runs-on: ubuntu-latest
timeout-minutes: 20
strategy:
fail-fast: false
matrix:
include:
- checker: ruff-strict
script: scripts/ruff_strict_gate.py
- checker: type-discipline
script: scripts/type_discipline_gate.py
- checker: test-quality
script: scripts/test_quality_gate.py
- checker: basedpyright
script: scripts/type_check_gate.py
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
ref: ${{ inputs.ref || github.sha }}
clean: true
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Cache the Rust build
if: matrix.checker == 'basedpyright'
uses: ./.github/actions/cache-cargo-build
- name: Cache Prisma binaries
if: matrix.checker == 'basedpyright'
uses: ./.github/actions/cache-prisma-binaries
# The three source scanners only need the pinned dev tools (ruff and the
# stdlib checkers), the same versions test-linting.yml's python job runs.
- name: Install the dev tools
if: matrix.checker != 'basedpyright'
run: |
uv sync --frozen --only-group dev --no-install-project
- name: Emit ${{ matrix.checker }} counts for HEAD
if: matrix.checker != 'basedpyright'
run: |
uv run --no-sync python ${{ matrix.script }} --emit-counts-dir "$RUNNER_TEMP/lint-counts"
# The basedpyright gate provisions its own measurement env (.venv-typecheck:
# a frozen uv sync of its canonical dependency groups plus a generated Prisma
# client), so no install step here can drift from what local runs measure.
- name: Emit basedpyright counts for HEAD
if: matrix.checker == 'basedpyright'
run: |
python scripts/type_check_gate.py --emit-counts-dir "$RUNNER_TEMP/lint-counts"
- name: Name the counts artifact
run: |
counts_file=$(ls "$RUNNER_TEMP"/lint-counts/${{ matrix.checker }}-counts-*.json)
echo "COUNTS_ARTIFACT_NAME=$(basename "$counts_file" .json)" >> "$GITHUB_ENV"
- name: Upload counts artifact
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
with:
name: ${{ env.COUNTS_ARTIFACT_NAME }}
path: ${{ runner.temp }}/lint-counts/
if-no-files-found: error

397
.github/workflows/required-checks-legacy.yml vendored Normal file
View file

@ -0,0 +1,397 @@
name: Required checks (legacy)
on:
pull_request:
branches:
- main
- "litellm_**"
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: true
jobs:
lint:
name: lint
permissions:
contents: read
pull-requests: read
actions: read
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
ref: ${{ github.event.pull_request.head.sha }}
fetch-depth: 1
clean: true
persist-credentials: false
- name: Detect relevant changes
id: changes
uses: ./.github/actions/detect-changes
- name: Fetch gate base (merge-base with target branch)
if: steps.changes.outputs.decision != 'skip'
env:
GH_TOKEN: ${{ github.token }}
BASE_SHA: ${{ github.event.pull_request.base.sha }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
run: |
retry() { "$@" || { sleep 15; "$@"; } || { sleep 30; "$@"; }; }
MERGE_BASE=$(retry gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
test -n "$MERGE_BASE"
retry git fetch --no-tags --depth=1 origin "$MERGE_BASE"
echo "GATE_BASE_SHA=$MERGE_BASE" >> "$GITHUB_ENV"
- name: Set up Python
if: steps.changes.outputs.decision != 'skip'
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
if: steps.changes.outputs.decision != 'skip'
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Cache uv dependencies
if: steps.changes.outputs.decision != 'skip'
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: |
~/.cache/uv
.venv
key: ${{ runner.os }}-uv-lint-${{ hashFiles('uv.lock') }}
restore-keys: |
${{ runner.os }}-uv-lint-
- name: Clean Python cache
if: steps.changes.outputs.decision != 'skip'
run: |
find . -type d -name "__pycache__" -exec rm -rf {} + || true
find . -name "*.pyc" -delete || true
- name: Check uv.lock is up to date
if: steps.changes.outputs.decision != 'skip'
run: |
uv lock --check || (echo "❌ uv.lock is out of sync with pyproject.toml. Run 'uv lock' locally and commit the result." && exit 1)
- name: Cache the Rust build
if: steps.changes.outputs.decision != 'skip'
uses: ./.github/actions/cache-cargo-build
- name: Install dependencies
if: steps.changes.outputs.decision != 'skip'
run: |
uv sync --frozen --group proxy-dev --group e2e-dev
- name: Cache Prisma binaries
if: steps.changes.outputs.decision != 'skip'
uses: ./.github/actions/cache-prisma-binaries
- name: Generate Prisma client
if: steps.changes.outputs.decision != 'skip'
run: |
uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
- name: Check ruff format
if: steps.changes.outputs.decision != 'skip'
run: |
git diff --name-only --diff-filter=ACMR "$GATE_BASE_SHA" HEAD -- ':(glob)litellm/**/*.py' | grep -v '^litellm/enterprise/' > "$RUNNER_TEMP/ruff_format_files.txt" || true
if [ ! -s "$RUNNER_TEMP/ruff_format_files.txt" ]; then
echo "No changed litellm Python files to check with ruff format."
exit 0
fi
xargs uv run --no-sync ruff format --check --exclude '/enterprise/' < "$RUNNER_TEMP/ruff_format_files.txt"
- name: Debug - Check file state
if: steps.changes.outputs.decision != 'skip'
run: |
echo "Current branch:"
git branch --show-current
echo "Last 3 commits:"
git log --oneline -3
echo "File content around line 43:"
head -50 litellm/litellm_core_utils/custom_logger_registry.py | tail -10
- name: Check MCP operation boundary
if: steps.changes.outputs.decision != 'skip'
run: uv run --no-sync python scripts/check_mcp_operation_boundary.py
- name: Run Ruff linting
if: steps.changes.outputs.decision != 'skip'
run: |
cd litellm
uv run --no-sync ruff check .
cd ..
- name: Run Ruff linting (test tree)
if: steps.changes.outputs.decision != 'skip'
run: |
uv run --no-sync ruff check --config ruff-tests.toml tests
- name: Check strict ruff rules (delta vs merge-base counts)
if: steps.changes.outputs.decision != 'skip'
env:
GH_TOKEN: ${{ github.token }}
run: |
uv run --no-sync python scripts/ruff_strict_gate.py --base "$GATE_BASE_SHA"
- name: Check type discipline (mutable collections / casts / type guards / kwargs / unexplained suppressions, delta vs merge-base counts)
if: steps.changes.outputs.decision != 'skip'
env:
GH_TOKEN: ${{ github.token }}
run: |
uv run --no-sync python scripts/type_discipline_gate.py --base "$GATE_BASE_SHA"
- name: Check test quality (zero-assert / mock-echo tests, sys.path.insert, raw env writes, litellm global mutation, credential-gated skips, conftest snapshot inventory, delta vs merge-base counts)
if: steps.changes.outputs.decision != 'skip'
env:
GH_TOKEN: ${{ github.token }}
run: |
uv run --no-sync python scripts/test_quality_gate.py --base "$GATE_BASE_SHA"
- name: Print OpenAI version
if: steps.changes.outputs.decision != 'skip'
run: |
uv run --no-sync python -c "import openai; print(f'OpenAI version: {openai.__version__}')"
- name: Check basedpyright (delta vs merge-base counts)
if: steps.changes.outputs.decision != 'skip'
env:
GH_TOKEN: ${{ github.token }}
run: |
uv run --no-sync python scripts/type_check_gate.py --base "$GATE_BASE_SHA"
- name: Check tests/e2e basedpyright (zero errors)
if: steps.changes.outputs.decision != 'skip'
run: |
if git diff --name-only --diff-filter=ACMRD "$GATE_BASE_SHA" HEAD -- ':(glob)tests/e2e/**/*.py' ':(glob)tests/e2e_harness/**/*.py' pyrightconfig.json | grep -q .; then
uv run --no-sync basedpyright tests/e2e tests/e2e_harness
else
echo "No changed tests/e2e Python files; skipping."
fi
- name: Run the e2e harness tests
if: steps.changes.outputs.decision != 'skip'
env:
LITELLM_MASTER_KEY: sk-e2e-harness-tests-reach-no-proxy
run: |
if ! git diff --name-only --diff-filter=ACMRD "$GATE_BASE_SHA" HEAD -- tests/e2e tests/e2e_harness ':(exclude)tests/e2e/ui' pyproject.toml uv.lock .github/workflows/test-linting.yml | grep -q .; then
echo "No changed e2e harness files; skipping."
exit 0
fi
retry() { "$@" || { sleep 15; "$@"; } || { sleep 30; "$@"; }; }
CLAUDE_VERSION="$(retry uv run --no-sync python tests/e2e/claude_code/pr_gate_version_resolver.py)"
tests/e2e/claude_code/cron_vm/install_claude_code.sh "$CLAUDE_VERSION" "$RUNNER_TEMP/claude-cli"
PATH="$RUNNER_TEMP/claude-cli:$PATH" uv run --no-sync pytest -q tests/e2e_harness
- name: Check for circular imports
if: steps.changes.outputs.decision != 'skip'
run: |
cd litellm
uv run --no-sync python ../tests/documentation_tests/test_circular_imports.py
cd ..
- name: Check import safety
if: steps.changes.outputs.decision != 'skip'
run: |
uv run --no-sync python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
frontend-lint:
name: frontend-lint
permissions:
contents: read
runs-on: ubuntu-latest
timeout-minutes: 8
defaults:
run:
working-directory: ui/litellm-dashboard
steps:
- name: Checkout repository
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
fetch-depth: 1
persist-credentials: false
- name: Collect changed files
id: changed
env:
GH_TOKEN: ${{ github.token }}
BASE_SHA: ${{ github.event.pull_request.base.sha }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
run: |
merge_base=$(gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
test -n "$merge_base"
git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
: > "$RUNNER_TEMP/prettier_files.txt"
: > "$RUNNER_TEMP/eslint_files.txt"
while IFS= read -r f; do
[ -f "$f" ] || continue
case "$f" in
*.js | *.jsx | *.ts | *.tsx | *.mjs | *.cjs)
printf '%s\n' "$f" >> "$RUNNER_TEMP/prettier_files.txt"
printf '%s\n' "$f" >> "$RUNNER_TEMP/eslint_files.txt" ;;
*.json | *.css | *.scss | *.md | *.mdx | *.yml | *.yaml | *.html)
printf '%s\n' "$f" >> "$RUNNER_TEMP/prettier_files.txt" ;;
esac
done < <(git diff --name-only --diff-filter=ACMR --relative "$merge_base" "$HEAD_SHA" -- .)
if [ -s "$RUNNER_TEMP/prettier_files.txt" ] || [ -s "$RUNNER_TEMP/eslint_files.txt" ]; then
echo "has_files=true" >> "$GITHUB_OUTPUT"
else
echo "has_files=false" >> "$GITHUB_OUTPUT"
echo "No lintable UI files changed in this PR; nothing to check."
fi
- name: Setup Node.js
if: steps.changed.outputs.has_files == 'true'
uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
with:
node-version-file: ui/litellm-dashboard/.nvmrc
cache: "npm"
cache-dependency-path: ui/litellm-dashboard/package-lock.json
- name: Install dependencies
if: steps.changed.outputs.has_files == 'true'
run: npm ci
- name: Lint changed files (prettier + eslint)
if: steps.changed.outputs.has_files == 'true'
run: |
prettier_files=()
eslint_files=()
while IFS= read -r f; do prettier_files+=("$f"); done < "$RUNNER_TEMP/prettier_files.txt"
while IFS= read -r f; do eslint_files+=("$f"); done < "$RUNNER_TEMP/eslint_files.txt"
status=0
if [ ${#prettier_files[@]} -gt 0 ]; then
echo "::group::Prettier (${#prettier_files[@]} files)"
npx prettier --check "${prettier_files[@]}" || { status=1; echo "::error::Unformatted files. Fix with: npm run format"; }
echo "::endgroup::"
fi
if [ ${#eslint_files[@]} -gt 0 ]; then
echo "::group::ESLint (${#eslint_files[@]} files)"
npx eslint --no-warn-ignored --pass-on-unpruned-suppressions "${eslint_files[@]}" || status=1
echo "::endgroup::"
fi
exit $status
- name: Check lint budgets
if: ${{ !cancelled() && steps.changed.outputs.has_files == 'true' }}
run: |
npx eslint . -f json -o "$RUNNER_TEMP/lint-report.json" || true
node scripts/check-lint-budgets.mjs "$RUNNER_TEMP/lint-report.json" eslint-budgets.json
- name: Check for dead code (knip)
if: ${{ !cancelled() && steps.changed.outputs.has_files == 'true' }}
run: npm run knip:ci
assert-ci-coverage:
name: assert-ci-coverage
permissions:
contents: read
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Assert every test file and Dockerfile is invoked by a job
run: |
python -m pip install "pyyaml==6.0.3"
python .github/scripts/assert_ci_coverage.py
- name: Assert no -k expression deselects a file from every job that globs it
run: python .github/scripts/assert_ci_coverage.py --slices
- name: Assert .github/workflows/ holds only workflows, correctly named
run: python .github/scripts/assert_workflow_dir_hygiene.py
dashboard-build:
name: Dashboard build
runs-on: ubuntu-24.04
timeout-minutes: 30
steps:
- name: Checkout
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Build the dashboard stage
run: docker build --target ui-builder -f Dockerfile .
core-checks:
name: Core checks (Python ${{ matrix.python-version }})
runs-on: ubuntu-24.04
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
env:
LITELLM_LOCAL_MODEL_COST_MAP: "True"
steps:
- name: Checkout
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: ${{ matrix.python-version }}
- name: Set up uv
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Install dependencies
run: .github/scripts/uv_sync_with_retries.sh --frozen --extra proxy --extra cli --group dev --group proxy-dev --python ${{ matrix.python-version }}
- name: Create the loopback-only network namespace
run: |
sudo ip netns add smoke
sudo ip netns exec smoke ip link set lo up
cat > "${RUNNER_TEMP}/in-netns" <<'WRAP'
#!/usr/bin/env bash
set -euo pipefail
exec sudo --preserve-env=LITELLM_LOCAL_MODEL_COST_MAP ip netns exec smoke setpriv --reuid "$(id -u)" --regid "$(id -g)" --init-groups -- env HOME="${HOME}" PATH="${PATH}" "$@"
WRAP
chmod +x "${RUNNER_TEMP}/in-netns"
echo "IN_NETNS=${RUNNER_TEMP}/in-netns" >> "${GITHUB_ENV}"
- name: Verify namespace isolation
run: $IN_NETNS .venv/bin/python .github/scripts/run_merge_smoke.py verify-isolation
- name: Verify interpreter version
run: $IN_NETNS .venv/bin/python .github/scripts/run_merge_smoke.py interpreter --expect ${{ matrix.python-version }}
- name: Import and CLI checks
run: $IN_NETNS .venv/bin/python .github/scripts/run_merge_smoke.py cli
- name: Proxy startup check
run: $IN_NETNS .venv/bin/python .github/scripts/run_merge_smoke.py proxy-startup --diagnostics-dir "${RUNNER_TEMP}/smoke-diagnostics"
- name: Upload smoke diagnostics
if: always()
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
with:
name: merge-smoke-diagnostics-py${{ matrix.python-version }}
path: ${{ runner.temp }}/smoke-diagnostics
if-no-files-found: ignore
- name: Remove the network namespace
if: always()
run: sudo ip netns delete smoke

220
.github/workflows/test-code-quality.yml vendored
View file

@ -1,220 +0,0 @@
name: Code Quality Checks
on:
pull_request:
branches:
- main
- "litellm_**"
push:
branches:
- main
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
code-quality:
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Checkout litellm-docs into docs/my-website (for documentation_tests)
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
repository: BerriAI/litellm-docs
path: docs/my-website
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Cache uv dependencies
if: github.ref == 'refs/heads/main'
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: |
~/.cache/uv
.venv
key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
restore-keys: |
${{ runner.os }}-uv-
- name: Cache uv dependencies
if: github.ref != 'refs/heads/main'
uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: |
~/.cache/uv
.venv
key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
restore-keys: |
${{ runner.os }}-uv-
- name: Cache the Rust build
uses: ./.github/actions/cache-cargo-build
- name: Install dependencies
run: uv sync --frozen --all-groups --all-extras
- name: check_licenses
run: uv run --no-sync python ./tests/code_coverage_tests/check_licenses.py
- name: check_provider_folders_documented
run: uv run --no-sync python ./tests/code_coverage_tests/check_provider_folders_documented.py
- name: check_prisma_binary_cache
run: uv run --no-sync python ./tests/code_coverage_tests/check_prisma_binary_cache.py
- name: check_workflow_startup_safety
run: uv run --no-sync python ./tests/code_coverage_tests/check_workflow_startup_safety.py
- name: check_workflow_job_name_collisions
run: uv run --no-sync python ./tests/code_coverage_tests/check_workflow_job_name_collisions.py
- name: test_workflow_job_name_collisions
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_workflow_job_name_collisions.py
- name: test_e2e_changed_gate
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_changed_gate.py tests/code_coverage_tests/test_e2e_idp_stack.py
- name: test_e2e_metadata
env:
PYTHONPATH: tests/e2e
run: |
export LITELLM_MASTER_KEY="sk-$(openssl rand -hex 16)"
uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_metadata.py tests/code_coverage_tests/test_e2e_junit_report.py
- name: Check merge smoke harness
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_merge_smoke.py
- name: router_code_coverage
run: uv run --no-sync python ./tests/code_coverage_tests/router_code_coverage.py
- name: test_chat_completion_imports
run: uv run --no-sync python ./tests/code_coverage_tests/test_chat_completion_imports.py
- name: info_log_check
run: uv run --no-sync python ./tests/code_coverage_tests/info_log_check.py
- name: check_guardrail_apply_decorator
run: uv run --no-sync python ./tests/code_coverage_tests/check_guardrail_apply_decorator.py
- name: test_ban_set_verbose
run: uv run --no-sync python ./tests/code_coverage_tests/test_ban_set_verbose.py
- name: code_qa_check_tests
run: uv run --no-sync python ./tests/code_coverage_tests/code_qa_check_tests.py
- name: check_get_model_cost_key_performance
run: uv run --no-sync python ./tests/code_coverage_tests/check_get_model_cost_key_performance.py
- name: test_proxy_types_import
run: uv run --no-sync python ./tests/code_coverage_tests/test_proxy_types_import.py
- name: callback_manager_test
run: uv run --no-sync python ./tests/code_coverage_tests/callback_manager_test.py
- name: recursive_detector
run: uv run --no-sync python ./tests/code_coverage_tests/recursive_detector.py
- name: test_router_strategy_async
run: uv run --no-sync python ./tests/code_coverage_tests/test_router_strategy_async.py
- name: litellm_logging_code_coverage
run: uv run --no-sync python ./tests/code_coverage_tests/litellm_logging_code_coverage.py
- name: ensure_async_clients_test
run: uv run --no-sync python ./tests/code_coverage_tests/ensure_async_clients_test.py
- name: enforce_llms_folder_style
run: uv run --no-sync python ./tests/code_coverage_tests/enforce_llms_folder_style.py
- name: prevent_key_leaks_in_exceptions
run: uv run --no-sync python ./tests/code_coverage_tests/prevent_key_leaks_in_exceptions.py
- name: check_unsafe_enterprise_import
run: uv run --no-sync python ./tests/code_coverage_tests/check_unsafe_enterprise_import.py
- name: ban_copy_deepcopy_kwargs
run: uv run --no-sync python ./tests/code_coverage_tests/ban_copy_deepcopy_kwargs.py
- name: check_fastuuid_usage
run: uv run --no-sync python ./tests/code_coverage_tests/check_fastuuid_usage.py
- name: check_py310_typing_imports
run: uv run --no-sync python ./tests/code_coverage_tests/check_py310_typing_imports.py
- name: check_e2e_no_raw_requests
run: uv run --no-sync python ./tests/code_coverage_tests/check_e2e_no_raw_requests.py
- name: check_migrations_no_data_rewrites
run: uv run --no-sync python ./tests/code_coverage_tests/check_migrations_no_data_rewrites.py
- name: check_no_publicly_known_master_key
run: uv run --no-sync python ./tests/code_coverage_tests/check_no_publicly_known_master_key.py
- name: test_check_no_publicly_known_master_key
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_check_no_publicly_known_master_key.py
- name: check_unbounded_in_lists (fails on findings not in the baseline)
run: uv run --no-sync python ./tests/code_coverage_tests/check_unbounded_in_lists.py
- name: memory_test
run: uv run --no-sync python ./tests/code_coverage_tests/memory_test.py
- name: documentation_test_env_keys
run: uv run --no-sync python ./tests/documentation_tests/test_env_keys.py
- name: documentation_test_router_settings
run: uv run --no-sync python ./tests/documentation_tests/test_router_settings.py
- name: documentation_test_api_docs
run: uv run --no-sync python ./tests/documentation_tests/test_api_docs.py
python-310-import-smoke:
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.10"
- name: Set up uv
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Install dependencies
run: uv sync --frozen --extra proxy --extra cli --python 3.10
- run: uv run --no-sync python --version
- name: Import litellm
run: uv run --no-sync python -c "import litellm"
- name: Check litellm CLI
run: uv run --no-sync litellm --version
- name: Check lite CLI
run: uv run --no-sync lite version

502
.github/workflows/test-linting.yml vendored
View file

@ -1,4 +1,4 @@
name: LiteLLM Linting
name: lint
on:
pull_request:
@ -14,22 +14,16 @@ concurrency:
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
lint:
runs-on: ubuntu-latest
timeout-minutes: 15
# actions: read lets scripts/type_check_gate.py download the base-counts
# artifact published by publish-basedpyright-base-counts.yml instead of
# re-running basedpyright over the merge-base tree.
python:
name: python
permissions:
contents: read
pull-requests: read
actions: read
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
# Check out the PR head, not the default refs/pull/N/merge: the merge ref
# folds in newer base commits, which the diff-based gates (ruff delta,
# Any-discipline) would otherwise blame on this branch.
with:
ref: ${{ github.event.pull_request.head.sha }}
fetch-depth: 1
@ -100,9 +94,6 @@ jobs:
if: steps.changes.outputs.decision != 'skip'
uses: ./.github/actions/cache-prisma-binaries
# basedpyright resolves Prisma's generated client (litellm/proxy/schema.prisma)
# only after `prisma generate` writes prisma/client.py et al. Without this the
# DB wrappers typed against the generated client would degrade to Unknown.
- name: Generate Prisma client
if: steps.changes.outputs.decision != 'skip'
run: |
@ -144,18 +135,24 @@ jobs:
run: |
uv run --no-sync ruff check --config ruff-tests.toml tests
- name: Check strict-rule budget (delta vs base)
- name: Check strict ruff rules (delta vs merge-base counts)
if: steps.changes.outputs.decision != 'skip'
env:
GH_TOKEN: ${{ github.token }}
run: |
uv run --no-sync python scripts/ruff_strict_gate.py --base "$GATE_BASE_SHA"
- name: Check type-discipline budget (mutable collections / casts / type guards / kwargs / unexplained suppressions, delta vs base)
- name: Check type discipline (mutable collections / casts / type guards / kwargs / unexplained suppressions, delta vs merge-base counts)
if: steps.changes.outputs.decision != 'skip'
env:
GH_TOKEN: ${{ github.token }}
run: |
uv run --no-sync python scripts/type_discipline_gate.py --base "$GATE_BASE_SHA"
- name: Check test-quality budget (zero-assert / mock-echo tests, sys.path.insert, raw env writes, litellm global mutation, credential-gated skips, conftest snapshot inventory, delta vs base)
- name: Check test quality (zero-assert / mock-echo tests, sys.path.insert, raw env writes, litellm global mutation, credential-gated skips, conftest snapshot inventory, delta vs merge-base counts)
if: steps.changes.outputs.decision != 'skip'
env:
GH_TOKEN: ${{ github.token }}
run: |
uv run --no-sync python scripts/test_quality_gate.py --base "$GATE_BASE_SHA"
@ -164,7 +161,7 @@ jobs:
run: |
uv run --no-sync python -c "import openai; print(f'OpenAI version: {openai.__version__}')"
- name: Check basedpyright budget (delta vs base)
- name: Check basedpyright (delta vs merge-base counts)
if: steps.changes.outputs.decision != 'skip'
env:
GH_TOKEN: ${{ github.token }}
@ -174,23 +171,25 @@ jobs:
- name: Check tests/e2e basedpyright (zero errors)
if: steps.changes.outputs.decision != 'skip'
run: |
if git diff --name-only --diff-filter=ACMRD "$GATE_BASE_SHA" HEAD -- ':(glob)tests/e2e/**/*.py' | grep -q .; then
uv run --no-sync basedpyright tests/e2e
if git diff --name-only --diff-filter=ACMRD "$GATE_BASE_SHA" HEAD -- ':(glob)tests/e2e/**/*.py' ':(glob)tests/e2e_harness/**/*.py' pyrightconfig.json | grep -q .; then
uv run --no-sync basedpyright tests/e2e tests/e2e_harness
else
echo "No changed tests/e2e Python files; skipping."
fi
- name: Run the claude_code harness unit tests
- name: Run the e2e harness tests
if: steps.changes.outputs.decision != 'skip'
env:
LITELLM_MASTER_KEY: sk-e2e-harness-tests-reach-no-proxy
run: |
if ! git diff --name-only --diff-filter=ACMRD "$GATE_BASE_SHA" HEAD -- ':(glob)tests/e2e/claude_code/**/*.py' ':(glob)tests/e2e/*.py' tests/e2e/claude_code/cron_vm/install_claude_code.sh pyproject.toml uv.lock .github/workflows/test-linting.yml | grep -q .; then
echo "No changed claude_code harness files; skipping."
if ! git diff --name-only --diff-filter=ACMRD "$GATE_BASE_SHA" HEAD -- tests/e2e tests/e2e_harness ':(exclude)tests/e2e/ui' pyproject.toml uv.lock .github/workflows/test-linting.yml | grep -q .; then
echo "No changed e2e harness files; skipping."
exit 0
fi
retry() { "$@" || { sleep 15; "$@"; } || { sleep 30; "$@"; }; }
CLAUDE_VERSION="$(retry uv run --no-sync python tests/e2e/claude_code/pr_gate_version_resolver.py)"
tests/e2e/claude_code/cron_vm/install_claude_code.sh "$CLAUDE_VERSION" "$RUNNER_TEMP/claude-cli"
PATH="$RUNNER_TEMP/claude-cli:$PATH" uv run --no-sync pytest -q --noconftest -o addopts= -o pythonpath=tests/e2e -p no:rerunfailures tests/e2e/claude_code/_*_unit_tests
PATH="$RUNNER_TEMP/claude-cli:$PATH" uv run --no-sync pytest -q tests/e2e_harness
- name: Check for circular imports
if: steps.changes.outputs.decision != 'skip'
@ -203,47 +202,11 @@ jobs:
if: steps.changes.outputs.decision != 'skip'
run: |
uv run --no-sync python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
# Intentionally NON-GATING. This job turns red when a *-budget.json ceiling is
# raised (or a rule/budget is dropped) so a loosening is obvious in review, but it
# must be kept OUT of the branch-protection required-checks list so a justified
# bump can still be merged by a human who has seen and accepted the red.
budget-ratchet:
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
contents: read
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
fetch-depth: 1
persist-credentials: false
- name: Fetch ratchet base
env:
BASE_SHA: ${{ github.event.pull_request.base.sha }}
run: |
retry() { "$@" || { sleep 15; "$@"; } || { sleep 30; "$@"; }; }
retry git fetch --no-tags --depth=1 origin "$BASE_SHA"
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Ratchet check (budgets may only decrease; non-gating)
env:
BASE_SHA: ${{ github.event.pull_request.base.sha }}
run: |
python scripts/budget_ratchet_check.py --base "$BASE_SHA"
secret-scan:
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
contents: read
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
@ -275,3 +238,418 @@ jobs:
else
echo "GITGUARDIAN_API_KEY not set, skipping ggshield scan"
fi
ui:
name: ui
permissions:
contents: read
runs-on: ubuntu-latest
timeout-minutes: 8
defaults:
run:
working-directory: ui/litellm-dashboard
steps:
- name: Checkout repository
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
fetch-depth: 1
persist-credentials: false
- name: Collect changed files
id: changed
env:
GH_TOKEN: ${{ github.token }}
BASE_SHA: ${{ github.event.pull_request.base.sha }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
run: |
# base.sha is the base branch tip from when the PR was opened, while
# actions/checkout leaves HEAD on a merge of the PR into the *current*
# base tip. "$BASE_SHA"...HEAD therefore spans every base-branch commit
# landed since, so a PR that touches no UI file still gets linted
# against hundreds of other people's files. Diff the PR head against its
# own merge base instead, which is exactly what this PR changed.
merge_base=$(gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
test -n "$merge_base"
git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
: > "$RUNNER_TEMP/prettier_files.txt"
: > "$RUNNER_TEMP/eslint_files.txt"
while IFS= read -r f; do
[ -f "$f" ] || continue
case "$f" in
*.js | *.jsx | *.ts | *.tsx | *.mjs | *.cjs)
printf '%s\n' "$f" >> "$RUNNER_TEMP/prettier_files.txt"
printf '%s\n' "$f" >> "$RUNNER_TEMP/eslint_files.txt" ;;
*.json | *.css | *.scss | *.md | *.mdx | *.yml | *.yaml | *.html)
printf '%s\n' "$f" >> "$RUNNER_TEMP/prettier_files.txt" ;;
esac
done < <(git diff --name-only --diff-filter=ACMR --relative "$merge_base" "$HEAD_SHA" -- .)
if [ -s "$RUNNER_TEMP/prettier_files.txt" ] || [ -s "$RUNNER_TEMP/eslint_files.txt" ]; then
echo "has_files=true" >> "$GITHUB_OUTPUT"
else
echo "has_files=false" >> "$GITHUB_OUTPUT"
echo "No lintable UI files changed in this PR; nothing to check."
fi
- name: Setup Node.js
if: steps.changed.outputs.has_files == 'true'
uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
with:
node-version-file: ui/litellm-dashboard/.nvmrc
cache: "npm"
cache-dependency-path: ui/litellm-dashboard/package-lock.json
- name: Install dependencies
if: steps.changed.outputs.has_files == 'true'
run: npm ci
- name: Lint changed files (prettier + eslint)
if: steps.changed.outputs.has_files == 'true'
run: |
prettier_files=()
eslint_files=()
while IFS= read -r f; do prettier_files+=("$f"); done < "$RUNNER_TEMP/prettier_files.txt"
while IFS= read -r f; do eslint_files+=("$f"); done < "$RUNNER_TEMP/eslint_files.txt"
status=0
if [ ${#prettier_files[@]} -gt 0 ]; then
echo "::group::Prettier (${#prettier_files[@]} files)"
npx prettier --check "${prettier_files[@]}" || { status=1; echo "::error::Unformatted files. Fix with: npm run format"; }
echo "::endgroup::"
fi
if [ ${#eslint_files[@]} -gt 0 ]; then
echo "::group::ESLint (${#eslint_files[@]} files)"
npx eslint --no-warn-ignored --pass-on-unpruned-suppressions "${eslint_files[@]}" || status=1
echo "::endgroup::"
fi
exit $status
- name: Check lint budgets
if: ${{ !cancelled() && steps.changed.outputs.has_files == 'true' }}
run: |
npx eslint . -f json -o "$RUNNER_TEMP/lint-report.json" || true
node scripts/check-lint-budgets.mjs "$RUNNER_TEMP/lint-report.json" eslint-budgets.json
- name: Check for dead code (knip)
if: ${{ !cancelled() && steps.changed.outputs.has_files == 'true' }}
run: npm run knip:ci
ui-api-types:
name: ui-api-types
permissions:
contents: read
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- name: Checkout repository
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
fetch-depth: 2
- name: Detect changes that can affect the generated types
id: changes
run: |
set -euo pipefail
if ! base="$(git rev-parse --verify --quiet HEAD^2 >/dev/null && git rev-parse HEAD^1)"; then
echo "Not a pull request merge commit, running the full check."
echo "relevant=true" >> "$GITHUB_OUTPUT"
exit 0
fi
files="$(git diff --name-only "$base" HEAD)"
if grep -Eq '^(litellm/(proxy|types)/|ui/litellm-dashboard/(src/lib/http/schema\.d\.ts|scripts/gen-api-types\.mjs|package(-lock)?\.json)$|\.github/workflows/test-linting\.yml$)' <<< "$files"; then
echo "relevant=true" >> "$GITHUB_OUTPUT"
else
echo "No proxy, types or generator changes in this pull request, nothing to verify."
echo "relevant=false" >> "$GITHUB_OUTPUT"
fi
- name: Set up Python
if: steps.changes.outputs.relevant == 'true'
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
if: steps.changes.outputs.relevant == 'true'
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Cache uv dependencies
if: steps.changes.outputs.relevant == 'true'
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: |
~/.cache/uv
.venv
key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
restore-keys: |
${{ runner.os }}-uv-
- name: Cache the Rust build
if: steps.changes.outputs.relevant == 'true'
uses: ./.github/actions/cache-cargo-build
- name: Install backend dependencies
if: steps.changes.outputs.relevant == 'true'
run: .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
- name: Cache Prisma binaries
if: steps.changes.outputs.relevant == 'true'
uses: ./.github/actions/cache-prisma-binaries
- name: Generate Prisma client
if: steps.changes.outputs.relevant == 'true'
run: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
- name: Regenerate the lazy OpenAPI snapshot
if: steps.changes.outputs.relevant == 'true'
run: uv run --no-sync python -m litellm.proxy._lazy_openapi_snapshot
- name: Fail if the lazy OpenAPI snapshot is stale
if: steps.changes.outputs.relevant == 'true'
run: |
if ! git diff --exit-code -- litellm/proxy/_lazy_openapi_snapshot.json; then
echo "::error file=litellm/proxy/_lazy_openapi_snapshot.json::The lazy OpenAPI snapshot is out of sync with the lazily loaded routes."
echo ""
echo "A lazily loaded route or model changed without regenerating the snapshot that /openapi.json serves for unloaded features."
echo "To fix, run from the repo root:"
echo " uv run python -m litellm.proxy._lazy_openapi_snapshot"
echo "then run npm run gen:api from ui/litellm-dashboard and commit both files."
exit 1
fi
echo "_lazy_openapi_snapshot.json is in sync with the lazily loaded routes."
- name: Set up Node.js
if: steps.changes.outputs.relevant == 'true'
uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
with:
node-version-file: ui/litellm-dashboard/.nvmrc
cache: "npm"
cache-dependency-path: ui/litellm-dashboard/package-lock.json
- name: Install dashboard dependencies
if: steps.changes.outputs.relevant == 'true'
working-directory: ui/litellm-dashboard
run: npm ci
- name: Regenerate types from the live spec
if: steps.changes.outputs.relevant == 'true'
working-directory: ui/litellm-dashboard
env:
LITELLM_PYTHON: "uv run --no-sync python"
run: npm run gen:api
- name: Fail if types are stale
if: steps.changes.outputs.relevant == 'true'
run: |
if ! git diff --exit-code -- ui/litellm-dashboard/src/lib/http/schema.d.ts; then
echo "::error file=ui/litellm-dashboard/src/lib/http/schema.d.ts::Generated API types are out of sync with the proxy OpenAPI spec."
echo ""
echo "A backend route or model changed without regenerating the dashboard types."
echo "To fix, run from ui/litellm-dashboard:"
echo " npm run gen:api"
echo "then commit the updated src/lib/http/schema.d.ts."
exit 1
fi
echo "schema.d.ts is in sync with the proxy OpenAPI spec."
code-quality:
name: code-quality
permissions:
contents: read
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Checkout litellm-docs into docs/my-website (for documentation_tests)
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
repository: BerriAI/litellm-docs
path: docs/my-website
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Cache uv dependencies
uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: |
~/.cache/uv
.venv
key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
restore-keys: |
${{ runner.os }}-uv-
- name: Cache the Rust build
uses: ./.github/actions/cache-cargo-build
- name: Install dependencies
run: uv sync --frozen --all-groups --all-extras
- name: check_licenses
run: uv run --no-sync python ./tests/code_coverage_tests/check_licenses.py
- name: check_provider_folders_documented
run: uv run --no-sync python ./tests/code_coverage_tests/check_provider_folders_documented.py
- name: check_prisma_binary_cache
run: uv run --no-sync python ./tests/code_coverage_tests/check_prisma_binary_cache.py
- name: check_workflow_startup_safety
run: uv run --no-sync python ./tests/code_coverage_tests/check_workflow_startup_safety.py
- name: check_workflow_job_name_collisions
run: uv run --no-sync python ./tests/code_coverage_tests/check_workflow_job_name_collisions.py
- name: test_workflow_job_name_collisions
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_workflow_job_name_collisions.py
- name: test_unit_passed_gate
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_unit_passed_gate.py
- name: test_e2e_changed_gate
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_changed_gate.py tests/code_coverage_tests/test_e2e_idp_stack.py
- name: test_e2e_metadata
env:
PYTHONPATH: tests/e2e
run: |
export LITELLM_MASTER_KEY="sk-$(openssl rand -hex 16)"
uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_metadata.py tests/code_coverage_tests/test_e2e_junit_report.py
- name: Check merge smoke harness
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_merge_smoke.py
- name: router_code_coverage
run: uv run --no-sync python ./tests/code_coverage_tests/router_code_coverage.py
- name: test_chat_completion_imports
run: uv run --no-sync python ./tests/code_coverage_tests/test_chat_completion_imports.py
- name: info_log_check
run: uv run --no-sync python ./tests/code_coverage_tests/info_log_check.py
- name: check_guardrail_apply_decorator
run: uv run --no-sync python ./tests/code_coverage_tests/check_guardrail_apply_decorator.py
- name: test_ban_set_verbose
run: uv run --no-sync python ./tests/code_coverage_tests/test_ban_set_verbose.py
- name: code_qa_check_tests
run: uv run --no-sync python ./tests/code_coverage_tests/code_qa_check_tests.py
- name: check_get_model_cost_key_performance
run: uv run --no-sync python ./tests/code_coverage_tests/check_get_model_cost_key_performance.py
- name: test_proxy_types_import
run: uv run --no-sync python ./tests/code_coverage_tests/test_proxy_types_import.py
- name: callback_manager_test
run: uv run --no-sync python ./tests/code_coverage_tests/callback_manager_test.py
- name: recursive_detector
run: uv run --no-sync python ./tests/code_coverage_tests/recursive_detector.py
- name: test_router_strategy_async
run: uv run --no-sync python ./tests/code_coverage_tests/test_router_strategy_async.py
- name: litellm_logging_code_coverage
run: uv run --no-sync python ./tests/code_coverage_tests/litellm_logging_code_coverage.py
- name: ensure_async_clients_test
run: uv run --no-sync python ./tests/code_coverage_tests/ensure_async_clients_test.py
- name: enforce_llms_folder_style
run: uv run --no-sync python ./tests/code_coverage_tests/enforce_llms_folder_style.py
- name: prevent_key_leaks_in_exceptions
run: uv run --no-sync python ./tests/code_coverage_tests/prevent_key_leaks_in_exceptions.py
- name: check_unsafe_enterprise_import
run: uv run --no-sync python ./tests/code_coverage_tests/check_unsafe_enterprise_import.py
- name: ban_copy_deepcopy_kwargs
run: uv run --no-sync python ./tests/code_coverage_tests/ban_copy_deepcopy_kwargs.py
- name: check_fastuuid_usage
run: uv run --no-sync python ./tests/code_coverage_tests/check_fastuuid_usage.py
- name: check_py310_typing_imports
run: uv run --no-sync python ./tests/code_coverage_tests/check_py310_typing_imports.py
- name: check_e2e_no_raw_requests
run: uv run --no-sync python ./tests/code_coverage_tests/check_e2e_no_raw_requests.py
- name: check_migrations_no_data_rewrites
run: uv run --no-sync python ./tests/code_coverage_tests/check_migrations_no_data_rewrites.py
- name: check_no_publicly_known_master_key
run: uv run --no-sync python ./tests/code_coverage_tests/check_no_publicly_known_master_key.py
- name: test_check_no_publicly_known_master_key
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_check_no_publicly_known_master_key.py
- name: check_unbounded_in_lists (fails on findings not in the baseline)
run: uv run --no-sync python ./tests/code_coverage_tests/check_unbounded_in_lists.py
- name: memory_test
run: uv run --no-sync python ./tests/code_coverage_tests/memory_test.py
- name: documentation_test_env_keys
run: uv run --no-sync python ./tests/documentation_tests/test_env_keys.py
- name: documentation_test_router_settings
run: uv run --no-sync python ./tests/documentation_tests/test_router_settings.py
- name: documentation_test_api_docs
run: uv run --no-sync python ./tests/documentation_tests/test_api_docs.py
ci-coverage:
name: ci-coverage
permissions:
contents: read
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Assert every test file and Dockerfile is invoked by a job
run: |
python -m pip install "pyyaml==6.0.3"
python .github/scripts/assert_ci_coverage.py
- name: Assert no -k expression deselects a file from every job that globs it
run: python .github/scripts/assert_ci_coverage.py --slices
- name: Assert .github/workflows/ holds only workflows, correctly named
run: python .github/scripts/assert_workflow_dir_hygiene.py
lint-passed:
name: lint passed
needs: [python, secret-scan, ui, ui-api-types, code-quality, ci-coverage]
if: always()
runs-on: ubuntu-latest
timeout-minutes: 2
permissions: {}
steps:
- name: Require every lint job to succeed
env:
NEEDS: ${{ toJSON(needs) }}
run: |
jq -r 'to_entries[] | "\(.key): \(.value.result)"' <<< "$NEEDS"
jq -e 'all(.[]; .result == "success")' <<< "$NEEDS" > /dev/null

105
.github/workflows/test-litellm-ui-lint.yml vendored
View file

@ -1,105 +0,0 @@
name: UI Lint
permissions:
contents: read
on:
pull_request:
branches:
- main
- "litellm_**"
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
frontend-lint:
runs-on: ubuntu-latest
timeout-minutes: 8
defaults:
run:
working-directory: ui/litellm-dashboard
steps:
- name: Checkout repository
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
fetch-depth: 1
persist-credentials: false
- name: Collect changed files
id: changed
env:
GH_TOKEN: ${{ github.token }}
BASE_SHA: ${{ github.event.pull_request.base.sha }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
run: |
# base.sha is the base branch tip from when the PR was opened, while
# actions/checkout leaves HEAD on a merge of the PR into the *current*
# base tip. "$BASE_SHA"...HEAD therefore spans every base-branch commit
# landed since, so a PR that touches no UI file still gets linted
# against hundreds of other people's files. Diff the PR head against its
# own merge base instead, which is exactly what this PR changed.
merge_base=$(gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
test -n "$merge_base"
git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
: > "$RUNNER_TEMP/prettier_files.txt"
: > "$RUNNER_TEMP/eslint_files.txt"
while IFS= read -r f; do
[ -f "$f" ] || continue
case "$f" in
*.js | *.jsx | *.ts | *.tsx | *.mjs | *.cjs)
printf '%s\n' "$f" >> "$RUNNER_TEMP/prettier_files.txt"
printf '%s\n' "$f" >> "$RUNNER_TEMP/eslint_files.txt" ;;
*.json | *.css | *.scss | *.md | *.mdx | *.yml | *.yaml | *.html)
printf '%s\n' "$f" >> "$RUNNER_TEMP/prettier_files.txt" ;;
esac
done < <(git diff --name-only --diff-filter=ACMR --relative "$merge_base" "$HEAD_SHA" -- .)
if [ -s "$RUNNER_TEMP/prettier_files.txt" ] || [ -s "$RUNNER_TEMP/eslint_files.txt" ]; then
echo "has_files=true" >> "$GITHUB_OUTPUT"
else
echo "has_files=false" >> "$GITHUB_OUTPUT"
echo "No lintable UI files changed in this PR; nothing to check."
fi
- name: Setup Node.js
if: steps.changed.outputs.has_files == 'true'
uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
with:
node-version-file: ui/litellm-dashboard/.nvmrc
cache: "npm"
cache-dependency-path: ui/litellm-dashboard/package-lock.json
- name: Install dependencies
if: steps.changed.outputs.has_files == 'true'
run: npm ci
- name: Lint changed files (prettier + eslint)
if: steps.changed.outputs.has_files == 'true'
run: |
prettier_files=()
eslint_files=()
while IFS= read -r f; do prettier_files+=("$f"); done < "$RUNNER_TEMP/prettier_files.txt"
while IFS= read -r f; do eslint_files+=("$f"); done < "$RUNNER_TEMP/eslint_files.txt"
status=0
if [ ${#prettier_files[@]} -gt 0 ]; then
echo "::group::Prettier (${#prettier_files[@]} files)"
npx prettier --check "${prettier_files[@]}" || { status=1; echo "::error::Unformatted files. Fix with: npm run format"; }
echo "::endgroup::"
fi
if [ ${#eslint_files[@]} -gt 0 ]; then
echo "::group::ESLint (${#eslint_files[@]} files)"
npx eslint --no-warn-ignored --pass-on-unpruned-suppressions "${eslint_files[@]}" || status=1
echo "::endgroup::"
fi
exit $status
- name: Check lint budgets
if: ${{ !cancelled() && steps.changed.outputs.has_files == 'true' }}
run: |
npx eslint . -f json -o "$RUNNER_TEMP/lint-report.json" || true
node scripts/check-lint-budgets.mjs "$RUNNER_TEMP/lint-report.json" eslint-budgets.json
- name: Check for dead code (knip)
if: ${{ !cancelled() && steps.changed.outputs.has_files == 'true' }}
run: npm run knip:ci

99
.github/workflows/test-litellm-ui-unit.yml vendored
View file

@ -1,99 +0,0 @@
name: UI Unit Tests
permissions:
contents: read
pull-requests: read
on:
pull_request:
branches:
- main
- "litellm_**"
push:
branches:
- main
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
ui-unit-tests:
runs-on: ubuntu-latest-16-cores
timeout-minutes: 20
defaults:
run:
working-directory: ui/litellm-dashboard
steps:
- name: Checkout repository
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
fetch-depth: 1
persist-credentials: false
- name: Detect relevant changes
id: changes
uses: ./.github/actions/detect-changes
with:
category: ui
- name: Setup Node.js
if: steps.changes.outputs.decision != 'skip'
uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
with:
node-version-file: ui/litellm-dashboard/.nvmrc
cache: "npm"
cache-dependency-path: ui/litellm-dashboard/package-lock.json
- name: Install dependencies
if: steps.changes.outputs.decision != 'skip'
run: npm ci
- name: Check UI production source types
if: steps.changes.outputs.decision != 'skip'
run: npm run typecheck
- name: Run UI type tests (Vitest)
if: steps.changes.outputs.decision != 'skip'
env:
CI: "true"
run: npm run test:types
- name: Run UI unit tests (Vitest)
if: steps.changes.outputs.decision != 'skip'
env:
CI: "true"
GH_TOKEN: ${{ github.token }}
BASE_SHA: ${{ github.event.pull_request.base.sha }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
run: |
full_suite() { npm run test -- --run --pool forks --maxWorkers=14; }
if [ -z "$BASE_SHA" ]; then
echo "Push to $GITHUB_REF_NAME: running the full suite"
full_suite
exit 0
fi
merge_base=$(gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
test -n "$merge_base"
git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
changed_files=()
while IFS= read -r f; do
changed_files+=("$f")
done < <(git diff --name-only --relative "$merge_base" "$HEAD_SHA" -- .)
if [ ${#changed_files[@]} -eq 0 ]; then
echo "No UI files changed in this PR; skipping unit tests."
exit 0
fi
scope=$(printf '%s\n' "${changed_files[@]}" | bash "$GITHUB_WORKSPACE/.github/scripts/select_ui_test_scope.sh")
if [ "$scope" != related ]; then
echo "Pull request: ${#changed_files[@]} changed UI files reach outside src/, so related would miss their dependents; running the full suite"
full_suite
exit 0
fi
echo "Pull request: running tests related to ${#changed_files[@]} changed UI files"
npm run test -- related "${changed_files[@]}" --run --passWithNoTests \
--pool forks --maxWorkers=14

64
.github/workflows/test-mcp-dependency-resolution.yml vendored
View file

@ -13,6 +13,9 @@ on:
- "rust-toolchain.toml"
- "litellm-rust/**"
- "litellm/__init__.py"
- "litellm/_version.py"
- "litellm/_lazy_imports_registry.py"
- "scripts/build_core_distribution.py"
- "litellm/proxy/proxy_server.py"
- "litellm/**/*mcp*"
- "litellm/**/*mcp*/**"
@ -86,6 +89,7 @@ jobs:
uv pip check --python ".venv-$extra"
if [ "$extra" = core ]; then
checker=("$GITHUB_WORKSPACE/tests/base_sdk_tests/check_base_sdk_install.py")
(cd "$RUNNER_TEMP" && "$GITHUB_WORKSPACE/.venv-$extra/bin/python" -I "$GITHUB_WORKSPACE/tests/base_sdk_tests/check_sdk_http.py")
else
checker=("$GITHUB_WORKSPACE/scripts/check_mcp_sdk_install.py" --extra "$extra")
fi
@ -115,3 +119,63 @@ jobs:
fi
(cd "$RUNNER_TEMP" && "$GITHUB_WORKSPACE/.venv-lowest-$extra/bin/python" "${checker[@]}")
done
core-distribution:
permissions:
contents: read
id-token: write
runs-on: ubuntu-latest
timeout-minutes: 30
env:
UV_PYTHON: ${{ matrix.python-version }}
LITELLM_LOCAL_MODEL_COST_MAP: "True"
strategy:
fail-fast: false
matrix:
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: ${{ matrix.python-version }}
- uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- uses: ./.github/actions/cache-cargo-build
- name: Build the core wheel and source distribution
run: >-
uv run --no-project --with coverage==7.14.0 python -m coverage run --rcfile=/dev/null
--branch --include="*/scripts/build_core_distribution.py"
scripts/build_core_distribution.py --out-dir dist/core
- name: Verify metadata, resources, and independent source rebuild
env:
CORE_DISTRIBUTION_DIR: ${{ github.workspace }}/dist/core
run: >-
uv run --no-project --with pytest==9.0.3 --with pytest-cov==5.0.0 --with coverage==7.14.0 --with 'tomli==2.4.1; python_version < "3.11"'
python -m pytest tests/base_sdk_tests/test_core_distribution.py -v
--cov-config=/dev/null --cov=scripts.build_core_distribution --cov-branch --cov-append --cov-report=
- name: Verify isolated core installations
run: |
wheel=$(realpath dist/core/litellm_core-*.whl)
for resolution in highest lowest-direct; do
uv venv --python ${{ matrix.python-version }} ".venv-core-$resolution"
uv pip install --python ".venv-core-$resolution" --resolution "$resolution" "$wheel"
uv pip check --python ".venv-core-$resolution"
(cd "$RUNNER_TEMP" && "$GITHUB_WORKSPACE/.venv-core-$resolution/bin/python" -I "$GITHUB_WORKSPACE/tests/base_sdk_tests/check_base_sdk_install.py" --profile core)
(cd "$RUNNER_TEMP" && "$GITHUB_WORKSPACE/.venv-core-$resolution/bin/python" -I "$GITHUB_WORKSPACE/tests/base_sdk_tests/check_sdk_http.py")
done
uv pip install --python .venv-core-highest coverage==7.14.0
.venv-core-highest/bin/python -I -m coverage run --rcfile=/dev/null --append --branch --include="*/tests/base_sdk_tests/check_base_sdk_install.py" tests/base_sdk_tests/check_base_sdk_install.py --profile core
uv pip install --python .venv-core-highest boto3 tokenizers huggingface-hub
(cd "$RUNNER_TEMP" && "$GITHUB_WORKSPACE/.venv-core-highest/bin/python" -I "$GITHUB_WORKSPACE/tests/base_sdk_tests/check_base_sdk_install.py" --profile dependencies)
(cd "$RUNNER_TEMP" && "$GITHUB_WORKSPACE/.venv-core-highest/bin/python" -I "$GITHUB_WORKSPACE/tests/base_sdk_tests/check_sdk_http.py")
uv run --no-project --with coverage==7.14.0 python -m coverage xml --rcfile=/dev/null -o coverage-core.xml
- name: Upload core packaging coverage
uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5.5.5
with:
use_oidc: true
files: coverage-core.xml
flags: core-packaging
fail_ci_if_error: false

24
.github/workflows/test-merge-smoke.yml vendored
View file

@ -1,4 +1,4 @@
name: Merge smoke checks
name: smoke
on:
pull_request:
@ -14,7 +14,7 @@ concurrency:
jobs:
dashboard-build:
name: Dashboard build
name: dashboard-build
runs-on: ubuntu-24.04
timeout-minutes: 30
steps:
@ -27,7 +27,7 @@ jobs:
run: docker build --target ui-builder -f Dockerfile .
core-checks:
name: Core checks (Python ${{ matrix.python-version }})
name: py${{ matrix.python-version }}
runs-on: ubuntu-24.04
timeout-minutes: 30
strategy:
@ -79,9 +79,6 @@ jobs:
- name: Proxy startup check
run: $IN_NETNS .venv/bin/python .github/scripts/run_merge_smoke.py proxy-startup --diagnostics-dir "${RUNNER_TEMP}/smoke-diagnostics"
- name: Run curated smoke cases
run: $IN_NETNS .venv/bin/python .github/scripts/run_merge_smoke.py pytest --manifest .github/merge-smoke-tests.json
- name: Upload smoke diagnostics
if: always()
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
@ -93,3 +90,18 @@ jobs:
- name: Remove the network namespace
if: always()
run: sudo ip netns delete smoke
smoke-passed:
name: smoke passed
needs: [dashboard-build, core-checks]
if: always()
runs-on: ubuntu-latest
timeout-minutes: 2
permissions: {}
steps:
- name: Require every smoke job to succeed
env:
NEEDS: ${{ toJSON(needs) }}
run: |
jq -r 'to_entries[] | "\(.key): \(.value.result)"' <<< "$NEEDS"
jq -e 'all(.[]; .result == "success")' <<< "$NEEDS" > /dev/null

11
.github/workflows/test-rust.yml vendored
View file

@ -6,6 +6,8 @@ on:
- "litellm-rust/**"
- "litellm/rust_bridge/**"
- "scripts/generate_trace_types.py"
- "scripts/generate_lens_contract.py"
- "litellm/proxy/lens/**"
- "scripts/trace_codegen/**"
- "tests/test_litellm_rust/**"
- "litellm/integrations/custom_logger.py"
@ -35,6 +37,8 @@ on:
- "litellm-rust/**"
- "litellm/rust_bridge/**"
- "scripts/generate_trace_types.py"
- "scripts/generate_lens_contract.py"
- "litellm/proxy/lens/**"
- "scripts/trace_codegen/**"
- "tests/test_litellm_rust/**"
- "litellm/integrations/custom_logger.py"
@ -94,6 +98,9 @@ jobs:
rust-test:
runs-on: ubuntu-latest
timeout-minutes: 30
# Trim debuginfo to keep the job's target dir within the runner's disk
env:
CARGO_PROFILE_DEV_DEBUG: line-tables-only
defaults:
run:
working-directory: litellm-rust
@ -132,6 +139,10 @@ jobs:
working-directory: .
run: uv run scripts/generate_trace_types.py --check
- name: Check generated Lens contracts
working-directory: .
run: uv run scripts/generate_lens_contract.py --check
- run: cargo nextest run --workspace --locked --features litellm-traces/schema,litellm-traces-clickhouse/schema
- run: cargo test --workspace --doc --locked

103
.github/workflows/test-unit-documentation.yml vendored
View file

@ -1,103 +0,0 @@
name: "Unit Tests: Documentation Validation"
on:
pull_request:
branches:
- main
- "litellm_**"
push:
branches:
- main
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
documentation:
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
contents: read
pull-requests: read
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Detect relevant changes
id: changes
uses: ./.github/actions/detect-changes
- name: Checkout litellm-docs into docs/my-website (for documentation_tests)
if: steps.changes.outputs.decision != 'skip'
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
repository: BerriAI/litellm-docs
path: docs/my-website
persist-credentials: false
- name: Set up Python
if: steps.changes.outputs.decision != 'skip'
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
if: steps.changes.outputs.decision != 'skip'
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Cache uv dependencies
if: steps.changes.outputs.decision != 'skip' && github.ref == 'refs/heads/main'
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: |
~/.cache/uv
.venv
key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
restore-keys: |
${{ runner.os }}-uv-
- name: Cache uv dependencies
if: steps.changes.outputs.decision != 'skip' && github.ref != 'refs/heads/main'
uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: |
~/.cache/uv
.venv
key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
restore-keys: |
${{ runner.os }}-uv-
- name: Cache the Rust build
if: steps.changes.outputs.decision != 'skip'
uses: ./.github/actions/cache-cargo-build
- name: Install dependencies
if: steps.changes.outputs.decision != 'skip'
run: |
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
- name: Cache Prisma binaries
if: steps.changes.outputs.decision != 'skip'
uses: ./.github/actions/cache-prisma-binaries
- name: Generate Prisma client
if: steps.changes.outputs.decision != 'skip'
run: |
uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
# Run the same documentation tests that CircleCI ran (as direct Python scripts)
- name: Run documentation validation tests
if: steps.changes.outputs.decision != 'skip'
run: |
uv run --no-sync python ./tests/documentation_tests/test_env_keys.py
uv run --no-sync python ./tests/documentation_tests/test_router_settings.py
uv run --no-sync python ./tests/documentation_tests/test_api_docs.py
uv run --no-sync python ./tests/documentation_tests/test_circular_imports.py

223
.github/workflows/test-unit-proxy-db.yml vendored
View file

@ -1,223 +0,0 @@
name: "Unit Tests: Proxy DB Operations"
on:
pull_request:
branches:
- main
- "litellm_**"
push:
branches:
- main
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
# Semantic matrix: each shard groups tests by concern (auth, server, logging, …)
# rather than alphabetical letter ranges. Adding a new test file means adding it
# to whichever group it belongs to, not reshuffling slices.
#
# Design targets:
# * Every shard runs in <= 7 minutes of wall-clock on the default runner.
# Most of a shard's time is pytest plugin load + xdist worker imports +
# pytest-cov instrumentation, not the tests themselves. Keeping per-shard
# work low and matching worker count to runner cores is what controls it.
# * `timeout` bounds the pytest step only. Checkout, dependency install, and
# Prisma client generation draw on a separate allowance in the base
# workflow, so slow setup shows up as a slow job rather than as a
# cancelled shard whose tests were passing.
# * workers: 4 matches the 4-core ubuntu-latest runner. -n 8 on 4 cores
# oversubscribes 2x and workers fight for CPU during their cold-start
# imports (measured ~441% CPU for -n 8 locally, i.e. ~55% effective).
# * test_key_generate_prisma.py stays serial (workers=0) — it has event-loop
# conflicts with the logging worker when run in parallel.
# * test_proxy_utils.py runs as a single shard with --dist=worksteal so
# xdist balances its 188 parametrized cases across workers instead of
# pinning the whole file to one worker (the default --dist=loadscope
# behavior for single-file targets).
jobs:
lens-python-310:
name: Lens Python 3.10
permissions:
contents: read
id-token: write
pull-requests: write
uses: ./.github/workflows/_test-unit-base.yml
with:
python-version: "3.10"
test-path: tests/unit/proxy/lens/test_inference.py
workers: 0
reruns: 0
timeout-minutes: 5
artifact-name: lens-python-310
# Fast guard — fails the workflow when a test directory or file inside a sharded
# tree is claimed by no shard. The semantic-shard design has no catch-all bucket,
# so an unassigned child runs nowhere; assert_ci_coverage.py holds the tree list
# and reads the same test-path keys the coverage census does.
assert-shard-coverage:
runs-on: ubuntu-latest
timeout-minutes: 2
permissions:
contents: read
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Assert every test directory and file is claimed by a shard
run: python3 .github/scripts/assert_ci_coverage.py --shards
proxy-db:
needs: assert-shard-coverage
# Display only the semantic shard name in the checks UI instead of GHA's
# default "proxy-db (key-generation, tests/unit/proxy/…, 0, loadscope, 20)"
# which includes every matrix field and gets truncated past the test-path.
name: ${{ matrix.test-group }}
permissions:
contents: read
id-token: write
pull-requests: write
strategy:
fail-fast: false
matrix:
include:
# Must run serially — event-loop conflict with the logging worker.
- test-group: key-generation
test-path: >-
tests/unit/proxy/management_endpoints/test_key_generate_prisma.py
workers: 0
dist: loadscope
timeout: 20
# ---- auth: split into 2 shards ----
- test-group: auth-checks
test-path: >-
tests/unit/proxy/auth/test_auth_checks.py
tests/unit/proxy/auth/test_user_api_key_auth.py
tests/unit/proxy/test_credential_slot_registry.py
tests/unit/proxy/test_deprecated_key_grace_period.py
workers: 4
dist: loadscope
timeout: 15
- test-group: jwt-and-keys
test-path: >-
tests/unit/proxy/auth/test_jwt.py
tests/unit/proxy/management_endpoints/test_jwt_key_mapping.py
tests/unit/proxy/test_proxy_custom_auth.py
workers: 4
dist: loadscope
timeout: 15
# ---- test_proxy_utils.py, single shard, worksteal distribution ----
- test-group: proxy-utils
test-path: >-
tests/unit/proxy/test_proxy_utils.py
workers: 4
dist: worksteal
timeout: 15
# ---- proxy server: split into 2 shards ----
- test-group: proxy-server-core
test-path: >-
tests/proxy_unit_tests/test_proxy_server_gemini_pass_through.py
tests/unit/proxy/test__lazy_features.py
tests/unit/proxy/test_aproxy_startup.py
tests/unit/proxy/test_proxy_server.py
workers: 4
dist: loadscope
timeout: 15
- test-group: proxy-runtime
test-path: >-
tests/unit/proxy/auth/test_multipart_bypass_repro.py
tests/unit/proxy/auth/test_proxy_routes.py
tests/unit/proxy/middleware/test_request_size_limit_middleware.py
tests/unit/proxy/test_proxy_config_unit_test.py
tests/unit/proxy/test_proxy_token_counter.py
tests/unit/proxy/test_server_root_path.py
workers: 4
dist: loadscope
timeout: 15
# ---- logging: split into 2 shards ----
- test-group: custom-logging
test-path: >-
tests/proxy_unit_tests/test_proxy_custom_logger.py
tests/unit/proxy/test_custom_callback_input.py
tests/unit/proxy/test_custom_logger_s3_gcs.py
workers: 4
dist: loadscope
timeout: 15
- test-group: logging-misc
test-path: >-
tests/unit/proxy/management_helpers/test_audit_logs_proxy.py
tests/unit/proxy/spend_tracking/test_search_api_logging.py
tests/unit/proxy/test_proxy_reject_logging.py
workers: 4
dist: loadscope
timeout: 15
- test-group: db-and-spend
test-path: >-
tests/unit/proxy/common_utils/test_proxy_encrypt_decrypt.py
tests/unit/proxy/db/db_transaction_queue/test_e2e_pod_lock_manager.py
tests/unit/proxy/db/test_update_daily_tag_spend.py
tests/unit/proxy/test_db_schema_changes.py
tests/unit/proxy/test_prisma_client_backoff_retry.py
tests/unit/proxy/test_update_spend.py
tests/unit/skills/test_skills_db.py
workers: 4
dist: loadscope
timeout: 15
# ---- guardrails + budget + hooks: split into 2 ----
- test-group: guardrails-hooks
test-path: >-
tests/unit/proxy/hooks/test_banned_keyword_list.py
tests/unit/proxy/test_proxy_setting_guardrails.py
tests/unit/proxy/test_unit_test_proxy_hooks.py
workers: 4
dist: loadscope
timeout: 15
- test-group: budgets
test-path: >-
tests/unit/proxy/auth/test_default_end_user_budget_simple.py
tests/unit/proxy/hooks/test_unit_test_max_model_budget_limiter.py
tests/unit/proxy/test_zero_cost_model_budget_bypass.py
workers: 4
dist: loadscope
timeout: 15
- test-group: endpoints-and-responses
test-path: >-
tests/proxy_unit_tests/test_proxy_exception_mapping.py
tests/unit/proxy/lens
tests/unit/proxy/auth/test_models_fallback_endpoint.py
tests/unit/proxy/common_utils/test_check_batch_cost.py
tests/unit/proxy/common_utils/test_check_responses_cost.py
tests/unit/proxy/common_utils/test_realtime_cache.py
tests/unit/proxy/google_endpoints/test_gemini_agents_endpoints.py
tests/unit/proxy/google_endpoints/test_google_endpoint_routing.py
tests/unit/proxy/google_endpoints/test_google_gemini_proxy_request.py
tests/unit/proxy/public_endpoints/test_blog_posts_endpoint.py
tests/unit/proxy/response_polling
tests/unit/proxy/test_custom_tokenizer_bug.py
tests/unit/proxy/test_get_favicon.py
tests/unit/proxy/test_get_image.py
tests/unit/proxy/test_prompt_test_endpoint.py
tests/unit/proxy/test_reducto_ocr_route.py
tests/unit/proxy/test_response_polling_pre_call_checks.py
tests/unit/proxy/test_ui_path_detection.py
workers: 4
dist: loadscope
timeout: 15
uses: ./.github/workflows/_test-unit-base.yml
with:
test-path: ${{ matrix.test-path }}
workers: ${{ matrix.workers }}
reruns: 2
timeout-minutes: ${{ matrix.timeout }}
dist: ${{ matrix.dist }}
artifact-name: proxy-db-${{ matrix.test-group }}

1009
.github/workflows/test-unit.yml vendored
View file

File diff suppressed because it is too large Load diff

9
AGENTS.md
View file

@ -54,13 +54,13 @@ Don't hesitate to use values in .env to get needed API keys and other secrets, a
Python max line length is 120, not 88
Never edit or commit `ruff-strict-budget.json`, `type-discipline-budget.json`, `basedpyright-code-budget.json`, or `test-quality-budget.json` on a PR branch, and don't run `make lint-budget-update` there. A scheduled Devin automation lowers the limits on the default branch in its own PR by exactly what landed since the last ratchet, so concurrent PRs don't fight over the same `"limit"` lines. Keep the hosted automation's target in sync when the repository default changes. If your branch already carries a budget edit, drop it before opening the PR
The four lint gates (`scripts/ruff_strict_gate.py`, `scripts/type_discipline_gate.py`, `scripts/test_quality_gate.py`, `scripts/type_check_gate.py`) compare each rule's codebase count on your branch against the count at its merge-base with the default branch, and a rule may not grow. There are no budget files to edit or ratchet: when a gate fails, fix the violations the branch introduced or remove at least as many of that rule elsewhere in the tree. The one exception is `reportAny` / `reportExplicitAny`, which share a fixed codebase-wide cap in `ANY_CAPS` in `scripts/type_check_gate.py` because Any spreads past the lines you touch. A branch may add Anys while the total stays under that cap. Only lower `ANY_CAPS`, and only in its own PR to the default branch, never raise it on a feature branch
`make check` (f.k.a. `make pre-commit`, which still works identically as an alias) saves its complete output to a log file in .git (overwriting previous logs) and prints that path as its first and last output lines. To inspect a run, read or grep that log instead of re-running the multi-minute checks just to see a different slice
`make check`, `make lint`, `scripts/pre_commit_lint.sh`, and the standalone budget gates (`scripts/ruff_strict_gate.py`, `scripts/type_discipline_gate.py`, `scripts/type_check_gate.py`) each hold one of 2 machine-wide slots, so when other sessions or worktrees on the same box are already running heavy work, yours prints "all N machine-wide slots are busy; queueing" and then stays quiet until a slot frees. Give the command a long timeout and let it wait rather than killing it, retrying it, or assuming it hung. Don't change the # of machine-wide slots or make it unlimited by setting `LITELLM_GATE_SLOTS=0`
`make check`, `make lint`, `scripts/pre_commit_lint.sh`, and the standalone lint gates (`scripts/ruff_strict_gate.py`, `scripts/type_discipline_gate.py`, `scripts/type_check_gate.py`) each hold one of 2 machine-wide slots, so when other sessions or worktrees on the same box are already running heavy work, yours prints "all N machine-wide slots are busy; queueing" and then stays quiet until a slot frees. Give the command a long timeout and let it wait rather than killing it, retrying it, or assuming it hung. Don't change the # of machine-wide slots or make it unlimited by setting `LITELLM_GATE_SLOTS=0`
If you're trying to create a new function that relies on untyped stuff, instead of adding more Any's and pushing `reportAny` / `reportExplicitAny` closer to their basedpyright ceilings, just validate it in the caller with Pydantic (a model or `TypeAdapter` that returns the typed thing or raises will do) and then pass the now typed variable in
If you're trying to create a new function that relies on untyped stuff, instead of adding more Any's and pushing `reportAny` / `reportExplicitAny` closer to their cap in `ANY_CAPS`, just validate it in the caller with Pydantic (a model or `TypeAdapter` that returns the typed thing or raises will do) and then pass the now typed variable in
If you get an LIT001 fail, refactor the code to follow functional programming best practices rather than introducing mutable data structures. For example, build values in one shot with comprehensions or generators wrapped in `tuple()` / `MappingProxyType()` / `frozenset()` instead of seeding an empty `list`/`dict`/`set` and mutating it over time. Ideally, `# mutable-ok` is never used; reach for it only as a genuine last resort when an immutable rewrite is truly impossible, and always pair it with a real reason
@ -97,6 +97,7 @@ Follow these coding conventions for new/updated code (a three-line fix in a lega
- Annotate every variable with `: Final` (LIT010). Unpacking and walrus targets cannot carry the annotation, so they are implicitly final. Don't rebind them. Never rebind or mutate function parameters (LIT011); `self`/`cls` attribute stores are the exception. If rebinding or in-place mutation is truly unavoidable, suppress with `# rebind-ok: <reason>`
- Qualify every TypedDict field with `ReadOnly[...]` (LIT012), which nests freely with `Required` / `NotRequired` / `Annotated` in any order. If making the key writable is truly unavoidable, suppress with `# writable-ok: <reason>`
- Comprehensions take at most one `for` clause and one `if` clause (LIT014); split stacked clauses into a helper generator, a named intermediate, or a plain loop. Suppress with `# comprehension-ok: <reason>` only when unavoidable
- Every pydantic model must be frozen (LIT015), set via `model_config = ConfigDict(frozen=True)`, a dict-literal `model_config`, an inner `class Config`, or the class keywords. Subclasses inherit it unless they override it. Replace in-place field writes with `model_copy(update=...)`. If making the model mutable is truly unavoidable, suppress with `# frozen-ok: <reason>` on the `class` line
- Use dependency injection
- Fully typed; no `Any` or coarse types like `dict[str, Any]` or just `dict`. Every function parameter must be strongly typed
- Use tagged unions + match
@ -129,4 +130,4 @@ Before implementing:
Ask yourself: "Would a senior engineer say this is overcomplicated?" If yes, simplify
Before requesting maintainer review, verify the current PR tip passes required CI and code coverage, meets Greptile confidence of at least 4/5, and has acceptable Veria and Bugbot reviews. Inspect warnings and findings, fix actionable issues, and rerun the affected checks and reviewers after changes. Record evidence for any false positive or unavailable review; never treat a pending or missing bot result as a pass. Do not lower coverage thresholds or lint budgets to satisfy a check
Before requesting maintainer review, verify the current PR tip passes required CI and code coverage, meets Greptile confidence of at least 4/5, and has acceptable Veria and Bugbot reviews. Inspect warnings and findings, fix actionable issues, and rerun the affected checks and reviewers after changes. Record evidence for any false positive or unavailable review; never treat a pending or missing bot result as a pass. Do not lower coverage thresholds or raise `ANY_CAPS` to satisfy a check

50
Makefile
View file

@ -6,9 +6,8 @@
test-proxy-unit-a test-proxy-unit-b test-integration test-unit-helm \
test-rust-extension rust-sqlx-prepare lens-dev \
info lint lint-inner lint-dev lint-checks format \
lint-basedpyright lint-e2e-basedpyright lint-basedpyright-budget-update lint-type-discipline lint-type-discipline-budget-update \
lint-ruff-budget lint-ruff-budget-update lint-budget-update lint-gate \
lint-test-quality lint-test-quality-budget-update \
lint-basedpyright lint-e2e-basedpyright lint-type-discipline \
lint-ruff-strict lint-gate lint-test-quality \
install-dev install-proxy-dev install-test-deps install-hooks \
install-helm-unittest check-circular-imports check-import-safety check check-inner pre-commit \
lint-install lint-fetch-base bootstrap
@ -31,14 +30,11 @@ help:
@echo " make lint - Run all linting (Ruff, basedpyright, format check, circular imports, import safety)"
@echo " make lint-ruff - Run Ruff linting only"
@echo " make lint-basedpyright - Run basedpyright strict, gated by per-rule error counts"
@echo " make lint-e2e-basedpyright - Run basedpyright over tests/e2e (zero errors allowed)"
@echo " make lint-basedpyright-budget-update - Ratchet basedpyright limits down by what this branch fixed"
@echo " make lint-e2e-basedpyright - Run basedpyright over tests/e2e and tests/e2e_harness (zero errors allowed)"
@echo " make lint-format - Check ruff format formatting (matches CI)"
@echo " make lint-ruff-budget - Gate the codebase total of each strict ruff rule against its limit"
@echo " make lint-ruff-strict - Gate each strict ruff rule's codebase total against its merge-base count"
@echo " make lint-gate - Strict ruff gate in CI-parity mode (fetches the default branch, simulates the merge)"
@echo " make lint-ruff-budget-update - Ratchet ruff-strict-budget.json limits down by what this branch fixed"
@echo " make lint-test-quality - Gate the test suite against test-quality-budget.json"
@echo " make lint-budget-update - Ratchet all budgets down (ruff + type-discipline + test quality + basedpyright)"
@echo " make lint-test-quality - Gate the test suite's TQ counts against their merge-base counts"
@echo " make check-circular-imports - Check for circular imports"
@echo " make check-import-safety - Check import safety"
@echo " make test - Run all tests"
@ -141,9 +137,9 @@ format-check: install-dev
lint-fetch-base:
@$(RESOLVE_BASE)
# Mirror test-linting.yml's lint job environment: the proxy-dev group plus a generated
# Mirror test-linting.yml's python job environment: the proxy-dev group plus a generated
# Prisma client, so `basedpyright tests/e2e` resolves the same modules CI does. The
# budget gate itself no longer measures here (scripts/type_check_gate.py provisions its
# basedpyright gate itself no longer measures here (scripts/type_check_gate.py provisions its
# own .venv-typecheck). --inexact tops up the venv instead of pruning the proxy extras
# gen:api and the running proxy need.
lint-install:
@ -211,27 +207,22 @@ lint-basedpyright: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
$(UV_RUN) python scripts/type_check_gate.py --base "$(BASE_REF)"
lint-e2e-basedpyright: $(LINT_E2E_DEP_INSTALL)
$(UV_RUN) basedpyright tests/e2e
$(UV_RUN) basedpyright tests/e2e tests/e2e_harness
# Type-discipline budget (mutable collections / casts / type guards / kwargs /
# Type-discipline gate (mutable collections / casts / type guards / kwargs /
# unexplained suppressions), the test-linting.yml step `make lint` used to omit.
lint-type-discipline: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
$(UV_RUN) python scripts/type_discipline_gate.py --base "$(BASE_REF)"
# Test-quality budget (zero-assert / mock-echo tests, sys.path.insert, raw env writes,
# Test-quality gate (zero-assert / mock-echo tests, sys.path.insert, raw env writes,
# litellm module-global mutation, credential-gated skips, conftest snapshot
# inventory), counted across tests/ the same delta-vs-base way.
lint-test-quality: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
$(UV_RUN) python scripts/test_quality_gate.py --base "$(BASE_REF)"
# --update lowers each limit by what this branch fixed since its branch point, so
# it needs the base ref fetched to resolve the merge-base.
lint-basedpyright-budget-update: install-dev
$(UV_RUN) python scripts/type_check_gate.py --update --base "$(BASE_REF)"
lint-format: format-check
lint-ruff-budget: install-dev
lint-ruff-strict: install-dev
$(UV_RUN) python scripts/ruff_strict_gate.py --base "$(BASE_REF)"
# Strict gate, invoked the same way CI does in test-linting.yml so a local pass
@ -239,28 +230,16 @@ lint-ruff-budget: install-dev
lint-gate: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
$(UV_RUN) python scripts/ruff_strict_gate.py --base "$(BASE_REF)"
lint-ruff-budget-update: install-dev
$(UV_RUN) python scripts/ruff_strict_gate.py --update --base "$(BASE_REF)"
lint-type-discipline-budget-update: install-dev
$(UV_RUN) python scripts/type_discipline_gate.py --update --base "$(BASE_REF)"
lint-test-quality-budget-update: install-dev
$(UV_RUN) python scripts/test_quality_gate.py --update --base "$(BASE_REF)"
# Ratchet all budgets in one shot (ruff strict + type-discipline + test quality + basedpyright)
lint-budget-update: lint-ruff-budget-update lint-type-discipline-budget-update lint-test-quality-budget-update lint-basedpyright-budget-update
check-circular-imports: $(LINT_DEP_INSTALL)
cd litellm && $(UV_RUN) python ../tests/documentation_tests/test_circular_imports.py && cd ..
check-import-safety: $(LINT_DEP_INSTALL)
@$(UV_RUN) python -c "from litellm import *; print('[from litellm import *] OK! no issues!');" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
# Combined linting, isomorphic to test-linting.yml's lint job so a local pass means a
# Combined linting, isomorphic to test-linting.yml's python job so a local pass means a
# green CI lint: it installs the same env (proxy-dev + generated Prisma client) and then
# runs the diff-scoped ruff format check, whole-tree ruff check, the strict-rule /
# type-discipline / basedpyright budgets as a delta vs the base, then the circular-import
# type-discipline / basedpyright gates as a delta vs the base, then the circular-import
# and import-safety checks. Steps that compare against the base resolve it the same way CI
# does (merge-base with origin's current default branch). Setup (env sync, Prisma client,
# base fetch) runs once up front; the checks themselves are independent, so a sub-make
@ -281,8 +260,7 @@ lint-dev: lint-format-changed check-circular-imports check-import-safety
# is staged (warning about changed files left unstaged); with nothing staged it falls
# back to the working tree's diff against the merge base with the base branch, so a
# fresh merge commit or an unstaged working tree still gets checked. Mirrors
# test-linting.yml (Python), test-litellm-ui-build.yml's frontend-lint (dashboard), and
# check-ui-api-types.yml (API-type drift), skipping any whose files aren't in scope.
# test-linting.yml (Python, UI, and API types), skipping any whose files aren't in scope.
# Not auto-installed as a git hook so it never slows an unrelated human commit.
check:
@$(GATE_SLOT_LOCK) $(MAKE) check-inner

20
README.md
View file

@ -90,6 +90,24 @@ Managing LLM calls across providers gets complicated fast — different SDKs, au
uv add litellm
```
An independent `litellm-core` distribution provides the Python SDK with the same
`import litellm` API and runtime dependencies. It has no optional extras, CLI entry
points, or bundled dashboard. Install one SDK distribution per environment because
`litellm` and `litellm-core` own overlapping Python files. Use `litellm` for the
proxy, CLI, and optional extras
Build and install core from this checkout while its release integration is pending:
```shell
python scripts/build_core_distribution.py --out-dir dist/core
python -m pip install dist/core/litellm_core-*.whl
```
The builder requires Git, `uv` and the Rust build toolchain. It reads the release version
from the root `pyproject.toml`, leaves source files unchanged, and produces a wheel
and a self-contained sdist in `dist/core`. Run installation in a fresh environment
without `litellm`
```python
from litellm import completion
import os
@ -401,7 +419,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th
| [Vercel AI Gateway (`vercel_ai_gateway`)](https://docs.litellm.ai/docs/providers/vercel_ai_gateway) | ✅ | ✅ | ✅ | | | | | | | |
| [VLLM (`vllm`)](https://docs.litellm.ai/docs/providers/vllm) | ✅ | ✅ | ✅ | | | | | | | |
| [Volcengine (`volcengine`)](https://docs.litellm.ai/docs/providers/volcano) | ✅ | ✅ | ✅ | | | | | | | |
| [Voyage AI (`voyage`)](https://docs.litellm.ai/docs/providers/voyage) | | | | ✅ | | | | | | |
| [VoyageAI by MongoDB (`voyage`)](https://docs.litellm.ai/docs/providers/voyage) | | | | ✅ | | | | | | |
| [WandB Inference (`wandb`)](https://docs.litellm.ai/docs/providers/wandb_inference) | ✅ | ✅ | ✅ | | | | | | | |
| [Watsonx Text (`watsonx_text`)](https://docs.litellm.ai/docs/providers/watsonx) | ✅ | ✅ | ✅ | | | | | | | |
| [xAI (`xai`)](https://docs.litellm.ai/docs/providers/xai) | ✅ | ✅ | ✅ | | | | | | | |

1
backend/routes/allowlist.py
View file

@ -23,6 +23,7 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = (
"/end_user/",
"/sso/",
"/liteadmin/slack/connect/",
"/moyai/connect/",
"/login",
"/v2/login",
"/v3/login",

146
basedpyright-code-budget.json
View file

@ -1,146 +0,0 @@
{
"reportAny": {
"limit": 13429
},
"reportArgumentType": {
"limit": 2198
},
"reportAssignmentType": {
"limit": 319
},
"reportAttributeAccessIssue": {
"limit": 480
},
"reportCallIssue": {
"limit": 112
},
"reportConstantRedefinition": {
"limit": 40
},
"reportDeprecated": {
"limit": 209
},
"reportDuplicateImport": {
"limit": 19
},
"reportExplicitAny": {
"limit": 3369
},
"reportFunctionMemberAccess": {
"limit": 7
},
"reportGeneralTypeIssues": {
"limit": 101
},
"reportIncompatibleMethodOverride": {
"limit": 56
},
"reportIncompatibleVariableOverride": {
"limit": 8
},
"reportInconsistentOverload": {
"limit": 12
},
"reportIndexIssue": {
"limit": 24
},
"reportInvalidTypeForm": {
"limit": 30
},
"reportInvalidTypeVarUse": {
"limit": 1
},
"reportMatchNotExhaustive": {
"limit": 0
},
"reportMissingParameterType": {
"limit": 5570
},
"reportMissingTypeArgument": {
"limit": 15281
},
"reportMissingTypeStubs": {
"limit": 40
},
"reportOperatorIssue": {
"limit": 0
},
"reportOptionalCall": {
"limit": 0
},
"reportOptionalIterable": {
"limit": 0
},
"reportOptionalMemberAccess": {
"limit": 0
},
"reportOptionalOperand": {
"limit": 0
},
"reportOptionalSubscript": {
"limit": 0
},
"reportPossiblyUnboundVariable": {
"limit": 56
},
"reportPrivateUsage": {
"limit": 1804
},
"reportRedeclaration": {
"limit": 8
},
"reportReturnType": {
"limit": 180
},
"reportTypedDictNotRequiredAccess": {
"limit": 22
},
"reportUndefinedVariable": {
"limit": 0
},
"reportUnknownArgumentType": {
"limit": 44802
},
"reportUnknownLambdaType": {
"limit": 109
},
"reportUnknownMemberType": {
"limit": 38269
},
"reportUnknownParameterType": {
"limit": 19584
},
"reportUnknownVariableType": {
"limit": 29814
},
"reportUnnecessaryCast": {
"limit": 110
},
"reportUnnecessaryComparison": {
"limit": 687
},
"reportUnnecessaryContains": {
"limit": 4
},
"reportUnnecessaryIsInstance": {
"limit": 816
},
"reportUntypedBaseClass": {
"limit": 0
},
"reportUntypedFunctionDecorator": {
"limit": 27
},
"reportUnusedClass": {
"limit": 21
},
"reportUnusedFunction": {
"limit": 136
},
"reportUnusedImport": {
"limit": 542
},
"reportUnusedVariable": {
"limit": 137
}
}

72
codecov.yaml
View file

@ -6,7 +6,7 @@ codecov:
ignore:
- "litellm-rust/**"
# Uploads are flagged per workflow/shard (GHA) or "circleci". carryforward makes
# Uploads are flagged per workflow tier (GHA) or "circleci". carryforward makes
# a re-upload of a flag replace its prior session instead of accumulating a
# conflicting one, and lets a commit reuse a flag from its parent when that flag
# was not re-uploaded. Required because the same commit can receive the
@ -27,6 +27,76 @@ flag_management:
carryforward: false
- name: circleci
carryforward: false
- name: core-utils
carryforward: false
- name: enterprise-routing
carryforward: false
- name: integrations
carryforward: false
- name: llm-vertex-ai
carryforward: false
- name: llm-other-providers
carryforward: false
- name: llm-openai-meta
carryforward: false
- name: misc
carryforward: false
- name: misc-dirs
carryforward: false
- name: proxy-auth
carryforward: false
- name: proxy-hooks-client
carryforward: false
- name: proxy-endpoints
carryforward: false
- name: proxy-feature-endpoints
carryforward: false
- name: proxy-server
carryforward: false
- name: mcp-elicitation
carryforward: false
- name: proxy-infra
carryforward: false
- name: proxy-infra-root
carryforward: false
- name: caching-local
carryforward: false
- name: proxy-extras
carryforward: false
- name: enterprise-package
carryforward: false
- name: enterprise-managed-files
carryforward: false
- name: responses-caching-types
carryforward: false
- name: lens-python-310
carryforward: false
- name: proxy-db-key-generation
carryforward: false
- name: proxy-db-auth-checks
carryforward: false
- name: proxy-db-jwt-and-keys
carryforward: false
- name: proxy-db-proxy-utils
carryforward: false
- name: proxy-db-proxy-server-core
carryforward: false
- name: proxy-db-proxy-runtime
carryforward: false
- name: proxy-db-mcp-oauth
carryforward: false
- name: proxy-db-custom-logging
carryforward: false
- name: proxy-db-logging-misc
carryforward: false
- name: proxy-db-db-and-spend
carryforward: false
- name: proxy-db-guardrails-hooks
carryforward: false
- name: proxy-db-budgets
carryforward: false
- name: proxy-db-endpoints-and-responses
carryforward: false
component_management:
individual_components:

127
cookbook/litellm_proxy_server/grafana_dashboard/dashboard_all_metrics/grafana_dashboard.json
View file

@ -6436,6 +6436,133 @@
],
"title": "litellm_zero_cost_requests rate",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 438
},
"id": 112,
"panels": [],
"title": "Project model rate limits",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "Configured rate limit for the Project on the requested model in the current window, by rate_limit_type",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"showPoints": "never",
"spanNulls": false
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 439
},
"id": 113,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "max by (project_id, project_alias, requested_model, rate_limit_type) (litellm_project_model_rate_limit_allowed_metric)",
"legendFormat": "{{project_alias}} ({{project_id}}) / {{requested_model}} / {{rate_limit_type}}",
"range": true,
"refId": "A"
}
],
"title": "litellm_project_model_rate_limit_allowed_metric",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "Requests or tokens the Project has consumed on the requested model in the current rate limit window, by rate_limit_type",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"showPoints": "never",
"spanNulls": false
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 439
},
"id": 114,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "max by (project_id, project_alias, requested_model, rate_limit_type) (litellm_project_model_rate_limit_used_metric)",
"legendFormat": "{{project_alias}} ({{project_id}}) / {{requested_model}} / {{rate_limit_type}}",
"range": true,
"refId": "A"
}
],
"title": "litellm_project_model_rate_limit_used_metric",
"type": "timeseries"
}
],
"preload": false,

51
deploy/lens/Dockerfile
View file

@ -1,36 +1,41 @@
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
FROM $UV_IMAGE AS uvbin
FROM $LITELLM_BUILD_IMAGE AS builder
COPY --from=uvbin /uv /usr/local/bin/uv
RUN apk add --no-cache python-3.13 build-base libseccomp-dev
ENV UV_PYTHON_DOWNLOADS=0 UV_LINK_MODE=copy
WORKDIR /app
COPY deploy/lens/requirements.lock /tmp/requirements.lock
RUN uv venv --python python3.13 /app/.venv && \
uv pip sync --python /app/.venv/bin/python --require-hashes --only-binary :all: /tmp/requirements.lock
RUN apk add --no-cache rust build-base cmake perl pkgconf openssl-dev libseccomp-dev python-3.13
WORKDIR /src
COPY .cargo/ .cargo/
COPY litellm-rust/ litellm-rust/
COPY litellm/proxy/lens/prompts/ litellm/proxy/lens/prompts/
WORKDIR /src/litellm-rust
ENV CARGO_PROFILE_RELEASE_DEBUG=0 CARGO_PROFILE_RELEASE_STRIP=symbols
RUN cargo build --locked --release -p litellm-lens
COPY deploy/lens/python_policy.c /tmp/python_policy.c
RUN cc -std=c11 -D_GNU_SOURCE -O2 -Wall -Wextra -Werror /tmp/python_policy.c -lseccomp -o /tmp/python-policy && \
/tmp/python-policy /app/python.seccomp
/tmp/python-policy /tmp/python.seccomp
FROM $LITELLM_RUNTIME_IMAGE AS runtime
FROM builder AS test-builder
RUN cargo test --locked --release -p litellm-lens --test sandbox --no-run --message-format=json > /tmp/test-artifacts.json && \
python3.13 -c 'import json, pathlib, shutil; rows = [json.loads(line) for line in pathlib.Path("/tmp/test-artifacts.json").read_text().splitlines()]; artifact, = [r["executable"] for r in rows if r.get("executable") and r["target"]["name"] == "sandbox"]; shutil.copyfile(artifact, "/tmp/lens-sandbox-tests")' && \
chmod 755 /tmp/lens-sandbox-tests
FROM $LITELLM_RUNTIME_IMAGE AS service
ARG LITELLM_RELEASE_TAG=""
RUN : "${LITELLM_RELEASE_TAG:?Pass --build-arg LITELLM_RELEASE_TAG matching the gateway}"
RUN apk add --no-cache python-3.13 setpriv
ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG} \
PATH="/app/.venv/bin:${PATH}" \
PYTHONDONTWRITEBYTECODE=1
RUN apk add --no-cache python-3.13 setpriv libgcc libstdc++ openssl ca-certificates
ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG} PYTHONDONTWRITEBYTECODE=1
WORKDIR /app
COPY --from=builder /app/.venv /app/.venv
COPY litellm/proxy/lens/__init__.py litellm/proxy/lens/models.py litellm/proxy/lens/trace_store.py litellm/proxy/lens/analysis.py litellm/proxy/lens/worker.py litellm/proxy/lens/release.py /app/lens/
COPY litellm/proxy/lens/context_pipeline.py litellm/proxy/lens/agent_review.py litellm/proxy/lens/agent_runtime.py litellm/proxy/lens/agent_workspace.py litellm/proxy/lens/python_tool.py litellm/proxy/lens/activity.py litellm/proxy/lens/agent_context.py /app/lens/
COPY litellm/proxy/lens/reviews.py litellm/proxy/lens/reconciliation.py /app/lens/
COPY litellm/proxy/lens/prompts/ /app/lens/prompts/
COPY --from=builder /app/python.seccomp /app/lens/python.seccomp
COPY --from=builder /src/litellm-rust/target/release/litellm-lens /usr/local/bin/litellm-lens
COPY --from=builder /tmp/python.seccomp /app/lens/python.seccomp
COPY deploy/lens/python_runtime.py /tmp/python_runtime.py
RUN python3.13 -S /tmp/python_runtime.py /app/lens/python-runtime.json && rm /tmp/python_runtime.py
USER 65532:65532
CMD ["python", "-m", "lens.worker"]
EXPOSE 4318
ENTRYPOINT ["/usr/local/bin/litellm-lens"]
FROM service AS smoke
COPY --from=test-builder /tmp/lens-sandbox-tests /usr/local/bin/lens-sandbox-tests
ENTRYPOINT ["/usr/local/bin/lens-sandbox-tests"]
CMD ["--ignored", "--nocapture", "--test-threads=1"]
FROM service AS runtime

7
deploy/lens/Dockerfile.dockerignore
View file

@ -1,12 +1,15 @@
**
!deploy/
!deploy/lens/
!deploy/lens/requirements.lock
!deploy/lens/python_policy.c
!deploy/lens/python_runtime.py
!litellm/
!litellm/proxy/
!litellm/proxy/lens/
!litellm/proxy/lens/*.py
!litellm/proxy/lens/prompts/
!litellm/proxy/lens/prompts/**
!.cargo/
!.cargo/**
!litellm-rust/
!litellm-rust/**
litellm-rust/target/

178
deploy/lens/README.md
View file

@ -1,112 +1,121 @@
# Lens worker
# Lens service
Lens reviews recorded activity and saves evidence-linked findings in the LiteLLM dashboard under Observability, Lens (`/ui/lens/`)
Lens records agent activity and investigates it in a separate Rust service. LiteLLM serves model requests, the dashboard, and investigation settings. Lens owns trace ingestion and ClickHouse access; PostgreSQL stays with LiteLLM
## Install
Agent exporters send traces directly to Lens. LiteLLM sends its optional request logs through a bounded background queue. If Lens or ClickHouse is unavailable, model requests continue; traces can be delayed or dropped according to the exporter's retry policy. The gateway never waits for ClickHouse during startup or inference
Build LiteLLM and its worker from the same source commit with the same release identity. The worker runs separately and connects to your gateway using a limited worker token
## New local installation
### New local installation
Install Docker with Compose and Git. This builds LiteLLM and its worker from the same checkout and starts the existing local tracing stack:
Install Docker with Compose, Python 3.10 or later, and Git. Clone LiteLLM, select a published release that includes Lens, and start the existing Compose stack:
```bash
git clone https://github.com/BerriAI/litellm.git
cd litellm
export LITELLM_RELEASE_TAG="sha-$(git rev-parse HEAD)"
export LENS_WORKER_IMAGE="litellm-lens-worker:${LITELLM_RELEASE_TAG}"
export OPENAI_API_KEY='sk-...'
docker build --build-arg LITELLM_RELEASE_TAG="$LITELLM_RELEASE_TAG" \
-f deploy/lens/Dockerfile -t "$LENS_WORKER_IMAGE" .
docker compose -f docker/docker-compose.tracing.yml up -d --build
python3 deploy/lens/configure.py --version <release-version>
docker compose --env-file deploy/lens/.env -f deploy/lens/stack.yaml up -d --wait
```
Open `http://localhost:4002/ui/` and sign in as `admin` with the key saved in `.lens-dev/master_key`. Go to **Lens > Investigations > Connect worker**, choose a model and monthly budget, then **Get install command**. Expand **Using Docker Compose or Helm?** and copy the worker token. In the same terminal, run:
The configuration command generates your keys and database passwords once, saves them in `deploy/lens/.env` with owner-only permissions, and preserves them on subsequent runs. Back up this file alongside your database volumes. Both images use the selected release; there is no local image build
```bash
export LITELLM_URL=http://litellm:4000
export LENS_WORKER_TOKEN='<paste-your-worker-token>'
docker compose -f docker/docker-compose.tracing.yml -f deploy/lens/compose.yaml up -d
```
Open `http://localhost:4000/ui/` and sign in as `admin` using `LITELLM_MASTER_KEY` from the saved file. Open **Lens**, select your framework, generate a tracing key, and copy the displayed configuration. The trace endpoint is already filled in. Keep your agent's existing model credentials; the tracing key only authorizes trace uploads
The worker joins the gateway's Docker network, and the dashboard shows **Worker connected**. Save the token privately for restarts and upgrades
PostgreSQL and ClickHouse use persistent Docker volumes and have no host ports. The dashboard and trace listener bind to localhost. Use your normal TLS and ingress for a hosted deployment. Stop the stack with `docker compose --env-file deploy/lens/.env -f deploy/lens/stack.yaml down`; omit `-v` to retain data
This stack is for local evaluation: it binds to localhost and uses development database credentials. For a hosted deployment, keep your normal database, keys, networking, and deployment process. Build both images from one source revision with the same `LITELLM_RELEASE_TAG`, publish the worker to your registry, and set `LENS_WORKER_IMAGE` on LiteLLM to that image
Under **Lens > Investigations > Connect worker**, choose an analysis model and monthly budget. The deployed service connects automatically after you save these settings. There is no worker command or second token to copy
### Existing LiteLLM installation
## Existing LiteLLM installation
Keep your deployment and PostgreSQL database. A working gateway/worker pair can stay as it is until you upgrade both. For a gateway built from source, use its exact commit and `LITELLM_RELEASE_TAG`; a release version or the latest commit on `main` is not a substitute for that source identity
Keep your gateway, PostgreSQL database, deployment tool, and existing encryption keys. Deploy the matching Lens image, give it access to ClickHouse, and configure the service connection on LiteLLM
The public development package is `ghcr.io/berriai/litellm-lens-worker-dev:sha-<full-commit>`. It publishes amd64 images on Lens-related changes, so an arbitrary source commit may have no image. Check the exact image exists before using it. If it is unavailable, your gateway uses a different release identity, or you need native arm64, build the worker from the gateway's checkout:
| Variable | LiteLLM | Lens service |
| --- | --- | --- |
| `LITELLM_LENS_SERVICE_TOKEN` | Same private random secret, at least 32 characters | Same secret |
| `LITELLM_LENS_URL` | Internal Lens URL, such as `http://lens-worker:4318` | Not needed |
| `LITELLM_LENS_PUBLIC_URL` | Ingestion base URL reachable by your agents | Not needed |
| `LITELLM_URL` | Not needed | LiteLLM URL reachable from Lens |
| `CLICKHOUSE_URL` | Remove it from Lens tracing configuration | ClickHouse HTTP URL with credentials |
| `CLICKHOUSE_DATABASE` | Not needed for Lens | Existing database name, defaults to `litellm` |
| `AGENT_TRACING_RETENTION_DAYS` | Not needed for Lens | Retention for traces and Lens request logs, defaults to `14` |
Remove the old `general_settings.tracing.store` configuration used for Lens from LiteLLM. Keep unrelated logging integrations and their configuration. Only Lens should reach its ClickHouse database. The shared service secret is an infrastructure credential: keep it out of browser code, agent exporters, screenshots, and public ingress headers
Expose the Lens HTTP listener on port 4318 through TLS. Route `/lens-ingest` on your existing hostname directly to Lens at the load balancer, then set `LITELLM_LENS_PUBLIC_URL=https://<your-host>/lens-ingest`. The gateway must not proxy these uploads. Alternatively use a separate hostname and forward `/v1/` to Lens. Keep `/internal/` private; it requires the service secret
### Standalone Docker or a container host
Build from the same source commit and `LITELLM_RELEASE_TAG` as your running gateway:
```bash
export LITELLM_RELEASE_TAG='<gateway-release-identity>'
export LENS_WORKER_IMAGE='<your-registry>/litellm-lens-worker:<your-image-tag>'
export LENS_WORKER_IMAGE='<your-registry>/litellm-lens-worker:<image-tag>'
docker build --build-arg LITELLM_RELEASE_TAG="$LITELLM_RELEASE_TAG" \
-f deploy/lens/Dockerfile -t "$LENS_WORKER_IMAGE" .
```
For a remote worker host, publish that image to a registry the host can pull from. Set the gateway's `LENS_WORKER_IMAGE` to the resulting image reference, restart the gateway using its normal deployment process, then copy its install command. Prefer the published image digest for hosted installations. Do not change the gateway's release identity just to accept another worker
Publish that image to a registry your host can pull from. Prefer a digest reference for hosted deployments. Public development images use `ghcr.io/berriai/litellm-lens-worker-dev:sha-<full-commit>`; check that the exact image exists before selecting it. An arbitrary commit may not have a published image
For Kubernetes or Render, run the standalone worker using `LITELLM_URL` and `LENS_WORKER_TOKEN` from setup. Keep existing databases and secrets. The worker needs no inbound port.
The image supports native amd64 and arm64. For worker-only Compose, use `deploy/lens/compose.yaml` with a private environment file containing `LENS_WORKER_IMAGE`, `LITELLM_URL`, `LITELLM_LENS_SERVICE_TOKEN`, and `CLICKHOUSE_URL`:
## Helm
```bash
docker compose --env-file /path/to/private/lens.env \
-f deploy/lens/compose.yaml up -d
```
The componentized source chart at `helm/litellm` includes an optional Lens worker. Use the chart from the same checkout as your gateway and keep your component image overrides in your values. Configure PostgreSQL and ClickHouse as usual, install the chart, then obtain a limited worker token from Lens setup. Store it in a Kubernetes Secret and enable the worker in your values:
The Compose listener binds to localhost. Your reverse proxy must reach it. On Render, run Lens as a web service with the same environment and listener port 4318, not an outbound-only background worker. Use `/health/live` for process health and `/health/ready` to check storage and tracing credentials
Lens does not need provider credentials, PostgreSQL credentials, a GPU, or the LiteLLM Python package. The image includes a small CPython runtime only for the investigator's confined calculation tool. Keep the shipped security settings, temporary filesystem, and resource limits
### Kubernetes with Helm
Both `helm/litellm` and `helm/litellm-helm` support Lens. Keep your existing chart, release name, namespace, and values. Add:
```yaml
lensWorker:
enabled: true
image:
repository: <your-worker-image-repository>
digest: sha256:<matching-worker-image-digest>
tokenSecret:
name: litellm-lens-worker
key: token
```
Set the worker repository and digest explicitly to an image built from the gateway's source commit and release identity. The chart connects the worker to the backend service. Keep these values and the Secret when upgrading the chart and update the gateway and worker image overrides together. `lensWorker.replicaCount` controls simultaneous investigations. To use a private registry or external proxy, set `lensWorker.image.repository`, `lensWorker.image.digest` (or `tag` for a source build), and `lensWorker.url`. A digest takes precedence over the tag. The dashboard uses the chart's worker image for standalone install commands too
Then run your usual Helm deployment command using the matching published chart. The chart supplies the matching Lens image, generates the shared service secret, starts a single ClickHouse instance with a persistent volume, and connects the services. Your cluster needs a default storage class, or set `lensWorker.clickhouse.storageClassName`. Bundled storage defaults to 20 GiB; set `lensWorker.clickhouse.storage` before installation to choose another size
## Standalone worker
Start with a source deployment that includes Lens, PostgreSQL, and agent tracing, and prepare its matching worker as described above. Configure one ClickHouse URL for trace writes, bounded reads, and Lens queries:
When your chart manages an ingress with one hostname, the chart fills in the public tracing address and routes `/lens-ingest` directly to Lens. TLS is detected from `ingress.tls` or an ALB certificate annotation. With custom ingress, multiple hostnames, or TLS terminated elsewhere, set the address explicitly:
```yaml
general_settings:
tracing:
store:
type: clickhouse
url: os.environ/CLICKHOUSE_URL
retention_days: 14
lensWorker:
enabled: true
publicUrl: https://<your-litellm-host>/lens-ingest
```
The URL, database, and retention settings can also come from `CLICKHOUSE_URL`, `CLICKHOUSE_DATABASE`, and `AGENT_TRACING_RETENTION_DAYS` when omitted from YAML. A YAML value wins when both are set. The database defaults to `litellm`. `retention_days` defaults to 14 and applies to both traces and spend logs
For a dedicated trace hostname, configure `lensWorker.ingress.enabled`, `host`, `className`, and `tls`. Its hostname supplies the public address unless you override `publicUrl`. Internal Lens routes stay private
Retention changes require a proxy restart. ClickHouse removes expired rows during background merges, not immediately at startup. Enable request/response logging to analyze LLM requests. Lens can only inspect content you actually retain
To use an existing ClickHouse database and secrets managed by your platform, keep these overrides:
In **Lens > Investigations**, click **Connect worker**, choose an analysis model and monthly limit, then **Get install command**. Use **Advanced options** to select an existing virtual key or change the proxy URL if the server running Docker needs a different network address. Copy the command and run it on your server. The dashboard shows **Worker connected** when the container checks in
The command already contains the compatible worker image and one worker token. The selected virtual key stays on the proxy; its secret is never sent to the worker. Once the matching image is available on the worker host, no second LiteLLM deployment is needed. Keep the command private because it includes the token. The LiteLLM release provides the dashboard and APIs; the container only runs background analysis
The dashboard uses the gateway's `LENS_WORKER_IMAGE` override when set. Public `:sha-<commit>` development images must match both the gateway commit and release identity. Build from source for the worker host's native architecture
After upgrading the gateway, update the worker image and redeploy it while keeping its proxy URL and token. Existing containers do not update automatically. If an investigation reports a worker compatibility error, update the image before retrying
For deployments managed with Compose, download `compose.yaml` and provide `LITELLM_URL`, `LENS_WORKER_TOKEN`, and an explicit `LENS_WORKER_IMAGE` in a private environment file:
```bash
docker compose --env-file /path/to/lens.env -f compose.yaml up -d
```yaml
lensWorker:
enabled: true
serviceTokenSecret:
name: litellm-lens-service
key: service-token
clickhouseSecret:
name: litellm-lens-clickhouse
key: url
clickhouseDatabase: litellm
retentionDays: 14
```
To work on Lens itself, `make lens-dev` runs the proxy, a worker from source and the hot-reload dashboard together; set `LENS_DEV_PROXY_PORT` / `LENS_DEV_UI_PORT` to move them off 4000/3000. For a local container build, set `LENS_WORKER_IMAGE=litellm-lens-worker:local` and `LITELLM_RELEASE_TAG` to the gateway's release tag, then use `docker compose -f deploy/lens/compose.yaml -f deploy/lens/compose.build.yaml up -d --build`
Supplying `clickhouseSecret.name` uses that database and disables bundled storage. Keep your database name and retention policy. For GitOps tools that render Helm without cluster access, supply both existing secrets so rendering cannot regenerate credentials
The generated command gives the worker 1 GiB of temporary memory-backed storage, shared across parallel reviews. Change `size=1g` in the Docker command or set `LENS_WORKER_TMP_SIZE` with Compose to fit your server and workload. Python reports storage failures to the reviewer and cleans up temporary files, so the reviewer can retry a smaller computation or report insufficient evidence. The worker remains available for other scans. Existing workers must be recreated with the new image and mount options
Normal Helm upgrades reuse the generated credentials. Secrets are retained on uninstall, and the ClickHouse volume is retained by Kubernetes. Back them up together. Treat changing the database, storage class, or secret reference as an infrastructure change, not a routine version update
The worker needs outbound HTTPS access to LiteLLM. It needs no inbound ports, provider keys, direct database access, or GPU. The proxy calls your selected model through its normal virtual-key authorization and inference pipeline; trace content reaches that model provider. Use a model with JSON output support and known token prices. One worker handles up to three investigations concurrently and can serve multiple lenses. For more throughput, start another worker with a separate credential
After deployment, open **Lens**. If it was already open, click **Check setup**. The setup section moves to your framework and tracing key when Lens is reachable and storage is ready. Investigation setup asks for the analysis model and budget; the installed service connects automatically
If your deployment restricts `allowed_ips`, allow the worker's address. For workers behind a reverse proxy with `use_x_forwarded_for: true`, also configure `mcp_trusted_proxy_ranges` with that proxy's CIDRs and, when needed, `mcp_xff_num_trusted_hops`. Lens reuses these existing trusted-proxy settings. Forwarded addresses without an established trust boundary are rejected by the allowlist; accepting them would let a worker impersonate an allowed address
## Upgrade
Setup, manual runs, feedback, and worker credentials are restricted to proxy administrators. Proxy-admin viewers can inspect results. Regular user and team keys cannot access the Lens API. Worker credentials can serve the administrator’s lenses. Revoke it in the connection dialog when retiring a worker. Redeploy the worker alongside proxy upgrades so their API versions match
Upgrade LiteLLM and Lens from the same source commit and release identity. For a coordinated published release, use its matching worker version; `deploy/lens/stack.yaml` starts LiteLLM, Lens, PostgreSQL, and ClickHouse for new installations. Standalone images remain available. Publishing an image does not update running containers
Keep the same databases, encryption keys, shared service secret, and public ingestion URL. Pause scheduled investigations and finish or cancel active runs, update both images through your usual deployment process, then check ingestion and run an investigation before resuming schedules. Do not run `docker compose down -v`
## Development
`make lens-dev` starts LiteLLM, Lens, and the hot-reload dashboard. Set `LENS_DEV_PROXY_PORT` and `LENS_DEV_UI_PORT` to change the local ports. For containers, pass the same release identity to both builds. Unversioned or incompatible workers are refused before claiming work
## Configure a lens
@ -116,7 +125,7 @@ Describe how the agent should behave and optionally add specific checks. Select
Choose your analysis model, parallelism and monthly budget. Parallelism controls simultaneous model calls, not the number of runs selected. New lenses run once by default. Turn on monitoring to repeat the same setup at a custom interval. **Run now** uses the same saved settings immediately, including the same lookback window and sampling. Each scan recalculates the window and reuses completed reviews when the selected trace content, expected behavior, enabled checks and analysis model are unchanged. Budget, name and schedule edits preserve reuse. Duplicate a lens when you want a separate investigation without changing an existing monitor
Pausing stops future scheduled scans; cancel the active scan separately if needed. The worker polls every two seconds; creating a lens or clicking Run now queues a scan, and due schedules are queued when the worker polls. Scans for the same lens never overlap, and its next interval starts after completion. Closing the browser does not stop the worker. A running scan retains its analysis settings and selected execution IDs across retries. Budget edits apply to subsequent model calls, including those in an active scan
Pausing stops future scheduled scans; cancel the active scan separately if needed. The worker polls every 2 to 15 seconds, backing off while idle; creating a lens or clicking Run now queues a scan, and due schedules are queued when the worker polls. Scans for the same lens never overlap, and its next interval starts after completion. Closing the browser does not stop the worker. A running scan retains its analysis settings and selected execution IDs across retries. Budget edits apply to subsequent model calls, including those in an active scan
## Read the results
@ -192,32 +201,14 @@ For local fixture data, run `make lens-dev ARGS=--seed`. Use `make lens-dev ARGS
Seeds append fresh IDs on every invocation and spread copies over recent timestamps. Restarts without `SEED` do not add data. Lens excludes activity received in the last two minutes, so wait two minutes after seeding before checking investigation previews. `LENS_DEV_SEED_COPIES` overrides total copies. Large seeds test data volume and pagination, rather than concurrent ingestion throughput or review accuracy. They can use substantial disk space; adjust `--copies` for your machine. Seeding expects the generated local tracing configuration. The old `run_tracing_proxy_local.sh --seed` command forwards to Lens dev, using its ports and saved master key
Local ingestion limits are explicit and configurable. Set OTLP and ClickHouse variables before starting the proxy and seeder so both processes use the same settings. Invalid, zero and negative values fail instead of silently falling back. Changing these limits does not require rebuilding Rust
| Environment variable | Default | Controls |
| --- | --- | --- |
| `LENS_DEV_SEED_COPIES` | 1 default, 2000 large | Total fixture copies |
| `LENS_DEV_SEED_TIMEOUT_SECONDS` | 120 | Seeder HTTP timeout |
| `OTLP_MAX_BODY_BYTES` | 16777216 | HTTP body and decompressed payload bytes |
| `OTLP_MAX_CONCURRENT_INGESTS` | 2 | Concurrent proxy ingestion requests |
| `OTLP_MAX_ATTRIBUTE_VALUE_BYTES` | 65536 | Stored attribute/content bytes |
| `OTLP_MAX_DECODE_DEPTH` | 32 | Nested decode depth |
| `OTLP_MAX_DECODE_NODES` | 65536 | JSON values or protobuf fields per export |
| `OTLP_MAX_SPANS` | 4096 | Spans per export |
| `OTLP_MAX_ATTRIBUTES` | 256 | Attributes per resource, scope, span, event or link |
| `OTLP_MAX_EVENTS` | 256 | Events per span |
| `OTLP_MAX_LINKS` | 256 | Links per span |
| `OTLP_MAX_DECODED_SPAN_BYTES` | 16777216 | Decoded span allocation budget |
| `CLICKHOUSE_TRACE_MAX_INSERT_BYTES` | 67108864 | Encoded trace or spend insert bytes |
| `CLICKHOUSE_INSERT_TIMEOUT_SECONDS` | 30 | ClickHouse insert HTTP timeout |
The wire parsers also enforce their library recursion limits (128 levels for JSON, 100 for protobuf). Raising the configured depth does not remove those parser limits.
The Rust receiver bounds each upload and its decompressed body to 16 MiB and permits two ingestion requests at once per replica. Exporters should split large batches and retry backpressure. `LENS_DEV_SEED_COPIES` and `LENS_DEV_SEED_TIMEOUT_SECONDS` control the seeder; the receiver's limits are compiled into the service
## Quality evaluation
Run the checked-in cases against a configured real model. Expected labels are used only for scoring, never passed to the model. Dev and held-out cases include missing outcomes, failed tools, recovery, handoffs, unsupported claims, repeated work, long evidence and prompt injection. The background option adds clean arithmetic traces to test rare-issue discovery at scale; those repeated synthetic cases do not establish accuracy on every production workload
```bash
cargo build --manifest-path litellm-rust/Cargo.toml -p litellm-lens --example worker_once --locked
python -m tests.proxy_behavior.lens.evaluate --api-base "$LITELLM_URL" \
--model your-model-alias --split all --background 1000 --concurrency 16 \
--output /tmp/lens-quality.json
@ -250,7 +241,7 @@ The hourly development pipeline pins all component images to the same selected c
## Worker dependencies
The worker uses the same digest-pinned Wolfi base and Python version as the component images. Python dependencies and their hashes are locked in `deploy/lens/requirements.lock`. To update them, edit `deploy/lens/requirements.in`, then run `uv pip compile --universal --python-version 3.13 --generate-hashes --no-emit-index-url deploy/lens/requirements.in -o deploy/lens/requirements.lock`. The image installs only the locked wheels with hash verification. CI builds and scans both native architectures
The service builds from the workspace Cargo.lock with a pinned Rust toolchain and a digest-pinned Wolfi runtime. It has no Python package dependencies. CPython and libseccomp support the confined calculation tool. CI builds, runs, and scans native amd64 and arm64 images
## Python analysis boundary
@ -260,14 +251,14 @@ The native worker image builds a syscall policy with libseccomp and includes the
Python execution requires a native Linux worker with Landlock ABI 3 or later and seccomp filtering. Build the image for the host architecture. Missing policy files, an incompatible kernel, or an unsupported host such as a macOS source worker returns a clear tool error. There is no unrestricted execution fallback. Keep the container's non-root user, dropped capabilities, no-new-privileges setting, read-only root and writable temporary mount
The worker permits two Python children at once across all investigations. Set `LENS_PYTHON_CONCURRENCY` to a positive integer to change this worker-wide pool. Queued calls consume no child process or scratch directory; cancelling a queued call does not start it. Model, read and search concurrency are separate
The worker permits two Python children at once across all investigations. Queued calls consume no child process or scratch directory; cancelling a queued call does not start it. Model, read and search concurrency are separate
| Per-call resource | Default |
| --- | --- |
| Elapsed execution time | 60 seconds |
| CPU time | 30 seconds |
| Process address space | 512 MiB |
| Captured stdout or stderr | 8 MiB per stream |
| Captured stdout or stderr | 4 MiB per stream |
| Individual scratch file size | 16 MiB |
| Monitored scratch storage | 64 MiB |
| Monitored scratch entries | 2,048 |
@ -281,12 +272,11 @@ Results include `stdout`, `stderr`, `exit_code`, `error` and `output_complete`.
This is a process boundary sharing the worker's Linux kernel. The checked-in smoke test verifies useful Python operations, filesystem and process restrictions, raw syscall attempts, resource failures, mapping accounting, cleanup and cancellation in the actual image. Run it on the deployment's native architecture and kernel:
```bash
docker build --build-arg LITELLM_RELEASE_TAG=lens-python-test \
-f deploy/lens/Dockerfile -t lens-worker:python-test .
docker run --rm --pull never --read-only --cap-drop ALL \
docker build --target smoke --build-arg LITELLM_RELEASE_TAG=lens-python-test \
-f deploy/lens/Dockerfile -t lens-worker:smoke .
docker run --rm --read-only --cap-drop ALL \
--security-opt no-new-privileges --network none \
--tmpfs /tmp:rw,noexec,nosuid,size=1g --entrypoint python -i \
lens-worker:python-test - < tests/proxy_behavior/lens/worker_python_smoke.py
--tmpfs /tmp:rw,noexec,nosuid,size=1g lens-worker:smoke
```
The same checks can run through pytest by setting `LENS_TEST_WORKER_IMAGE` to an already-built native image. The worker image CI runs the standalone smoke without adding pytest to the production image
The smoke target runs the Rust sandbox integration tests. The production image contains neither Cargo nor the test executable

12
deploy/lens/compose.yaml
View file

@ -3,8 +3,16 @@ services:
image: ${LENS_WORKER_IMAGE:-${LITELLM_VERSION:+ghcr.io/berriai/litellm-lens-worker:v}${LITELLM_VERSION:-}}
environment:
LITELLM_URL: ${LITELLM_URL:?Set the URL reachable from this container}
LENS_WORKER_TOKEN: ${LENS_WORKER_TOKEN:?Create a worker credential in the Lens UI}
LENS_PYTHON_CONCURRENCY: ${LENS_PYTHON_CONCURRENCY:-2}
LENS_WORKER_TOKEN: ${LENS_WORKER_TOKEN:-}
LITELLM_LENS_SERVICE_TOKEN: ${LITELLM_LENS_SERVICE_TOKEN:?Set the same secret on LiteLLM and Lens}
CLICKHOUSE_URL: ${CLICKHOUSE_URL:?Set the ClickHouse URL reachable from Lens}
CLICKHOUSE_DATABASE: ${CLICKHOUSE_DATABASE:-litellm}
AGENT_TRACING_RETENTION_DAYS: ${AGENT_TRACING_RETENTION_DAYS:-14}
ports:
- "127.0.0.1:${LENS_PORT:-4318}:4318"
mem_limit: 2g
cpus: 2
pids_limit: 64
restart: unless-stopped
read_only: true
tmpfs:

6
deploy/lens/config.yaml
View file

@ -1,7 +1,7 @@
model_list: []
general_settings:
master_key: os.environ/LITELLM_MASTER_KEY
tracing:
store:
type: clickhouse
url: os.environ/CLICKHOUSE_URL
retention_days: 14
type: lens

73
deploy/lens/configure.py Normal file
View file

@ -0,0 +1,73 @@
from __future__ import annotations
import argparse
import os
import re
import secrets
import shlex
import sys
from pathlib import Path
from typing import Final
SECRET_NAMES: Final = (
"LITELLM_MASTER_KEY",
"LITELLM_SALT_KEY",
"LITELLM_LENS_SERVICE_TOKEN",
"POSTGRES_PASSWORD",
"CLICKHOUSE_PASSWORD",
)
def environment_content(path: Path) -> str:
if not path.exists():
return "".join(
f"{name}={'sk-' if name.endswith('KEY') else ''}{secrets.token_hex(32)}\n" for name in SECRET_NAMES
)
saved: Final = path.read_text().splitlines()
values: Final = dict(line.split("=", 1) for line in saved if "=" in line)
if any(not values.get(name) for name in SECRET_NAMES):
raise ValueError(f"{path} is incomplete. Restore your saved credentials before continuing")
return "\n".join(line for line in saved if not line.startswith("LITELLM_VERSION=")) + "\n"
def configure(path: Path, version: str) -> None:
release: Final = version.removeprefix("v")
if not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+(?:[-.][a-zA-Z0-9.-]+)?", release):
raise ValueError("Use a published release version, such as 1.82.0 or 1.82.0-nightly")
if path.is_symlink():
raise ValueError(f"Refusing to replace a symlink: {path}")
existing: Final = path.exists()
content: Final = environment_content(path)
temporary: Final = path.with_name(f".{path.name}.{secrets.token_hex(8)}")
descriptor: Final = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
try:
with os.fdopen(descriptor, "w") as output:
output.write(content + f"LITELLM_VERSION={release}\n")
if existing:
os.replace(temporary, path)
else:
os.link(temporary, path)
finally:
temporary.unlink(missing_ok=True)
def main() -> None:
parser: Final = argparse.ArgumentParser(description="Create or update the configuration for the Lens Compose stack")
parser.add_argument("--version", required=True, help="Published LiteLLM release; Lens uses the matching version")
parser.add_argument("--env-file", type=Path, default=Path(__file__).with_name(".env"))
arguments: Final = parser.parse_args()
try:
configure(arguments.env_file, arguments.version)
except (OSError, ValueError) as error:
parser.exit(1, f"Could not configure Lens: {error}\n")
sys.stdout.write(
f"Saved {arguments.env_file}. Existing keys and database passwords are preserved\n"
f"Start with: docker compose --env-file {shlex.quote(str(arguments.env_file))} "
"-f deploy/lens/stack.yaml up -d --wait\n"
"Open http://localhost:4000/ui/ and sign in as admin with LITELLM_MASTER_KEY from the saved file\n"
"Back up this file with your database volumes. Do not commit it\n"
)
if __name__ == "__main__":
main()

2
deploy/lens/requirements.in
View file

@ -1,2 +0,0 @@
httpx==0.28.1
pydantic==2.13.4

172
deploy/lens/requirements.lock
View file

@ -1,172 +0,0 @@
# This file was autogenerated by uv via the following command:
# uv pip compile --universal --python-version 3.13 --generate-hashes --no-emit-index-url deploy/lens/requirements.in -o deploy/lens/requirements.lock
annotated-types==0.8.0 \
--hash=sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7 \
--hash=sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0
# via pydantic
anyio==4.15.1 \
--hash=sha256:6152fdbbf9a77fdec97731721bebf7c4c44f7c29b424b0065826173efc7ed101 \
--hash=sha256:9f28306018cbd6d329e64a36d58256edff76dd996fe423bc957326e578b82a94
# via httpx
certifi==2026.7.22 \
--hash=sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775 \
--hash=sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55
# via
# httpcore
# httpx
h11==0.16.0 \
--hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \
--hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86
# via httpcore
httpcore==1.0.9 \
--hash=sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 \
--hash=sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8
# via httpx
httpx==0.28.1 \
--hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \
--hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad
# via -r deploy/lens/requirements.in
idna==3.20 \
--hash=sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44 \
--hash=sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c
# via
# anyio
# httpx
pydantic==2.13.4 \
--hash=sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba \
--hash=sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6
# via -r deploy/lens/requirements.in
pydantic-core==2.46.4 \
--hash=sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0 \
--hash=sha256:0186750b482eefa11d7f435892b09c5c606193ef3375bcf94aa00ae6bfb66262 \
--hash=sha256:041bde0a48fd37cf71cab1c9d56d3e8625a3793fef1f7dd232b3ff37e978ecda \
--hash=sha256:0c563b08bca408dc7f65f700633d8442fffb2421fc47b8101377e9fd65051ff0 \
--hash=sha256:0cbe8b01f948de4286c74cdd6c667aceb38f5c1e26f0693b3983d9d74887c65e \
--hash=sha256:0ce40cd7b21210e99342afafbd4d0f76d784eb5b1d60f3bdc566be4983c6c73b \
--hash=sha256:0e96592440881c74a213e5ad528e2b24d3d4f940de2766bed9010ab1d9e51594 \
--hash=sha256:10e17cbb10a330363733efc4d7c4d0dd827ac0909b8f6a6542298fed1ea62f29 \
--hash=sha256:133878133d271ade3d41d1bfb2a45ec38dbdbda40bc065921c6b04e4630127e2 \
--hash=sha256:14d4edf427bdcf950a8a02d7cb44a08614388dd6e1bdcbf4f67504fa7887da9c \
--hash=sha256:14f4c5d6db102bd796a627bbb3a17b4cf4574b9ae861d8b7c9a9661c6dd3362d \
--hash=sha256:17299feefe090f2caa5b8e37222bb5f663e4935a8bfa6931d4102e5df1a9f398 \
--hash=sha256:184c081504d17f1c1066e430e117142b2c77d9448a97f7b65c6ac9fd9aee238d \
--hash=sha256:18e5ceec2ab67e6d5f1a9085e5a24c9c4e2ac4545730bfe668680bca05e555f3 \
--hash=sha256:19e51f073cd3df251856a8a4189fbdf1de4012c3ebacfb1884f94f1eb406079f \
--hash=sha256:1a7dd0b3ee80d90150e3495a3a13ac34dbcbfd4f012996a6a1d8900e91b5c0fb \
--hash=sha256:1d8ba486450b14f3b1d63bc521d410ec7565e52f887b9fb671791886436a42f7 \
--hash=sha256:2108ba5c1c1eca18030634489dc544844144ee36357f2f9f780b93e7ddbb44b5 \
--hash=sha256:228ee9bae8bef5b1e97ec58302f80357c37199e0d0a99174e138d28e6957b9d9 \
--hash=sha256:23ace664830ee0bfe014a0c7bc248b1f7f25ed7ad103852c317624a1083af462 \
--hash=sha256:2412e734dcb48da14d4e4006b82b46b74f2518b8a26ee7e58c6844a6cd6d03c4 \
--hash=sha256:29c61fc04a3d840155ff08e475a04809278972fe6aef51e2720554e96367e34b \
--hash=sha256:2f84c03c8607173d16b5a854ec68a2f9079ae03237a54fb506d13af47e1d018d \
--hash=sha256:3009f12e4e90b7f88b4f9adb1b0c4a3d58fe7820f3238c190047209d148026df \
--hash=sha256:3245406455a5d98187ec35530fd772b1d799b26667980872c8d4614991e2c4a2 \
--hash=sha256:3447661d99f75a3683a4cf5c87da72f2161964611864dbbeac7fbb118bb4bfc0 \
--hash=sha256:372429a130e469c9cd698925ce5fc50940b7a1336b0d82038e63d5bbc4edc519 \
--hash=sha256:395aebd9183f9d112f569aeb5b2214d1a10a33bec8456447f7fbdfa51d38d4cd \
--hash=sha256:3a233125ac121aa3ffba9a2b59edfc4a985a76092dc8279586ab4b71390875e7 \
--hash=sha256:3be77f45df024d789a672ae34f8b06fb346c4f9f46ea714956660ea4862e89ac \
--hash=sha256:3bf92c5d0e00fefaab325a4d27828fe6b6e2a21848686b5b60d2d9eeb09d76c6 \
--hash=sha256:3ecbc122d18468d06ca279dc26a8c2e2d5acb10943bb35e36ae92096dc3b5565 \
--hash=sha256:3fb702cd90b0446a3a1c5e470bfa0dd23c0233b676a9099ddcc964fa6ca13898 \
--hash=sha256:428e04521a40150c85216fc8b85e8d39fece235a9cf5e383761238c7fa9b96fb \
--hash=sha256:432c179df7874eeb73307aad2df0755e1ae0efa61ff0ea89b93e194411ae3928 \
--hash=sha256:4a05d69cba51d852c5c3e92758653245a50c0b646ced0cf05bd793ed592839d6 \
--hash=sha256:4c63ebc82684aa89d9a3bcbd13d515b3be44250dc68dd3bd81526c1cb31286c3 \
--hash=sha256:4fc73cb559bdb54b1134a706a2802a4cddd27a0633f5abb7e53056268751ac6a \
--hash=sha256:4fcbe087dbc2068af7eda3aa87634eba216dbda64d1ae73c8684b621d33f6596 \
--hash=sha256:56cb4851bcaf3d117eddcef4fe66afd750a50274b0da8e22be256d10e5611987 \
--hash=sha256:5855698a4856556d86e8e6cd8434bc3ac0314ee8e12089ae0e143f64c6256e4e \
--hash=sha256:5a4330cdbc57162e4b3aa303f588ba752257694c9c9be3e7ebb11b4aca659b5d \
--hash=sha256:5b712b53160b79a5850310b912a5ef8e57e56947c8ad690c227f5c9d7e561712 \
--hash=sha256:5d5902252db0d3cedf8d4a1bc68f70eeb430f7e4c7104c8c476753519b423008 \
--hash=sha256:617d7e2ca7dcb8c5cf6bcb8c59b8832c94b36196bbf1cbd1bfb56ed341905edd \
--hash=sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1 \
--hash=sha256:633147d34cf4550417f12e2b1a0383973bdf5cdfde212cb09e9a581cf10820be \
--hash=sha256:66ce7632c22d837c95301830e111ad0128a32b8207533b60896a96c4915192ea \
--hash=sha256:6b3ace8194b0e5204818c92802dcdca7fc6d88aabbb799d7c795540d9cd6d292 \
--hash=sha256:6f2eeda33a839975441c86a4119e1383c50b47faf0cbb5176985565c6bb02c33 \
--hash=sha256:7027560ee92211647d0d34e3f7cd6f50da56399d26a9c8ad0da286d3869a53f3 \
--hash=sha256:7283d57845ecf5a163403eb0702dfc220cc4fbdd18919cb5ccea4f95ee1cdab4 \
--hash=sha256:7a5f930472650a82629163023e630d160863fce524c616f4e5186e5de9d9a49b \
--hash=sha256:7bfb192b3f4b9e8a89b6277b6ce787564f62cfd272055f6e685726b111dc7826 \
--hash=sha256:811ff8e9c313ab425368bcbb36e5c4ebd7108c2bbf4e4089cfbb0b01eff63fac \
--hash=sha256:8233f2947cf85404441fd7e0085f53b10c93e0ee78611099b5c7237e36aacbf7 \
--hash=sha256:82cf5301172168103724d49a1444d3378cb20cdee30b116a1bd6031236298a5d \
--hash=sha256:8358a950c8909158e3df31538a7e4edc2d7265a7c54b47f0864d9e5bae9dcebf \
--hash=sha256:85bb3611ff1802f3ee7fdd7dbff26b56f343fb432d57a4728fdd49b6ef35e2f4 \
--hash=sha256:86e1a4418c6cd97d60c95c71164158eaf7324fae7b0923264016baa993eba6fc \
--hash=sha256:8b9bab013d1c7a79d3501ff86d0bc9c31bf587db4551677b96bec07df78c6b15 \
--hash=sha256:8c5dac79fa1614d1e06ca695109c6105923bd9c7d1d6c918d4e637b7e6b32fd3 \
--hash=sha256:8d0820e8192167f80d88d64038e609c31452eeca865b4e1d9950a27a4609b00b \
--hash=sha256:8daafc69c93ee8a0204506a3b6b30f586ef54028f52aeeeb5c4cfc5184fd5914 \
--hash=sha256:9037063db01f09b09e237c282b6792bd4da634b5402c4e7f0c61effed7701a04 \
--hash=sha256:905a0ed8ea6f2d61c1738835f99b699348d7857379083e5fc497fa0c967a407c \
--hash=sha256:90884113d8b48f760e9587002789ddd741e76ab9f89518cd1e43b1f1a52ec44b \
--hash=sha256:91a06d2e259ecfbd8c901d70c3c507900458498142b3026a296b7de4d1322cc9 \
--hash=sha256:926c9541b14b12b1681dca8a0b75feb510b06c6341b70a8e500c2fdcff837cce \
--hash=sha256:9401557acd873c3a7f3eb9383edef8ac4968f9510e340f4808d427e75667e7b4 \
--hash=sha256:9551187363ffc0de2a00b2e47c25aeaeb1020b69b668762966df15fc5659dd5a \
--hash=sha256:962ccbab7b642487b1d8b7df90ef677e03134cf1fd8880bf698649b22a69371f \
--hash=sha256:97e7cf2be5c77b7d1a9713a05605d49460d02c6078d38d8bef3cbe323c548424 \
--hash=sha256:9aa768456404a8bf48a4406685ac2bec8e72b62c69313734fa3b73cf33b3a894 \
--hash=sha256:9bc519fbf2b7578398853d815009ae5e4d4603d12f4e3f91da8c06852d3da3e9 \
--hash=sha256:9d56801be94b86a9da183e5f3766e6310752b99ff647e38b09a9500d88e46e76 \
--hash=sha256:9f444c499b3eefd3a92e348059471ea0c3a6e303d9c1cec09fa748fd9f895201 \
--hash=sha256:9fa8ae11da9e2b3126c6426f147e0fba88d96d65921799bb30c6abd1cb2c97fb \
--hash=sha256:a0f62d0a58f4e7da165457e995725421e0064f2255d8eccebc49f41bbc23b109 \
--hash=sha256:a396dcc17e5a0b164dbe026896245a4fa9ff402edca1dff0be3d53a517f74de4 \
--hash=sha256:aaa2a54443eff1950ba5ddc6b6ccda0d9c84a364276a62f969bdf2a390650848 \
--hash=sha256:ad785e92e6dc634c21555edc8bd6b64957ab844541bcb96a1366c202951ae526 \
--hash=sha256:af8244b2bef6aaad6d92cda81372de7f8c8d36c9f0c3ea36e827c60e7d9467a0 \
--hash=sha256:b078afbc25f3a1436c7a1d2cd3e322497ee99615ba97c563566fdf46aff1ee01 \
--hash=sha256:b2f69dec1725e79a012d920df1707de5caf7ed5e08f3be4435e25803efc47458 \
--hash=sha256:b8458003118a712e66286df6a707db01c52c0f52f7db8e4a38f0da1d3b94fc4e \
--hash=sha256:bb63e0198ca18aad131c089b9204c23079c3afa95487e561f4c522d519e55aba \
--hash=sha256:bfec22eab3c8cc2ceec0248aec886624116dc079afa027ecc8ad4a7e62010f8a \
--hash=sha256:c1747f85cee84c26985853c6f3d9bd3e75da5212912443fa111c113b9c246f39 \
--hash=sha256:c1b3f518abeca3aa13c712fd202306e145abf59a18b094a6bafb2d2bbf59192c \
--hash=sha256:c50f2528cf200c5eed56faf3f4e22fcd5f38c157a8b78576e6ba3168ec35f000 \
--hash=sha256:c68fcd102d71ea85c5b2dfac3f4f8476eff42a9e078fd5faefff6d145063536b \
--hash=sha256:c7a7bd4e39e8e4c12c39cd480356842b6a8a06e41b23a55a5e3e191718838ddf \
--hash=sha256:c94f0688e7b8d0a67abf40e57a7eaaecd17cc9586706a31b76c031f63df052b4 \
--hash=sha256:cbaf13819775b7f769bf4a1f066cb6df7a28d4480081a589828ef190226881cd \
--hash=sha256:cd2213145bcc2ba85884d0ac63d222fece9209678f77b9b4d76f054c561adb28 \
--hash=sha256:ce5c1d2a8b27468f433ca974829c44060b8097eedc39933e3c206a90ee49c4a9 \
--hash=sha256:d396ec2b979760aaf3218e76c24e65bd0aca24983298653b3a9d7a45f9e47b30 \
--hash=sha256:d51026d73fcfd93610abc7b27789c26b313920fcfb20e27462d74a7f8b06e983 \
--hash=sha256:d80ee3d731373b24cebbc10d689ca4ee1875caf0d5703a245db18efd4dd37fc1 \
--hash=sha256:d995260fdf4e1db774581b4900e0f832abe3c7c84996726bbc161b19c8f29e76 \
--hash=sha256:da4b951fe36dc7c3a1ccb4e3cd1747c3542b8c9ceede8fc86cae054e764485f5 \
--hash=sha256:daa27d92c36f24388fe3ad306b174781c747627f134452e4f128ea00ce1fe8c4 \
--hash=sha256:db06ffe51636ffe9ca531fe9023dd64bdd794be8754cb5df57c5498ae5b518a7 \
--hash=sha256:e0d65b8c354be7fb5f720c3caa8bc940bc2d20ce749c8e06135f07f8ed95dd7c \
--hash=sha256:e68b7a074f65a2fd746c52a7ce6142ab7006074ac269ace0c25cd8ba171f8066 \
--hash=sha256:e739fee756ba1010f8bcccb534252e85a35fe45ae92c295a06059ce58b74ccd3 \
--hash=sha256:e846ae7835bf0703ae43f534ab79a867146dadd59dc9ca5c8b53d5c8f7c9ef02 \
--hash=sha256:e9c26f834c65f5752f3f06cb08cb86a913ceb7274d0db6e267808a708b46bc89 \
--hash=sha256:ea793e075b70290d89d8142074262885d3f7da19634845135751bd6344f73b50 \
--hash=sha256:f027324c56cd5406ca49c124b0db10e56c69064fec039acc571c29020cc87c76 \
--hash=sha256:f13a646d65d09fbf1bc6b3a9635d30095c8e7e5cc419ff35ecc563c5fd04cd49 \
--hash=sha256:f47286a97f0bc9b8859519809077b91b2cefe4ae47fcbf5e466a009c1c5d742b \
--hash=sha256:f747929cf940cddb5b3668a390056ddd5ba2e5010615ea2dcf4f9c4f3ab8791d \
--hash=sha256:f99626688942fb746e545232e7726926f3be91b5975f8b55327665fafda991c7 \
--hash=sha256:f9fa868638bf362d3d138ea55829cefb3d5f4b0d7f142234382a15e2485dbec4 \
--hash=sha256:fbdb89b3e1c94a30cc5edfce477c6e6a5dc4d8f84665b455c27582f211a1c72c \
--hash=sha256:fc010ab034c8c7452522748bf937df58020d256ccae0874463d1f4d01758af8e \
--hash=sha256:fc3e9034a63de20e15e8ade85358bc6efc614008cab72898b4b4952bea0509ff \
--hash=sha256:fd8b3d9fd264be37976686c7f65cd52a83f5e84f4bfd2adf9c1d469676bbb6ae
# via pydantic
typing-extensions==4.16.0 \
--hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \
--hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5
# via
# anyio
# pydantic
# pydantic-core
# typing-inspection
typing-inspection==0.4.4 \
--hash=sha256:547274fa6b0a561ccf549cc9524b999a578e737d015d8709d021f9d0d13bea47 \
--hash=sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147
# via pydantic

40
deploy/lens/smoke.sh Normal file
View file

@ -0,0 +1,40 @@
#!/usr/bin/env bash
set -euo pipefail
image="${1:?pass the built image reference}"
release="${2:?pass the expected release tag}"
version="$(docker run --rm --network none --read-only --cap-drop ALL --security-opt no-new-privileges "$image" --version)"
test "$version" = "litellm-lens $release protocol=7"
container="$(docker run -d --network none --read-only --cap-drop ALL \
--security-opt no-new-privileges --pids-limit 64 --memory 2g --cpus 2 \
--tmpfs /tmp:rw,noexec,nosuid,size=256m \
-e LITELLM_URL=http://127.0.0.1:1 \
-e CLICKHOUSE_URL=http://127.0.0.1:1 \
-e LITELLM_LENS_SERVICE_TOKEN=isolated-runtime-smoke-secret-32-characters \
"$image")"
trap 'docker rm -f "$container" >/dev/null' EXIT
test "$(docker exec "$container" id -u)" = 65532
docker exec -i "$container" python3.13 -I -S - <<'PY'
import time
import urllib.error
import urllib.request
for attempt in range(50):
try:
with urllib.request.urlopen("http://127.0.0.1:4318/health/live", timeout=1) as response:
assert response.status == 200
break
except urllib.error.URLError:
if attempt == 49:
raise
time.sleep(0.1)
for path, expected in (("health/ready", 503), ("internal/status", 401)):
try:
urllib.request.urlopen(f"http://127.0.0.1:4318/{path}", timeout=1)
except urllib.error.HTTPError as error:
assert error.code == expected, (path, error.code)
else:
raise AssertionError(f"{path} should return {expected}")
print("Unprivileged Lens service remains live with unavailable dependencies")
PY

28
deploy/lens/stack.yaml
View file

@ -10,9 +10,7 @@ services:
import os, sys
from urllib.parse import quote
postgres_password = quote(os.environ["POSTGRES_PASSWORD"], safe="")
clickhouse_password = quote(os.environ["CLICKHOUSE_PASSWORD"], safe="")
os.environ["DATABASE_URL"] = f"postgresql://litellm:{postgres_password}@db:5432/litellm"
os.environ["CLICKHOUSE_URL"] = f"http://default:{clickhouse_password}@clickhouse:8123"
os.execv("docker/prod_entrypoint.sh", ["docker/prod_entrypoint.sh", *sys.argv[1:]])
command: ["--config", "/app/lens-config.yaml", "--port", "4000"]
environment:
@ -20,29 +18,37 @@ services:
LITELLM_SALT_KEY: ${LITELLM_SALT_KEY:?Set a permanent encryption key and keep it across upgrades}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set a permanent database password}
STORE_MODEL_IN_DB: "True"
CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:?Set a permanent ClickHouse password}
LITELLM_LENS_URL: http://lens-worker:4318
LITELLM_LENS_PUBLIC_URL: ${LITELLM_LENS_PUBLIC_URL:-http://localhost:4318}
LITELLM_LENS_SERVICE_TOKEN: ${LITELLM_LENS_SERVICE_TOKEN:?Set the shared Lens service secret}
LENS_WORKER_IMAGE: ghcr.io/berriai/litellm-lens-worker:v${LITELLM_VERSION}
volumes:
- ./config.yaml:/app/lens-config.yaml:ro
ports:
- "127.0.0.1:${LITELLM_PORT:-4000}:4000"
networks: [proxy, storage]
networks: [proxy, database]
depends_on:
db:
condition: service_healthy
clickhouse:
condition: service_healthy
restart: unless-stopped
lens-worker:
profiles: [lens]
image: ghcr.io/berriai/litellm-lens-worker:v${LITELLM_VERSION}
environment:
LITELLM_URL: http://litellm:4000
LENS_WORKER_TOKEN: ${LENS_WORKER_TOKEN:-}
LENS_PYTHON_CONCURRENCY: ${LENS_PYTHON_CONCURRENCY:-2}
LITELLM_LENS_SERVICE_TOKEN: ${LITELLM_LENS_SERVICE_TOKEN}
CLICKHOUSE_HOST: clickhouse
CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:?Set a permanent ClickHouse password}
CLICKHOUSE_DATABASE: ${CLICKHOUSE_DATABASE:-litellm}
AGENT_TRACING_RETENTION_DAYS: ${AGENT_TRACING_RETENTION_DAYS:-14}
depends_on: [litellm]
networks: [proxy]
networks: [proxy, storage]
ports:
- "127.0.0.1:${LENS_PORT:-4318}:4318"
mem_limit: 2g
cpus: 2
pids_limit: 64
restart: unless-stopped
read_only: true
tmpfs:
@ -56,7 +62,7 @@ services:
POSTGRES_DB: litellm
POSTGRES_USER: litellm
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
networks: [storage]
networks: [database]
volumes:
- postgres_data:/var/lib/postgresql/data
healthcheck:
@ -84,6 +90,8 @@ services:
networks:
proxy:
database:
internal: true
storage:
internal: true

52
deploy/lens/test_configure.py Normal file
View file

@ -0,0 +1,52 @@
import tempfile
import unittest
from pathlib import Path
from configure import SECRET_NAMES, configure
class ComposeConfigurationTests(unittest.TestCase):
def test_restart_and_upgrade_preserve_private_credentials_and_custom_settings(self) -> None:
with tempfile.TemporaryDirectory() as directory:
path = Path(directory) / ".env"
configure(path, "v1.2.3")
original = dict(line.split("=", 1) for line in path.read_text().splitlines())
self.assertEqual(path.stat().st_mode & 0o777, 0o600)
self.assertEqual(len({original[name] for name in SECRET_NAMES}), len(SECRET_NAMES))
self.assertTrue(all(len(original[name]) >= 64 for name in SECRET_NAMES))
with path.open("a") as output:
output.write("LITELLM_LENS_PUBLIC_URL=https://traces.example/prefix\n")
configure(path, "1.2.3")
configure(path, "v1.2.4-nightly")
updated = dict(line.split("=", 1) for line in path.read_text().splitlines())
self.assertEqual({name: updated[name] for name in SECRET_NAMES}, {name: original[name] for name in SECRET_NAMES})
self.assertEqual(updated["LITELLM_VERSION"], "1.2.4-nightly")
self.assertEqual(updated["LITELLM_LENS_PUBLIC_URL"], "https://traces.example/prefix")
self.assertEqual(path.stat().st_mode & 0o777, 0o600)
def test_incomplete_configuration_is_never_replaced_with_new_database_passwords(self) -> None:
with tempfile.TemporaryDirectory() as directory:
path = Path(directory) / ".env"
original = "POSTGRES_PASSWORD=existing\n"
path.write_text(original)
with self.assertRaisesRegex(ValueError, "incomplete"):
configure(path, "1.2.3")
self.assertEqual(path.read_text(), original)
def test_invalid_release_and_symlink_leave_existing_files_untouched(self) -> None:
with tempfile.TemporaryDirectory() as directory:
path = Path(directory) / ".env"
target = Path(directory) / "saved"
target.write_text("preserve")
path.symlink_to(target)
with self.assertRaisesRegex(ValueError, "symlink"):
configure(path, "1.2.3")
self.assertEqual(target.read_text(), "preserve")
path.unlink()
with self.assertRaisesRegex(ValueError, "published release"):
configure(path, "1.2.3\nPOSTGRES_PASSWORD=replaced")
self.assertFalse(path.exists())
if __name__ == "__main__":
unittest.main()

30
docker/docker-compose.tracing.yml
View file

@ -13,8 +13,9 @@ services:
LITELLM_SALT_KEY: sk-local-tracing-salt-key
DATABASE_URL: postgresql://litellm:litellm@db:5432/litellm
STORE_MODEL_IN_DB: "True"
CLICKHOUSE_URL: http://default:local-tracing@clickhouse:8123
CLICKHOUSE_DATABASE: litellm
LITELLM_LENS_URL: http://lens-worker:4318
LITELLM_LENS_PUBLIC_URL: http://localhost:4318
LITELLM_LENS_SERVICE_TOKEN: ${LITELLM_LENS_SERVICE_TOKEN:?set LITELLM_LENS_SERVICE_TOKEN}
OPENAI_API_KEY: ${OPENAI_API_KEY:-}
LENS_WORKER_IMAGE: ${LENS_WORKER_IMAGE:-}
volumes:
@ -24,8 +25,29 @@ services:
depends_on:
db:
condition: service_healthy
clickhouse:
condition: service_healthy
lens-worker:
build:
context: ..
dockerfile: deploy/lens/Dockerfile
args:
LITELLM_RELEASE_TAG: ${LITELLM_RELEASE_TAG:?set LITELLM_RELEASE_TAG to the source commit}
environment:
LITELLM_URL: http://litellm:4000
LITELLM_LENS_SERVICE_TOKEN: ${LITELLM_LENS_SERVICE_TOKEN:?set LITELLM_LENS_SERVICE_TOKEN}
CLICKHOUSE_URL: http://default:local-tracing@clickhouse:8123
CLICKHOUSE_DATABASE: litellm
ports:
- "127.0.0.1:4318:4318"
read_only: true
cap_drop: [ALL]
security_opt: [no-new-privileges:true]
tmpfs:
- /tmp:rw,noexec,nosuid,nodev,size=${LENS_WORKER_TMP_SIZE:-1g},mode=1777
mem_limit: 2g
cpus: 2
pids_limit: 64
restart: unless-stopped
db:
image: postgres:16

4
docker/tracing-config.yaml
View file

@ -8,6 +8,4 @@ general_settings:
master_key: os.environ/LITELLM_MASTER_KEY
tracing:
store:
type: clickhouse
url: os.environ/CLICKHOUSE_URL
retention_days: 14
type: lens

28
enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
View file

@ -732,6 +732,7 @@ class CheckBatchCost:
another pod claimed it. Raises on results-fetch or cost-computation
failures so the caller can leave the job unprocessed and retry it on a
later poll.
"""
from litellm.batches.batch_utils import (
count_error_file_failed_requests,
@ -815,15 +816,14 @@ class CheckBatchCost:
custom_llm_provider=custom_llm_provider,
)
# CheckBatchCost bypasses async_post_call_success_hook, so convert raw
# output/error file IDs to managed base64 IDs before the DB write here.
managed_files_hook = self.proxy_logging_obj.get_proxy_hook("managed_files")
if managed_files_hook is not None:
from litellm.proxy.openai_files_endpoints.common_utils import ManagedBatchOutputFileWriter
managed_files_hook: Final = self.proxy_logging_obj.get_proxy_hook("managed_files")
if isinstance(managed_files_hook, ManagedBatchOutputFileWriter):
managed_file_writer: Final = managed_files_hook
from litellm.proxy._types import UserAPIKeyAuth
managed_file_model_name = self._get_managed_file_model_name(
job=job, deployment_info=deployment_info
)
managed_file_model_name = self._get_managed_file_model_name(job=job, deployment_info=deployment_info)
_minimal_auth = UserAPIKeyAuth(
user_id=job.created_by or "default-user-id",
team_id=getattr(job, "team_id", None),
@ -832,17 +832,19 @@ class CheckBatchCost:
_raw_file_id = cast(str | None, getattr(response, _file_attr, None))
if _raw_file_id and not _is_base64_encoded_unified_file_id(_raw_file_id):
try:
_unified_file_id = managed_files_hook.get_unified_output_file_id(
_unified_file_id = managed_file_writer.get_unified_output_file_id(
output_file_id=_raw_file_id,
model_id=model_id,
model_name=managed_file_model_name,
)
await managed_files_hook.store_unified_file_id(
file_id=_unified_file_id,
file_object=None,
await managed_file_writer.store_batch_output_file(
unified_file_id=_unified_file_id,
provider_file_id=_raw_file_id,
model_id=model_id,
model_name=managed_file_model_name,
owner=_minimal_auth,
litellm_parent_otel_span=None,
model_mappings={model_id: _raw_file_id},
user_api_key_dict=_minimal_auth,
size_bytes=len(content_bytes) if _file_attr == "output_file_id" else None,
)
setattr(response, _file_attr, _unified_file_id)
verbose_proxy_logger.info(

57
enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py
View file

@ -6,7 +6,7 @@ same route are non-inference and free.
"""
from datetime import datetime, timedelta, timezone
from typing import TYPE_CHECKING, Dict, Final, Optional, Protocol, cast
from typing import TYPE_CHECKING, Dict, Final, Protocol, cast
import litellm
from litellm._logging import verbose_proxy_logger
@ -47,6 +47,10 @@ def _managed_object_table(prisma_client: "PrismaClient") -> "TableActions[_Manag
return ManagedObjectRepository(prisma_client).table
def _is_response_gone_at_provider(error: Exception, provider_response_id: str) -> bool:
return getattr(error, "status_code", None) == 404 and provider_response_id in str(error)
class CheckResponsesCost:
def __init__(
self,
@ -61,10 +65,15 @@ class CheckResponsesCost:
self.prisma_client: PrismaClient = prisma_client
self.llm_router: Router = llm_router
def _resolve_deployment(self, response_id: str) -> bool:
model_id: str | None = ResponsesAPIRequestUtils.get_model_id_from_response_id(response_id)
return model_id is not None and self.llm_router.get_deployment(model_id=model_id) is not None
async def _get_response(
self,
response_id: str,
litellm_metadata: Dict[str, str],
via_router: bool,
) -> ResponsesAPIResponse:
"""Fetch the upstream response, using deployment credentials when available.
@ -75,8 +84,7 @@ class CheckResponsesCost:
sees provider env vars, so it fails for every deployment whose credentials
live in the config; the row then never leaves ``queued``.
"""
model_id: Optional[str] = ResponsesAPIRequestUtils.get_model_id_from_response_id(response_id)
if model_id is None or self.llm_router.get_deployment(model_id=model_id) is None:
if not via_router:
return await litellm.aget_responses(response_id=response_id, litellm_metadata=litellm_metadata)
router_response = await self.llm_router.aget_responses(
response_id=response_id, litellm_metadata=litellm_metadata
@ -140,6 +148,8 @@ class CheckResponsesCost:
- Cost is tracked by the get-responses call, billed because the poll is stamped
with BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN
- Mark responses in a terminal state as complete in the database
- Mark responses the provider no longer has (404 through a resolved
deployment) as stale_expired
"""
try:
await self._cleanup_stale_managed_objects()
@ -159,6 +169,7 @@ class CheckResponsesCost:
verbose_proxy_logger.debug(f"Found {len(jobs)} response jobs to check")
completed_jobs: Final[list[_ManagedObjectRow]] = []
expired_jobs: Final[list[_ManagedObjectRow]] = []
for job in jobs:
unified_object_id = job.unified_object_id
@ -171,31 +182,48 @@ class CheckResponsesCost:
# Get the stored response object to extract model information
stored_response = job.file_object
model_name = stored_response.get("model", None)
# Decrypt the response ID
responses_id_security, _, _ = ResponsesIDSecurity()._decrypt_response_id(unified_object_id)
# Prepare metadata with model information for cost tracking
litellm_metadata = {
"user_api_key_user_id": job.created_by or "default-user-id",
INTERNAL_CALL_ORIGIN_METADATA_KEY: BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN,
}
# Add model information if available
if model_name:
litellm_metadata["model"] = model_name
litellm_metadata["model_group"] = model_name # Use same value for model_group
via_router = self._resolve_deployment(responses_id_security)
except Exception as e:
verbose_proxy_logger.warning(
f"Skipping job {unified_object_id} due to error: {e}"
)
continue
provider_response_id = ResponsesAPIRequestUtils.decode_responses_api_response_id(
responses_id_security
).get("response_id", responses_id_security)
try:
response = await self._get_response(
response_id=responses_id_security,
litellm_metadata=litellm_metadata,
via_router=via_router,
)
verbose_proxy_logger.debug(
f"Response {unified_object_id} status: {response.status}, model: {model_name}"
)
except Exception as e:
if via_router and _is_response_gone_at_provider(e, provider_response_id):
verbose_proxy_logger.info(
f"Response {unified_object_id} no longer available at provider (404), marking stale_expired: {e}"
)
expired_jobs.append(job)
continue
verbose_proxy_logger.warning(
f"Skipping job {unified_object_id} due to error: {e}"
)
@ -210,6 +238,7 @@ class CheckResponsesCost:
# Mark completed jobs in the database
if len(completed_jobs) > 0:
await _managed_object_table(self.prisma_client).update_many(
# bounded-ok: at most MAX_OBJECTS_PER_POLL_CYCLE rows per cycle, the find_many take above
where={"id": {"in": [job.id for job in completed_jobs]}},
data={"status": "completed"},
)
@ -217,3 +246,13 @@ class CheckResponsesCost:
f"Marked {len(completed_jobs)} response jobs as completed"
)
if len(expired_jobs) > 0:
await _managed_object_table(self.prisma_client).update_many(
# bounded-ok: at most MAX_OBJECTS_PER_POLL_CYCLE rows per cycle, the find_many take above
where={"id": {"in": [job.id for job in expired_jobs]}},
data={"status": "stale_expired"},
)
verbose_proxy_logger.info(
f"Marked {len(expired_jobs)} response jobs as stale_expired"
)

461
enterprise/litellm_enterprise/proxy/hooks/managed_files.py
View file

@ -1,9 +1,11 @@
# What is this?
## This hook is used to check for LiteLLM managed files in the request body, and replace them with model-specific file id
import asyncio
import base64
import json
from collections.abc import Iterator, Mapping, Sequence
import time
from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
from types import MappingProxyType
from typing import (
TYPE_CHECKING,
@ -23,21 +25,25 @@ from uuid import NAMESPACE_URL, uuid5
import httpx
from fastapi import HTTPException
from pydantic import ValidationError
from typing_extensions import ReadOnly
from typing_extensions import ReadOnly, Unpack
import litellm
from litellm import Router, verbose_logger
from litellm._internal_context import with_service_target
from litellm._uuid import uuid
from litellm.caching.caching import DualCache
from litellm.constants import MAX_FILE_LIST_LIMIT
from litellm.constants import (
BATCH_OUTPUT_FILE_FALLBACK_RETRY_AFTER_SECONDS,
BATCH_OUTPUT_FILE_LOOKUP_TIMEOUT_SECONDS,
MAX_FILE_LIST_LIMIT,
)
from litellm.files.types import FileRetrieveCallOptions, FileRetrieveProvider
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR
from litellm.litellm_core_utils.prompt_templates.common_utils import (
extract_file_metadata,
)
from openai import AsyncOpenAI
from openai import APIConnectionError, AsyncOpenAI
from openai.types.file_deleted import FileDeleted
from litellm.llms.base_llm.files.storage_backend_factory import get_storage_backend
@ -55,7 +61,10 @@ from litellm.proxy._types import (
ProxyException,
UserAPIKeyAuth,
)
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
from litellm.proxy.litellm_pre_call_utils import (
LiteLLMProxyRequestSetup,
sanitize_for_log,
)
from litellm.proxy.openai_files_endpoints.common_utils import (
BATCH_CREATE_HIDDEN_PARAM,
FILE_LIST_CONTINUATION_CHUNK_SIZE,
@ -77,6 +86,7 @@ from litellm.proxy.openai_files_endpoints.common_utils import (
from litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attribution import (
request_tags_from_metadata,
)
from litellm.repositories.managed_file_repository import ManagedFileRepository
from litellm.types.llms.openai import ( # pyright: ignore[reportAttributeAccessIssue]
AllMessageValues,
AsyncCursorPage,
@ -96,6 +106,17 @@ from litellm.types.utils import (
SpecialEnums,
)
class _ManagedFileRetrieve(Protocol):
async def __call__(
self,
*,
file_id: str,
_litellm_internal_model_credentials: Mapping[str, object] | None = None,
**kwargs: Unpack[FileRetrieveCallOptions],
) -> OpenAIFileObject: ...
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
from prisma.models import (
@ -146,6 +167,91 @@ def _parse_managed_file_object(raw_file_object: object, unified_file_id: str) ->
return None
def _batch_output_file_object(
unified_file_id: str, raw_file_id: str, size_bytes: int, *, fallback: bool
) -> OpenAIFileObject:
filename: Final = raw_file_id.rsplit("/", 1)[-1] or raw_file_id
return OpenAIFileObject(
id=unified_file_id,
object="file",
purpose="batch_output",
filename=filename,
created_at=int(time.time()),
bytes=size_bytes,
status="processed",
litellm_details_fallback=True if fallback else None,
)
def _public_file_object(file_object: OpenAIFileObject, unified_file_id: str) -> OpenAIFileObject:
return file_object.model_copy(update={"id": unified_file_id, "litellm_details_fallback": None})
def _is_transient_file_retrieve_error(error: Exception) -> bool:
status_code: Final = getattr(error, "status_code", None)
if isinstance(status_code, int):
return status_code in {408, 429} or status_code >= 500
return isinstance(error, (httpx.TransportError, APIConnectionError, asyncio.TimeoutError))
def _proxy_llm_router() -> Router | None:
import litellm.proxy.proxy_server as proxy_server_module
return cast(Router | None, getattr(proxy_server_module, "llm_router", None))
def _provider_file_retrieve_credentials(
*,
llm_router: Router | None,
model_id: str | None,
) -> Mapping[str, object] | None:
if llm_router is None or model_id is None:
return None
try:
credentials: Final = llm_router.get_deployment_credentials_with_provider(model_id)
except Exception as error:
verbose_logger.warning(
"Failed to retrieve credentials for provider file "
f"model_id={sanitize_for_log(model_id)}: {sanitize_for_log(error)}"
)
return None
return cast(Mapping[str, object], credentials) if credentials else None
_PROVIDER_FILE_RETRIEVE_PROVIDERS: Final[frozenset[str]] = frozenset(
{
"openai",
"azure",
"gemini",
"vertex_ai",
"bedrock",
"hosted_vllm",
"litellm_proxy",
"manus",
"anthropic",
"mistral",
"xai",
}
)
def _model_name_file_retrieve_provider(model_name: str | None) -> FileRetrieveProvider | None:
if model_name is None:
return None
provider, separator, _ = model_name.partition("/")
if not separator or provider not in _PROVIDER_FILE_RETRIEVE_PROVIDERS:
return None
return cast(FileRetrieveProvider, provider)
def _has_provider_file_retrieve_route(
*,
router_credentials: Mapping[str, object] | None,
model_name: str | None,
) -> bool:
return bool(router_credentials) or _model_name_file_retrieve_provider(model_name) is not None
class _ManagedFileRow(Protocol):
unified_file_id: str
file_object: OpenAIFileObject
@ -161,6 +267,8 @@ class _ManagedFileRow(Protocol):
class _ManagedFileTableActions(Protocol):
async def find_first(self, where: Mapping[str, object]) -> Optional[_ManagedFileRow]: ...
async def update_many(self, where: Mapping[str, object], data: Mapping[str, object]) -> int: ...
async def find_many(
self,
where: Mapping[str, object],
@ -253,13 +361,21 @@ def _storage_metadata_of(file_object: OpenAIFileObject | None) -> Mapping[str, s
_MANAGED_FILES_TARGET: Final = "managed_files"
_PROVIDER_FILE_RETRIEVE_RETRY_DELAYS_SECONDS: Final = (0.5, 1.0, 2.0)
class PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
# Class variables or attributes
def __init__(self, internal_usage_cache: InternalUsageCache, prisma_client: PrismaClient):
def __init__(
self,
internal_usage_cache: InternalUsageCache,
prisma_client: PrismaClient,
*,
sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,
):
self.internal_usage_cache = internal_usage_cache
self.prisma_client = prisma_client
self._sleep = sleep
@staticmethod
def _get_prometheus_logger():
@ -595,14 +711,231 @@ class PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
for file_id in provider_file_ids:
model_name = decode_model_from_file_id(file_id)
raw_file_id = get_original_file_id(file_id)
await self.store_unified_file_id(
file_id=file_id,
file_object=None,
await self.store_batch_output_file(
unified_file_id=file_id,
provider_file_id=raw_file_id,
model_id=model_name or None,
model_name=model_name,
litellm_parent_otel_span=litellm_parent_otel_span,
model_mappings={model_name: raw_file_id} if model_name else {},
user_api_key_dict=owner_identity,
owner=owner_identity,
)
async def _afile_retrieve_with_retries(
self,
*,
provider_file_id: str,
call_options: Mapping[str, object],
internal_model_credentials: Mapping[str, object] | None = None,
) -> OpenAIFileObject:
"""Retrieve provider file details with transient retries and SDK retries disabled."""
retrieve_file: Final = cast(_ManagedFileRetrieve, litellm.afile_retrieve)
retrieve_options: Final = cast(
FileRetrieveCallOptions,
{**call_options, "max_retries": 0},
)
for attempt in range(len(_PROVIDER_FILE_RETRIEVE_RETRY_DELAYS_SECONDS) + 1):
if attempt > 0:
await self._sleep(_PROVIDER_FILE_RETRIEVE_RETRY_DELAYS_SECONDS[attempt - 1])
try:
if internal_model_credentials is None:
return await retrieve_file(
file_id=provider_file_id,
**retrieve_options,
)
return await retrieve_file(
file_id=provider_file_id,
_litellm_internal_model_credentials=MappingProxyType(dict(internal_model_credentials)),
**retrieve_options,
)
except Exception as error:
if not _is_transient_file_retrieve_error(error) or attempt == len(
_PROVIDER_FILE_RETRIEVE_RETRY_DELAYS_SECONDS
):
raise
raise RuntimeError("Provider file retrieve retry loop ended without a result")
async def _fetch_provider_file_object(
self,
*,
unified_file_id: str,
provider_file_id: str,
model_id: str | None,
model_name: str | None,
llm_router: Router | None = None,
raise_on_failure: bool = False,
allow_default_provider: bool = False,
) -> tuple[OpenAIFileObject | None, bool]:
"""Fetch provider file details through the configured route under a total timeout."""
route_llm_router: Final = llm_router if llm_router is not None else _proxy_llm_router()
router_credentials: Final = _provider_file_retrieve_credentials(
llm_router=route_llm_router,
model_id=model_id,
)
model_name_provider: Final = _model_name_file_retrieve_provider(model_name)
fetch_route_available: Final = _has_provider_file_retrieve_route(
router_credentials=router_credentials,
model_name=model_name,
)
default_provider_route: Final = allow_default_provider and route_llm_router is not None
if not fetch_route_available and not default_provider_route:
return None, False
try:
if router_credentials is not None:
provider_file_object: Final = await asyncio.wait_for(
self._afile_retrieve_with_retries(
provider_file_id=provider_file_id,
call_options=router_credentials,
internal_model_credentials=router_credentials,
),
timeout=BATCH_OUTPUT_FILE_LOOKUP_TIMEOUT_SECONDS,
)
return provider_file_object.model_copy(update={"id": unified_file_id}), True
if model_name_provider is not None:
provider_file_object_by_model_name: Final = await asyncio.wait_for(
self._afile_retrieve_with_retries(
provider_file_id=provider_file_id,
call_options={"custom_llm_provider": model_name_provider},
),
timeout=BATCH_OUTPUT_FILE_LOOKUP_TIMEOUT_SECONDS,
)
return (
provider_file_object_by_model_name.model_copy(update={"id": unified_file_id}),
True,
)
if default_provider_route:
provider_file_object_by_default_route: Final = await asyncio.wait_for(
self._afile_retrieve_with_retries(
provider_file_id=provider_file_id,
call_options=router_credentials or {},
),
timeout=BATCH_OUTPUT_FILE_LOOKUP_TIMEOUT_SECONDS,
)
return (
provider_file_object_by_default_route.model_copy(update={"id": unified_file_id}),
True,
)
return None, False
except Exception as error:
verbose_logger.warning(
"Failed to retrieve batch file object for "
f"provider_file_id={sanitize_for_log(provider_file_id)}: "
f"{type(error).__name__} {sanitize_for_log(error)}"
)
if raise_on_failure:
if isinstance(error, TimeoutError) and not str(error):
raise TimeoutError(
"Provider file retrieve timed out "
f"after {BATCH_OUTPUT_FILE_LOOKUP_TIMEOUT_SECONDS} seconds"
) from error
raise
return None, True
async def _save_refreshed_file_object(
self,
stored: LiteLLM_ManagedFileTable,
file_object: OpenAIFileObject,
) -> None:
if not await ManagedFileRepository(self.prisma_client).update_file_object(stored.unified_file_id, file_object):
return
refreshed_row: Final = stored.model_copy(update={"file_object": file_object})
await self.internal_usage_cache.async_set_cache(
key=stored.unified_file_id,
value=refreshed_row.model_dump(),
litellm_parent_otel_span=None,
)
async def store_batch_output_file(
self,
*,
unified_file_id: str,
provider_file_id: str,
model_id: str | None,
model_name: str | None = None,
owner: UserAPIKeyAuth,
litellm_parent_otel_span: Span | None,
size_bytes: int | None = None,
fetch_provider_details: bool = True,
) -> None:
"""Register batch output or error file metadata, optionally fetching provider details."""
stored_file: Final = await self.get_unified_file_id(unified_file_id, litellm_parent_otel_span)
stored_object: Final = stored_file.file_object if stored_file is not None else None
if not fetch_provider_details:
if stored_file is not None:
return
router_credentials: Final = _provider_file_retrieve_credentials(
llm_router=_proxy_llm_router(),
model_id=model_id,
)
file_object_without_provider_details: Final = _batch_output_file_object(
unified_file_id,
provider_file_id,
size_bytes or 0,
fallback=_has_provider_file_retrieve_route(
router_credentials=router_credentials,
model_name=model_name,
),
)
await self.store_unified_file_id(
file_id=unified_file_id,
file_object=file_object_without_provider_details,
litellm_parent_otel_span=litellm_parent_otel_span,
model_mappings={model_id: provider_file_id} if model_id else {},
user_api_key_dict=owner,
)
return
if stored_object is not None and not stored_object.litellm_details_fallback:
return
fallback_written_recently: Final = (
stored_object is not None
and time.time() - stored_object.created_at < BATCH_OUTPUT_FILE_FALLBACK_RETRY_AFTER_SECONDS
)
provider_fetch_result: Final = (
(None, True)
if fallback_written_recently
else await self._fetch_provider_file_object(
unified_file_id=unified_file_id,
provider_file_id=provider_file_id,
model_id=model_id,
model_name=model_name,
)
)
provider_object, fetch_route_available = provider_fetch_result
if (
stored_file is not None
and provider_object is None
and (size_bytes is None or (stored_object is not None and stored_object.bytes == size_bytes))
):
return
file_object: Final = (
provider_object
if provider_object is not None
else (
stored_object.model_copy(update={"bytes": size_bytes})
if stored_object is not None and size_bytes is not None
else _batch_output_file_object(
unified_file_id,
provider_file_id,
size_bytes or 0,
fallback=fetch_route_available,
)
)
)
if stored_file is not None:
await self._save_refreshed_file_object(stored_file, file_object)
return
await self.store_unified_file_id(
file_id=unified_file_id,
file_object=file_object,
litellm_parent_otel_span=litellm_parent_otel_span,
model_mappings={model_id: provider_file_id} if model_id else {},
user_api_key_dict=owner,
)
async def list_user_batches(
self,
user_api_key_dict: UserAPIKeyAuth,
@ -745,6 +1078,7 @@ class PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
user_api_key_dict=user_api_key_dict,
db_batch_object=row,
unified_batch_id=_is_base64_encoded_unified_file_id(row.unified_object_id),
fetch_provider_details=False,
)
except Exception as e:
verbose_logger.warning(f"Failed to resolve managed file ids for batch {row.unified_object_id}: {e}")
@ -806,7 +1140,7 @@ class PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
}
)
return [
parsed_file_object.model_copy(update={"id": row.unified_file_id})
_public_file_object(parsed_file_object, row.unified_file_id)
for row in file_ids
if (parsed_file_object := _parse_managed_file_object(row.file_object, row.unified_file_id)) is not None
]
@ -1474,42 +1808,13 @@ class PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
)
setattr(response, file_attr, unified_file_id)
# Use llm_router credentials when available. Without credentials,
# Azure and other auth-required providers return 500/401.
file_object = None
try:
# Import module and use getattr for better testability with mocks
import litellm.proxy.proxy_server as proxy_server_module
_llm_router = getattr(proxy_server_module, "llm_router", None)
if _llm_router is not None and model_id:
_creds = _llm_router.get_deployment_credentials_with_provider(model_id) or {}
file_object = await litellm.afile_retrieve(
file_id=provider_file_id,
**cast(FileRetrieveCallOptions, _creds),
)
else:
file_object = await litellm.afile_retrieve(
custom_llm_provider=cast(
FileRetrieveProvider,
model_name.split("/")[0] if model_name and "/" in model_name else "openai",
),
file_id=provider_file_id,
)
verbose_logger.debug(
f"Successfully retrieved file object for {file_attr}={provider_file_id}"
)
except Exception as e:
verbose_logger.warning(
f"Failed to retrieve file object for {file_attr}={provider_file_id}: {str(e)}. Storing with None and will fetch on-demand."
)
await self.store_unified_file_id(
file_id=unified_file_id,
file_object=file_object,
await self.store_batch_output_file(
unified_file_id=unified_file_id,
provider_file_id=provider_file_id,
model_id=model_id,
model_name=resolved_model_name,
owner=user_api_key_dict,
litellm_parent_otel_span=user_api_key_dict.parent_otel_span,
model_mappings={model_id: provider_file_id},
user_api_key_dict=user_api_key_dict,
)
request_metadata: Final = data.get("litellm_metadata")
await self.store_unified_object_id(
@ -1612,6 +1917,7 @@ class PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
async def afile_retrieve(
self, file_id: str, litellm_parent_otel_span: Optional[Span], llm_router: Optional[Router] = None
) -> OpenAIFileObject:
"""Return public details for a managed file ID, refreshing a basic entry when possible."""
stored_file_object = await self.get_unified_file_id(file_id, litellm_parent_otel_span)
# Case 1 : This is not a managed file
@ -1621,30 +1927,57 @@ class PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
# Case 2: Managed file and the file object exists in the database
# The stored file_object has the raw provider ID. Replace with the unified ID
# so callers see a consistent ID (matching Case 3 which does response.id = file_id).
if stored_file_object and stored_file_object.file_object:
# Use model_copy to ensure the ID update persists (Pydantic v2 compatibility)
response = stored_file_object.file_object.model_copy(update={"id": file_id})
return response
if stored_file_object.file_object is not None:
file_object: Final = stored_file_object.file_object
if file_object.litellm_details_fallback and stored_file_object.model_mappings:
try:
model_id, provider_file_id = next(iter(stored_file_object.model_mappings.items()))
refreshed_file_object, _ = await self._fetch_provider_file_object(
unified_file_id=file_id,
provider_file_id=provider_file_id,
model_id=model_id,
model_name=model_id,
llm_router=llm_router,
)
if refreshed_file_object is None:
return _public_file_object(file_object, file_id)
await self._save_refreshed_file_object(stored_file_object, refreshed_file_object)
return _public_file_object(refreshed_file_object, file_id)
except Exception as error:
verbose_logger.warning(
"Failed to refresh batch file object for "
f"file_id={sanitize_for_log(file_id)}: {sanitize_for_log(error)}"
)
return _public_file_object(file_object, file_id)
# Case 3: Managed file exists in the database but not the file object (for. e.g the batch task might not have run)
# So we fetch the file object from the provider. We deliberately do not store the result to avoid interfering with batch cost tracking code.
if not llm_router:
model_mapping: Final = next(iter(stored_file_object.model_mappings.items()), None)
if model_mapping is None:
raise Exception(
f"LiteLLM Managed File object with id={file_id} has no file_object "
f"and llm_router is required to fetch from provider"
f"LiteLLM Managed File object with id={file_id} has no file_object and no provider route to fetch it"
)
model_id, model_file_id = model_mapping
try:
model_id, model_file_id = next(iter(stored_file_object.model_mappings.items()))
credentials: Final = llm_router.get_deployment_credentials_with_provider(model_id) or {}
response = await litellm.afile_retrieve(
file_id=model_file_id,
**cast(FileRetrieveCallOptions, credentials),
response, fetch_route_available = await self._fetch_provider_file_object(
unified_file_id=file_id,
provider_file_id=model_file_id,
model_id=model_id,
model_name=model_id,
llm_router=llm_router,
raise_on_failure=True,
allow_default_provider=True,
)
response.id = file_id # Replace with unified ID
return response
except Exception as e:
raise Exception(f"Failed to retrieve file {file_id} from provider: {str(e)}") from e
if not fetch_route_available:
raise Exception(
f"LiteLLM Managed File object with id={file_id} has no file_object and no provider route to fetch it"
)
if response is None:
raise ValueError("Provider file details could not be retrieved")
return _public_file_object(response, file_id)
async def afile_list(
self,
@ -1707,7 +2040,7 @@ class PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
**cursor_args,
)
matches.extend(
parsed_file_object.model_copy(update={"id": row.unified_file_id})
_public_file_object(parsed_file_object, row.unified_file_id)
for row in chunk
if (parsed_file_object := _parse_managed_file_object(row.file_object, row.unified_file_id)) is not None
and (purpose is None or parsed_file_object.purpose == purpose)
@ -2110,4 +2443,4 @@ class PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
verbose_logger.debug(
f"Converted file {file_id} from storage backend to base64 with format {content_type}"
)
_PROXY_LiteLLMManagedFiles = PROXY_LiteLLMManagedFiles
PROXY_LiteLLMManagedFiles = _PROXY_LiteLLMManagedFiles

4
enterprise/litellm_enterprise/proxy/hooks/managed_vector_stores.py
View file

@ -38,7 +38,7 @@ else:
PrismaClient = Any
class PROXY_LiteLLMManagedVectorStores(
class _PROXY_LiteLLMManagedVectorStores(
CustomLogger, BaseManagedResource[VectorStoreCreateResponse]
):
"""
@ -462,4 +462,4 @@ class PROXY_LiteLLMManagedVectorStores(
parent_otel_span=parent_otel_span,
resource_id_key="vector_store_id",
)
_PROXY_LiteLLMManagedVectorStores = PROXY_LiteLLMManagedVectorStores
PROXY_LiteLLMManagedVectorStores = _PROXY_LiteLLMManagedVectorStores

4
enterprise/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-enterprise"
version = "0.1.74"
version = "0.1.76"
description = "Package for LiteLLM Enterprise features"
readme = "README.md"
requires-python = ">=3.9"
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.1.74"
version = "0.1.76"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-enterprise==",

2
gateway/routes/allowlist.py
View file

@ -62,6 +62,8 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = (
"/rerank",
"/v1/decisions",
"/decisions",
"/v1/systemone",
"/systemone",
"/v1/ocr",
"/ocr",
"/v1/rag/",

77
helm/litellm-helm/templates/_helpers.tpl
View file

@ -321,3 +321,80 @@ through an emptyDir. Empty when the sidecar is off or uses 127.0.0.1 TCP.
- name: LITELLM_COLLECTOR_DRAIN_TIMEOUT_SECONDS
value: {{ .Values.collector.drainTimeoutSeconds | quote }}
{{- end -}}
{{- define "litellm.lensWorker.image" -}}
{{- if .Values.lensWorker.image.digest -}}
{{- if not (regexMatch "^sha256:[0-9a-f]{64}$" .Values.lensWorker.image.digest) -}}
{{- fail "lensWorker.image.digest must be sha256 followed by 64 lowercase hex characters" -}}
{{- end -}}
{{- printf "%s@%s" .Values.lensWorker.image.repository .Values.lensWorker.image.digest -}}
{{- else -}}
{{- $backendTag := .Values.image.tag | default .Chart.AppVersion -}}
{{- $releaseTag := ternary (printf "v%s" $backendTag) $backendTag (regexMatch "^[0-9]" $backendTag) -}}
{{- $tag := .Values.lensWorker.image.tag | default $releaseTag -}}
{{- $repository := .Values.lensWorker.image.repository -}}
{{- if and (hasPrefix "sha-" $tag) (eq $repository "ghcr.io/berriai/litellm-lens-worker") -}}
{{- $repository = "ghcr.io/berriai/litellm-lens-worker-dev" -}}
{{- end -}}
{{- printf "%s:%s" $repository $tag -}}
{{- end -}}
{{- end -}}
{{- define "litellm.gateway.collectorSocketDir" -}}
{{- if and .Values.gateway.collector.enabled (hasPrefix "unix://" .Values.gateway.collector.address) -}}
{{- dir (trimPrefix "unix://" .Values.gateway.collector.address) -}}
{{- end -}}
{{- end -}}
{{/*
LITELLM_COLLECTOR_* env shared by the producer (gateway container) and the
consumer (collector container), so both agree on the transport and the
shutdown drain window.
*/}}
{{- define "litellm.gateway.collectorEnv" -}}
{{- with .Values.gateway.collector }}
- name: LITELLM_COLLECTOR_ENABLED
value: "true"
- name: LITELLM_COLLECTOR_ADDRESS
value: {{ .address | quote }}
- name: LITELLM_COLLECTOR_BUFFER_SIZE
value: {{ .bufferSize | quote }}
- name: LITELLM_COLLECTOR_ON_UNAVAILABLE
value: {{ .onUnavailable | quote }}
- name: LITELLM_COLLECTOR_DRAIN_TIMEOUT_SECONDS
value: {{ .drainTimeoutSeconds | quote }}
{{- end }}
{{- end -}}
{{- define "litellm.lensWorker.labels" -}}
{{- $labels := include "litellm.labels" . | fromYaml -}}
{{- $_ := set $labels "app.kubernetes.io/name" (printf "%s-lens-worker" (include "litellm.name" . | trunc 51 | trimSuffix "-")) -}}
{{- toYaml $labels -}}
{{- end -}}
{{- define "litellm.lensWorker.serviceTokenSecretName" -}}
{{- .Values.lensWorker.serviceTokenSecret.name | default (printf "%s-lens-service" (include "litellm.fullname" .)) -}}
{{- end -}}
{{- define "litellm.lensWorker.bundledClickhouse" -}}
{{- if and .Values.lensWorker.enabled .Values.lensWorker.clickhouse.enabled (not .Values.lensWorker.clickhouseSecret.name) -}}true{{- end -}}
{{- end -}}
{{- define "litellm.lensWorker.publicUrl" -}}
{{- if .Values.lensWorker.publicUrl -}}
{{- .Values.lensWorker.publicUrl -}}
{{- else if .Values.lensWorker.ingress.enabled -}}
{{- $tls := or (not (empty .Values.lensWorker.ingress.tls)) (hasKey .Values.lensWorker.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
{{- printf "%s://%s" (ternary "https" "http" $tls) (required "lensWorker.ingress.host is required" .Values.lensWorker.ingress.host) -}}
{{- else if and .Values.ingress.enabled (eq (len .Values.ingress.hosts) 1) -}}
{{- $host := required "ingress.hosts[0].host is required" (first .Values.ingress.hosts).host -}}
{{- $tls := or (not (empty .Values.ingress.tls)) (hasKey .Values.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
{{- printf "%s://%s/lens-ingest" (ternary "https" "http" $tls) $host -}}
{{- else -}}
{{- fail "lensWorker.publicUrl is required when there is no single ingress hostname" -}}
{{- end -}}
{{- end -}}
{{- define "litellm.lensWorker.clickhouseName" -}}
{{- printf "%s-lens-clickhouse" (include "litellm.fullname" . | trunc 47 | trimSuffix "-") -}}
{{- end -}}

11
helm/litellm-helm/templates/deployment.yaml
View file

@ -56,6 +56,17 @@ spec:
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.image.pullPolicy }}
env:
{{- if .Values.lensWorker.enabled }}
- name: LITELLM_LENS_URL
value: {{ printf "http://%s-lens-worker:%v" (include "litellm.fullname" .) .Values.lensWorker.service.port | quote }}
- name: LITELLM_LENS_PUBLIC_URL
value: {{ include "litellm.lensWorker.publicUrl" . | quote }}
- name: LITELLM_LENS_SERVICE_TOKEN
valueFrom:
secretKeyRef:
name: {{ include "litellm.lensWorker.serviceTokenSecretName" . | quote }}
key: {{ .Values.lensWorker.serviceTokenSecret.key | quote }}
{{- end }}
{{- include "litellm.proxyEnv" . | nindent 12 }}
{{- if .Values.liteadmin.enabled }}
- name: LITELLM_ADMIN_AGENT_URL

14
helm/litellm-helm/templates/ingress.yaml
View file

@ -44,6 +44,20 @@ spec:
- host: {{ .host | quote }}
http:
paths:
{{- if $.Values.lensWorker.enabled }}
- path: /lens-ingest
pathType: Prefix
backend:
{{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
service:
name: {{ $fullName }}-lens-worker
port:
number: {{ $.Values.lensWorker.service.port }}
{{- else }}
serviceName: {{ $fullName }}-lens-worker
servicePort: {{ $.Values.lensWorker.service.port }}
{{- end }}
{{- end }}
{{- range .paths }}
- path: {{ .path }}
{{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }}

98
helm/litellm-helm/templates/lens/clickhouse.yaml Normal file
View file

@ -0,0 +1,98 @@
{{- if include "litellm.lensWorker.bundledClickhouse" . }}
{{- $name := include "litellm.lensWorker.clickhouseName" . }}
apiVersion: v1
kind: Service
metadata:
name: {{ $name }}
spec:
clusterIP: None
selector:
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/component: lens-clickhouse
ports:
- name: http
port: 8123
targetPort: http
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: {{ $name }}
spec:
serviceName: {{ $name }}
replicas: 1
selector:
matchLabels:
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/component: lens-clickhouse
template:
metadata:
labels:
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/component: lens-clickhouse
spec:
automountServiceAccountToken: false
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
securityContext:
runAsNonRoot: true
runAsUser: 101
runAsGroup: 101
fsGroup: 101
seccompProfile:
type: RuntimeDefault
containers:
- name: clickhouse
image: {{ .Values.lensWorker.clickhouse.image | quote }}
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: [ALL]
env:
- name: CLICKHOUSE_USER
value: default
- name: CLICKHOUSE_PASSWORD
valueFrom:
secretKeyRef:
name: {{ $name }}
key: password
- name: CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT
value: "1"
ports:
- name: http
containerPort: 8123
startupProbe:
httpGet:
path: /ping
port: http
periodSeconds: 5
timeoutSeconds: 3
failureThreshold: 60
readinessProbe:
httpGet:
path: /ping
port: http
livenessProbe:
httpGet:
path: /ping
port: http
timeoutSeconds: 3
resources:
{{- toYaml .Values.lensWorker.clickhouse.resources | nindent 12 }}
volumeMounts:
- name: data
mountPath: /var/lib/clickhouse
volumeClaimTemplates:
- metadata:
name: data
spec:
accessModes: [ReadWriteOnce]
{{- if ne .Values.lensWorker.clickhouse.storageClassName nil }}
storageClassName: {{ .Values.lensWorker.clickhouse.storageClassName | quote }}
{{- end }}
resources:
requests:
storage: {{ .Values.lensWorker.clickhouse.storage | quote }}
{{- end }}

109
helm/litellm-helm/templates/lens/deployment.yaml Normal file
View file

@ -0,0 +1,109 @@
{{- if .Values.lensWorker.enabled }}
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "litellm.fullname" . }}-lens-worker
labels:
{{- include "litellm.lensWorker.labels" . | nindent 4 }}
app.kubernetes.io/component: lens-worker
spec:
replicas: {{ .Values.lensWorker.replicaCount }}
selector:
matchLabels:
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/component: lens-worker
template:
metadata:
labels:
{{- include "litellm.lensWorker.labels" . | nindent 8 }}
app.kubernetes.io/component: lens-worker
spec:
automountServiceAccountToken: false
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
securityContext:
runAsNonRoot: true
runAsUser: 65532
runAsGroup: 65532
fsGroup: 65532
seccompProfile:
type: RuntimeDefault
containers:
- name: lens-worker
image: {{ include "litellm.lensWorker.image" . | quote }}
imagePullPolicy: {{ .Values.lensWorker.image.pullPolicy }}
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop: [ALL]
env:
- name: LITELLM_URL
value: {{ .Values.lensWorker.url | default (printf "http://%s:%v" (include "litellm.fullname" .) .Values.service.port) | quote }}
- name: LITELLM_LENS_SERVICE_TOKEN
valueFrom:
secretKeyRef:
name: {{ include "litellm.lensWorker.serviceTokenSecretName" . | quote }}
key: {{ .Values.lensWorker.serviceTokenSecret.key | quote }}
{{- if include "litellm.lensWorker.bundledClickhouse" . }}
- name: CLICKHOUSE_HOST
value: {{ include "litellm.lensWorker.clickhouseName" . | quote }}
- name: CLICKHOUSE_PASSWORD
valueFrom:
secretKeyRef:
name: {{ include "litellm.lensWorker.clickhouseName" . | quote }}
key: password
{{- else }}
- name: CLICKHOUSE_URL
valueFrom:
secretKeyRef:
name: {{ required "lensWorker.clickhouseSecret.name is required" .Values.lensWorker.clickhouseSecret.name | quote }}
key: {{ .Values.lensWorker.clickhouseSecret.key | quote }}
{{- end }}
- name: CLICKHOUSE_DATABASE
value: {{ .Values.lensWorker.clickhouseDatabase | quote }}
- name: AGENT_TRACING_RETENTION_DAYS
value: {{ .Values.lensWorker.retentionDays | quote }}
{{- if .Values.lensWorker.tokenSecret.name }}
- name: LENS_WORKER_TOKEN
valueFrom:
secretKeyRef:
name: {{ .Values.lensWorker.tokenSecret.name | quote }}
key: {{ .Values.lensWorker.tokenSecret.key | quote }}
{{- end }}
ports:
- name: otlp
containerPort: 4318
livenessProbe:
httpGet:
path: /health/live
port: otlp
readinessProbe:
httpGet:
path: /health/ready
port: otlp
resources:
{{- toYaml .Values.lensWorker.resources | nindent 12 }}
volumeMounts:
- name: tmp
mountPath: /tmp
volumes:
- name: tmp
emptyDir:
medium: Memory
sizeLimit: {{ .Values.lensWorker.tmpSizeLimit }}
{{- with .Values.lensWorker.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.lensWorker.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.lensWorker.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- end }}

29
helm/litellm-helm/templates/lens/ingress.yaml Normal file
View file

@ -0,0 +1,29 @@
{{- if and .Values.lensWorker.enabled .Values.lensWorker.ingress.enabled }}
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: {{ include "litellm.fullname" . }}-lens-worker
{{- with .Values.lensWorker.ingress.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
{{- with .Values.lensWorker.ingress.className }}
ingressClassName: {{ . | quote }}
{{- end }}
{{- with .Values.lensWorker.ingress.tls }}
tls:
{{- toYaml . | nindent 4 }}
{{- end }}
rules:
- host: {{ required "lensWorker.ingress.host is required" .Values.lensWorker.ingress.host | quote }}
http:
paths:
- path: /v1/
pathType: Prefix
backend:
service:
name: {{ include "litellm.fullname" . }}-lens-worker
port:
name: otlp
{{- end }}

27
helm/litellm-helm/templates/lens/secrets.yaml Normal file
View file

@ -0,0 +1,27 @@
{{- if and .Values.lensWorker.enabled (not .Values.lensWorker.serviceTokenSecret.name) }}
{{- $name := include "litellm.lensWorker.serviceTokenSecretName" . }}
{{- $existing := lookup "v1" "Secret" .Release.Namespace $name }}
apiVersion: v1
kind: Secret
metadata:
name: {{ $name }}
annotations:
helm.sh/resource-policy: keep
type: Opaque
data:
{{ .Values.lensWorker.serviceTokenSecret.key }}: {{ if $existing }}{{ required "Saved Lens service secret is missing its key" (index $existing.data .Values.lensWorker.serviceTokenSecret.key) | quote }}{{ else }}{{ randAlphaNum 64 | b64enc | quote }}{{ end }}
{{- end }}
{{- if include "litellm.lensWorker.bundledClickhouse" . }}
{{- $name := include "litellm.lensWorker.clickhouseName" . }}
{{- $existing := lookup "v1" "Secret" .Release.Namespace $name }}
---
apiVersion: v1
kind: Secret
metadata:
name: {{ $name }}
annotations:
helm.sh/resource-policy: keep
type: Opaque
data:
password: {{ if $existing }}{{ required "Saved Lens ClickHouse secret is missing its password" (index $existing.data "password") | quote }}{{ else }}{{ randAlphaNum 64 | b64enc | quote }}{{ end }}
{{- end }}

18
helm/litellm-helm/templates/lens/service.yaml Normal file
View file

@ -0,0 +1,18 @@
{{- if .Values.lensWorker.enabled }}
apiVersion: v1
kind: Service
metadata:
name: {{ include "litellm.fullname" . }}-lens-worker
{{- with .Values.lensWorker.service.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
selector:
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/component: lens-worker
ports:
- name: otlp
port: {{ .Values.lensWorker.service.port }}
targetPort: otlp
{{- end }}

18
helm/litellm-helm/templates/migrations-job.yaml
View file

@ -6,6 +6,9 @@ metadata:
name: {{ include "litellm.fullname" . }}-migrations
labels:
{{- include "litellm.labels" . | nindent 4 }}
{{- with .Values.migrationJob.jobLabels }}
{{- toYaml . | nindent 4 }}
{{- end }}
annotations:
{{- if .Values.migrationJob.hooks.argocd.enabled }}
argocd.argoproj.io/hook: PreSync
@ -17,6 +20,9 @@ metadata:
helm.sh/hook-weight: {{ .Values.migrationJob.hooks.helm.weight | default "1" | quote }}
{{- end }}
checksum/config: {{ toYaml .Values | sha256sum }}
{{- with .Values.migrationJob.jobAnnotations }}
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
template:
metadata:
@ -25,6 +31,9 @@ spec:
{{- with .Values.podLabels }}
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.migrationJob.podLabels }}
{{- toYaml . | nindent 8 }}
{{- end }}
annotations:
{{- with .Values.migrationJob.annotations }}
{{- toYaml . | nindent 8 }}
@ -47,7 +56,16 @@ spec:
imagePullPolicy: {{ .Values.image.pullPolicy }}
securityContext:
{{- toYaml .Values.securityContext | nindent 12 }}
{{- if .Values.migrationJob.command }}
command: {{ toYaml .Values.migrationJob.command | nindent 12 }}
{{- else }}
command: ["python", "litellm/proxy/prisma_migration.py"]
{{- end }}
{{- if .Values.migrationJob.args }}
args: {{ toYaml .Values.migrationJob.args | nindent 12 }}
{{- else if .Values.migrationJob.command }}
args: []
{{- end }}
workingDir: "/app"
env:
{{- if .Values.db.useExisting }}

84
helm/litellm-helm/tests/lens_endpoint_defaults_tests.yaml Normal file
View file

@ -0,0 +1,84 @@
suite: Lens endpoint defaults
templates:
- deployment.yaml
- configmap-litellm.yaml
set:
lensWorker.enabled: true
ingress.enabled: true
ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
ingress.tls:
- hosts: [gateway.example]
secretName: tls
tests:
- it: derives the trace endpoint from the deployment hostname and TLS
template: deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://gateway.example/lens-ingest
- it: preserves an explicitly configured public address
set:
lensWorker.publicUrl: https://custom.example/prefix
template: deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://custom.example/prefix
- it: uses the dedicated Lens ingress when configured
set:
lensWorker.ingress.enabled: true
lensWorker.ingress.host: traces.example
lensWorker.ingress.tls:
- hosts: [traces.example]
secretName: traces-tls
template: deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://traces.example
- it: uses HTTPS for a dedicated ALB ingress with certificate annotations
set:
lensWorker.ingress.enabled: true
lensWorker.ingress.host: traces.example
lensWorker.ingress.className: alb
lensWorker.ingress.tls: []
lensWorker.ingress.annotations:
alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-west-2:123456789012:certificate/test
alb.ingress.kubernetes.io/listen-ports: '[{"HTTP":80},{"HTTPS":443}]'
alb.ingress.kubernetes.io/ssl-redirect: "443"
template: deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://traces.example
- it: uses HTTP for a dedicated ingress without its own TLS
set:
lensWorker.ingress.enabled: true
lensWorker.ingress.host: traces.example
lensWorker.ingress.tls: []
lensWorker.ingress.annotations: {}
template: deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: http://traces.example
- it: uses HTTP when ingress has no TLS
set:
ingress.tls: []
template: deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: http://gateway.example/lens-ingest

45
helm/litellm-helm/tests/lens_saved_secrets_tests.yaml Normal file
View file

@ -0,0 +1,45 @@
suite: Lens preserves credentials across Helm upgrades
templates:
- lens/secrets.yaml
set:
fullnameOverride: lens-test
lensWorker.enabled: true
release:
namespace: lens
name: lens-test
upgrade: true
kubernetesProvider:
scheme:
v1/Secret:
gvr:
version: v1
resource: secrets
namespaced: true
objects:
- apiVersion: v1
kind: Secret
metadata:
name: lens-test-lens-service
namespace: lens
data:
service-token: c2F2ZWQtc2VydmljZS10b2tlbg==
- apiVersion: v1
kind: Secret
metadata:
name: lens-test-lens-clickhouse
namespace: lens
data:
password: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=
tests:
- it: reuses the service credential instead of breaking running services
documentIndex: 0
asserts:
- equal:
path: data.service-token
value: c2F2ZWQtc2VydmljZS10b2tlbg==
- it: reuses the database password instead of locking out stored traces
documentIndex: 1
asserts:
- equal:
path: data.password
value: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=

167
helm/litellm-helm/tests/lens_service_tests.yaml Normal file
View file

@ -0,0 +1,167 @@
suite: Lens service isolation and ingestion routing
templates:
- configmap-litellm.yaml
- deployment.yaml
- ingress.yaml
- lens/ingress.yaml
- lens/service.yaml
- lens/deployment.yaml
tests:
- it: connects deployment.yaml to the shared Lens service
template: deployment.yaml
set: &id001
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: https://gateway.example/lens-ingest
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_URL
value: http://lens-test-lens-worker:4318
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://gateway.example/lens-ingest
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_SERVICE_TOKEN
valueFrom:
secretKeyRef:
name: lens-service
key: service-token
- it: routes uploads directly to Lens instead of the gateway
template: ingress.yaml
set:
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: https://gateway.example/lens-ingest
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
ingress.enabled: true
ingress.hosts:
- host: gateway.example
paths:
- path: /
pathType: Prefix
asserts:
- contains:
path: spec.rules[0].http.paths
content:
path: /lens-ingest
pathType: Prefix
backend:
service:
name: lens-test-lens-worker
port:
number: 4318
- it: keeps internal routes out of a dedicated ingestion hostname
template: lens/ingress.yaml
set:
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: https://gateway.example/lens-ingest
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
lensWorker.ingress.enabled: true
lensWorker.ingress.host: traces.example
asserts:
- equal:
path: spec.rules[0].http.paths
value:
- path: /v1/
pathType: Prefix
backend:
service:
name: lens-test-lens-worker
port:
name: otlp
- it: maps the Lens service to the ingestion listener
template: lens/service.yaml
set: *id001
asserts:
- equal:
path: spec.selector
value:
app.kubernetes.io/instance: RELEASE-NAME
app.kubernetes.io/component: lens-worker
- equal:
path: spec.ports
value:
- name: otlp
port: 4318
targetPort: otlp
- it: gives only Lens the ClickHouse secret
template: lens/deployment.yaml
set: *id001
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_URL
valueFrom:
secretKeyRef:
name: lens-storage
key: url
- it: requires an agent reachable ingestion URL
template: deployment.yaml
set:
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: ''
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
asserts:
- failedTemplate:
errorMessage: lensWorker.publicUrl is required when there is no single ingress hostname
- it: omits Lens connection settings when disabled in deployment.yaml
template: deployment.yaml
asserts:
- notContains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_URL
any: true
- it: preserves an existing ClickHouse database and retention
template: lens/deployment.yaml
set:
lensWorker.enabled: true
lensWorker.publicUrl: https://traces.example
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
lensWorker.clickhouseDatabase: existing_traces
lensWorker.retentionDays: 45
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_DATABASE
value: existing_traces
- contains:
path: spec.template.spec.containers[0].env
content:
name: AGENT_TRACING_RETENTION_DAYS
value: '45'
- it: keeps Lens pods outside the gateway autoscaling selector
template: lens/deployment.yaml
set: &id002
nameOverride: inference
lensWorker.enabled: true
lensWorker.publicUrl: https://traces.example
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
asserts:
- equal:
path: spec.template.metadata.labels["app.kubernetes.io/name"]
value: inference-lens-worker
- it: preserves the existing gateway deployment selector
template: deployment.yaml
set: *id002
asserts:
- equal:
path: spec.selector.matchLabels["app.kubernetes.io/name"]
value: inference

160
helm/litellm-helm/tests/lens_setup_tests.yaml Normal file
View file

@ -0,0 +1,160 @@
suite: Lens managed setup
set:
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: https://traces.example
release:
namespace: lens
name: lens-test
templates:
- lens/deployment.yaml
- lens/clickhouse.yaml
- lens/secrets.yaml
tests:
- it: generates both private credentials for a new installation
template: lens/secrets.yaml
asserts:
- hasDocuments:
count: 2
- matchRegex:
path: data.service-token
pattern: '^[A-Za-z0-9+/]{86}==$'
documentIndex: 0
- matchRegex:
path: data.password
pattern: '^[A-Za-z0-9+/]{86}==$'
documentIndex: 1
- equal:
path: metadata.annotations["helm.sh/resource-policy"]
value: keep
- it: connects Lens to its private bundled storage
template: lens/deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_HOST
value: lens-test-lens-clickhouse
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_PASSWORD
valueFrom:
secretKeyRef:
name: lens-test-lens-clickhouse
key: password
- it: stores traces on a persistent volume with the chosen storage class
template: lens/clickhouse.yaml
documentIndex: 1
set:
lensWorker.clickhouse.storage: 40Gi
lensWorker.clickhouse.storageClassName: fast
asserts:
- equal:
path: spec.volumeClaimTemplates[0].spec
value:
accessModes: [ReadWriteOnce]
storageClassName: fast
resources:
requests:
storage: 40Gi
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_PASSWORD
valueFrom:
secretKeyRef:
name: lens-test-lens-clickhouse
key: password
- it: keeps an existing external database instead of creating a new one
template: lens/clickhouse.yaml
set:
lensWorker.clickhouseSecret.name: external-clickhouse
asserts:
- hasDocuments:
count: 0
- it: leaves supplied secrets under their existing manager
template: lens/secrets.yaml
set:
lensWorker.clickhouseSecret.name: external-clickhouse
lensWorker.serviceTokenSecret.name: external-service
asserts:
- hasDocuments:
count: 0
- it: creates no credentials or database when Lens is disabled
templates:
- lens/secrets.yaml
- lens/clickhouse.yaml
set:
lensWorker.enabled: false
asserts:
- hasDocuments:
count: 0
- it: requires an external database when bundled storage is explicitly disabled
template: lens/deployment.yaml
set:
lensWorker.clickhouse.enabled: false
asserts:
- failedTemplate:
errorMessage: lensWorker.clickhouseSecret.name is required
- it: keeps storage names valid and uses the same name for the connection
set:
fullnameOverride: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
asserts:
- matchRegex:
path: metadata.name
pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
template: lens/clickhouse.yaml
documentIndex: 0
- matchRegex:
path: metadata.name
pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
template: lens/clickhouse.yaml
documentIndex: 1
- equal:
path: metadata.name
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
template: lens/clickhouse.yaml
documentIndex: 0
- equal:
path: spec.serviceName
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
template: lens/clickhouse.yaml
documentIndex: 1
- equal:
path: metadata.name
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
template: lens/secrets.yaml
documentIndex: 1
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_HOST
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
template: lens/deployment.yaml
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_PASSWORD
valueFrom:
secretKeyRef:
name: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
key: password
template: lens/deployment.yaml
- it: allows cold storage startup and inherits registry credentials
template: lens/clickhouse.yaml
documentIndex: 1
set:
imagePullSecrets: [{name: registry-auth}]
asserts:
- equal:
path: spec.template.spec.imagePullSecrets
value: [{name: registry-auth}]
- equal:
path: spec.template.spec.containers[0].startupProbe
value:
httpGet: {path: /ping, port: http}
periodSeconds: 5
timeoutSeconds: 3
failureThreshold: 60

70
helm/litellm-helm/tests/migrations-job_tests.yaml
View file

@ -360,3 +360,73 @@ tests:
asserts:
- notExists:
path: spec.activeDeadlineSeconds
- it: should set custom jobLabels and jobAnnotations on Job metadata
template: migrations-job.yaml
set:
migrationJob:
enabled: true
jobLabels:
environment: production
team: platform
jobAnnotations:
example.com/cost-center: "1234"
asserts:
- equal:
path: metadata.labels.environment
value: production
- equal:
path: metadata.labels.team
value: platform
- equal:
path: metadata.annotations['example.com/cost-center']
value: "1234"
- it: should set custom podLabels on Pod template
template: migrations-job.yaml
set:
migrationJob:
enabled: true
podLabels:
custom.io/pod-role: migration
asserts:
- equal:
path: spec.template.metadata.labels['custom.io/pod-role']
value: migration
- it: should override container command and args
template: migrations-job.yaml
set:
migrationJob:
enabled: true
command:
- sh
args:
- -c
- echo migrating
asserts:
- equal:
path: spec.template.spec.containers[0].command
value:
- sh
- equal:
path: spec.template.spec.containers[0].args
value:
- -c
- echo migrating
- it: should clear container args when only command is specified
template: migrations-job.yaml
set:
migrationJob:
enabled: true
command:
- sh
asserts:
- equal:
path: spec.template.spec.containers[0].command
value:
- sh
- equal:
path: spec.template.spec.containers[0].args
value: []

57
helm/litellm-helm/values.yaml
View file

@ -572,6 +572,11 @@ migrationJob:
# In that case, pre-install/pre-upgrade hooks run before normal resources, so this defaults to "default".
serviceAccountName: ""
annotations: {}
jobLabels: {} # Custom labels for the Job metadata
jobAnnotations: {} # Custom annotations for the Job metadata
podLabels: {} # Custom labels for the Job pod template
command: [] # Override container command (defaults to ["python", "litellm/proxy/prisma_migration.py"])
args: [] # Override container args
ttlSecondsAfterFinished: 120
resources: {}
# Unset by default. This job runs the database migration and exits, so it does not
@ -647,3 +652,55 @@ serviceMonitor:
namespaceSelector:
matchNames: []
# - test-namespace
lensWorker:
enabled: false
replicaCount: 1
image:
repository: ghcr.io/berriai/litellm-lens-worker
tag: ""
digest: ""
pullPolicy: IfNotPresent
tokenSecret:
name: ""
key: token
serviceTokenSecret:
name: ""
key: service-token
clickhouse:
enabled: true
image: clickhouse/clickhouse-server:26.9.6.6
storage: 20Gi
storageClassName: null
resources:
requests:
cpu: 100m
memory: 512Mi
limits:
memory: 2Gi
clickhouseDatabase: litellm
retentionDays: 14
clickhouseSecret:
name: ""
key: url
publicUrl: ""
service:
port: 4318
annotations: {}
ingress:
enabled: false
className: ""
host: ""
annotations: {}
tls: []
url: ""
tmpSizeLimit: 1Gi
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
memory: 2Gi
nodeSelector: {}
tolerations: []
affinity: {}

46
helm/litellm/templates/_helpers.tpl
View file

@ -514,3 +514,49 @@ shutdown drain window.
value: {{ .drainTimeoutSeconds | quote }}
{{- end }}
{{- end -}}
{{- define "litellm.lensConnectionEnv" -}}
{{- if .Values.lensWorker.enabled }}
- name: LITELLM_LENS_URL
value: {{ printf "http://%s-lens-worker:%v" (include "litellm.fullname" .) .Values.lensWorker.service.port | quote }}
- name: LITELLM_LENS_PUBLIC_URL
value: {{ include "litellm.lensWorker.publicUrl" . | quote }}
- name: LITELLM_LENS_SERVICE_TOKEN
valueFrom:
secretKeyRef:
name: {{ include "litellm.lensWorker.serviceTokenSecretName" . | quote }}
key: {{ .Values.lensWorker.serviceTokenSecret.key | quote }}
{{- end }}
{{- end -}}
{{- define "litellm.lensWorker.labels" -}}
{{- $labels := include "litellm.commonLabels" . | fromYaml -}}
{{- $_ := set $labels "app.kubernetes.io/name" (printf "%s-lens-worker" (include "litellm.name" . | trunc 51 | trimSuffix "-")) -}}
{{- toYaml $labels -}}
{{- end -}}
{{- define "litellm.lensWorker.serviceTokenSecretName" -}}
{{- .Values.lensWorker.serviceTokenSecret.name | default (printf "%s-lens-service" (include "litellm.fullname" .)) -}}
{{- end -}}
{{- define "litellm.lensWorker.bundledClickhouse" -}}
{{- if and .Values.lensWorker.enabled .Values.lensWorker.clickhouse.enabled (not .Values.lensWorker.clickhouseSecret.name) -}}true{{- end -}}
{{- end -}}
{{- define "litellm.lensWorker.publicUrl" -}}
{{- if .Values.lensWorker.publicUrl -}}
{{- .Values.lensWorker.publicUrl -}}
{{- else if .Values.lensWorker.ingress.enabled -}}
{{- $tls := or (not (empty .Values.lensWorker.ingress.tls)) (hasKey .Values.lensWorker.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
{{- printf "%s://%s" (ternary "https" "http" $tls) (required "lensWorker.ingress.host is required" .Values.lensWorker.ingress.host) -}}
{{- else if and .Values.ingress.enabled .Values.ingress.host -}}
{{- $tls := or (not (empty .Values.ingress.tls)) (hasKey .Values.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
{{- printf "%s://%s/lens-ingest" (ternary "https" "http" $tls) .Values.ingress.host -}}
{{- else -}}
{{- fail "lensWorker.publicUrl is required when there is no single ingress hostname" -}}
{{- end -}}
{{- end -}}
{{- define "litellm.lensWorker.clickhouseName" -}}
{{- printf "%s-lens-clickhouse" (include "litellm.fullname" . | trunc 47 | trimSuffix "-") -}}
{{- end -}}

1
helm/litellm/templates/backend/deployment.yaml
View file

@ -57,6 +57,7 @@ spec:
containerPort: 4001
protocol: TCP
env:
{{- include "litellm.lensConnectionEnv" . | nindent 12 }}
- name: LENS_WORKER_IMAGE
value: {{ include "litellm.lensWorker.image" . | quote }}
{{- include "litellm.serverEnv" (dict "root" $ "component" .Values.backend) | nindent 12 }}

1
helm/litellm/templates/gateway/deployment.yaml
View file

@ -55,6 +55,7 @@ spec:
containerPort: 4000
protocol: TCP
env:
{{- include "litellm.lensConnectionEnv" . | nindent 12 }}
{{- include "litellm.serverEnv" (dict "root" $ "component" .Values.gateway) | nindent 12 }}
{{- if .Values.gateway.config.create }}
- name: CONFIG_FILE_PATH

10
helm/litellm/templates/ingress.yaml
View file

@ -156,6 +156,16 @@ spec:
port:
number: {{ $gatewayPort }}
{{- end }}
{{- if .Values.lensWorker.enabled }}
{{- $builtinPathKeys = append $builtinPathKeys "/lens-ingest|Prefix" }}
- path: /lens-ingest
pathType: Prefix
backend:
service:
name: {{ include "litellm.fullname" . }}-lens-worker
port:
number: {{ .Values.lensWorker.service.port }}
{{- end }}
{{- /*
--- Operator-supplied extra paths (ingress.extraPaths) ---
Rendered after every built-in path so an entry can never take

98
helm/litellm/templates/lens/clickhouse.yaml Normal file
View file

@ -0,0 +1,98 @@
{{- if include "litellm.lensWorker.bundledClickhouse" . }}
{{- $name := include "litellm.lensWorker.clickhouseName" . }}
apiVersion: v1
kind: Service
metadata:
name: {{ $name }}
spec:
clusterIP: None
selector:
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/component: lens-clickhouse
ports:
- name: http
port: 8123
targetPort: http
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: {{ $name }}
spec:
serviceName: {{ $name }}
replicas: 1
selector:
matchLabels:
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/component: lens-clickhouse
template:
metadata:
labels:
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/component: lens-clickhouse
spec:
automountServiceAccountToken: false
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
securityContext:
runAsNonRoot: true
runAsUser: 101
runAsGroup: 101
fsGroup: 101
seccompProfile:
type: RuntimeDefault
containers:
- name: clickhouse
image: {{ .Values.lensWorker.clickhouse.image | quote }}
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: [ALL]
env:
- name: CLICKHOUSE_USER
value: default
- name: CLICKHOUSE_PASSWORD
valueFrom:
secretKeyRef:
name: {{ $name }}
key: password
- name: CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT
value: "1"
ports:
- name: http
containerPort: 8123
startupProbe:
httpGet:
path: /ping
port: http
periodSeconds: 5
timeoutSeconds: 3
failureThreshold: 60
readinessProbe:
httpGet:
path: /ping
port: http
livenessProbe:
httpGet:
path: /ping
port: http
timeoutSeconds: 3
resources:
{{- toYaml .Values.lensWorker.clickhouse.resources | nindent 12 }}
volumeMounts:
- name: data
mountPath: /var/lib/clickhouse
volumeClaimTemplates:
- metadata:
name: data
spec:
accessModes: [ReadWriteOnce]
{{- if ne .Values.lensWorker.clickhouse.storageClassName nil }}
storageClassName: {{ .Values.lensWorker.clickhouse.storageClassName | quote }}
{{- end }}
resources:
requests:
storage: {{ .Values.lensWorker.clickhouse.storage | quote }}
{{- end }}

43
helm/litellm/templates/lens/deployment.yaml
View file

@ -4,7 +4,7 @@ kind: Deployment
metadata:
name: {{ include "litellm.fullname" . }}-lens-worker
labels:
{{- include "litellm.commonLabels" . | nindent 4 }}
{{- include "litellm.lensWorker.labels" . | nindent 4 }}
app.kubernetes.io/component: lens-worker
spec:
replicas: {{ .Values.lensWorker.replicaCount }}
@ -15,7 +15,7 @@ spec:
template:
metadata:
labels:
{{- include "litellm.commonLabels" . | nindent 8 }}
{{- include "litellm.lensWorker.labels" . | nindent 8 }}
app.kubernetes.io/component: lens-worker
spec:
automountServiceAccountToken: false
@ -42,11 +42,48 @@ spec:
env:
- name: LITELLM_URL
value: {{ .Values.lensWorker.url | default (printf "http://%s:%v" (include "litellm.backend.fullname" .) .Values.backend.service.port) | quote }}
- name: LITELLM_LENS_SERVICE_TOKEN
valueFrom:
secretKeyRef:
name: {{ include "litellm.lensWorker.serviceTokenSecretName" . | quote }}
key: {{ .Values.lensWorker.serviceTokenSecret.key | quote }}
{{- if include "litellm.lensWorker.bundledClickhouse" . }}
- name: CLICKHOUSE_HOST
value: {{ include "litellm.lensWorker.clickhouseName" . | quote }}
- name: CLICKHOUSE_PASSWORD
valueFrom:
secretKeyRef:
name: {{ include "litellm.lensWorker.clickhouseName" . | quote }}
key: password
{{- else }}
- name: CLICKHOUSE_URL
valueFrom:
secretKeyRef:
name: {{ required "lensWorker.clickhouseSecret.name is required" .Values.lensWorker.clickhouseSecret.name | quote }}
key: {{ .Values.lensWorker.clickhouseSecret.key | quote }}
{{- end }}
- name: CLICKHOUSE_DATABASE
value: {{ .Values.lensWorker.clickhouseDatabase | quote }}
- name: AGENT_TRACING_RETENTION_DAYS
value: {{ .Values.lensWorker.retentionDays | quote }}
{{- if .Values.lensWorker.tokenSecret.name }}
- name: LENS_WORKER_TOKEN
valueFrom:
secretKeyRef:
name: {{ required "lensWorker.tokenSecret.name must reference a Lens worker token" .Values.lensWorker.tokenSecret.name | quote }}
name: {{ .Values.lensWorker.tokenSecret.name | quote }}
key: {{ .Values.lensWorker.tokenSecret.key | quote }}
{{- end }}
ports:
- name: otlp
containerPort: 4318
livenessProbe:
httpGet:
path: /health/live
port: otlp
readinessProbe:
httpGet:
path: /health/ready
port: otlp
resources:
{{- toYaml .Values.lensWorker.resources | nindent 12 }}
volumeMounts:

29
helm/litellm/templates/lens/ingress.yaml Normal file
View file

@ -0,0 +1,29 @@
{{- if and .Values.lensWorker.enabled .Values.lensWorker.ingress.enabled }}
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: {{ include "litellm.fullname" . }}-lens-worker
{{- with .Values.lensWorker.ingress.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
{{- with .Values.lensWorker.ingress.className }}
ingressClassName: {{ . | quote }}
{{- end }}
{{- with .Values.lensWorker.ingress.tls }}
tls:
{{- toYaml . | nindent 4 }}
{{- end }}
rules:
- host: {{ required "lensWorker.ingress.host is required" .Values.lensWorker.ingress.host | quote }}
http:
paths:
- path: /v1/
pathType: Prefix
backend:
service:
name: {{ include "litellm.fullname" . }}-lens-worker
port:
name: otlp
{{- end }}

27
helm/litellm/templates/lens/secrets.yaml Normal file
View file

@ -0,0 +1,27 @@
{{- if and .Values.lensWorker.enabled (not .Values.lensWorker.serviceTokenSecret.name) }}
{{- $name := include "litellm.lensWorker.serviceTokenSecretName" . }}
{{- $existing := lookup "v1" "Secret" .Release.Namespace $name }}
apiVersion: v1
kind: Secret
metadata:
name: {{ $name }}
annotations:
helm.sh/resource-policy: keep
type: Opaque
data:
{{ .Values.lensWorker.serviceTokenSecret.key }}: {{ if $existing }}{{ required "Saved Lens service secret is missing its key" (index $existing.data .Values.lensWorker.serviceTokenSecret.key) | quote }}{{ else }}{{ randAlphaNum 64 | b64enc | quote }}{{ end }}
{{- end }}
{{- if include "litellm.lensWorker.bundledClickhouse" . }}
{{- $name := include "litellm.lensWorker.clickhouseName" . }}
{{- $existing := lookup "v1" "Secret" .Release.Namespace $name }}
---
apiVersion: v1
kind: Secret
metadata:
name: {{ $name }}
annotations:
helm.sh/resource-policy: keep
type: Opaque
data:
password: {{ if $existing }}{{ required "Saved Lens ClickHouse secret is missing its password" (index $existing.data "password") | quote }}{{ else }}{{ randAlphaNum 64 | b64enc | quote }}{{ end }}
{{- end }}

18
helm/litellm/templates/lens/service.yaml Normal file
View file

@ -0,0 +1,18 @@
{{- if .Values.lensWorker.enabled }}
apiVersion: v1
kind: Service
metadata:
name: {{ include "litellm.fullname" . }}-lens-worker
{{- with .Values.lensWorker.service.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
selector:
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/component: lens-worker
ports:
- name: otlp
port: {{ .Values.lensWorker.service.port }}
targetPort: otlp
{{- end }}

86
helm/litellm/tests/lens_endpoint_defaults_tests.yaml Normal file
View file

@ -0,0 +1,86 @@
suite: Lens endpoint defaults
templates:
- gateway/deployment.yaml
- gateway/configmap.yaml
set:
lensWorker.enabled: true
ingress.enabled: true
ingress.host: gateway.example
ingress.tls:
- hosts: [gateway.example]
secretName: tls
tests:
- it: derives the trace endpoint from the deployment hostname and TLS
template: gateway/deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://gateway.example/lens-ingest
- it: preserves an explicitly configured public address
set:
lensWorker.publicUrl: https://custom.example/prefix
template: gateway/deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://custom.example/prefix
- it: uses the dedicated Lens ingress when configured
set:
lensWorker.ingress.enabled: true
lensWorker.ingress.host: traces.example
lensWorker.ingress.tls:
- hosts: [traces.example]
secretName: traces-tls
template: gateway/deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://traces.example
- it: uses HTTPS for a dedicated ALB ingress with certificate annotations
set:
lensWorker.ingress.enabled: true
lensWorker.ingress.host: traces.example
lensWorker.ingress.className: alb
lensWorker.ingress.tls: []
lensWorker.ingress.annotations:
alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-west-2:123456789012:certificate/test
alb.ingress.kubernetes.io/listen-ports: '[{"HTTP":80},{"HTTPS":443}]'
alb.ingress.kubernetes.io/ssl-redirect: "443"
template: gateway/deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://traces.example
- it: uses HTTP for a dedicated ingress without its own TLS
set:
lensWorker.ingress.enabled: true
lensWorker.ingress.host: traces.example
lensWorker.ingress.tls: []
lensWorker.ingress.annotations: {}
template: gateway/deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: http://traces.example
- it: uses HTTP when ingress has no TLS
set:
ingress.tls: []
template: gateway/deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: http://gateway.example/lens-ingest
values:
- ./values/required.yaml

47
helm/litellm/tests/lens_saved_secrets_tests.yaml Normal file
View file

@ -0,0 +1,47 @@
suite: Lens preserves credentials across Helm upgrades
templates:
- lens/secrets.yaml
set:
fullnameOverride: lens-test
lensWorker.enabled: true
release:
namespace: lens
name: lens-test
upgrade: true
kubernetesProvider:
scheme:
v1/Secret:
gvr:
version: v1
resource: secrets
namespaced: true
objects:
- apiVersion: v1
kind: Secret
metadata:
name: lens-test-lens-service
namespace: lens
data:
service-token: c2F2ZWQtc2VydmljZS10b2tlbg==
- apiVersion: v1
kind: Secret
metadata:
name: lens-test-lens-clickhouse
namespace: lens
data:
password: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=
tests:
- it: reuses the service credential instead of breaking running services
documentIndex: 0
asserts:
- equal:
path: data.service-token
value: c2F2ZWQtc2VydmljZS10b2tlbg==
- it: reuses the database password instead of locking out stored traces
documentIndex: 1
asserts:
- equal:
path: data.password
value: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=
values:
- ./values/required.yaml

196
helm/litellm/tests/lens_service_tests.yaml Normal file
View file

@ -0,0 +1,196 @@
suite: Lens service isolation and ingestion routing
templates:
- gateway/configmap.yaml
- gateway/deployment.yaml
- backend/deployment.yaml
- ingress.yaml
- lens/ingress.yaml
- lens/service.yaml
- lens/deployment.yaml
tests:
- it: connects gateway/deployment.yaml to the shared Lens service
template: gateway/deployment.yaml
set: &id001
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: https://gateway.example/lens-ingest
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_URL
value: http://lens-test-lens-worker:4318
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://gateway.example/lens-ingest
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_SERVICE_TOKEN
valueFrom:
secretKeyRef:
name: lens-service
key: service-token
- it: connects backend/deployment.yaml to the shared Lens service
template: backend/deployment.yaml
set: *id001
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_URL
value: http://lens-test-lens-worker:4318
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://gateway.example/lens-ingest
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_SERVICE_TOKEN
valueFrom:
secretKeyRef:
name: lens-service
key: service-token
- it: routes uploads directly to Lens instead of the gateway
template: ingress.yaml
set:
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: https://gateway.example/lens-ingest
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
ingress.enabled: true
ingress.host: gateway.example
asserts:
- contains:
path: spec.rules[0].http.paths
content:
path: /lens-ingest
pathType: Prefix
backend:
service:
name: lens-test-lens-worker
port:
number: 4318
- it: keeps internal routes out of a dedicated ingestion hostname
template: lens/ingress.yaml
set:
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: https://gateway.example/lens-ingest
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
lensWorker.ingress.enabled: true
lensWorker.ingress.host: traces.example
asserts:
- equal:
path: spec.rules[0].http.paths
value:
- path: /v1/
pathType: Prefix
backend:
service:
name: lens-test-lens-worker
port:
name: otlp
- it: maps the Lens service to the ingestion listener
template: lens/service.yaml
set: *id001
asserts:
- equal:
path: spec.selector
value:
app.kubernetes.io/instance: RELEASE-NAME
app.kubernetes.io/component: lens-worker
- equal:
path: spec.ports
value:
- name: otlp
port: 4318
targetPort: otlp
- it: gives only Lens the ClickHouse secret
template: lens/deployment.yaml
set: *id001
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_URL
valueFrom:
secretKeyRef:
name: lens-storage
key: url
- it: requires an agent reachable ingestion URL
template: gateway/deployment.yaml
set:
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: ''
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
asserts:
- failedTemplate:
errorMessage: lensWorker.publicUrl is required when there is no single ingress hostname
- it: omits Lens connection settings when disabled in gateway/deployment.yaml
template: gateway/deployment.yaml
asserts:
- notContains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_URL
any: true
- it: omits Lens connection settings when disabled in backend/deployment.yaml
template: backend/deployment.yaml
asserts:
- notContains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_URL
any: true
- it: preserves an existing ClickHouse database and retention
template: lens/deployment.yaml
set:
lensWorker.enabled: true
lensWorker.publicUrl: https://traces.example
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
lensWorker.clickhouseDatabase: existing_traces
lensWorker.retentionDays: 45
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_DATABASE
value: existing_traces
- contains:
path: spec.template.spec.containers[0].env
content:
name: AGENT_TRACING_RETENTION_DAYS
value: '45'
- it: keeps Lens pods outside the gateway autoscaling selector
template: lens/deployment.yaml
set: &id002
nameOverride: inference
lensWorker.enabled: true
lensWorker.publicUrl: https://traces.example
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
asserts:
- equal:
path: spec.template.metadata.labels["app.kubernetes.io/name"]
value: inference-lens-worker
- it: preserves the existing gateway deployment selector
template: gateway/deployment.yaml
set: *id002
asserts:
- equal:
path: spec.selector.matchLabels["app.kubernetes.io/name"]
value: inference
values:
- ./values/required.yaml

162
helm/litellm/tests/lens_setup_tests.yaml Normal file
View file

@ -0,0 +1,162 @@
suite: Lens managed setup
set:
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: https://traces.example
release:
namespace: lens
name: lens-test
templates:
- lens/deployment.yaml
- lens/clickhouse.yaml
- lens/secrets.yaml
tests:
- it: generates both private credentials for a new installation
template: lens/secrets.yaml
asserts:
- hasDocuments:
count: 2
- matchRegex:
path: data.service-token
pattern: '^[A-Za-z0-9+/]{86}==$'
documentIndex: 0
- matchRegex:
path: data.password
pattern: '^[A-Za-z0-9+/]{86}==$'
documentIndex: 1
- equal:
path: metadata.annotations["helm.sh/resource-policy"]
value: keep
- it: connects Lens to its private bundled storage
template: lens/deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_HOST
value: lens-test-lens-clickhouse
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_PASSWORD
valueFrom:
secretKeyRef:
name: lens-test-lens-clickhouse
key: password
- it: stores traces on a persistent volume with the chosen storage class
template: lens/clickhouse.yaml
documentIndex: 1
set:
lensWorker.clickhouse.storage: 40Gi
lensWorker.clickhouse.storageClassName: fast
asserts:
- equal:
path: spec.volumeClaimTemplates[0].spec
value:
accessModes: [ReadWriteOnce]
storageClassName: fast
resources:
requests:
storage: 40Gi
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_PASSWORD
valueFrom:
secretKeyRef:
name: lens-test-lens-clickhouse
key: password
- it: keeps an existing external database instead of creating a new one
template: lens/clickhouse.yaml
set:
lensWorker.clickhouseSecret.name: external-clickhouse
asserts:
- hasDocuments:
count: 0
- it: leaves supplied secrets under their existing manager
template: lens/secrets.yaml
set:
lensWorker.clickhouseSecret.name: external-clickhouse
lensWorker.serviceTokenSecret.name: external-service
asserts:
- hasDocuments:
count: 0
- it: creates no credentials or database when Lens is disabled
templates:
- lens/secrets.yaml
- lens/clickhouse.yaml
set:
lensWorker.enabled: false
asserts:
- hasDocuments:
count: 0
- it: requires an external database when bundled storage is explicitly disabled
template: lens/deployment.yaml
set:
lensWorker.clickhouse.enabled: false
asserts:
- failedTemplate:
errorMessage: lensWorker.clickhouseSecret.name is required
- it: keeps storage names valid and uses the same name for the connection
set:
fullnameOverride: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
asserts:
- matchRegex:
path: metadata.name
pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
template: lens/clickhouse.yaml
documentIndex: 0
- matchRegex:
path: metadata.name
pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
template: lens/clickhouse.yaml
documentIndex: 1
- equal:
path: metadata.name
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
template: lens/clickhouse.yaml
documentIndex: 0
- equal:
path: spec.serviceName
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
template: lens/clickhouse.yaml
documentIndex: 1
- equal:
path: metadata.name
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
template: lens/secrets.yaml
documentIndex: 1
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_HOST
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
template: lens/deployment.yaml
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_PASSWORD
valueFrom:
secretKeyRef:
name: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
key: password
template: lens/deployment.yaml
- it: allows cold storage startup and inherits registry credentials
template: lens/clickhouse.yaml
documentIndex: 1
set:
imagePullSecrets: [{name: registry-auth}]
asserts:
- equal:
path: spec.template.spec.imagePullSecrets
value: [{name: registry-auth}]
- equal:
path: spec.template.spec.containers[0].startupProbe
value:
httpGet: {path: /ping, port: http}
periodSeconds: 5
timeoutSeconds: 3
failureThreshold: 60
values:
- ./values/required.yaml

49
helm/litellm/tests/lens_worker_tests.yaml
View file

@ -11,7 +11,9 @@ tests:
set:
backend.image.tag: sha-0123456789abcdef
lensWorker.enabled: true
lensWorker.tokenSecret.name: lens-credential
lensWorker.publicUrl: https://traces.example
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
asserts:
- equal:
path: spec.template.spec.containers[0].image
@ -31,7 +33,9 @@ tests:
set:
backend.image.tag: sha-0123456789abcdef
lensWorker.enabled: true
lensWorker.tokenSecret.name: lens-credential
lensWorker.publicUrl: https://traces.example
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
lensWorker.image.repository: registry.example/lens-worker
asserts:
- equal:
@ -41,7 +45,9 @@ tests:
template: lens/deployment.yaml
set:
lensWorker.enabled: true
lensWorker.tokenSecret.name: lens-credential
lensWorker.publicUrl: https://traces.example
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
lensWorker.image.tag: replaced-release
lensWorker.image.digest: sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
asserts:
@ -71,20 +77,29 @@ tests:
asserts:
- hasDocuments:
count: 0
- it: requires a limited worker credential when enabled
- it: uses the managed service secret when none is supplied
template: lens/deployment.yaml
set:
lensWorker.enabled: true
lensWorker.publicUrl: https://traces.example
asserts:
- failedTemplate:
errorMessage: lensWorker.tokenSecret.name must reference a Lens worker token
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_SERVICE_TOKEN
valueFrom:
secretKeyRef:
name: RELEASE-NAME-litellm-lens-service
key: service-token
- it: uses the chart release and a secret without granting Kubernetes access
template: lens/deployment.yaml
chart:
appVersion: v1.2.3
set:
lensWorker.enabled: true
lensWorker.tokenSecret.name: lens-credential
lensWorker.publicUrl: https://traces.example
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
asserts:
- equal:
path: spec.template.spec.containers[0].image
@ -92,8 +107,8 @@ tests:
- equal:
path: spec.template.spec.containers[0].env[1].valueFrom.secretKeyRef
value:
name: lens-credential
key: token
name: lens-service
key: service-token
- equal:
path: spec.template.spec.automountServiceAccountToken
value: false
@ -120,7 +135,9 @@ tests:
template: lens/deployment.yaml
set:
lensWorker.enabled: true
lensWorker.tokenSecret.name: lens-credential
lensWorker.publicUrl: https://traces.example
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
lensWorker.url: https://gateway.example/proxy
lensWorker.image.repository: registry.example/lens-worker
lensWorker.image.tag: branch-main-1234567
@ -137,7 +154,9 @@ tests:
appVersion: 1.2.3-rc.4
set:
lensWorker.enabled: true
lensWorker.tokenSecret.name: lens-credential
lensWorker.publicUrl: https://traces.example
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
asserts:
- equal:
path: spec.template.spec.containers[0].image
@ -147,7 +166,9 @@ tests:
set:
backend.image.tag: branch-main-1234567
lensWorker.enabled: true
lensWorker.tokenSecret.name: lens-credential
lensWorker.publicUrl: https://traces.example
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
asserts:
- equal:
path: spec.template.spec.containers[0].image
@ -167,7 +188,9 @@ tests:
set:
backend.image.tag: 1.2.3-dev.4
lensWorker.enabled: true
lensWorker.tokenSecret.name: lens-credential
lensWorker.publicUrl: https://traces.example
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
asserts:
- equal:
path: spec.template.spec.containers[0].image

29
helm/litellm/values.yaml
View file

@ -641,6 +641,35 @@ lensWorker:
tokenSecret:
name: ""
key: token
serviceTokenSecret:
name: ""
key: service-token
clickhouse:
enabled: true
image: clickhouse/clickhouse-server:26.9.6.6
storage: 20Gi
storageClassName: null
resources:
requests:
cpu: 100m
memory: 512Mi
limits:
memory: 2Gi
clickhouseDatabase: litellm
retentionDays: 14
clickhouseSecret:
name: ""
key: url
publicUrl: ""
service:
port: 4318
annotations: {}
ingress:
enabled: false
className: ""
host: ""
annotations: {}
tls: []
url: ""
tmpSizeLimit: 1Gi
resources:

2
litellm-proxy-extras/litellm_proxy_extras/migrations/20261005000000_add_mcp_server_rpm/migration.sql Normal file
View file

@ -0,0 +1,2 @@
-- AlterTable
ALTER TABLE "LiteLLM_MCPServerTable" ADD COLUMN IF NOT EXISTS "rpm" INTEGER;

5
litellm-proxy-extras/litellm_proxy_extras/migrations/20261007000100_lens_ingestion_keys/migration.sql Normal file
View file

@ -0,0 +1,5 @@
CREATE TABLE IF NOT EXISTS "LiteLLM_LensIngestionKey" (
"id" TEXT NOT NULL,
"data" JSONB NOT NULL,
CONSTRAINT "LiteLLM_LensIngestionKey_pkey" PRIMARY KEY ("id")
);

19
litellm-proxy-extras/litellm_proxy_extras/migrations/20261007120000_add_autorouter_daily_tokens/migration.sql Normal file
View file

@ -0,0 +1,19 @@
DO $$
BEGIN
IF NOT EXISTS (
SELECT 1 FROM pg_attribute
WHERE attrelid = to_regclass('"LiteLLM_AutoRouterDailySpend"')
AND attname = 'total_tokens' AND NOT attisdropped
) THEN
ALTER TABLE "LiteLLM_AutoRouterDailySpend"
ADD COLUMN IF NOT EXISTS "total_tokens" BIGINT NOT NULL DEFAULT 0;
END IF;
IF NOT EXISTS (
SELECT 1 FROM pg_attribute
WHERE attrelid = to_regclass('"LiteLLM_AutoRouterDailySpend"')
AND attname = 'token_recorded_turns' AND NOT attisdropped
) THEN
ALTER TABLE "LiteLLM_AutoRouterDailySpend"
ADD COLUMN IF NOT EXISTS "token_recorded_turns" INTEGER NOT NULL DEFAULT 0;
END IF;
END $$;

1
litellm-proxy-extras/litellm_proxy_extras/migrations/20261008000000_lens_due_at/migration.sql Normal file
View file

@ -0,0 +1 @@
ALTER TABLE "LiteLLM_Lens" ADD COLUMN IF NOT EXISTS "due_at" TIMESTAMP(3) DEFAULT '1970-01-01 00:00:00';

2
litellm-proxy-extras/litellm_proxy_extras/migrations/20261008000100_lens_due_at_index/migration.sql Normal file
View file

@ -0,0 +1,2 @@
CREATE INDEX CONCURRENTLY IF NOT EXISTS "LiteLLM_Lens_due_at_idx"
ON "LiteLLM_Lens" ("due_at");

16
litellm-proxy-extras/litellm_proxy_extras/migrations/20261008000200_lens_trace_signals/migration.sql Normal file
View file

@ -0,0 +1,16 @@
CREATE TABLE IF NOT EXISTS "LiteLLM_LensSignalConfig" (
"id" TEXT NOT NULL,
"data" JSONB NOT NULL,
CONSTRAINT "LiteLLM_LensSignalConfig_pkey" PRIMARY KEY ("id")
);
CREATE TABLE IF NOT EXISTS "LiteLLM_LensTraceSignal" (
"trace_id" TEXT NOT NULL,
"trace_ref" TEXT NOT NULL DEFAULT '',
"config_key" TEXT NOT NULL,
"span_count" INTEGER NOT NULL,
"claimed_until" TIMESTAMP(3),
"classified_at" TIMESTAMP(3),
"data" JSONB NOT NULL,
CONSTRAINT "LiteLLM_LensTraceSignal_pkey" PRIMARY KEY ("trace_id", "trace_ref")
);

2
litellm-proxy-extras/litellm_proxy_extras/migrations/20261009000000_add_credential_display_name/migration.sql Normal file
View file

@ -0,0 +1,2 @@
-- AlterTable
ALTER TABLE "LiteLLM_CredentialsTable" ADD COLUMN IF NOT EXISTS "display_name" TEXT;

29
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -42,6 +42,7 @@ model LiteLLM_BudgetTable {
model LiteLLM_CredentialsTable {
credential_id String @id @default(uuid())
credential_name String @unique
display_name String?
credential_values Json
credential_info Json?
created_at DateTime @default(now()) @map("created_at")
@ -417,6 +418,7 @@ model LiteLLM_MCPServerTable {
source_url String?
timeout Float?
max_concurrent_requests Int?
rpm Int?
// BYOM submission lifecycle
approval_status String? @default("active")
submitted_by String?
@ -1756,6 +1758,8 @@ model LiteLLM_AutoRouterDailySpend {
router_name String
router_type String
turns Int @default(0)
total_tokens BigInt @default(0)
token_recorded_turns Int @default(0)
spend Float @default(0)
saved_spend Float @default(0)
savings_estimated_turns Int @default(0)
@ -1939,6 +1943,9 @@ model LiteLLM_Lens {
id String @id
version Int @default(0)
data Json
due_at DateTime? @default(dbgenerated("'1970-01-01 00:00:00'::timestamp without time zone"))
@@index([due_at], map: "LiteLLM_Lens_due_at_idx")
}
model LiteLLM_LensRun {
@ -1966,6 +1973,11 @@ model LiteLLM_LensWorker {
data Json
}
model LiteLLM_LensIngestionKey {
id String @id
data Json
}
model LiteLLM_LensDataset {
id String
revision Int
@ -1974,3 +1986,20 @@ model LiteLLM_LensDataset {
@@id([id, revision])
}
model LiteLLM_LensSignalConfig {
id String @id
data Json
}
model LiteLLM_LensTraceSignal {
trace_id String
trace_ref String @default("")
config_key String
span_count Int
claimed_until DateTime?
classified_at DateTime?
data Json
@@id([trace_id, trace_ref])
}

22
litellm-proxy-extras/litellm_proxy_extras/utils.py
View file

@ -11,7 +11,7 @@ import time
from collections.abc import Callable
from dataclasses import dataclass, replace
from pathlib import Path
from typing import TYPE_CHECKING, Final, Optional, Union
from typing import TYPE_CHECKING, Final, Optional, Protocol, Union
from urllib.parse import unquote, urlsplit
from litellm_proxy_extras import prisma_toolchain
@ -35,6 +35,12 @@ if TYPE_CHECKING:
import psycopg.sql
class LensCheckConnect(Protocol):
def __call__(
self, conninfo: str, *, connect_timeout: int, autocommit: bool
) -> "psycopg.Connection[tuple[object, ...]]": ...
def str_to_bool(value: Optional[str]) -> bool:
if value is None:
return False
@ -243,11 +249,9 @@ def _redact_credentials(text: str) -> str:
passwords: Final = sorted(_configured_database_passwords(), key=len, reverse=True)
alternation: Final = "|".join(re.escape(password) for password in passwords)
password_pattern: Final = (
re.compile(rf"(?P<lead>:|password=)(?:{alternation})(?=@|&|$|[\s'\"\]),])", re.IGNORECASE)
if passwords
else None
re.compile(rf"(?<![A-Za-z0-9_])(?:{alternation})(?![A-Za-z0-9_])", re.IGNORECASE) if passwords else None
)
result: Final = password_pattern.sub(rf"\g<lead>{_REDACTED}", text) if password_pattern is not None else text
result: Final = password_pattern.sub(_REDACTED, text) if password_pattern is not None else text
return _secret_shape_redactor()(result)
@ -690,7 +694,7 @@ class ProxyExtrasDBManager:
)
@staticmethod
def raise_if_lens_rename_pending() -> None:
def raise_if_lens_rename_pending(connect: LensCheckConnect | None = None) -> None:
database_url: Final = os.environ.get("DATABASE_URL")
if not database_url:
return
@ -698,8 +702,9 @@ class ProxyExtrasDBManager:
import psycopg
except ImportError as exc:
raise RuntimeError("Install psycopg to verify Lens data safety before prisma db push.") from exc
open_connection: Final = connect if connect is not None else psycopg.connect
try:
with psycopg.connect(
with open_connection(
ProxyExtrasDBManager._strip_prisma_query_params(database_url), connect_timeout=10, autocommit=True
) as connection:
legacy: Final = connection.execute(
@ -710,7 +715,8 @@ class ProxyExtrasDBManager:
).fetchone()
except psycopg.Error as exc:
raise RuntimeError(
"Cannot verify Lens data safety; refusing prisma db push. Check database connectivity and psycopg installation."
"Cannot verify Lens data safety; refusing prisma db push. "
f"The database check failed: {_redact_credentials(str(exc)).strip()}"
) from exc
if legacy is not None:
raise RuntimeError(

4
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-proxy-extras"
version = "0.4.106"
version = "0.4.108"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
readme = "README.md"
requires-python = ">=3.9"
@ -30,7 +30,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.4.106"
version = "0.4.108"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-proxy-extras==",

4
litellm-rust/.agents/skills/rust-string-enums/SKILL.md
View file

@ -35,7 +35,7 @@ pub enum EventType {
}
```
Use `#[strum(serialize_all = "snake_case")]` or another supported case style when it exactly matches the contract. Use explicit variant spellings otherwise. With multiple accepted aliases, set `to_string` to the existing canonical output: Strum Display otherwise selects the longest `serialize` spelling
Spell each variant explicitly with `#[strum(serialize = "...")]`; do not use `serialize_all`. With multiple accepted aliases, set `to_string` to the existing canonical output: Strum Display otherwise selects the longest `serialize` spelling
Use only the derives the contract needs. A deserialize-only type should remain deserialize-only. Do not add an unknown variant to a closed enum, or derive Serde for a type that currently has no serialization contract
@ -43,7 +43,7 @@ Plain Serde derives with `rename` or `rename_all` remain appropriate for closed
## Preserve behavior during migration
Read the type, its callers, and existing tests before changing it. Preserve canonical output, accepted aliases, case sensitivity, whitespace handling, unknown values, malformed-input rejection, and existing public conversion APIs. Keep conversions needed by callers or compatibility even when Serde no longer uses them
Read the type, its callers, and existing tests before changing it. Preserve canonical output, accepted aliases, case sensitivity, whitespace handling, unknown values, malformed-input rejection, and existing public conversion contracts. Wrappers that only delegate to a Strum-derived trait are removed rather than kept: callers use `FromStr` and `From<Enum> for &'static str` directly. Keep conversions that add behavior needed by callers or compatibility even when Serde no longer uses them
Use the workspace dependencies and enable `serde_with.workspace = true` in a crate only when needed. Check the versions and enabled features in `Cargo.toml` and `Cargo.lock` rather than upgrading dependencies for this refactor

7
litellm-rust/AGENTS.md
View file

@ -4,6 +4,10 @@ For diagnostic tracing changes, follow [.agents/skills/rust-tracing/SKILL.md](.a
For string-valued enums and their Serde conversions, follow [.agents/skills/rust-string-enums/SKILL.md](.agents/skills/rust-string-enums/SKILL.md)
For fieldless enums, derive `strum::VariantArray` and use `VARIANTS` instead of a hand-listed `ALL` array; derive Strum string conversions instead of hand-written variant-to-string matches
Use the derived conversions directly (`<&'static str>::from(x)` / `.into()`, `str::parse`) with no `as_str`/`parse` wrapper that only delegates, and spell each variant with explicit `#[strum(serialize = "...")]` instead of `serialize_all`
## Test placement
- Never create a `tests.rs` (or `test.rs`) file under `src/`, and never `#[path = "tests.rs"] mod tests;`
@ -17,6 +21,9 @@ For string-valued enums and their Serde conversions, follow [.agents/skills/rust
Use [`#[rstest]`](https://docs.rs/rstest/latest/rstest/attr.rstest.html) for new and updated tests and [`#[fixture]`](https://docs.rs/rstest/latest/rstest/attr.fixture.html) for reusable setup, injected through typed test arguments. Express input variations as named `#[case::name(...)]` cases instead of loops or duplicated tests so each failure identifies its case. Keep behavior assertions in the test body and fixtures focused on setup. Use the workspace `rstest` dependency
- Never loop over inputs (`for`, `.iter().for_each`, `.all`) inside a test body; give each input its own `#[case::name(...)]`, or use `#[values(...)]` for a cross product
- Exception: a test pinning a Rust table against a repo-owned data file (for example `include_str!` of a JSON config) may iterate that file's entries
## Error definitions
- A crate's errors live in `src/error.rs`, defined with `thiserror`, and re-exported from `lib.rs`

140
litellm-rust/Cargo.lock generated
View file

@ -3485,6 +3485,7 @@ dependencies = [
"rstest",
"serde",
"serde_json",
"strum",
"thiserror 2.0.19",
"tokio",
]
@ -3692,7 +3693,9 @@ dependencies = [
"litellm-auth-types",
"rstest",
"serde",
"serde_json",
"serde_yaml_ng",
"strum",
"tempfile",
"thiserror 2.0.19",
]
@ -3990,6 +3993,7 @@ dependencies = [
"rustls-native-certs",
"serde",
"serde_json",
"strum",
"tempfile",
"thiserror 2.0.19",
"tokio",
@ -4171,6 +4175,45 @@ dependencies = [
"wiremock",
]
[[package]]
name = "litellm-lens"
version = "0.1.0"
dependencies = [
"axum",
"bytes",
"chrono",
"flate2",
"futures-util",
"http 1.4.2",
"jsonschema",
"libc",
"litellm-http",
"litellm-storage-clickhouse",
"litellm-traces",
"litellm-traces-cache",
"litellm-traces-clickhouse",
"litellm-tracing",
"prettyplease",
"prost",
"reqwest 0.12.28",
"rstest",
"serde",
"serde_json",
"sha2 0.10.9",
"subtle",
"syn 2.0.119",
"tempfile",
"thiserror 2.0.19",
"tokio",
"tower-http",
"tracing",
"typify",
"unicode-casefold",
"url",
"uuid",
"wiremock",
]
[[package]]
name = "litellm-llms"
version = "0.1.0"
@ -4209,6 +4252,7 @@ dependencies = [
name = "litellm-llms-types"
version = "0.1.0"
dependencies = [
"indexmap 2.14.0",
"macro_rules_attribute",
"rstest",
"schemars 1.2.2",
@ -4499,6 +4543,7 @@ dependencies = [
"serde",
"serde_json",
"sha2 0.10.9",
"strum",
"tar",
"target-lexicon",
"tempfile",
@ -5409,6 +5454,16 @@ dependencies = [
"zerocopy",
]
[[package]]
name = "prettyplease"
version = "0.2.37"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b"
dependencies = [
"proc-macro2",
"syn 2.0.119",
]
[[package]]
name = "primeorder"
version = "0.13.6"
@ -5975,6 +6030,16 @@ version = "0.8.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
[[package]]
name = "regress"
version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "158a764437582235e3501f683b93a0a6f8d825d04a789dbe5ed30b8799b8908a"
dependencies = [
"hashbrown 0.16.1",
"memchr",
]
[[package]]
name = "relative-path"
version = "1.9.3"
@ -6421,6 +6486,18 @@ dependencies = [
"parking_lot",
]
[[package]]
name = "schemars"
version = "0.8.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3fbf2ae1b8bc8e02df939598064d22402220cd5bbcca1c76f7d6a310974d5615"
dependencies = [
"dyn-clone",
"schemars_derive 0.8.22",
"serde",
"serde_json",
]
[[package]]
name = "schemars"
version = "0.9.0"
@ -6441,12 +6518,25 @@ checksum = "687274d293b6cdc6e73e0fee520bf2049650090d7164f87672d212a3c530cf4a"
dependencies = [
"chrono",
"dyn-clone",
"indexmap 2.14.0",
"ref-cast",
"schemars_derive",
"schemars_derive 1.2.2",
"serde",
"serde_json",
]
[[package]]
name = "schemars_derive"
version = "0.8.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32e265784ad618884abaea0600a9adf15393368d840e0222d101a072f3f7534d"
dependencies = [
"proc-macro2",
"quote",
"serde_derive_internals 0.29.1",
"syn 2.0.119",
]
[[package]]
name = "schemars_derive"
version = "1.2.2"
@ -6455,7 +6545,7 @@ checksum = "d98c67716b46af2f0b8cf752abc930f6f9aecfbf671ecfb531db8a31dbe4e2ba"
dependencies = [
"proc-macro2",
"quote",
"serde_derive_internals",
"serde_derive_internals 0.30.0",
"syn 3.0.6",
]
@ -6561,6 +6651,17 @@ dependencies = [
"syn 3.0.6",
]
[[package]]
name = "serde_derive_internals"
version = "0.29.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.119",
]
[[package]]
name = "serde_derive_internals"
version = "0.30.0"
@ -7927,6 +8028,35 @@ dependencies = [
"syn 2.0.119",
]
[[package]]
name = "typify"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b715573a376585888b742ead9be5f4826105e622169180662e2c81bed4a149c3"
dependencies = [
"typify-impl",
]
[[package]]
name = "typify-impl"
version = "0.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa7b026f540b148b81043c720889dbb942b08659aa8a43f624ac4f04dbfc1861"
dependencies = [
"heck",
"log",
"proc-macro2",
"quote",
"regress",
"schemars 0.8.22",
"semver",
"serde",
"serde_json",
"syn 2.0.119",
"thiserror 2.0.19",
"unicode-ident",
]
[[package]]
name = "ucd-trie"
version = "0.1.7"
@ -7951,6 +8081,12 @@ version = "0.3.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5"
[[package]]
name = "unicode-casefold"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b7f66b1c8f8caa2ab31dc6d3f35386f16efdab89668f93411e565ac368908e8f"
[[package]]
name = "unicode-general-category"
version = "1.1.0"

1
litellm-rust/crates/cache/Cargo.toml vendored
View file

@ -8,6 +8,7 @@ repository.workspace = true
[dependencies]
serde.workspace = true
serde_json = { workspace = true, features = ["preserve_order"] }
strum.workspace = true
thiserror.workspace = true
[dev-dependencies]

Some files were not shown because too many files have changed in this diff Show more