From b401ac6aafd9df4e3242b881291ad52974c3c536 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Wed, 4 Nov 2026 15:47:35 -0800 Subject: [PATCH 01/39] fix(cost): bill Batch cached tokens at the Standard cache price when a row has no Batch cache price (#50062) * fix(cost): bill Batch cached tokens at the Standard cache price when a row has no Batch cache price * fix(cost): resolve the Standard cache price for the prompt length in the Batch fallback --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/cost_calculator.py | 28 +++++++- tests/unit/test_cost_calculator.py | 101 ++++++++++++++++++++++++++++- 2 files changed, 125 insertions(+), 4 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 9154a6fdc30..100c84fc0fc 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -3323,15 +3323,35 @@ def batch_cost_calculator( return total_prompt_cost * vertex_uplift, total_completion_cost * vertex_uplift +def _standard_rate(model_info: ModelInfo, key: str, prompt_tokens: int) -> float | None: + return get_cost_per_unit(model_info, get_modality_token_cost_key(model_info, key, prompt_tokens), None) + + +def _cached_audio_batch_fallback( + model_info: ModelInfo, batch_cache_read_rate: float | None, cache_read_rate: float | None, prompt_tokens: int +) -> float: + if batch_cache_read_rate is not None: + return batch_cache_read_rate + standard_audio_rate: Final = _standard_rate(model_info, "cache_read_input_audio_token_cost", prompt_tokens) + if standard_audio_rate is not None: + return standard_audio_rate + return 0.0 if cache_read_rate is None else cache_read_rate + + def _batch_prompt_cost( usage: Usage, model_info: ModelInfo, input_rate: float, - cache_read_rate: float | None, + batch_cache_read_rate: float | None, cache_creation_rate: float | None, cache_creation_rates_by_ttl: Mapping[str, float], ) -> float: details: Final = parse_prompt_tokens_details(usage) + cache_read_rate: Final = ( + _standard_rate(model_info, "cache_read_input_token_cost", usage.prompt_tokens) + if batch_cache_read_rate is None + else batch_cache_read_rate + ) cached_tokens: Final = details["cache_hit_tokens"] if cache_read_rate is not None else 0 cached_audio_tokens: Final = details["cache_hit_audio_tokens"] if cache_read_rate is not None else 0 bills_cache_writes: Final = cache_creation_rate is not None or bool(cache_creation_rates_by_ttl) @@ -3352,7 +3372,11 @@ def _batch_prompt_cost( + video_tokens * _batch_rate(model_info, "input_cost_per_video_token_batches", input_rate) + (cached_tokens - cached_audio_tokens) * (cache_read_rate or 0.0) + cached_audio_tokens - * _batch_rate(model_info, "cache_read_input_audio_token_cost_batches", cache_read_rate or 0.0) + * _batch_rate( + model_info, + "cache_read_input_audio_token_cost_batches", + _cached_audio_batch_fallback(model_info, batch_cache_read_rate, cache_read_rate, usage.prompt_tokens), + ) + ( calculate_cache_writing_cost( written_tokens, diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index 6a63a502dd3..6de368fbf00 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -4531,6 +4531,103 @@ def test_batch_cost_calculator_falls_back_to_text_batch_rate_for_modalities(): assert prompt_cost == pytest.approx(100 * 1e-7) +def _cached_audio_batch_usage() -> Usage: + return Usage( + prompt_tokens=1000, + completion_tokens=0, + total_tokens=1000, + prompt_tokens_details=PromptTokensDetailsWrapper( + cached_tokens=1000, audio_tokens=1000, cached_tokens_details=CachedTokensDetails(audio_tokens=1000) + ), + ) + + +def _cached_text_batch_usage() -> Usage: + return Usage( + prompt_tokens=1000, + completion_tokens=0, + total_tokens=1000, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=1000), + ) + + +@pytest.mark.parametrize( + ("audio_batch_cache_price", "usage", "expected_rate"), + [ + pytest.param(None, _cached_audio_batch_usage(), 1e-7, id="cached-audio-bills-the-standard-audio-cache-price"), + pytest.param(None, _cached_text_batch_usage(), 3e-8, id="cached-text-bills-the-standard-cache-price"), + pytest.param(8e-8, _cached_audio_batch_usage(), 8e-8, id="an-explicit-audio-batch-cache-price-still-wins"), + ], +) +def test_batch_cost_calculator_bills_cached_tokens_at_the_standard_cache_price_when_the_row_has_no_batch_cache_price( + audio_batch_cache_price: float | None, usage: Usage, expected_rate: float +) -> None: + model_info: Final[ModelInfo] = { + "input_cost_per_token": 3e-7, + "input_cost_per_audio_token": 1e-6, + "cache_read_input_token_cost": 3e-8, + "cache_read_input_audio_token_cost": 1e-7, + "input_cost_per_token_batches": 1.5e-7, + "input_cost_per_audio_token_batches": 5e-7, + "cache_read_input_audio_token_cost_batches": audio_batch_cache_price, + } + + prompt_cost, _ = batch_cost_calculator( + usage=usage, model="gemini-2.5-flash", custom_llm_provider="vertex_ai", model_info=model_info + ) + + assert prompt_cost == pytest.approx(1000 * expected_rate) + + +def test_batch_cost_calculator_bills_long_context_cached_tokens_at_the_standard_long_context_cache_price() -> None: + model_info: Final[ModelInfo] = { + "input_cost_per_token": 1.25e-6, + "input_cost_per_token_above_200k_tokens": 2.5e-6, + "cache_read_input_token_cost": 1.25e-7, + "cache_read_input_token_cost_above_200k_tokens": 2.5e-7, + "input_cost_per_token_batches": 6.25e-7, + "input_cost_per_token_above_200k_tokens_batches": 1.25e-6, + } + usage: Final = Usage( + prompt_tokens=300_000, + completion_tokens=0, + total_tokens=300_000, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=300_000), + ) + + prompt_cost, _ = batch_cost_calculator( + usage=usage, model="gemini-2.5-pro", custom_llm_provider="vertex_ai", model_info=model_info + ) + + assert prompt_cost == pytest.approx(300_000 * 2.5e-7) + + +def _batch_and_standard_cached_audio_cost(model: str, provider: str) -> tuple[str, float, float]: + batch_prompt_cost, _ = batch_cost_calculator(_cached_audio_batch_usage(), model, provider) + standard_prompt_cost, _ = litellm.cost_per_token( + model=model, custom_llm_provider=provider, usage_object=_cached_audio_batch_usage() + ) + return model, batch_prompt_cost, standard_prompt_cost + + +def test_batch_cached_audio_never_costs_more_than_standard_on_any_row_with_a_batch_input_price( + _local_model_cost_map: None, +) -> None: + rows: Final = tuple( + (name, entry["litellm_provider"]) + for name, entry in litellm.model_cost.items() + if isinstance(entry, dict) + and entry.get("input_cost_per_token_batches") is not None + and entry.get("cache_read_input_audio_token_cost") is not None + ) + assert rows, "no cost map row prices both Batch input and Standard cached audio" + + costs: Final = tuple(_batch_and_standard_cached_audio_cost(name, provider) for name, provider in rows) + overpriced: Final = tuple(cost for cost in costs if cost[1] > cost[2]) + + assert overpriced == () + + @pytest.mark.parametrize( ("image_batch_rate", "image_tokens", "expected_completion_cost"), ( @@ -6254,7 +6351,7 @@ def test_batch_cost_calculator_bills_cached_tokens_at_the_flat_batch_cached_rate assert prompt_cost == pytest.approx(100 * 1e-6 + 900 * 1e-7) -def test_batch_cost_calculator_bills_cached_tokens_at_the_batch_input_rate_without_a_cached_batch_rate( +def test_batch_cost_calculator_bills_cached_tokens_at_the_standard_cache_rate_without_a_cached_batch_rate( _tiered_batch_models: None, ) -> None: from litellm.cost_calculator import batch_cost_calculator @@ -6263,7 +6360,7 @@ def test_batch_cost_calculator_bills_cached_tokens_at_the_batch_input_rate_witho usage=_cached_usage(300_048, 300_045, 11), model=_FLAT_CACHE_BATCH_MODEL, custom_llm_provider="openai" ) - assert prompt_cost == pytest.approx(300_048 * 3e-6) + assert prompt_cost == pytest.approx(3 * 3e-6 + 300_045 * 2e-7) def _cache_write_usage(prompt_tokens: int, cache_write_tokens: int, completion_tokens: int) -> Usage: From 44b85d0c5c3093543b1f911c6808836f16dd3a58 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Wed, 4 Nov 2026 15:47:57 -0800 Subject: [PATCH 02/39] fix(cost_calculator): keep regional and speed multipliers for per-call custom prices and log the billed per-call entry (#50063) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/cost_calculator.py | 6 ++ litellm/litellm_core_utils/litellm_logging.py | 4 ++ .../test_litellm_logging.py | 47 ++++++++++++++ tests/unit/test_cost_calculator.py | 65 +++++++++++++++++++ 4 files changed, 122 insertions(+) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 100c84fc0fc..3910a554905 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -616,11 +616,17 @@ def cost_per_token( ) if per_second is not None: return per_second + if custom_llm_provider == "anthropic": + return anthropic_cost_per_token( + model=model, usage=usage_block, service_tier=service_tier, model_info=requested_model_info_override + ) return generic_cost_per_token( model=model, usage=usage_block, custom_llm_provider=custom_llm_provider or "", service_tier=service_tier, + data_residency=data_residency, + vertex_location=vertex_location, model_info=requested_model_info_override, ) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 1fcb06b80e8..ce57a9bd4a5 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -7143,6 +7143,7 @@ class StandardLoggingPayloadSetup: init_response_obj: object, api_base: str | None = None, router_model_id: str | None = None, + custom_model_info: ModelInfo | None = None, ) -> StandardLoggingModelInformation: model_cost_name: Final = select_model_name_for_cost_calc( model=base_model if custom_pricing else None, @@ -7161,6 +7162,8 @@ class StandardLoggingPayloadSetup: if router_model_id is not None else model_cost_name ) + if model_cost_name is not None and custom_model_info is not None: + return StandardLoggingModelInformation(model_map_key=model_cost_name, model_map_value=custom_model_info) if model_cost_name is None: model_cost_information = StandardLoggingModelInformation(model_map_key="", model_map_value=None) else: @@ -7911,6 +7914,7 @@ def get_standard_logging_object_payload( init_response_obj=init_response_obj, api_base=litellm_params.get("api_base"), router_model_id=logging_obj.get_router_model_id(), + custom_model_info=logging_obj.custom_model_info if custom_pricing else None, ) error_information, error_str = StandardLoggingPayloadSetup.get_error_information_for_logging_payload( diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index a014d1d352d..e8ea0c04f6f 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -14594,3 +14594,50 @@ async def test_a_recovered_attempt_bills_a_failed_agentic_follow_up_but_not_a_fa assert own_cost > 0 assert after_a_dropped_stream == pytest.approx(own_cost) assert after_a_failed_follow_up == pytest.approx(own_cost + _FAILED_ATTEMPT[1]) + + +def test_standard_logging_payload_reports_the_per_call_prices_the_request_was_billed_from() -> None: + """A direct call that sets its own prices is logged at those prices, not at the catalog row's rates.""" + from litellm.litellm_core_utils.litellm_logging import get_standard_logging_object_payload + from litellm.litellm_core_utils.per_call_pricing import per_call_model_info + + rates: Final = {"input_cost_per_token": 3e-06, "output_cost_per_token": 7e-06} + now: Final = datetime_unit_test(2026, 1, 1, 12, 0, 0) + logging_obj: Final = LitellmLogging( + model="gpt-5.4-nano", + messages=[{"role": "user", "content": "Hi"}], + stream=False, + call_type="completion", + start_time=now, + litellm_call_id="per-call-prices", + function_id="fn", + ) + logging_obj.update_environment_variables( + model="gpt-5.4-nano", user="", optional_params={}, litellm_params=rates, custom_llm_provider="openai" + ) + logging_obj.set_per_call_pricing(per_call_model_info("gpt-5.4-nano", "openai", rates, "chat")) + response: Final = ModelResponse( + model="gpt-5.4-nano", + choices=[litellm.Choices(message=litellm.Message(role="assistant", content="hi"))], + usage=litellm.Usage(prompt_tokens=10, completion_tokens=6, total_tokens=16), + ) + + payload: Final = get_standard_logging_object_payload( + kwargs={ + "model": "gpt-5.4-nano", + "custom_llm_provider": "openai", + "litellm_params": rates, + "response_cost": logging_obj.response_cost_calculator(result=response), + }, + init_response_obj=response, + start_time=now, + end_time=now, + logging_obj=logging_obj, + status="success", + ) + + assert payload is not None + assert payload["response_cost"] == pytest.approx(10 * 3e-06 + 6 * 7e-06) + model_map_value: Final = payload["model_map_information"]["model_map_value"] + assert model_map_value is not None + assert (model_map_value["input_cost_per_token"], model_map_value["output_cost_per_token"]) == (3e-06, 7e-06) diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index 6de368fbf00..49d51e99790 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -11645,3 +11645,68 @@ def test_upstream_cost_header_total_on_an_unmapped_alias_is_split_by_the_served_ input_share: Final = (10 * 0.001) / (10 * 0.001 + 20 * 0.002) assert breakdown["input_cost"] == pytest.approx(0.25 * input_share) assert breakdown["output_cost"] == pytest.approx(0.25 * (1 - input_share)) + + +_PER_CALL_RATES: Final = MappingProxyType({"input_cost_per_token": 2e-06, "output_cost_per_token": 4e-06}) + + +def _per_call_priced_reply(model: str, usage_extras: Mapping[str, str]) -> ModelResponse: + usage: Final = Usage(prompt_tokens=1000, completion_tokens=1000, total_tokens=2000) + for field, value in usage_extras.items(): + setattr(usage, field, value) + return ModelResponse(model=model, choices=[Choices(message=Message(content="hi"))], usage=usage) + + +@pytest.mark.parametrize( + ("row", "call_kwargs", "usage_extras", "multiplier"), + [ + pytest.param( + {"litellm_provider": "openai", "mode": "chat", "regional_processing_uplift_multiplier_eu": 1.25}, + { + "model": "openai/lit-eu-residency-test", + "api_key": "fake-key", + "api_base": "https://eu.api.openai.com/v1", + }, + {}, + 1.25, + id="openai-data-residency", + ), + pytest.param( + {"litellm_provider": "vertex_ai", "mode": "chat", "regional_endpoint_uplift_multiplier": 1.5}, + { + "model": "vertex_ai/lit-regional-endpoint-test", + "vertex_project": "lit-project", + "vertex_location": "us-east5", + "vertex_credentials": "{}", + }, + {}, + 1.5, + id="vertex-regional-endpoint", + ), + pytest.param( + {"litellm_provider": "anthropic", "mode": "chat", "provider_specific_entry": {"fast": 2.0}}, + {"model": "anthropic/lit-fast-mode-test", "api_key": "fake-key"}, + {"speed": "fast"}, + 2.0, + id="anthropic-fast-mode", + ), + ], +) +def test_per_call_custom_prices_keep_the_regional_and_speed_multipliers( + monkeypatch: pytest.MonkeyPatch, + row: Mapping[str, object], + call_kwargs: Mapping[str, str], + usage_extras: Mapping[str, str], + multiplier: float, +) -> None: + monkeypatch.setitem(litellm.model_cost, call_kwargs["model"], {**row, **_PER_CALL_RATES}) + _invalidate_model_cost_lowercase_map() + + response: Final = litellm.completion( + messages=[{"role": "user", "content": "hello"}], + mock_response=_per_call_priced_reply(call_kwargs["model"].split("/", 1)[1], usage_extras), + **call_kwargs, + **_PER_CALL_RATES, + ) + + assert response._hidden_params["response_cost"] == pytest.approx((1000 * 2e-06 + 1000 * 4e-06) * multiplier) From 513fe4ac8e175cde810d72094a0e71436adc310f Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 5 Nov 2026 00:28:25 +0000 Subject: [PATCH 03/39] chore(cost-map): add openai gpt-image-2.6-sunburst and gpt-image-2.6-flare from the pricing page (#50134) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 32 +++++++++++++++++++ model_prices_and_context_window.json | 32 +++++++++++++++++++ 2 files changed, 64 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 25c7c3593ed..1aa2ac27152 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -85880,5 +85880,37 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true + }, + "gpt-image-2.6-sunburst": { + "cache_read_input_image_token_cost": 2e-06, + "cache_read_input_token_cost": 1.25e-06, + "input_cost_per_image_token": 8e-06, + "input_cost_per_token": 5e-06, + "litellm_provider": "openai", + "mode": "image_generation", + "output_cost_per_image_token": 3e-05, + "source": "https://developers.openai.com/api/docs/pricing", + "supported_endpoints": [ + "/v1/images/generations", + "/v1/images/edits" + ], + "supports_pdf_input": true, + "supports_vision": true + }, + "gpt-image-2.6-flare": { + "cache_read_input_image_token_cost": 2e-06, + "cache_read_input_token_cost": 1.25e-06, + "input_cost_per_image_token": 8e-06, + "input_cost_per_token": 5e-06, + "litellm_provider": "openai", + "mode": "image_generation", + "output_cost_per_image_token": 3e-05, + "source": "https://developers.openai.com/api/docs/pricing", + "supported_endpoints": [ + "/v1/images/generations", + "/v1/images/edits" + ], + "supports_pdf_input": true, + "supports_vision": true } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 25c7c3593ed..1aa2ac27152 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -85880,5 +85880,37 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true + }, + "gpt-image-2.6-sunburst": { + "cache_read_input_image_token_cost": 2e-06, + "cache_read_input_token_cost": 1.25e-06, + "input_cost_per_image_token": 8e-06, + "input_cost_per_token": 5e-06, + "litellm_provider": "openai", + "mode": "image_generation", + "output_cost_per_image_token": 3e-05, + "source": "https://developers.openai.com/api/docs/pricing", + "supported_endpoints": [ + "/v1/images/generations", + "/v1/images/edits" + ], + "supports_pdf_input": true, + "supports_vision": true + }, + "gpt-image-2.6-flare": { + "cache_read_input_image_token_cost": 2e-06, + "cache_read_input_token_cost": 1.25e-06, + "input_cost_per_image_token": 8e-06, + "input_cost_per_token": 5e-06, + "litellm_provider": "openai", + "mode": "image_generation", + "output_cost_per_image_token": 3e-05, + "source": "https://developers.openai.com/api/docs/pricing", + "supported_endpoints": [ + "/v1/images/generations", + "/v1/images/edits" + ], + "supports_pdf_input": true, + "supports_vision": true } } From 733b7764039711cc9f7ee103c8e17b7e9f4a47d3 Mon Sep 17 00:00:00 2001 From: Oliver Yu <57586652+tc3oliver@users.noreply.github.com> Date: Thu, 5 Nov 2026 08:30:12 +0800 Subject: [PATCH 04/39] fix(least-busy): reserve the picked deployment so a burst of requests spreads out (#47484) * fix(least-busy): reserve the picked deployment so a burst of requests spreads out The in-flight count only went up in log_pre_api_call, which runs after selection and the pre-call hooks. Requests selected in that window all saw the same counts and landed on the same deployment (16 simultaneous requests over 4 deployments went as lopsided as 12/4/0/0 in the proxy). When the router selects for a request it is about to send, least-busy now records a reservation for the chosen deployment in the same synchronous step that reads the reservations, and log_pre_api_call turns it into the normal in-flight count. Selections without request kwargs (token counting, probes) do not reserve. A reservation whose request never reaches the API call expires after 30 s. The shared and per-worker counts and their success/failure decrements are unchanged. * fix(least-busy): prune expired and empty selection reservations The reservation map kept a key per model group forever, so a wildcard route with caller-chosen model names grew it without bound. Drop expired and empty entries on every write. Signed-off-by: Oliver Yu * fix(least-busy): keep pick reservations in the shared router cache Selectors that share a router cache dedupe to one registered callback, so only that instance saw log_pre_api_call. Reservations held on the other instances were never released and skewed picks for the full TTL. Storing them in the router cache's in-memory layer lets any selector release them, and the per-key TTL replaces the full-map rebuild on every write. Tests now hold the burst with a barrier instead of a sleep and use a manual clock object. * fix(least-busy): restart the reservation TTL on every write InMemoryCache keeps the TTL a live key was first given, so a reservation key that kept being rewritten still expired 30 s after its first pick and took the newer reservations with it. Drop the key before each write so the TTL counts from the latest one, and leave it deleted once it is empty. Signed-off-by: Oliver Yu * fix(router): count least-busy decisions calls and release reservations on cache hits Least-busy now reads the model group from litellm_metadata before metadata, so /v1/decisions calls that carry caller metadata are counted in flight and release their reservation at pre-call. A response cache hit releases its reservation instead of decrementing another request's in-flight count, and Responses WebSocket turn admission selects without reserving. The reservation store is reached through a typed protocol. * fix(decisions): count proxy /v1/decisions calls in least-busy's in-flight totals * fix(least-busy): keep reservations out of the shared router cache and count requests that start during a Redis read Least-busy now reads a call's deployment from the routing metadata the router sets, so /v1/decisions logging params stay as on main and failed decisions calls no longer start cooldowns. * style(router): wrap the least-busy reservation store line * fix(least-busy): lock the reservation read-and-update so threaded sync calls keep each other's picks Router.completion from a thread pool and sync success callbacks can read and rewrite a deployment's reservations at the same time, so one thread's pick or release could be lost. * fix(least-busy): release a pick whose call ends before reaching the provider A call that fails or returns before the provider handoff (a 400 for an unsupported param, token counting, a proxy-gate rejection without a start log, batch create) now drops its reservation instead of decrementing a count it never added. Router.select picks and batch cancel no longer reserve, since neither makes a call that logs. * test(least-busy): give callback fixtures a fixed call-start time The routing-group tests fed least-busy a finished call without the start time litellm stamps before counting it, so the count stayed up after the fix. Fixtures now use a fixed timestamp instead of the clock. * fix(least-busy): free tier-group picks, stop lookups reserving, and count only started attempts Reservations are keyed by deployment id, so a complexity, auto, adaptive or quality router frees its tier pick when the call starts. Each started attempt is recorded under its call id, and only a recorded start lowers the count when the attempt ends, so a fallback hop rejected before the provider frees its pick. Whether a pick reserves now comes from a context variable set by the routed call sites, so get_available_deployment and the deprecated async_get_available_deployment no longer hold a deployment, and custom routing strategies and Router subclasses keep their method signatures. * fix(least-busy): free a request's unstarted picks when it ends Each pick now carries its call id, so a start frees only its own call's pick, and the router frees every pick a request never started once that request ends. A fallback hop rejected before the provider no longer leaves its pick reserved for 30 seconds. * fix(router): free a least-busy pick when a vector store create ends * fix(router): free least-busy picks of calls made without a call id * fix(router): free the least-busy pick of a sync text completion * test(router): use fixed start times in least-busy fixtures * fix(router): free the least-busy pick of a sync generic call that fails before its provider * fix(router): read in-flight counts and start attempts under one lock so a busy deployment never looks free * fix(router): keep pass-through lookups from reserving least-busy picks * test(least-busy): wait for the helper thread to start before the timed join * fix(least-busy): keep realtime sessions out of in-flight counts --------- Signed-off-by: Oliver Yu Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/router_deprecated.py | 20 +- litellm/router_python_core.py | 441 ++++++------ litellm/router_strategy/least_busy.py | 341 +++++++-- .../test_least_busy_concurrent_decisions.py | 90 +++ tests/unit/decisions/test_main.py | 39 + tests/unit/router_strategy/test_least_busy.py | 672 +++++++++++++++++- .../test_router_routing_groups.py | 37 + tests/unit/test_router/test_router.py | 12 +- .../test_router/test_router_helper_utils.py | 28 + 9 files changed, 1416 insertions(+), 264 deletions(-) create mode 100644 tests/integration/routing/test_least_busy_concurrent_decisions.py diff --git a/litellm/router_deprecated.py b/litellm/router_deprecated.py index 83a1251d3c0..0a02408a717 100644 --- a/litellm/router_deprecated.py +++ b/litellm/router_deprecated.py @@ -27,11 +27,12 @@ import inspect import sys import warnings from collections import Counter -from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence +from collections.abc import Awaitable, Callable, Coroutine, Iterator, Mapping, Sequence from types import MappingProxyType -from typing import TYPE_CHECKING, Final, Generic, TypeVar, overload +from typing import TYPE_CHECKING, Final, Generic, ParamSpec, TypeVar, overload from litellm.router_api import ASSIGNABLE_ALIASES, SETTING_FIELDS, ChangeSet +from litellm.router_strategy.least_busy import picks_reserve if TYPE_CHECKING: from typing import Any # noqa: TID251 # only to repeat two old signatures that had Any, unchanged @@ -58,6 +59,7 @@ if TYPE_CHECKING: from litellm.types.utils import ModelInfo as ModelMapInfo _T: Final = TypeVar("_T") +_P: Final = ParamSpec("_P") DEPRECATED_ROUTER_MEMBER_USES: Final[Counter[str]] = Counter() # mutable-ok: the live tally _note increments @@ -164,6 +166,18 @@ def _attr(replacement: str, read: Callable[["Router"], _T]) -> _DeprecatedAttrib return _DeprecatedAttribute(replacement, read) +def _without_reserving(lookup: Callable[_P, Awaitable[_T]]) -> Callable[_P, Coroutine[object, object, _T]]: + @functools.wraps(lookup) + async def looked_up( + *args: _P.args, + **kwargs: _P.kwargs, # kwargs-ok: forwards the wrapped lookup's ParamSpec unchanged + ) -> _T: + with picks_reserve(False): + return await lookup(*args, **kwargs) + + return looked_up + + # (id(router), member name) pairs whose subclass override is running in this context. _OVERRIDES_RUNNING: Final[contextvars.ContextVar[frozenset[tuple[int, str]]]] = contextvars.ContextVar( "router_overrides_running", default=frozenset() @@ -333,7 +347,7 @@ class DeprecatedRouterMembers: deployment_latency_map = _attr("health().latency", lambda r: r._core_deployment_latency_map) async_get_available_deployment = _attr( "select(model, request_kwargs)", - lambda r: r._core_async_get_available_deployment, + lambda r: _without_reserving(r._core_async_get_available_deployment), ) # -- observers ------------------------------------------------------------------------------ diff --git a/litellm/router_python_core.py b/litellm/router_python_core.py index c0c5d74a88a..4b5ecc82581 100644 --- a/litellm/router_python_core.py +++ b/litellm/router_python_core.py @@ -282,7 +282,15 @@ from litellm.router_strategy.complexity_router.context_compaction import ( reject_recursive_compactor, surface_for_call, ) -from litellm.router_strategy.least_busy import LeastBusyLoggingHandler +from litellm.router_strategy.least_busy import ( + CALL_IDENTITY, + SELECTION_RESERVATION_MAX_KEYS, + LeastBusyLoggingHandler, + call_id_of, + owning_picks, + pick_reserves, + picks_reserve, +) from litellm.router_strategy.lowest_cost import LowestCostLoggingHandler from litellm.router_strategy.lowest_latency import LowestLatencyLoggingHandler from litellm.router_strategy.lowest_tpm_rpm import LowestTPMLoggingHandler @@ -1462,13 +1470,14 @@ class PythonCore(DeprecatedRouterMembers): (``pass_through``). """ if kind == "pick": - return await self._core_async_get_available_deployment( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the selection cores return untyped deployment dicts - model=model, - request_kwargs=request_kwargs if request_kwargs is not None else {}, - messages=messages, - input=input, - specific_deployment=specific_deployment, - ) + with picks_reserve(False): + return await self._core_async_get_available_deployment( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the selection cores return untyped deployment dicts + model=model, + request_kwargs=request_kwargs if request_kwargs is not None else {}, + messages=messages, + input=input, + specific_deployment=specific_deployment, + ) if kind == "healthy": return await self._core_async_get_healthy_deployments( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the selection cores return untyped deployment dicts model=model, @@ -1480,13 +1489,14 @@ class PythonCore(DeprecatedRouterMembers): health_check_probe=health_check_probe, ) if kind == "pass_through": - return self._core_get_available_deployment_for_pass_through( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the selection cores return untyped deployment dicts - model=model, - messages=messages, - input=input, - specific_deployment=specific_deployment, - request_kwargs=request_kwargs, - ) + with picks_reserve(False): + return self._core_get_available_deployment_for_pass_through( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the selection cores return untyped deployment dicts + model=model, + messages=messages, + input=input, + specific_deployment=specific_deployment, + request_kwargs=request_kwargs, + ) raise ValueError(f"unknown select kind {kind!r}") def apply(self, changes: ChangeSet) -> ApplyResult: @@ -1768,6 +1778,9 @@ class PythonCore(DeprecatedRouterMembers): self._core_cache = DualCache( redis_cache=redis_cache, in_memory_cache=InMemoryCache() ) # use a dual cache (Redis+In-Memory) for tracking cooldowns, usage, etc. + self._core_least_busy_reservations: InMemoryCache = InMemoryCache( + max_size_in_memory=SELECTION_RESERVATION_MAX_KEYS + ) self._claude_code_session_router_cache: DualCache = DualCache( redis_cache=redis_cache, in_memory_cache=InMemoryCache(), @@ -2271,7 +2284,9 @@ class PythonCore(DeprecatedRouterMembers): selector: RouterStrategySelector | None = None match self._normalize_strategy(strategy): case RoutingStrategy.LEAST_BUSY.value: - selector = LeastBusyLoggingHandler(router_cache=self._core_cache) + selector = LeastBusyLoggingHandler( + router_cache=self._core_cache, reservation_store=self._core_least_busy_reservations + ) case RoutingStrategy.USAGE_BASED_ROUTING.value: selector = LowestTPMLoggingHandler( router_cache=self._core_cache, @@ -2736,6 +2751,7 @@ class PythonCore(DeprecatedRouterMembers): messages: list[dict[str, str]] | None, input: str | list | None, request_kwargs: dict | None, + reserve: bool, ) -> Any | None: """ Asks the strategy selector for a deployment. Caller handles @@ -2749,6 +2765,8 @@ class PythonCore(DeprecatedRouterMembers): return await selector.async_get_available_deployments( model_group=model, healthy_deployments=healthy_deployments, + reserve=reserve, + call_id=call_id_of(CALL_IDENTITY.validate_python(request_kwargs or {})), ) case "usage-based-routing": # `LowestTPMLoggingHandler` (v1) only exposes the sync @@ -2789,6 +2807,7 @@ class PythonCore(DeprecatedRouterMembers): messages: list[dict[str, str]] | None, input: str | list | None, request_kwargs: dict | None, + reserve: bool, ) -> Any | None: """ Sync sibling of `_select_deployment_async`. Caller handles @@ -2805,6 +2824,8 @@ class PythonCore(DeprecatedRouterMembers): return selector.get_available_deployments( model_group=model, healthy_deployments=healthy_deployments, + reserve=reserve, + call_id=call_id_of(CALL_IDENTITY.validate_python(request_kwargs or {})), ) case "usage-based-routing" | "usage-based-routing-v2": return selector.get_available_deployments( @@ -3524,12 +3545,13 @@ class PythonCore(DeprecatedRouterMembers): ) # pick the one that is available (lowest TPM/RPM) - deployment = self.get_available_deployment( - model=model, - messages=messages, - specific_deployment=kwargs.pop("specific_deployment", None), - request_kwargs=kwargs, - ) + with picks_reserve(True): + deployment = self.get_available_deployment( + model=model, + messages=messages, + specific_deployment=kwargs.pop("specific_deployment", None), + request_kwargs=kwargs, + ) self._drop_unsupported_classifier_reasoning_effort( deployment=cast(DeploymentTypedDict, deployment), # cast-ok: selection returns a router deployment model=model, @@ -6104,12 +6126,13 @@ class PythonCore(DeprecatedRouterMembers): deployment = None # rebind-ok: pre-init so the except block can stamp a failure with no deployment picked try: verbose_router_logger.debug("Inside _image_generation()- model: %s", model) - deployment = self.get_available_deployment( # rebind-ok: set on success, see pre-init - model=model, - messages=[{"role": "user", "content": "prompt"}], - specific_deployment=kwargs.pop("specific_deployment", None), - request_kwargs=kwargs, - ) + with picks_reserve(True): + deployment = self.get_available_deployment( # rebind-ok: set on success, see pre-init + model=model, + messages=[{"role": "user", "content": "prompt"}], + specific_deployment=kwargs.pop("specific_deployment", None), + request_kwargs=kwargs, + ) self._update_kwargs_with_deployment(deployment=deployment, kwargs=kwargs) data: Final = deployment["litellm_params"].copy() @@ -6496,37 +6519,38 @@ class PythonCore(DeprecatedRouterMembers): **kwargs, ): messages: Final = [{"role": "user", "content": prompt}] - try: - kwargs["model"] = model - kwargs["prompt"] = prompt - kwargs["num_retries"] = kwargs.get("num_retries", self.num_retries) - kwargs.setdefault("metadata", {}).update({"model_group": model}) + with owning_picks(self._core_least_busy_reservations, call_id_of(CALL_IDENTITY.validate_python(kwargs))): + try: + kwargs["model"] = model + kwargs["prompt"] = prompt + kwargs["num_retries"] = kwargs.get("num_retries", self.num_retries) + kwargs.setdefault("metadata", {}).update({"model_group": model}) - # pick the one that is available (lowest TPM/RPM) - with endpoint_selection("text_completion", model=model): - deployment: Final = self.get_available_deployment( - model=model, - messages=messages, - specific_deployment=kwargs.pop("specific_deployment", None), - request_kwargs=kwargs, + # pick the one that is available (lowest TPM/RPM) + with endpoint_selection("text_completion", model=model), picks_reserve(True): + deployment: Final = self.get_available_deployment( + model=model, + messages=messages, + specific_deployment=kwargs.pop("specific_deployment", None), + request_kwargs=kwargs, + ) + + data: Final = deployment["litellm_params"].copy() + for k, v in self._core_default_litellm_params.items(): + if k not in kwargs: # prioritize model-specific params > default router params + kwargs[k] = v + elif k == "metadata": + kwargs[k].update(v) + pinned_model_id: Final = deployment_model_id_pin(_LITELLM_PARAMS_ADAPTER.validate_python(data)) + if pinned_model_id is not None: + kwargs["model_id"] = pinned_model_id # rebind-ok: this call's own kwargs, filled like the defaults + + # call via litellm.completion() + return self._host.invoke("text_completion")( + **{**data, "prompt": prompt, "caching": self._core_cache_responses, **kwargs} ) - - data: Final = deployment["litellm_params"].copy() - for k, v in self._core_default_litellm_params.items(): - if k not in kwargs: # prioritize model-specific params > default router params - kwargs[k] = v - elif k == "metadata": - kwargs[k].update(v) - pinned_model_id: Final = deployment_model_id_pin(_LITELLM_PARAMS_ADAPTER.validate_python(data)) - if pinned_model_id is not None: - kwargs["model_id"] = pinned_model_id # rebind-ok: this call's own kwargs, filled like the defaults - - # call via litellm.completion() - return self._host.invoke("text_completion")( - **{**data, "prompt": prompt, "caching": self._core_cache_responses, **kwargs} - ) - except Exception as e: - raise e + except Exception as e: + raise e async def atext_completion( self, @@ -7671,64 +7695,69 @@ class PythonCore(DeprecatedRouterMembers): handler_name: Final = original_function.__name__ metadata_variable_name: Final = get_router_metadata_variable_name(function_name="generic_api_call") deployment = None # rebind-ok: pre-init so the except block can stamp a failure with no deployment picked - try: - verbose_router_logger.debug("Inside _generic_api_call() - handler: %s, model: %s", handler_name, model) - self._update_kwargs_before_fallbacks( - model=model, - kwargs=kwargs, - metadata_variable_name=metadata_variable_name, - ) - with self._video_call_pin(handler_name, kwargs), self._route_capability_pin(handler_name): - deployment = self.get_available_deployment( # rebind-ok: set on success, see pre-init + with owning_picks(self._core_least_busy_reservations, call_id_of(CALL_IDENTITY.validate_python(kwargs))): + try: + verbose_router_logger.debug("Inside _generic_api_call() - handler: %s, model: %s", handler_name, model) + self._update_kwargs_before_fallbacks( model=model, - messages=kwargs.get("messages", None), - input=kwargs.get("input", None), - specific_deployment=kwargs.pop("specific_deployment", None), - request_kwargs=kwargs, + kwargs=kwargs, + metadata_variable_name=metadata_variable_name, + ) + with ( + self._video_call_pin(handler_name, kwargs), + self._route_capability_pin(handler_name), + picks_reserve(True), + ): + deployment = self.get_available_deployment( # rebind-ok: set on success, see pre-init + model=model, + messages=kwargs.get("messages", None), + input=kwargs.get("input", None), + specific_deployment=kwargs.pop("specific_deployment", None), + request_kwargs=kwargs, + ) + self._update_kwargs_with_deployment( + deployment=deployment, + kwargs=kwargs, + function_name="generic_api_call", + splits_stream_timeout=handler_name in _STREAM_TIMEOUT_AWARE_GENERIC_HANDLERS, ) - self._update_kwargs_with_deployment( - deployment=deployment, - kwargs=kwargs, - function_name="generic_api_call", - splits_stream_timeout=handler_name in _STREAM_TIMEOUT_AWARE_GENERIC_HANDLERS, - ) - data: Final = deployment["litellm_params"].copy() - model_name: Final = data["model"] + data: Final = deployment["litellm_params"].copy() + model_name: Final = data["model"] - self.total_calls[model_name] += 1 + self.total_calls[model_name] += 1 - # For passthrough routes, use the actual model from deployment - # and swap model name in endpoint if present - if "endpoint" in kwargs and kwargs["endpoint"]: - kwargs["endpoint"] = kwargs["endpoint"].replace(model, model_name) - kwargs["model"] = model_name + # For passthrough routes, use the actual model from deployment + # and swap model name in endpoint if present + if "endpoint" in kwargs and kwargs["endpoint"]: + kwargs["endpoint"] = kwargs["endpoint"].replace(model, model_name) + kwargs["model"] = model_name - # Perform pre-call checks for routing strategy - self.routing_strategy_pre_call_checks(deployment=deployment) + # Perform pre-call checks for routing strategy + self.routing_strategy_pre_call_checks(deployment=deployment) - custom_llm_provider: Final = provider_for_generic_call(data) + custom_llm_provider: Final = provider_for_generic_call(data) - response: Final = original_function( - **{ - **data, - "custom_llm_provider": custom_llm_provider, - "caching": self._core_cache_responses, - **kwargs, - } - ) + response: Final = original_function( + **{ + **data, + "custom_llm_provider": custom_llm_provider, + "caching": self._core_cache_responses, + **kwargs, + } + ) - self.success_calls[model_name] += 1 - verbose_router_logger.info("%s(model=%s)\x1b[32m 200 OK\x1b[0m", handler_name, model_name) - return response - except Exception as e: - verbose_router_logger.info("%s(model=%s)\x1b[31m Exception %s\x1b[0m", handler_name, model, e) - if model is not None: - self.fail_calls[model] += 1 - self._stamp_failed_attempt( - e, _REQUEST_KWARGS_ADAPTER.validate_python(kwargs), picked=deployment is not None - ) - raise e + self.success_calls[model_name] += 1 + verbose_router_logger.info("%s(model=%s)\x1b[32m 200 OK\x1b[0m", handler_name, model_name) + return response + except Exception as e: + verbose_router_logger.info("%s(model=%s)\x1b[31m Exception %s\x1b[0m", handler_name, model, e) + if model is not None: + self.fail_calls[model] += 1 + self._stamp_failed_attempt( + e, _REQUEST_KWARGS_ADAPTER.validate_python(kwargs), picked=deployment is not None + ) + raise e def embedding( self, @@ -7752,7 +7781,7 @@ class PythonCore(DeprecatedRouterMembers): deployment = None # rebind-ok: pre-init so the except block can stamp a failure with no deployment picked try: verbose_router_logger.debug("Inside embedding()- model: %s", model) - with embedding_deployment_selection(): + with embedding_deployment_selection(), picks_reserve(True): deployment = self.get_available_deployment( # rebind-ok: set on success, see pre-init model=model, input=input, @@ -8061,68 +8090,71 @@ class PythonCore(DeprecatedRouterMembers): Returns: VectorStoreCreateResponse """ - try: - # If model is None, use the factory function approach (direct SDK call) - if model is None: - from litellm.vector_stores.main import acreate + with owning_picks(self._core_least_busy_reservations, call_id_of(CALL_IDENTITY.validate_python(kwargs))): + try: + # If model is None, use the factory function approach (direct SDK call) + if model is None: + from litellm.vector_stores.main import acreate - # Use the factory function to handle the call - factory_fn: Final = self.factory_function(acreate, call_type="avector_store_create") - return await factory_fn(**kwargs) + # Use the factory function to handle the call + factory_fn: Final = self.factory_function(acreate, call_type="avector_store_create") + return await factory_fn(**kwargs) - from litellm.vector_stores import acreate as avector_store_create_sdk + from litellm.vector_stores import acreate as avector_store_create_sdk - parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) - deployment: Final = await self._core_async_get_available_deployment( - model=model, - messages=[{"role": "user", "content": "vector-store-api-fake-text"}], - specific_deployment=kwargs.pop("specific_deployment", None), - request_kwargs=kwargs, - ) - data: Final = deployment["litellm_params"].copy() - model_name: Final = data["model"] - self._update_kwargs_with_deployment( - deployment=deployment, - kwargs=kwargs, - function_name="avector_store_create", - ) - - model_client: Final = self._get_async_openai_model_client( - deployment=deployment, - kwargs=kwargs, - ) - self.total_calls[model_name] += 1 - - # Get custom provider from deployment params - custom_llm_provider = data.get("custom_llm_provider") - _, inferred_custom_llm_provider, _, _ = get_llm_provider( - model=data["model"], - custom_llm_provider=custom_llm_provider, - ) - custom_llm_provider = custom_llm_provider or inferred_custom_llm_provider - - async with self._deployment_slot(deployment=deployment, kwargs=kwargs, parent_otel_span=parent_otel_span): - response = await avector_store_create_sdk( - **{ - **data, - "custom_llm_provider": custom_llm_provider, - "caching": self._core_cache_responses, - "client": model_client, - **kwargs, - } + parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) + deployment: Final = await self._core_async_get_available_deployment( + model=model, + messages=[{"role": "user", "content": "vector-store-api-fake-text"}], + specific_deployment=kwargs.pop("specific_deployment", None), + request_kwargs=kwargs, + ) + data: Final = deployment["litellm_params"].copy() + model_name: Final = data["model"] + self._update_kwargs_with_deployment( + deployment=deployment, + kwargs=kwargs, + function_name="avector_store_create", ) - self.success_calls[model_name] += 1 - verbose_router_logger.info("litellm.avector_store_create(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) + model_client: Final = self._get_async_openai_model_client( + deployment=deployment, + kwargs=kwargs, + ) + self.total_calls[model_name] += 1 - return response - except Exception as e: - verbose_router_logger.exception( - "litellm.avector_store_create(model=%s)\x1b[31m Exception %s\x1b[0m", model, e - ) - if model is not None: - self.fail_calls[model] += 1 - raise e + # Get custom provider from deployment params + custom_llm_provider = data.get("custom_llm_provider") + _, inferred_custom_llm_provider, _, _ = get_llm_provider( + model=data["model"], + custom_llm_provider=custom_llm_provider, + ) + custom_llm_provider = custom_llm_provider or inferred_custom_llm_provider + + async with self._deployment_slot( + deployment=deployment, kwargs=kwargs, parent_otel_span=parent_otel_span + ): + response = await avector_store_create_sdk( + **{ + **data, + "custom_llm_provider": custom_llm_provider, + "caching": self._core_cache_responses, + "client": model_client, + **kwargs, + } + ) + + self.success_calls[model_name] += 1 + verbose_router_logger.info("litellm.avector_store_create(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) + + return response + except Exception as e: + verbose_router_logger.exception( + "litellm.avector_store_create(model=%s)\x1b[31m Exception %s\x1b[0m", model, e + ) + if model is not None: + self.fail_calls[model] += 1 + raise e def _override_vector_store_methods_for_router(self): """ @@ -8370,12 +8402,13 @@ class PythonCore(DeprecatedRouterMembers): try: verbose_router_logger.debug("Inside _acancel_batch()- model: %s", model) parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) - deployment: Final = await self._core_async_get_available_deployment( - model=model, - messages=[{"role": "user", "content": "batch-api-fake-text"}], - specific_deployment=kwargs.pop("specific_deployment", None), - request_kwargs=kwargs, - ) + with picks_reserve(False): + deployment: Final = await self._core_async_get_available_deployment( + model=model, + messages=[{"role": "user", "content": "batch-api-fake-text"}], + specific_deployment=kwargs.pop("specific_deployment", None), + request_kwargs=kwargs, + ) selected_deployment_id: Final = (deployment.get("model_info") or {}).get("id") data: Final = deployment["litellm_params"].copy() @@ -9520,38 +9553,39 @@ class PythonCore(DeprecatedRouterMembers): mock_timeout: Final = kwargs.pop("mock_timeout", None) - try: - self._handle_mock_testing_fallbacks( - kwargs=kwargs, - model_group=model_group, - fallbacks=fallbacks, - context_window_fallbacks=context_window_fallbacks, - content_policy_fallbacks=content_policy_fallbacks, - ) + with owning_picks(self._core_least_busy_reservations, call_id_of(CALL_IDENTITY.validate_python(kwargs))): + try: + self._handle_mock_testing_fallbacks( + kwargs=kwargs, + model_group=model_group, + fallbacks=fallbacks, + context_window_fallbacks=context_window_fallbacks, + content_policy_fallbacks=content_policy_fallbacks, + ) - if mock_timeout is not None: - response = await self.async_function_with_retries(*args, **kwargs, mock_timeout=mock_timeout) - else: - response = await self.async_function_with_retries(*args, **kwargs) - if verbose_router_logger.isEnabledFor(logging.DEBUG): - verbose_router_logger.debug("Async Response: %s", response) - response = add_fallback_headers_to_response( - response=response, - attempted_fallbacks=0, - ) - return response - except Exception as e: - return await self.async_function_with_fallbacks_common_utils( - e, - disable_fallbacks, - fallbacks, - context_window_fallbacks, - content_policy_fallbacks, - model_group, - args, - kwargs, - include_fallback_errors=include_fallback_errors, - ) + if mock_timeout is not None: + response = await self.async_function_with_retries(*args, **kwargs, mock_timeout=mock_timeout) + else: + response = await self.async_function_with_retries(*args, **kwargs) + if verbose_router_logger.isEnabledFor(logging.DEBUG): + verbose_router_logger.debug("Async Response: %s", response) + response = add_fallback_headers_to_response( + response=response, + attempted_fallbacks=0, + ) + return response + except Exception as e: + return await self.async_function_with_fallbacks_common_utils( + e, + disable_fallbacks, + fallbacks, + context_window_fallbacks, + content_policy_fallbacks, + model_group, + args, + kwargs, + include_fallback_errors=include_fallback_errors, + ) def _handle_mock_testing_fallbacks( self, @@ -10856,6 +10890,7 @@ class PythonCore(DeprecatedRouterMembers): messages=None, input=None, request_kwargs=routing_kwargs, + reserve=False, ) if selected is None: no_deployment: Final = await async_raise_no_deployment_exception( @@ -16214,6 +16249,7 @@ class PythonCore(DeprecatedRouterMembers): messages=messages, input=input, specific_deployment=specific_deployment, + reserve=pick_reserves(default=True), ) async def _async_get_available_deployment( @@ -16223,6 +16259,7 @@ class PythonCore(DeprecatedRouterMembers): messages: list[dict[str, str]] | None, input: str | list | None, specific_deployment: bool | None, + reserve: bool, ): if ( self._core_routing_strategy != "usage-based-routing-v2" @@ -16319,6 +16356,7 @@ class PythonCore(DeprecatedRouterMembers): messages=messages, input=input, request_kwargs=request_kwargs, + reserve=reserve and request_kwargs not in (None, {}), ) if deployment is None: exception: Final = await async_raise_no_deployment_exception( @@ -16488,6 +16526,7 @@ class PythonCore(DeprecatedRouterMembers): messages=messages, input=input, request_kwargs=request_kwargs, + reserve=pick_reserves(default=False) and request_kwargs not in (None, {}), ) if deployment is None: @@ -17365,6 +17404,7 @@ class PythonCore(DeprecatedRouterMembers): messages=messages, input=input, request_kwargs=request_kwargs, + reserve=pick_reserves(default=False) and request_kwargs not in (None, {}), ) if deployment is None: @@ -17545,6 +17585,7 @@ class PythonCore(DeprecatedRouterMembers): messages=messages, input=input, request_kwargs=request_kwargs, + reserve=pick_reserves(default=False) and request_kwargs not in (None, {}), ) if deployment is None: diff --git a/litellm/router_strategy/least_busy.py b/litellm/router_strategy/least_busy.py index 8e0c3d6a7e7..22159079f98 100644 --- a/litellm/router_strategy/least_busy.py +++ b/litellm/router_strategy/least_busy.py @@ -1,6 +1,11 @@ +import contextlib import logging -from collections.abc import Mapping, Sequence -from typing import Final +import threading +import time +import uuid +from collections.abc import Callable, Generator, Mapping, Sequence +from contextvars import ContextVar +from typing import Final, Protocol, TypeAlias from pydantic import TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict @@ -8,11 +13,31 @@ from typing_extensions import ReadOnly, TypedDict from litellm._internal_context import with_service_target from litellm._logging import verbose_router_logger from litellm.caching.caching import DualCache +from litellm.caching.in_memory_cache import InMemoryCache from litellm.caching.redis_cache import log_redis_failure from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type +from litellm.types.utils import CallTypes IN_FLIGHT_COUNT_TTL_SECONDS: Final = 60 * 60 +SELECTION_RESERVATION_TTL_SECONDS: Final = 30.0 +SELECTION_RESERVATION_MAX_KEYS: Final = 10_000 +_RESERVATIONS_LOCK: Final = threading.Lock() +_PICK_RESERVES: Final[ContextVar[bool]] = ContextVar("least_busy_pick_reserves") +_PICK_OWNER: Final[ContextVar[str | None]] = ContextVar("least_busy_pick_owner", default=None) + + +@contextlib.contextmanager +def picks_reserve(reserve: bool) -> Generator[None, None, None]: + token: Final = _PICK_RESERVES.set(reserve) + try: + yield + finally: + _PICK_RESERVES.reset(token) + + +def pick_reserves(default: bool) -> bool: + return _PICK_RESERVES.get(default) class _ModelInfo(TypedDict, total=False): @@ -21,10 +46,12 @@ class _ModelInfo(TypedDict, total=False): class _Metadata(TypedDict, total=False): model_group: ReadOnly[str | None] + model_info: ReadOnly[_ModelInfo | None] class _LitellmParams(TypedDict, total=False): metadata: ReadOnly[_Metadata | None] + litellm_metadata: ReadOnly[_Metadata | None] model_info: ReadOnly[_ModelInfo | None] @@ -32,6 +59,10 @@ class _CallKwargs(TypedDict, total=False): litellm_params: ReadOnly[_LitellmParams | None] +class CallIdentity(TypedDict, total=False): + litellm_call_id: ReadOnly[object] + + class _DeploymentModelInfo(TypedDict): id: ReadOnly[str | int] @@ -41,38 +72,85 @@ class _Deployment(TypedDict): _CALL_KWARGS: Final = TypeAdapter(_CallKwargs) +CALL_IDENTITY: Final = TypeAdapter(CallIdentity) _DEPLOYMENTS: Final = TypeAdapter(list[_Deployment]) -_MEMORY_COUNTS: Final = TypeAdapter(tuple[float | None, ...] | None) +_MEMORY_COUNTS: Final[TypeAdapter[tuple[float | None, ...] | None]] = TypeAdapter(tuple[float | None, ...] | None) +_ATTEMPT_TIMES: Final[TypeAdapter[tuple[float, ...] | None]] = TypeAdapter(tuple[float, ...] | None) +_Pick: TypeAlias = tuple[float, str | None] +_PICKS: Final[TypeAdapter[tuple[_Pick, ...] | None]] = TypeAdapter(tuple[_Pick, ...] | None) +_PICKED_DEPLOYMENTS: Final[TypeAdapter[tuple[str, ...] | None]] = TypeAdapter(tuple[str, ...] | None) def _request_count_key(model_group: str, deployment_id: str) -> str: return f"{model_group}_request_count:{deployment_id}" +def _model_group(metadata: _Metadata | None) -> str | None: + return metadata.get("model_group") if metadata else None + + +def _deployment_id(model_info: _ModelInfo | None) -> str | int | None: + return model_info.get("id") if model_info else None + + +def _routed_deployment_id(metadata: _Metadata | None) -> str | int | None: + return _deployment_id(metadata.get("model_info")) if metadata else None + + def _deployment_ref(kwargs: Mapping[str, object]) -> tuple[str, str] | None: - if is_batch_retrieve_call_type(kwargs.get("call_type")): + call_type: Final = kwargs.get("call_type") + if is_batch_retrieve_call_type(call_type) or call_type == CallTypes.arealtime.value: return None try: call: Final = _CALL_KWARGS.validate_python(kwargs) except ValidationError: return None - litellm_params: Final = call.get("litellm_params") - metadata: Final = litellm_params.get("metadata") if litellm_params else None - model_info: Final = litellm_params.get("model_info") if litellm_params else None - model_group: Final = metadata.get("model_group") if metadata else None - deployment_id: Final = model_info.get("id") if model_info else None + litellm_params: Final = call.get("litellm_params") or _LitellmParams() + routing_metadata: Final = (litellm_params.get("litellm_metadata"), litellm_params.get("metadata")) + model_group: Final = next((group for group in map(_model_group, routing_metadata) if group is not None), None) + deployment_id: Final = next( + ( + deployment_id + for deployment_id in ( + _deployment_id(litellm_params.get("model_info")), + *map(_routed_deployment_id, routing_metadata), + ) + if deployment_id is not None + ), + None, + ) if model_group is None or deployment_id is None: return None return model_group, str(deployment_id) -def _request_count_keys(model_group: str, healthy_deployments: Sequence[Mapping[str, object]]) -> tuple[str, ...]: +def _reservation_key(deployment_id: str) -> str: + return f"least_busy_reservations:{deployment_id}" + + +def _started_attempts_key(call_id: str, ref: tuple[str, str]) -> str: + return f"least_busy_started:{call_id}:{_request_count_key(*ref)}" + + +def _picked_deployments_key(call_id: str) -> str: + return f"least_busy_picked:{call_id}" + + +def call_id_of(kwargs: Mapping[str, object]) -> str | None: + call_id: Final = kwargs.get("litellm_call_id") + return call_id if isinstance(call_id, str) else None + + +def _deployment_ids(healthy_deployments: Sequence[Mapping[str, object]]) -> tuple[str, ...]: return tuple( - _request_count_key(model_group, str(deployment["model_info"]["id"])) - for deployment in _DEPLOYMENTS.validate_python(healthy_deployments) + str(deployment["model_info"]["id"]) for deployment in _DEPLOYMENTS.validate_python(healthy_deployments) ) +def _request_count_keys(model_group: str, deployment_ids: tuple[str, ...]) -> tuple[str, ...]: + return tuple(_request_count_key(model_group, deployment_id) for deployment_id in deployment_ids) + + def _as_counts(values: Sequence[float | None]) -> tuple[int, ...]: return tuple(0 if value is None else int(value) for value in values) @@ -84,12 +162,21 @@ def _local_counts(raw: object, keys: tuple[str, ...]) -> tuple[int, ...]: return _as_counts(values) -def _least_busy( - healthy_deployments: Sequence[Mapping[str, object]], counts: tuple[int, ...] -) -> Mapping[str, object] | None: - if not healthy_deployments: - return None - return healthy_deployments[min(range(len(healthy_deployments)), key=lambda index: counts[index])] +def _unexpired(picks: Sequence[_Pick], now: float) -> tuple[_Pick, ...]: + return tuple(pick for pick in picks if now - pick[0] < SELECTION_RESERVATION_TTL_SECONDS) + + +def _without_one_pick(picks: tuple[_Pick, ...], call_id: str | None) -> tuple[_Pick, ...]: + tags: Final = tuple(tag for _, tag in picks) + owner: Final = next((tag for tag in (call_id, _PICK_OWNER.get()) if tag is not None and tag in tags), None) + if owner not in tags: + return picks + index: Final = tags.index(owner) + return (*picks[:index], *picks[index + 1 :]) + + +def _least_busy_index(counts: tuple[int, ...]) -> int: + return min(range(len(counts)), key=lambda index: counts[index]) def _warn_unreadable(model_group: str, error: Exception) -> None: @@ -111,24 +198,156 @@ def _warn_unwritable(key: str, error: Exception) -> None: ) +class _ReservationStore(Protocol): + def get_cache(self, key: str) -> object: ... + + def set_cache(self, key: str, value: tuple[object, ...], *, ttl: float) -> None: ... + + def delete_cache(self, key: str) -> None: ... + + +def _picks(store: _ReservationStore, deployment_id: str) -> tuple[_Pick, ...]: + return _PICKS.validate_python(store.get_cache(_reservation_key(deployment_id))) or () + + +def _attempt_times(store: _ReservationStore, key: str) -> tuple[float, ...]: + return _ATTEMPT_TIMES.validate_python(store.get_cache(key)) or () + + +def _picked_deployments(store: _ReservationStore, call_id: str) -> tuple[str, ...]: + return _PICKED_DEPLOYMENTS.validate_python(store.get_cache(_picked_deployments_key(call_id))) or () + + +def _put(store: _ReservationStore, key: str, values: tuple[object, ...], ttl: float) -> None: + # InMemoryCache keeps a live key's first TTL, so drop the key to restart it + store.delete_cache(key) + if values: + store.set_cache(key, values, ttl=ttl) + + +def _release_call_on(store: _ReservationStore, deployment_id: str, call_id: str) -> None: + kept: Final = tuple(pick for pick in _picks(store, deployment_id) if pick[1] != call_id) + _put(store, _reservation_key(deployment_id), kept, SELECTION_RESERVATION_TTL_SECONDS) + + +def release_unstarted_picks(store: _ReservationStore, call_id: str | None) -> None: + if call_id is None: + return + if not _picked_deployments(store, call_id): + return + with _RESERVATIONS_LOCK: + for deployment_id in frozenset(_picked_deployments(store, call_id)): + _release_call_on(store, deployment_id, call_id) + store.delete_cache(_picked_deployments_key(call_id)) + + +@contextlib.contextmanager +def owning_picks(store: _ReservationStore, call_id: str | None) -> Generator[None, None, None]: + owner: Final = call_id or _PICK_OWNER.get() or uuid.uuid4().hex + token: Final = _PICK_OWNER.set(owner) + try: + yield + finally: + _PICK_OWNER.reset(token) + release_unstarted_picks(store, owner) + + class LeastBusyLoggingHandler(CustomLogger): test_flag: bool = False logged_success: int = 0 logged_failure: int = 0 - def __init__(self, router_cache: DualCache): + def __init__( + self, + router_cache: DualCache, + clock: Callable[[], float] = time.monotonic, + reservation_store: _ReservationStore | None = None, + ): self.router_cache = router_cache self.router_cache_id = str(id(router_cache)) + self._clock = clock + self._reservations: Final[_ReservationStore] = ( + reservation_store + if reservation_store is not None + else InMemoryCache(max_size_in_memory=SELECTION_RESERVATION_MAX_KEYS, clock=clock) + ) + + def _live_reservations(self, deployment_id: str, now: float) -> tuple[_Pick, ...]: + return _unexpired(_picks(self._reservations, deployment_id), now) + + def _store_reservations(self, deployment_id: str, picks: tuple[_Pick, ...]) -> None: + _put(self._reservations, _reservation_key(deployment_id), picks, SELECTION_RESERVATION_TTL_SECONDS) + + def _take_reservation(self, deployment_id: str, call_id: str | None, now: float) -> None: + self._store_reservations(deployment_id, _without_one_pick(self._live_reservations(deployment_id, now), call_id)) + + def _record_pick(self, call_id: str, deployment_id: str) -> None: + picked: Final = (*_picked_deployments(self._reservations, call_id), deployment_id) + _put(self._reservations, _picked_deployments_key(call_id), picked, SELECTION_RESERVATION_TTL_SECONDS) + + def _select( + self, + healthy_deployments: Sequence[Mapping[str, object]], + deployment_ids: tuple[str, ...], + keys: tuple[str, ...], + shared: tuple[int, ...] | None, + reserve: bool, + call_id: str | None, + ) -> Mapping[str, object] | None: + if not healthy_deployments: + return None + with _RESERVATIONS_LOCK: + now: Final = self._clock() + counts: Final = ( + self._this_workers_counts(keys) if shared is None else self._at_least_this_workers(keys, shared) + ) + live: Final = tuple(self._live_reservations(deployment_id, now) for deployment_id in deployment_ids) + index: Final = _least_busy_index(tuple(count + len(r) for count, r in zip(counts, live, strict=True))) + if reserve: + tag: Final = call_id if call_id is not None else _PICK_OWNER.get() + self._store_reservations(deployment_ids[index], (*live[index], (now, tag))) + if tag is not None: + self._record_pick(tag, deployment_ids[index]) + return healthy_deployments[index] + + def _start_attempt(self, ref: tuple[str, str], call_id: str | None) -> None: + with _RESERVATIONS_LOCK: + now: Final = self._clock() + self._take_reservation(ref[1], call_id, now) + self._count_locally(_request_count_key(*ref), 1) + if call_id is not None: + key: Final = _started_attempts_key(call_id, ref) + started: Final = (*_attempt_times(self._reservations, key), now) + _put(self._reservations, key, started, IN_FLIGHT_COUNT_TTL_SECONDS) + + def _end_attempt(self, ref: tuple[str, str], call_id: str | None) -> bool: + if call_id is None: + with _RESERVATIONS_LOCK: + self._count_locally(_request_count_key(*ref), -1) + return True + key: Final = _started_attempts_key(call_id, ref) + with _RESERVATIONS_LOCK: + started: Final = _attempt_times(self._reservations, key) + if started: + _put(self._reservations, key, started[1:], IN_FLIGHT_COUNT_TTL_SECONDS) + self._count_locally(_request_count_key(*ref), -1) + else: + self._take_reservation(ref[1], call_id, self._clock()) + return bool(started) @with_service_target("router_usage") def log_pre_api_call(self, model: str, messages: object, kwargs: Mapping[str, object]) -> None: - self._increment(kwargs, 1) + ref: Final = _deployment_ref(kwargs) + if ref is None: + return + self._start_attempt(ref, call_id_of(kwargs)) + self._count_shared(_request_count_key(*ref), 1) @with_service_target("router_usage") def log_success_event( self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object ) -> None: - self._increment(kwargs, -1) + self._settle(kwargs) if self.test_flag: self.logged_success += 1 @@ -136,7 +355,7 @@ class LeastBusyLoggingHandler(CustomLogger): def log_failure_event( self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object ) -> None: - self._increment(kwargs, -1) + self._settle(kwargs) if self.test_flag: self.logged_failure += 1 @@ -144,7 +363,7 @@ class LeastBusyLoggingHandler(CustomLogger): async def async_log_success_event( self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object ) -> None: - await self._async_increment(kwargs, -1) + await self._async_settle(kwargs) if self.test_flag: self.logged_success += 1 @@ -152,15 +371,20 @@ class LeastBusyLoggingHandler(CustomLogger): async def async_log_failure_event( self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object ) -> None: - await self._async_increment(kwargs, -1) + await self._async_settle(kwargs) if self.test_flag: self.logged_failure += 1 @with_service_target("router_usage") def get_available_deployments( - self, model_group: str, healthy_deployments: Sequence[Mapping[str, object]] + self, + model_group: str, + healthy_deployments: Sequence[Mapping[str, object]], + reserve: bool = False, + call_id: str | None = None, ) -> Mapping[str, object] | None: - keys: Final = _request_count_keys(model_group, healthy_deployments) + deployment_ids: Final = _deployment_ids(healthy_deployments) + keys: Final = _request_count_keys(model_group, deployment_ids) redis_cache: Final = self.router_cache.redis_cache if redis_cache is not None: try: @@ -168,15 +392,19 @@ class LeastBusyLoggingHandler(CustomLogger): except Exception as e: _warn_unreadable(model_group, e) else: - return _least_busy(healthy_deployments, shared) - local: Final = _local_counts(self.router_cache.batch_get_cache(list(keys), local_only=True), keys) - return _least_busy(healthy_deployments, local) + return self._select(healthy_deployments, deployment_ids, keys, shared, reserve, call_id) + return self._select(healthy_deployments, deployment_ids, keys, None, reserve, call_id) @with_service_target("router_usage") async def async_get_available_deployments( - self, model_group: str, healthy_deployments: Sequence[Mapping[str, object]] + self, + model_group: str, + healthy_deployments: Sequence[Mapping[str, object]], + reserve: bool = False, + call_id: str | None = None, ) -> Mapping[str, object] | None: - keys: Final = _request_count_keys(model_group, healthy_deployments) + deployment_ids: Final = _deployment_ids(healthy_deployments) + keys: Final = _request_count_keys(model_group, deployment_ids) redis_cache: Final = self.router_cache.redis_cache if redis_cache is not None: try: @@ -184,42 +412,51 @@ class LeastBusyLoggingHandler(CustomLogger): except Exception as e: _warn_unreadable(model_group, e) else: - return _least_busy(healthy_deployments, shared) - local: Final = _local_counts(await self.router_cache.async_batch_get_cache(list(keys), local_only=True), keys) - return _least_busy(healthy_deployments, local) + return self._select(healthy_deployments, deployment_ids, keys, shared, reserve, call_id) + return self._select(healthy_deployments, deployment_ids, keys, None, reserve, call_id) - def _increment(self, kwargs: Mapping[str, object], delta: int) -> None: + def _settle(self, kwargs: Mapping[str, object]) -> None: ref: Final = _deployment_ref(kwargs) - if ref is None: + if ref is None or not self._end_attempt(ref, call_id_of(kwargs)): return - key: Final = _request_count_key(*ref) - redis_cache: Final = self.router_cache.redis_cache + self._count_shared(_request_count_key(*ref), -1) + + async def _async_settle(self, kwargs: Mapping[str, object]) -> None: + ref: Final = _deployment_ref(kwargs) + if ref is None or not self._end_attempt(ref, call_id_of(kwargs)): + return + await self._async_count_shared(_request_count_key(*ref), -1) + + def _this_workers_counts(self, keys: tuple[str, ...]) -> tuple[int, ...]: + return _local_counts(self.router_cache.batch_get_cache(list(keys), local_only=True), keys) + + def _at_least_this_workers(self, keys: tuple[str, ...], shared: tuple[int, ...]) -> tuple[int, ...]: + return tuple(max(counts) for counts in zip(shared, self._this_workers_counts(keys), strict=True)) + + def _count_locally(self, key: str, delta: int) -> None: try: local: Final = self.router_cache.increment_cache( key, delta, local_only=True, ttl=IN_FLIGHT_COUNT_TTL_SECONDS ) if local < 0: self.router_cache.set_cache(key, 0, local_only=True, ttl=IN_FLIGHT_COUNT_TTL_SECONDS) - if redis_cache is None: - return + except TypeError as e: + _warn_unwritable(key, e) + + def _count_shared(self, key: str, delta: int) -> None: + redis_cache: Final = self.router_cache.redis_cache + if redis_cache is None: + return + try: redis_cache.increment_with_floor(key, delta, IN_FLIGHT_COUNT_TTL_SECONDS) except Exception as e: _warn_unwritable(key, e) - async def _async_increment(self, kwargs: Mapping[str, object], delta: int) -> None: - ref: Final = _deployment_ref(kwargs) - if ref is None: - return - key: Final = _request_count_key(*ref) + async def _async_count_shared(self, key: str, delta: int) -> None: redis_cache: Final = self.router_cache.redis_cache + if redis_cache is None: + return try: - local: Final = await self.router_cache.async_increment_cache( - key, delta, local_only=True, ttl=IN_FLIGHT_COUNT_TTL_SECONDS - ) - if local is not None and local < 0: - await self.router_cache.async_set_cache(key, 0, local_only=True, ttl=IN_FLIGHT_COUNT_TTL_SECONDS) - if redis_cache is None: - return await redis_cache.async_increment_with_floor(key, delta, IN_FLIGHT_COUNT_TTL_SECONDS) except Exception as e: _warn_unwritable(key, e) diff --git a/tests/integration/routing/test_least_busy_concurrent_decisions.py b/tests/integration/routing/test_least_busy_concurrent_decisions.py new file mode 100644 index 00000000000..bb923534db4 --- /dev/null +++ b/tests/integration/routing/test_least_busy_concurrent_decisions.py @@ -0,0 +1,90 @@ +import json +import threading +from collections import Counter +from collections.abc import Callable +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from typing import Final + +import httpx +import pytest +import yaml +from integration._support.client import Gateway, gateway_from_environment +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, wire_server + +pytestmark = pytest.mark.timeout(240) + +_GROUP: Final = "least-busy-decider" +_DEPLOYMENTS: Final = ("least-busy-decider-a", "least-busy-decider-b") +_BURST: Final = 4 +_REPLY: Final = json.dumps( + { + "model": "Qwen/Qwen3-0.6B", + "answers": {"billing": {"type": "noul", "noul": 0.91}}, + "usage": {"input_tokens": 367, "output_tokens": 3}, + } +).encode() +_DECISION: Final = { + "model": _GROUP, + "input": "I was charged twice for order 4417.", + "questions": [{"type": "predicate", "name": "billing", "instructions": "Is this a billing issue?"}], +} + + +def _held_until_the_burst_arrives(arrived: threading.Barrier) -> Callable[[Request], Reply]: + def respond(request: Request) -> Reply: + arrived.wait() + return Reply(body=_REPLY) + + return respond + + +def _write_config(path: Path, provider_url: str) -> Path: + path.write_text( + yaml.safe_dump( + { + "model_list": [ + { + "model_name": _GROUP, + "litellm_params": {"model": "hosted_vllm/Qwen/Qwen3-0.6B", "api_base": provider_url}, + "model_info": {"id": deployment_id}, + } + for deployment_id in _DEPLOYMENTS + ], + "general_settings": { + "master_key": "os.environ/LITELLM_MASTER_KEY", + "database_url": "os.environ/DATABASE_URL", + "disable_model_info_refresh": True, + }, + "router_settings": {"routing_strategy": "least-busy", "num_retries": 0, "disable_cooldowns": True}, + } + ) + ) + return path + + +def _decide(proxy: Gateway) -> httpx.Response: + return proxy.client.post( + "/v1/decisions", json=_DECISION, headers={"Authorization": f"Bearer {proxy.key}"}, timeout=120 + ) + + +def test_least_busy_spreads_a_burst_of_decisions_calls_across_the_group(tmp_path: Path) -> None: + arrived: Final = threading.Barrier(_BURST, timeout=60) + with ( + wire_server(_held_until_the_burst_arrives(arrived)) as upstream, + gateway_from_environment() as rig, + owned_proxy( + rig, + tmp_path, + {"STORE_MODEL_IN_DB": "False"}, + config=_write_config(tmp_path / "least-busy.yaml", upstream.url), + ) as proxy, + ThreadPoolExecutor(max_workers=_BURST) as pool, + ): + responses: Final = tuple(pool.map(lambda _: _decide(proxy), range(_BURST))) + assert [response.status_code for response in responses] == [200] * _BURST, [r.text for r in responses] + assert Counter(response.headers["x-litellm-model-id"] for response in responses) == Counter( + {deployment_id: _BURST // len(_DEPLOYMENTS) for deployment_id in _DEPLOYMENTS} + ) diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index 1e42e74806d..0e2813e0a0d 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -6,8 +6,10 @@ import copy import json import re import threading +import uuid from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence from concurrent.futures import wait +from datetime import datetime from functools import partial from types import MappingProxyType from typing import Final, TypeAlias @@ -300,6 +302,43 @@ async def test_router_dispatches_typesafe_decisions_without_api_base( assert response.answers["sentiment"].choice == "positive" +@pytest.mark.asyncio +async def test_least_busy_frees_a_decisions_deployment_once_a_proxy_logged_call_finishes( + respx_mock: respx.MockRouter, +) -> None: + respx_mock.post("https://api.typesafe.ai/v1/systemone").respond(json=_RESPONSE) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "jev", + "litellm_params": {"model": "typesafe/jev-latest", "api_key": "k"}, + "model_info": {"id": deployment_id}, + } + for deployment_id in ("jev-a", "jev-b") + ], + routing_strategy="least-busy", + ) + + async def call_through_proxy_logging() -> object: + logging_obj, _ = litellm.utils.function_setup( + original_function="adecisions", + rules_obj=litellm.utils.Rules(), + start_time=datetime(2026, 1, 1), + model="jev", + litellm_call_id=str(uuid.uuid4()), + ) + response: Final = await router.adecisions( + model="jev", + input="Is this a defect?", + questions=[{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}], + litellm_logging_obj=logging_obj, + ) + await _drain_logging_worker() + return response._hidden_params["model_id"] + + assert (await call_through_proxy_logging(), await call_through_proxy_logging()) == ("jev-a", "jev-a") + + def test_decisions_uses_the_same_wire_contract_for_sync_calls(respx_mock: respx.MockRouter) -> None: route: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) diff --git a/tests/unit/router_strategy/test_least_busy.py b/tests/unit/router_strategy/test_least_busy.py index 5b3af388ae5..f156a5ee405 100644 --- a/tests/unit/router_strategy/test_least_busy.py +++ b/tests/unit/router_strategy/test_least_busy.py @@ -1,9 +1,16 @@ import asyncio +import contextlib +import contextvars +import datetime import json import logging +import threading +from collections.abc import Callable, Generator +from concurrent.futures import ThreadPoolExecutor from typing import Final import httpx +import openai import pytest import respx @@ -11,11 +18,19 @@ import litellm from litellm import Router from litellm.caching.caching import DualCache -from litellm.caching.in_memory_cache import InMemoryCache +from litellm.caching.in_memory_cache import DEFAULT_MAX_SIZE_IN_MEMORY, InMemoryCache from litellm.caching.redis_cache import RedisCircuitBreakerOpenError +from litellm.constants import LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER -from litellm.router_strategy.least_busy import IN_FLIGHT_COUNT_TTL_SECONDS, LeastBusyLoggingHandler -from litellm.types.utils import ModelResponse +from litellm.router_strategy.least_busy import ( + IN_FLIGHT_COUNT_TTL_SECONDS, + SELECTION_RESERVATION_TTL_SECONDS, + LeastBusyLoggingHandler, + owning_picks, + picks_reserve, + release_unstarted_picks, +) +from litellm.types.utils import CallTypes, ModelResponse GROUP: Final = "least-busy-group" DEPLOYMENT_A: Final[dict[str, object]] = {"model_info": {"id": "dep-a"}} @@ -23,8 +38,11 @@ DEPLOYMENT_B: Final[dict[str, object]] = {"model_info": {"id": "dep-b"}} HEALTHY: Final = [DEPLOYMENT_A, DEPLOYMENT_B] -def _call_kwargs(deployment_id: str) -> dict[str, object]: - return {"litellm_params": {"metadata": {"model_group": GROUP}, "model_info": {"id": deployment_id}}} +def _call_kwargs(deployment_id: str, call_id: str = "call-1") -> dict[str, object]: + return { + "litellm_params": {"metadata": {"model_group": GROUP}, "model_info": {"id": deployment_id}}, + "litellm_call_id": call_id, + } class SharedRedisCounters: @@ -58,6 +76,20 @@ class SharedRedisCounters: return self.batch_get_counts(key_list) +class SlowRedisRead(SharedRedisCounters): + """Answers a count read with the counts from when the read started, after letting other work run.""" + + def __init__(self) -> None: + super().__init__() + self.while_reading: Callable[[], None] = lambda: None + + async def async_batch_get_counts(self, key_list: list[str]) -> tuple[int | None, ...]: + counts_when_asked: Final = self.batch_get_counts(key_list) + await asyncio.sleep(0) + self.while_reading() + return counts_when_asked + + def _worker(shared: SharedRedisCounters | None) -> LeastBusyLoggingHandler: cache: Final = DualCache(in_memory_cache=InMemoryCache(), redis_cache=shared) # pyright: ignore[reportArgumentType] # duck-typed Redis double return LeastBusyLoggingHandler(router_cache=cache) @@ -81,6 +113,16 @@ async def test_worker_routes_around_a_request_another_worker_started() -> None: assert await picking_worker.async_get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_A +@pytest.mark.asyncio +async def test_a_request_that_starts_while_redis_is_read_still_counts_on_its_deployment() -> None: + redis: Final = SlowRedisRead() + worker: Final = _worker(redis) + assert await worker.async_get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A + redis.while_reading = lambda: worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a")) + + assert await worker.async_get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_B + + def test_sync_pick_reads_the_shared_counts() -> None: shared: Final = SharedRedisCounters() streaming_worker: Final = _worker(shared) @@ -98,6 +140,44 @@ def test_sync_pick_reads_the_shared_counts() -> None: assert picking_worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_A +def _routed_through_litellm_metadata(deployment_id: str) -> dict[str, object]: + return { + "litellm_params": { + "metadata": {"tags": ["caller-tag"]}, + "litellm_metadata": {"model_group": GROUP}, + "model_info": {"id": deployment_id}, + }, + "litellm_call_id": "call-1", + } + + +def test_a_request_routed_through_litellm_metadata_counts_while_in_flight() -> None: + shared: Final = SharedRedisCounters() + calling_worker: Final = _worker(shared) + picking_worker: Final = _worker(shared) + + assert calling_worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A + calling_worker.log_pre_api_call(model="m", messages=[], kwargs=_routed_through_litellm_metadata("dep-a")) + + assert picking_worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_B + + calling_worker.log_success_event(_routed_through_litellm_metadata("dep-a"), None, None, None) + + assert calling_worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_A + + +def test_a_closed_realtime_session_leaves_its_deployment_idle() -> None: + shared: Final = SharedRedisCounters() + worker: Final = _worker(shared) + session_event: Final = {**_routed_through_litellm_metadata("dep-a"), "call_type": CallTypes.arealtime.value} + + for _ in range(5): + worker.log_pre_api_call(model="m", messages=[], kwargs=session_event) + worker.log_success_event(session_event, None, None, None) + + assert worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_A + + def test_the_handler_never_pushes_a_counters_ttl_forward() -> None: """A worker that dies mid-request leaves a +1 nobody will ever decrement. Redis expires that stuck count an hour after the key was created, which only works while nothing writes the TTL @@ -259,12 +339,16 @@ ROUTER_GROUP: Final = "least-busy-router-group" ROUTER_DEPLOYMENT_IDS: Final = ("1", "2", "3") -def _least_busy_router() -> Router: +def _least_busy_router(group: str = ROUTER_GROUP, *, use_in_pass_through: bool = False) -> Router: return Router( model_list=[ { - "model_name": ROUTER_GROUP, - "litellm_params": {"model": "openai/gpt-4.1-mini", "api_key": f"key-{deployment_id}"}, + "model_name": group, + "litellm_params": { + "model": "openai/gpt-4.1-mini", + "api_key": f"key-{deployment_id}", + "use_in_pass_through": use_in_pass_through, + }, "model_info": {"id": deployment_id}, } for deployment_id in ROUTER_DEPLOYMENT_IDS @@ -391,3 +475,575 @@ async def test_router_picks_least_busy_deployment_and_completion_restores_counts assert response._hidden_params["model_id"] == "1" assert route.calls.last.request.headers["authorization"] == "Bearer key-1" assert _router_counts(router) == seeded + + +@pytest.mark.asyncio +async def test_router_spreads_requests_that_start_together( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + burst: Final = 2 * len(ROUTER_DEPLOYMENT_IDS) + all_started: Final = asyncio.Barrier(burst) + + async def completion_after_all_started(_: httpx.Request) -> httpx.Response: + await all_started.wait() + return httpx.Response( + 200, + json={ + "id": "chatcmpl-least-busy", + "object": "chat.completion", + "created": 1, + "model": "gpt-4.1-mini", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "done"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + ) + + route: Final = respx_mock.post("https://api.openai.com/v1/chat/completions").mock( + side_effect=completion_after_all_started + ) + router: Final = _least_busy_router() + + await asyncio.gather( + *(router.acompletion(model=ROUTER_GROUP, messages=[{"role": "user", "content": "hi"}]) for _ in range(burst)) + ) + + one_per_deployment: Final = tuple(f"Bearer key-{deployment_id}" for deployment_id in ROUTER_DEPLOYMENT_IDS) + assert sorted(call.request.headers["authorization"] for call in route.calls) == sorted(one_per_deployment * 2) + + +class _ManualClock: + def __init__(self) -> None: + self.now = 0.0 + + def __call__(self) -> float: + return self.now + + +def test_a_reservation_that_never_reaches_the_api_call_expires() -> None: + clock: Final = _ManualClock() + worker: Final = LeastBusyLoggingHandler(router_cache=DualCache(), clock=clock) + + assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A + clock.now = 1.0 + assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_B + clock.now = 2.0 + assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A + clock.now = SELECTION_RESERVATION_TTL_SECONDS + 0.5 + assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A + + +def test_a_fresh_reservation_outlives_the_ttl_of_the_first_one_on_its_deployment() -> None: + clock: Final = _ManualClock() + cache: Final = DualCache(in_memory_cache=InMemoryCache(clock=clock)) + worker: Final = LeastBusyLoggingHandler(router_cache=cache, clock=clock) + + assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A + worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a")) + worker.log_success_event(kwargs=_call_kwargs("dep-a"), response_obj=None, start_time=None, end_time=None) + clock.now = SELECTION_RESERVATION_TTL_SECONDS - 0.1 + assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A + clock.now = SELECTION_RESERVATION_TTL_SECONDS + 0.1 + + assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_B + + +def test_a_started_request_moves_its_reservation_into_the_in_flight_count() -> None: + worker: Final = _worker(None) + + assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A + worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a")) + + assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_B + assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A + + +def test_a_request_started_through_another_selector_of_the_router_releases_the_reservation() -> None: + cache: Final = DualCache() + reservations: Final = InMemoryCache() + picker: Final = LeastBusyLoggingHandler(router_cache=cache, reservation_store=reservations) + registered: Final = LeastBusyLoggingHandler(router_cache=cache, reservation_store=reservations) + + assert picker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A + registered.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a")) + + assert picker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_B + assert picker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A + + +class _StoreWhereThreadsReadBeforeWriting: + def __init__(self) -> None: + self._store: Final = InMemoryCache() + self._first_reads: Final = threading.Barrier(2, timeout=0.5) + self._thread_state: Final = threading.local() + self._meeting: Final = threading.Event() + + @contextlib.contextmanager + def first_reads_meet(self) -> Generator[None, None, None]: + self._meeting.set() + yield + self._meeting.clear() + + def get_cache(self, key: str) -> object: + value: Final = self._store.get_cache(key) + if self._meeting.is_set() and not getattr(self._thread_state, "has_read", False): + self._thread_state.has_read = True + with contextlib.suppress(threading.BrokenBarrierError): + self._first_reads.wait() + return value + + def set_cache(self, key: str, value: tuple[object, ...], *, ttl: float) -> None: + self._store.set_cache(key, value, ttl=ttl) + + def delete_cache(self, key: str) -> None: + self._store.delete_cache(key) + + +def test_two_threads_picking_at_once_reserve_different_deployments() -> None: + store: Final = _StoreWhereThreadsReadBeforeWriting() + worker: Final = LeastBusyLoggingHandler(router_cache=DualCache(), reservation_store=store) + + with store.first_reads_meet(), ThreadPoolExecutor(max_workers=2) as pool: + picks: Final = tuple(pool.map(lambda _: worker.get_available_deployments(GROUP, HEALTHY, reserve=True), range(2))) + + assert sorted(pick["model_info"]["id"] for pick in picks if pick is not None) == ["dep-a", "dep-b"] + + +def test_a_reservation_and_a_release_on_one_deployment_at_once_both_count() -> None: + store: Final = _StoreWhereThreadsReadBeforeWriting() + worker: Final = LeastBusyLoggingHandler(router_cache=DualCache(), reservation_store=store) + assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A + worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-b")) + worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-b")) + + with store.first_reads_meet(), ThreadPoolExecutor(max_workers=2) as pool: + reserved: Final = pool.submit(worker.get_available_deployments, GROUP, HEALTHY, True) + started: Final = pool.submit(worker.log_pre_api_call, "m", [], _call_kwargs("dep-a")) + assert reserved.result() is DEPLOYMENT_A + started.result() + + assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A + assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_B + + + +class _SlowLocalRead(InMemoryCache): + """Answers a count read with the counts from when the read started, once another thread has begun its request.""" + + def __init__(self) -> None: + super().__init__() + self.while_reading: Callable[[], None] = lambda: None + self.other_thread: threading.Thread | None = None + + def batch_get_cache(self, keys: list[str], **kwargs: object) -> list[object]: + counts_when_asked: Final = super().batch_get_cache(keys, **kwargs) + began: Final = threading.Event() + + def begin_then_run() -> None: + began.set() + self.while_reading() + + self.other_thread = threading.Thread(target=begin_then_run) + self.other_thread.start() + began.wait() + self.other_thread.join(timeout=0.2) + return counts_when_asked + + +def test_a_request_that_starts_on_another_thread_while_counts_are_read_still_counts_on_its_deployment() -> None: + local: Final = _SlowLocalRead() + worker: Final = LeastBusyLoggingHandler(router_cache=DualCache(in_memory_cache=local)) + assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A + local.while_reading = lambda: worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a")) + + picked_while_the_other_thread_started: Final = worker.get_available_deployments(GROUP, HEALTHY, reserve=True) + if local.other_thread is not None: + local.other_thread.join() + + assert picked_while_the_other_thread_started is DEPLOYMENT_B + + +def _router_pick(router: Router, group: str = ROUTER_GROUP) -> str: + deployment: Final = router.get_available_deployment(group, request_kwargs={"metadata": {}}) + return deployment["model_info"]["id"] + + +def test_a_pick_stays_reserved_while_other_router_keys_fill_its_cache() -> None: + router: Final = _least_busy_router() + + with picks_reserve(True): + first: Final = _router_pick(router) + for index in range(DEFAULT_MAX_SIZE_IN_MEMORY): + router.health().store.set_cache(f"deployment-usage-{index}", 1, local_only=True, ttl=60) + + assert _router_pick(router) != first + + +@pytest.mark.asyncio +async def test_looking_up_a_deployment_does_not_hold_it_for_later_picks() -> None: + router: Final = _least_busy_router() + + sync_picks: Final = [_router_pick(router) for _ in range(3)] + async_picks: Final = [ + (await router.async_get_available_deployment(ROUTER_GROUP, request_kwargs={"metadata": {}}))["model_info"]["id"] + for _ in range(3) + ] + + assert sync_picks + async_picks == [ROUTER_DEPLOYMENT_IDS[0]] * 6 + + +@pytest.mark.asyncio +async def test_looking_up_a_pass_through_deployment_does_not_hold_it_for_later_picks() -> None: + router: Final = _least_busy_router(use_in_pass_through=True) + + selected: Final = [ + (await router.select(ROUTER_GROUP, {"metadata": {}}, kind="pass_through"))["model_info"]["id"] for _ in range(2) + ] + async_picks: Final = [ + (await router.async_get_available_deployment_for_pass_through(ROUTER_GROUP, request_kwargs={"metadata": {}}))[ + "model_info" + ]["id"] + for _ in range(2) + ] + sync_picks: Final = [ + router.get_available_deployment_for_pass_through(ROUTER_GROUP, request_kwargs={"metadata": {}})["model_info"]["id"] + for _ in range(2) + ] + + assert selected + async_picks + sync_picks == [ROUTER_DEPLOYMENT_IDS[0]] * 6 + + +@pytest.mark.asyncio +async def test_a_cache_hit_releases_the_pick_it_reserved() -> None: + worker: Final = _worker(None) + worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a")) + + assert await worker.async_get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_B + await worker.async_log_success_event({**_call_kwargs("dep-b"), "cache_hit": True}, None, None, None) + + assert await worker.async_get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_B + + +def test_a_cache_hit_leaves_requests_in_flight_on_its_deployment_counted() -> None: + worker: Final = _worker(None) + worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a")) + + worker.log_success_event({**_call_kwargs("dep-a", "cached-call"), "cache_hit": True}, None, None, None) + + assert worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_B + + +@pytest.mark.parametrize("outcome", [LeastBusyLoggingHandler.log_success_event, LeastBusyLoggingHandler.log_failure_event]) +def test_a_call_that_ends_before_reaching_the_provider_frees_its_pick( + outcome: Callable[[LeastBusyLoggingHandler, dict[str, object], None, None, None], None], +) -> None: + worker: Final = _worker(None) + assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A + + outcome(worker, _call_kwargs("dep-a"), None, None, None) + + assert worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_A + + +def test_a_call_the_proxy_rejected_after_logging_its_start_leaves_no_count_behind() -> None: + worker: Final = _worker(None) + rejected: Final = {**_call_kwargs("dep-a"), LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL: True} + + worker.log_pre_api_call(model="m", messages=[], kwargs=rejected) + worker.log_failure_event(rejected, None, None, None) + + assert worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_A + + +def test_a_call_logged_under_its_callers_group_name_frees_the_pick_made_for_another_group() -> None: + worker: Final = _worker(None) + assert worker.get_available_deployments("tier-group", HEALTHY, reserve=True) is DEPLOYMENT_A + + worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a")) + worker.log_success_event(_call_kwargs("dep-a"), None, None, None) + + assert worker.get_available_deployments("tier-group", HEALTHY) is DEPLOYMENT_A + + +def test_a_fallback_attempt_that_fails_before_reaching_its_provider_frees_its_pick() -> None: + worker: Final = _worker(None) + worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a", "busy-call")) + first_attempt_started_at: Final = datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc) + first_attempt: Final = {**_call_kwargs("dep-a", "fallback-call"), "api_call_start_time": first_attempt_started_at} + worker.log_pre_api_call(model="m", messages=[], kwargs=first_attempt) + worker.log_failure_event(first_attempt, None, None, None) + + assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_B + second_attempt: Final = {**_call_kwargs("dep-b", "fallback-call"), "api_call_start_time": first_attempt_started_at} + worker.log_failure_event(second_attempt, None, None, None) + + assert worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_B + + +def test_ending_one_request_frees_only_the_picks_it_never_started() -> None: + reservations: Final = InMemoryCache() + worker: Final = LeastBusyLoggingHandler(router_cache=DualCache(), reservation_store=reservations) + worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-b", "busy-call")) + assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True, call_id="ended-call") is DEPLOYMENT_A + assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True, call_id="running-call") is DEPLOYMENT_A + + release_unstarted_picks(reservations, "ended-call") + + assert [worker.get_available_deployments(GROUP, HEALTHY, reserve=True) for _ in range(2)] == [ + DEPLOYMENT_A, + DEPLOYMENT_B, + ] + + +def test_a_call_without_a_call_id_turns_its_own_pick_into_its_in_flight_count() -> None: + reservations: Final = InMemoryCache() + worker: Final = LeastBusyLoggingHandler(router_cache=DualCache(), reservation_store=reservations) + worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-b", "busy-call")) + + with owning_picks(reservations, None): + assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A + worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a", "attempt-call")) + + assert worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_A + + +def test_another_call_starting_leaves_the_pick_of_a_call_without_a_call_id_reserved() -> None: + reservations: Final = InMemoryCache() + worker: Final = LeastBusyLoggingHandler(router_cache=DualCache(), reservation_store=reservations) + worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-b", "busy-call")) + other_call: Final = contextvars.copy_context() + + with owning_picks(reservations, None): + assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A + other_call.run(worker.log_pre_api_call, model="m", messages=[], kwargs=_call_kwargs("dep-a", "other-call")) + + assert worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_B + + +def test_a_call_started_without_a_pick_of_its_own_leaves_another_calls_pick_reserved() -> None: + worker: Final = _worker(None) + assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True, call_id="picked-call") is DEPLOYMENT_A + + worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a", "looked-up-call")) + + assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_B + assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_B + + +def _fallback_router() -> Router: + return Router( + model_list=[ + { + "model_name": "unreachable-group", + "litellm_params": {"model": "openai/gpt-4.1-mini", "api_key": "key-unreachable"}, + "model_info": {"id": "unreachable"}, + }, + *( + { + "model_name": ROUTER_GROUP, + "litellm_params": {"model": "openai/gpt-5.4-nano", "api_key": f"key-{deployment_id}"}, + "model_info": {"id": deployment_id}, + } + for deployment_id in ROUTER_DEPLOYMENT_IDS + ), + ], + routing_strategy="least-busy", + num_retries=0, + fallbacks=[{"unreachable-group": [ROUTER_GROUP]}], + ) + + +@pytest.mark.asyncio +async def test_a_fallback_hop_rejected_before_its_provider_frees_its_pick_when_every_hop_shares_one_logging_object( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + route: Final = respx_mock.post("https://api.openai.com/v1/chat/completions").mock( + return_value=httpx.Response(500, json={"error": {"message": "upstream down"}}) + ) + router: Final = _fallback_router() + logging_obj, request = litellm.utils.function_setup( + original_function="acompletion", + rules_obj=litellm.utils.Rules(), + start_time=datetime.datetime(2026, 1, 1), + model="unreachable-group", + messages=[{"role": "user", "content": "hi"}], + logit_bias={"1": 1}, + litellm_call_id="fallback-call", + ) + + with pytest.raises(litellm.InternalServerError): + await router.acompletion(**request, litellm_logging_obj=logging_obj) + await _drain_logging() + + assert {call.request.headers["authorization"] for call in route.calls} == {"Bearer key-unreachable"} + assert _router_pick(router) == ROUTER_DEPLOYMENT_IDS[0] + + +def test_a_retry_that_starts_before_the_failed_attempt_is_logged_counts_each_attempt_once() -> None: + worker: Final = _worker(None) + attempt: Final = _call_kwargs("dep-a", "retried-call") + + worker.log_pre_api_call(model="m", messages=[], kwargs=attempt) + worker.log_pre_api_call(model="m", messages=[], kwargs=attempt) + worker.log_failure_event(attempt, None, None, None) + worker.log_success_event(attempt, None, None, None) + + assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A + assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_B + + +async def _drain_logging() -> None: + for _ in range(10): + await asyncio.sleep(0) + await GLOBAL_LOGGING_WORKER.flush() + + +def _called_deployment(route: respx.Route) -> str: + return route.calls.last.request.headers["authorization"].removeprefix("Bearer key-") + + +_BATCH: Final = { + "id": "batch_least_busy", + "object": "batch", + "endpoint": "/v1/chat/completions", + "input_file_id": "file-least-busy", + "completion_window": "24h", + "status": "validating", + "created_at": 1, +} +_REJECTED_BATCH: Final = {"error": {"message": "Invalid input file", "type": "invalid_request_error"}} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("provider_response", [httpx.Response(200, json=_BATCH), httpx.Response(400, json=_REJECTED_BATCH)]) +async def test_a_batch_create_does_not_keep_its_deployment_busy( + provider_response: httpx.Response, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + route: Final = respx_mock.post("https://api.openai.com/v1/batches").mock(return_value=provider_response) + router: Final = _least_busy_router() + + with contextlib.suppress(openai.BadRequestError): + await router.acreate_batch( + model=ROUTER_GROUP, completion_window="24h", endpoint="/v1/chat/completions", input_file_id="file-least-busy" + ) + await _drain_logging() + + assert _router_pick(router) == _called_deployment(route) + + +_VECTOR_STORE: Final = { + "id": "vs_least_busy", + "object": "vector_store", + "created_at": 1, + "name": "least-busy", + "usage_bytes": 0, + "file_counts": {"in_progress": 0, "completed": 0, "failed": 0, "cancelled": 0, "total": 0}, + "status": "completed", +} +_REJECTED_VECTOR_STORE: Final = {"error": {"message": "Invalid name", "type": "invalid_request_error"}} +_CALL_IDENTITIES: Final = ({"litellm_call_id": "least-busy-call"}, {}) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("call_identity", _CALL_IDENTITIES) +@pytest.mark.parametrize( + "provider_response", [httpx.Response(200, json=_VECTOR_STORE), httpx.Response(400, json=_REJECTED_VECTOR_STORE)] +) +async def test_a_vector_store_create_does_not_keep_its_deployment_busy( + provider_response: httpx.Response, + call_identity: dict[str, str], + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + route: Final = respx_mock.post("https://api.openai.com/v1/vector_stores").mock(return_value=provider_response) + router: Final = _least_busy_router() + + with contextlib.suppress(openai.BadRequestError): + await router.avector_store_create(model=ROUTER_GROUP, name="least-busy", **call_identity) + await _drain_logging() + + assert _router_pick(router) == _called_deployment(route) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("call_identity", _CALL_IDENTITIES) +async def test_listing_fine_tuning_jobs_does_not_keep_its_deployment_busy( + call_identity: dict[str, str], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + route: Final = respx_mock.get(url__startswith="https://api.openai.com/v1/fine_tuning/jobs").mock( + return_value=httpx.Response(200, json={"object": "list", "data": [], "has_more": False}) + ) + router: Final = _least_busy_router() + + await router.alist_fine_tuning_jobs(model=ROUTER_GROUP, **call_identity) + await _drain_logging() + + assert _router_pick(router) == _called_deployment(route) + + +TEXT_GROUP: Final = "openai/least-busy-text-group" + + +@pytest.mark.parametrize("call_identity", _CALL_IDENTITIES) +def test_a_sync_text_completion_does_not_keep_its_deployment_busy( + call_identity: dict[str, str], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + route: Final = respx_mock.post("https://api.openai.com/v1/completions").mock( + return_value=httpx.Response( + 200, + json={ + "id": "cmpl-least-busy", + "object": "text_completion", + "created": 1, + "model": "least-busy-text-group", + "choices": [{"text": "ok", "index": 0, "finish_reason": "stop", "logprobs": None}], + "usage": {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3}, + }, + ) + ) + router: Final = _least_busy_router(TEXT_GROUP) + + router.text_completion(model=TEXT_GROUP, prompt="Say ok", **call_identity) + + assert _router_pick(router, TEXT_GROUP) == _called_deployment(route) + + + +@pytest.mark.parametrize("call_identity", _CALL_IDENTITIES) +def test_a_sync_responses_call_that_fails_before_reaching_its_provider_frees_its_pick( + call_identity: dict[str, str], +) -> None: + router: Final = _least_busy_router() + + with pytest.raises(litellm.InternalServerError): + router.responses(model=ROUTER_GROUP, input="hi", api_base="not-a-url", **call_identity) + + assert _router_pick(router) == ROUTER_DEPLOYMENT_IDS[0] + +@pytest.mark.asyncio +async def test_a_batch_cancel_does_not_keep_its_deployment_busy( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + route: Final = respx_mock.post("https://api.openai.com/v1/batches/batch_least_busy/cancel").mock( + return_value=httpx.Response(200, json={**_BATCH, "status": "cancelling"}) + ) + router: Final = _least_busy_router() + + await router.acancel_batch(model=ROUTER_GROUP, batch_id="batch_least_busy") + + assert _router_pick(router) == _called_deployment(route) + + +@pytest.mark.asyncio +async def test_picking_a_deployment_without_calling_it_does_not_keep_it_busy() -> None: + router: Final = _least_busy_router() + + picked: Final = await router.select(ROUTER_GROUP, {"metadata": {}}) + + assert _router_pick(router) == picked["model_info"]["id"] diff --git a/tests/unit/router_strategy/test_router_routing_groups.py b/tests/unit/router_strategy/test_router_routing_groups.py index afcd1e2a7ff..951182051c4 100644 --- a/tests/unit/router_strategy/test_router_routing_groups.py +++ b/tests/unit/router_strategy/test_router_routing_groups.py @@ -21,6 +21,7 @@ from litellm import Router from litellm.caching.redis_cache import RedisPipelineIncrementOperation from litellm.integrations.custom_logger import CustomLogger from litellm.router_api import RoutingContext +from litellm.router_strategy.least_busy import picks_reserve from litellm.types.router import DeploymentTypedDict, FallbackAccessCheck, RoutingGroup, RoutingStrategy from litellm.utils import Rules, function_setup @@ -482,6 +483,38 @@ def test_two_least_busy_groups_count_a_request_once(monkeypatch): assert router.cache.get_cache("filtered-model_request_count:deploy-1") == 0 +def _filtered_model_pick(router: Router) -> str: + deployment: Final = router.get_available_deployment( + "filtered-model", request_kwargs=_proxy_shaped_request(model="filtered-model") + ) + return deployment["model_info"]["id"] + + +def test_a_least_busy_group_pick_stops_counting_once_its_request_finishes(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "callbacks", []) + monkeypatch.setattr(litellm, "input_callback", []) + router: Final = _build_router( + routing_strategy="least-busy", + routing_groups=[{"group_name": "fast", "models": ["filtered-model"], "routing_strategy": "least-busy"}], + ) + + request: Final = _proxy_shaped_request(model="filtered-model") + with picks_reserve(True): + first: Final = router.get_available_deployment("filtered-model", request_kwargs=request)["model_info"]["id"] + kwargs: Final = { + "litellm_params": {"metadata": {"model_group": "filtered-model"}, "model_info": {"id": first}}, + "litellm_call_id": request["litellm_call_id"], + } + for callback in litellm.input_callback: + if isinstance(callback, CustomLogger): + callback.log_pre_api_call(model="filtered-model", messages=[], kwargs=kwargs) + for callback in litellm.callbacks: + if isinstance(callback, CustomLogger): + callback.log_success_event(kwargs, None, None, None) + + assert _filtered_model_pick(router) == first + + def test_two_routers_in_one_process_each_count_their_own_requests(monkeypatch): """ Least-busy hangs its counting off litellm's global callback lists, and those lists keep one @@ -614,6 +647,7 @@ async def test_select_deployment_async_returns_none_without_selector(): messages=None, input=None, request_kwargs=None, + reserve=False, ) assert result is None @@ -628,6 +662,7 @@ def test_select_deployment_sync_returns_none_without_selector(): messages=None, input=None, request_kwargs=None, + reserve=False, ) assert result is None @@ -662,6 +697,7 @@ def test_select_deployment_sync_does_not_dispatch_cost_based_routing(): messages=None, input=None, request_kwargs=None, + reserve=False, ) assert result is None @@ -708,6 +744,7 @@ async def test_async_dispatch_falls_back_to_sync_for_usage_based_routing_v1(): messages=[{"role": "user", "content": "hi"}], input=None, request_kwargs={}, + reserve=False, ) assert v1_spy.called, "async dispatch must route v1 strategy through sync method" diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index c16c1c62202..f5962cb4933 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -30102,11 +30102,21 @@ def test_reset_custom_routing_strategy(): assert "get_available_deployment" not in router._core.__dict__ assert "_core_async_get_available_deployment" not in router._core.__dict__ - assert router.async_get_available_deployment.__func__ is Router.async_get_available_deployment + assert router.async_get_available_deployment.__wrapped__.__func__ is Router.async_get_available_deployment router._reset_custom_routing_strategy() +@pytest.mark.asyncio +async def test_select_picks_through_a_custom_routing_strategy() -> None: + router: Final = _create_custom_routing_router() + router.set_custom_routing_strategy(SpecialEndpointRoutingStrategy(router)) + + picked: Final = await router.select("azure-model", {"metadata": {}}) + + assert picked["model_info"]["id"] == "very-special-endpoint" + + @pytest.mark.asyncio async def test_azure_router_acompletion_passes_deployment_credentials_for_plain_and_stream( monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter diff --git a/tests/unit/test_router/test_router_helper_utils.py b/tests/unit/test_router/test_router_helper_utils.py index 5ad6eb819ef..3bbb69413d9 100644 --- a/tests/unit/test_router/test_router_helper_utils.py +++ b/tests/unit/test_router/test_router_helper_utils.py @@ -844,6 +844,34 @@ async def test_responses_websocket_turn_is_refused_once_its_deployment_crosses_b await admit_turn() +@pytest.mark.asyncio +async def test_responses_websocket_turns_do_not_make_their_deployment_look_busy_to_least_busy(): + router: Final = Router( + model_list=[ + { + "model_name": "ws-least-busy", + "litellm_params": {"model": "openai/gpt-5-mini", "api_key": "sk-fake"}, + "model_info": {"id": deployment_id}, + } + for deployment_id in ("ws-least-busy-a", "ws-least-busy-b") + ], + routing_strategy="least-busy", + ) + admit_turn: Final = (await _kwargs_routed_to(router, "ws-least-busy", "_aresponses_websocket"))["turn_admission"] + assert callable(admit_turn) + for _ in range(3): + await admit_turn() + + picks: Final = [ + (await router._core_async_get_available_deployment(model="ws-least-busy", request_kwargs={"input": "hi"}))[ + "model_info" + ]["id"] + for _ in range(2) + ] + + assert sorted(picks) == ["ws-least-busy-a", "ws-least-busy-b"] + + @pytest.mark.asyncio async def test_responses_websocket_turn_after_the_first_counts_toward_deployment_rpm(): router = Router( From 7a641e5172564a2173c8968637d985007aeffccd Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 4 Nov 2026 16:37:55 -0800 Subject: [PATCH 05/39] fix(proxy): answer an upstream overload with 503 server_is_overloaded on OpenAI routes (#49993) * fix(anthropic): answer a 503 as 529 overloaded_error on /v1/messages Anthropic's error contract has no 503, so a client built against it (Claude Code) treats a 503 from /v1/messages as a plain 5xx and retries it ten times instead of taking its overload path. A Bedrock 503 now leaves /v1/messages and /v1/messages/count_tokens as 529 overloaded_error, for both a JSON answer and a stream that fails before its first byte. An Anthropic 529 overloaded_error now maps to litellm.ServiceUnavailableError (503) instead of InternalServerError (500), so OpenAI-shaped routes answer it as 503, the status OpenAI documents for an overloaded engine, and router retry and allowed-fails policies key it as ServiceUnavailableError. * test(proxy): expect overloaded_error on a 503 stream error after a keepalive ping * fix(proxy): record the 529 the Anthropic routes answer and type the chat stream error frame The messages route stamps the answered status on the exception before post_call_failure_hook runs, so the spend log row, the OTel server span, the failed-request metric, and alerting record the 529 the client saw instead of the deployment's 503, while the deployment failure metric and cooldown keep the deployment's own status. A chat completions stream error frame names the type its status stands for instead of the null .type ServiceUnavailableError carries. An Anthropic-shaped 503 envelope passes through as 529 overloaded_error through a validated TypedDict, and count_tokens answers every ProxyException as the Anthropic envelope. Five integration tests that pinned 503 on the messages route now expect 529 * test(proxy): expect the top-level Anthropic envelope from count_tokens errors count_tokens now answers a ProxyException as a JSONResponse carrying the Anthropic error envelope instead of raising HTTPException, which FastAPI wrapped in {"detail": ...}. The three legacy error-format tests now read the response body and check that no detail wrap is left. * test(proxy): cover the 529 overloaded_error answer across routes, cooldown, and retry budgets * fix(anthropic): keep Anthropic overloads as InternalServerError and answer 529 from the provider status An upstream Anthropic 529 stays litellm.InternalServerError (status 500) so InternalServerErrorRetries, InternalServerErrorAllowedFails, SDK handlers, and the OpenAI-shaped routes behave as on main. The error now carries provider_status_code, and the /v1/messages routes answer 529 overloaded_error from it. * refactor(proxy): unwrap a mid-stream fallback error once instead of recursing for the provider status * fix(anthropic): keep count_tokens' detail body, end keepalive streams with an Anthropic error event, and log any 529 as an overload * test(integration): cover the audit round 3 cells and expect 529 on the count_tokens 503 cases * fix(proxy): re-raise an unchanged stream HTTPException as itself and walk nested mid-stream fallbacks for the provider status * fix(proxy): walk nested mid-stream fallbacks without recursion * fix(anthropic): preserve 503s and answer upstream 529s on messages routes Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(anthropic): answer an upstream Anthropic 529 as 529 on count_tokens Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(anthropic): use integration coverage for keepalive errors Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): answer an upstream overload with 503 server_is_overloaded on OpenAI routes Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): pass answered overload status to failure hooks Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): preserve Anthropic route exception handling Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(proxy): bind the transformed failure without rebinding the parameter Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): observe the overload failure hook through a real ProxyLogging Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../anthropic_endpoints/answered_status.py | 25 ++- litellm/proxy/common_request_processing.py | 78 +++++---- .../common_utils/openai_error_payload.py | 26 +++ litellm/proxy/proxy_server.py | 8 + ...test_anthropic_messages_overloaded_wire.py | 44 +++-- ...st_anthropic_overloaded_failure_signals.py | 8 +- .../proxy/test_common_request_processing.py | 158 +++++++++++++++++- 7 files changed, 291 insertions(+), 56 deletions(-) diff --git a/litellm/proxy/anthropic_endpoints/answered_status.py b/litellm/proxy/anthropic_endpoints/answered_status.py index 8a3469f9119..26eeb08c8c9 100644 --- a/litellm/proxy/anthropic_endpoints/answered_status.py +++ b/litellm/proxy/anthropic_endpoints/answered_status.py @@ -1,10 +1,29 @@ from typing import Final, TypeVar -from litellm.exceptions import InternalServerError, MidStreamFallbackError +from litellm.exceptions import APIError, InternalServerError, MidStreamFallbackError + +_PROVIDER_OVERLOAD_STATUS_CODE: Final = 529 +_OPENAI_OVERLOAD_ANSWERED_STATUS_CODE: Final = 503 _Raised = TypeVar("_Raised", bound=Exception) +def is_provider_overload(exc: Exception) -> bool: + return _provider_status_code(exc) == _PROVIDER_OVERLOAD_STATUS_CODE + + +def with_openai_overload_status(exc: _Raised) -> _Raised: + return ( + _copy_carrying( + _copy_carrying(exc, "provider_status_code", _PROVIDER_OVERLOAD_STATUS_CODE), + "status_code", + _OPENAI_OVERLOAD_ANSWERED_STATUS_CODE, + ) + if is_provider_overload(exc) + else exc + ) + + def _provider_status_code(exc: Exception) -> int | None: unwrapped = exc # rebind-ok: the recursion gate requires iterative wrapper traversal while isinstance(unwrapped, MidStreamFallbackError) and unwrapped.original_exception is not None: @@ -12,6 +31,10 @@ def _provider_status_code(exc: Exception) -> int | None: match unwrapped: case InternalServerError(provider_status_code=int() as provider_status): return provider_status + case APIError(provider_status_code=int() as provider_status): + return provider_status + case APIError(status_code=int() as provider_status): + return provider_status case _: return None diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 17e6d3ee411..e812b03385d 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -95,6 +95,7 @@ from litellm.litellm_core_utils.streaming_handler import ( ) from litellm.litellm_core_utils.tool_fields import DropPolicy, MalformedField, drop_policy, resolve_tool_fields from litellm.proxy._types import LiteLLMRoutes, ProxyErrorTypes, ProxyException, UserAPIKeyAuth +from litellm.proxy.anthropic_endpoints.answered_status import is_provider_overload, with_openai_overload_status from litellm.proxy.auth.auth_checks import ( can_key_call_resolved_model, is_model_cost_zero, @@ -115,6 +116,7 @@ from litellm.proxy.common_utils.http_parsing_utils import ( ) from litellm.proxy.common_utils.openai_error_payload import ( LITELLM_CALL_ID_HEADER, + OverloadedProxyException, attribute_of, error_status_code, openai_error_param, @@ -164,6 +166,7 @@ _ChunkT: Final = TypeVar("_ChunkT") KNOWN_PROXY_ROUTES: Final = frozenset( route for member in LiteLLMRoutes for route in member.value if route.startswith("/") ) +_OPENAI_OVERLOAD_ROUTE_TYPES: Final = frozenset({"acompletion", "atext_completion", "aresponses"}) ProxyRouteType: TypeAlias = Literal[ "acompletion", @@ -4398,28 +4401,28 @@ class ProxyBaseLLMRequestProcessing: version: str | None = None, ) -> Never: """Raises ProxyException (OpenAI API compatible) if an exception is raised""" - log_llm_api_exception(e, self.litellm_call_id) + answered: Final = with_openai_overload_status(e) if self.route_type in _OPENAI_OVERLOAD_ROUTE_TYPES else e + log_llm_api_exception(answered, self.litellm_call_id) billing_logging_obj: Final = _REQUEST_FIELDS.validate_python(self.data).get("litellm_logging_obj") failed_in_agentic_follow_up: Final = is_agentic_followup_failure(e) # Allow callbacks to transform the error response transformed_exception: Final = await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, - original_exception=e, + original_exception=answered, request_data=self.data, dispatched_call_type=self.route_type, ) # Use transformed exception if callback returned one, otherwise use original - if transformed_exception is not None: - e = transformed_exception - litellm_debug_info: Final = getattr(e, "litellm_debug_info", "") + failure: Final = transformed_exception if transformed_exception is not None else answered + litellm_debug_info: Final = getattr(failure, "litellm_debug_info", "") verbose_proxy_logger.debug( "An error occurred: %s %s", - e, + failure, litellm_debug_info, ) timeout: Final = getattr( - e, "timeout", None + failure, "timeout", None ) # returns the timeout set by the wrapper. Used for testing if model-specific timeout are set correctly _litellm_logging_obj: Final = ( billing_logging_obj if isinstance(billing_logging_obj, LiteLLMLoggingObj) else None @@ -4451,11 +4454,11 @@ class ProxyBaseLLMRequestProcessing: # a failed request reports no timing, matching /v1/chat/completions read_timing_from_logging_obj=False, ) - headers = getattr(e, "headers", None) or {} + headers = getattr(failure, "headers", None) or {} if not headers: - _response: Final = attribute_of(e, "response") + _response: Final = attribute_of(failure, "response") _response_headers: Final = getattr(_response, "headers", None) if _response is not None else None - _provider_headers: Final = _response_headers or getattr(e, "litellm_response_headers", None) + _provider_headers: Final = _response_headers or getattr(failure, "litellm_response_headers", None) if _provider_headers: headers = get_response_headers(dict(_provider_headers)) headers.update(custom_headers) @@ -4475,21 +4478,21 @@ class ProxyBaseLLMRequestProcessing: safe_headers: Final = {k: v for k, v in headers.items() if k.lower() not in UNSAFE_PROXY_RESPONSE_HEADERS} - self._apply_router_cooldown_retry_after(safe_headers, e) + self._apply_router_cooldown_retry_after(safe_headers, failure) - if isinstance(e, ProxyException): - e.headers = { - **{k: v for k, v in e.headers.items() if k.lower() not in UNSAFE_PROXY_RESPONSE_HEADERS}, + if isinstance(failure, ProxyException): + failure.headers = { + **{k: v for k, v in failure.headers.items() if k.lower() not in UNSAFE_PROXY_RESPONSE_HEADERS}, **{k: v if isinstance(v, str) else str(v) for k, v in safe_headers.items()}, } - raise e + raise failure - if isinstance(e, HTTPException): - raise proxy_exception_from_http_exception(e, safe_headers) - elif isinstance(e, httpx.HTTPStatusError): + if isinstance(failure, HTTPException): + raise proxy_exception_from_http_exception(failure, safe_headers) + elif isinstance(failure, httpx.HTTPStatusError): # Handle httpx.HTTPStatusError - extract actual error from response # This matches the original behavior before the refactor in commit 511d435f6f - http_status_error: Final[httpx.HTTPStatusError] = e + http_status_error: Final[httpx.HTTPStatusError] = failure error_body: Final = await http_status_error.response.aread() error_text: Final = error_body.decode("utf-8") @@ -4499,12 +4502,12 @@ class ProxyBaseLLMRequestProcessing: detail={"error": error_text}, headers=error_headers, ) - error_msg: Final = f"{e}" + error_msg: Final = f"{failure}" # Check for AttributeError in the exception chain. # The AttributeError may be wrapped in multiple layers # (e.g. AttributeError -> OpenAIException -> APIConnectionError), # so walk __cause__, __context__, and original_exception recursively. - has_attribute_error: Final = _has_attribute_error_in_chain(e) + has_attribute_error: Final = _has_attribute_error_in_chain(failure) if has_attribute_error: raise ProxyException( @@ -4519,12 +4522,12 @@ class ProxyBaseLLMRequestProcessing: # VertexAIError, etc.) all have a status_code attribute reflecting # the upstream API response. Use it to return the correct HTTP code # instead of defaulting to 500. - _exc_status_code: Final = getattr(e, "status_code", None) + _exc_status_code: Final = getattr(failure, "status_code", None) if _exc_status_code is not None and isinstance(_exc_status_code, int) and 400 <= _exc_status_code <= 599: _code = _exc_status_code else: _code = status.HTTP_500_INTERNAL_SERVER_ERROR - if should_report_bug(e): + if should_report_bug(failure): proxy_server_request: Final = self.data.get("proxy_server_request") request_url: Final = ( proxy_server_request.get("url") if isinstance(proxy_server_request, Mapping) else None @@ -4533,24 +4536,33 @@ class ProxyBaseLLMRequestProcessing: verbose_proxy_logger.error( bug_report_notice( build_proxy_bug_report( - e, + failure, call_type=allowlisted(request_path, KNOWN_PROXY_ROUTES), custom_llm_provider=self.data.get("custom_llm_provider"), stream=self.data.get("stream"), ) ) ) - client_message: Final = getattr(e, "message", error_msg) + client_message: Final = getattr(failure, "message", error_msg) + redacted_client_message: Final = redact_internal_details_from_client_message( + strip_bug_report_notice(client_message) if isinstance(client_message, str) else error_msg + ) + provider_specific_fields: Final = getattr( + failure, "provider_specific_fields", None + ) or relayed_provider_specific_fields(failure) + if self.route_type in _OPENAI_OVERLOAD_ROUTE_TYPES and is_provider_overload(failure): + raise OverloadedProxyException( + message=redacted_client_message, + headers=cast(dict[str, str], safe_headers), # cast-ok: ProxyException coerces header values to strings + provider_specific_fields=provider_specific_fields, + ) raise ProxyException( - message=redact_internal_details_from_client_message( - strip_bug_report_notice(client_message) if isinstance(client_message, str) else error_msg - ), - type=openai_error_type(e, _code), - param=openai_error_param(e), - openai_code=getattr(e, "code", None), + message=redacted_client_message, + type=openai_error_type(failure, _code), + param=openai_error_param(failure), + openai_code=getattr(failure, "code", None), code=_code, - provider_specific_fields=getattr(e, "provider_specific_fields", None) - or relayed_provider_specific_fields(e), + provider_specific_fields=provider_specific_fields, headers=safe_headers, ) diff --git a/litellm/proxy/common_utils/openai_error_payload.py b/litellm/proxy/common_utils/openai_error_payload.py index a948c390375..26369bf3f87 100644 --- a/litellm/proxy/common_utils/openai_error_payload.py +++ b/litellm/proxy/common_utils/openai_error_payload.py @@ -17,6 +17,7 @@ from litellm.proxy.db.db_lookup_gate import DBLookupDeadlineExceeded from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler, exception_chain LITELLM_CALL_ID_HEADER: Final = "x-litellm-call-id" +_SERVER_IS_OVERLOADED_CODE: Final = "server_is_overloaded" _OPENAI_ERROR_TYPE_BY_STATUS: Final[Mapping[int, str]] = MappingProxyType( { @@ -44,6 +45,31 @@ _FAILOVER_REFUSALS: Final = ( ) +class OverloadedProxyException(ProxyException): + def __init__( + self, + message: str, + headers: dict[str, str] | None, # mutable-ok: matches ProxyException's constructor + provider_specific_fields: dict[str, object] | None, # mutable-ok: matches ProxyException's constructor + ) -> None: + super().__init__( # pyright: ignore[reportUnknownMemberType] # ProxyException omits generic arguments + message=message, + type="service_unavailable_error", + param=None, + code=status.HTTP_503_SERVICE_UNAVAILABLE, + headers=None if headers is None else dict(headers), + provider_specific_fields=provider_specific_fields, + ) + + def to_dict(self) -> dict[str, object]: # mutable-ok: matches ProxyException.to_dict + return { + **_ERROR_BODY.validate_python( + super().to_dict() # pyright: ignore[reportUnknownMemberType] # ProxyException omits generic arguments + ), + "code": _SERVER_IS_OVERLOADED_CODE, + } + + def attribute_of(value: object, name: str, default: object = None) -> object: return getattr(value, name, default) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 87bc78730b5..b64c88521dc 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -426,6 +426,7 @@ from litellm.proxy.agent_endpoints.auth.agent_access_groups import CeilingResolv from litellm.proxy.analytics_endpoints.analytics_endpoints import ( router as analytics_router, ) +from litellm.proxy.anthropic_endpoints.answered_status import is_provider_overload from litellm.proxy.auth.auth_checks import ( ROLE_BASED_PERMISSIONS_ADAPTER, ExperimentalUIJWTToken, @@ -13375,6 +13376,13 @@ async def completion( _response.choices[0].text = e.message return _response except Exception as e: + if is_provider_overload(e): + raise await base_llm_response_processor.handle_llm_api_exception( + e=e, + user_api_key_dict=user_api_key_dict, + proxy_logging_obj=proxy_logging_obj, + version=version, + ) await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, diff --git a/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_messages_overloaded_wire.py b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_messages_overloaded_wire.py index bae775164dd..9d0e31058e4 100644 --- a/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_messages_overloaded_wire.py +++ b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_messages_overloaded_wire.py @@ -140,6 +140,16 @@ def _assert_internal_server_error(response: httpx.Response) -> None: assert "InternalServerError" in string_value(error["message"]), response.text +def _assert_openai_overloaded(response: httpx.Response) -> None: + assert response.status_code == 503, response.text + error: Final = _openai_error(response) + assert set(error) == {"message", "type", "param", "code"}, response.text + assert string_value(error["message"]), response.text + assert error["type"] == "service_unavailable_error", response.text + assert error["param"] is None, response.text + assert error["code"] == "server_is_overloaded", response.text + + def _one_post(wire: Wire, target: str) -> None: assert [(request.method, unquote(request.target)) for request in wire.drain()] == [("POST", target)] @@ -361,7 +371,7 @@ def test_messages_overloaded_frame_before_content_without_a_retry_budget_is_rela ] -def test_chat_completions_overloaded_frame_before_content_answers_500(gateway: Gateway) -> None: +def test_chat_completions_overloaded_frame_before_content_answers_503(gateway: Gateway) -> None: prompt: Final = _prompt() with wire_server(_always(_overloaded_before_content(prompt))) as wire, gateway.scenario() as scenario: model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY) @@ -370,24 +380,24 @@ def test_chat_completions_overloaded_frame_before_content_answers_500(gateway: G "/v1/chat/completions", {"model": model, "messages": [{"role": "user", "content": prompt}], "stream": True, "num_retries": 0}, ) - _assert_internal_server_error(response) + _assert_openai_overloaded(response) _one_post(wire, "/v1/messages") -def test_chat_completions_529_answers_500_to_the_sync_sdk(gateway: Gateway) -> None: +def test_chat_completions_529_answers_503_to_the_sync_sdk(gateway: Gateway) -> None: with wire_server(_always(status_reply(529))) as wire, gateway.scenario() as scenario: model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY) with pytest.raises(openai.APIStatusError) as caught: _openai_sdk(gateway).chat.completions.create( model=model, messages=[{"role": "user", "content": _prompt()}], extra_body={"num_retries": 0} ) - assert caught.value.status_code == 500, caught.value.response.text - _assert_internal_server_error(caught.value.response) + assert caught.value.status_code == 503, caught.value.response.text + _assert_openai_overloaded(caught.value.response) _one_post(wire, "/v1/messages") - _assert_logged_failure(model, rows=1, error_class="InternalServerError", code="500") + _assert_logged_failure(model, rows=1, error_class="InternalServerError", code="503") -async def test_chat_completions_stream_529_answers_500_to_the_async_sdk(gateway: Gateway) -> None: +async def test_chat_completions_stream_529_answers_503_to_the_async_sdk(gateway: Gateway) -> None: with wire_server(_always(status_reply(529))) as wire, gateway.scenario() as scenario: model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY) with pytest.raises(openai.APIStatusError) as caught: @@ -397,19 +407,31 @@ async def test_chat_completions_stream_529_answers_500_to_the_async_sdk(gateway: stream=True, extra_body={"num_retries": 0}, ) - assert caught.value.status_code == 500, caught.value.response.text - _assert_internal_server_error(caught.value.response) + assert caught.value.status_code == 503, caught.value.response.text + _assert_openai_overloaded(caught.value.response) + _one_post(wire, "/v1/messages") + + +def test_completions_529_answers_503(gateway: Gateway) -> None: + with wire_server(_always(status_reply(529))) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY) + response: Final = gateway.request( + "POST", + "/v1/completions", + {"model": model, "prompt": _prompt(), "max_tokens": 16, "num_retries": 0}, + ) + _assert_openai_overloaded(response) _one_post(wire, "/v1/messages") @pytest.mark.parametrize("shape", ("non-stream", "stream")) -def test_responses_529_answers_500(gateway: Gateway, shape: Shape) -> None: +def test_responses_529_answers_503(gateway: Gateway, shape: Shape) -> None: with wire_server(_always(status_reply(529))) as wire, gateway.scenario() as scenario: model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY) response: Final = gateway.request( "POST", "/v1/responses", {"model": model, "input": _prompt(), "stream": shape == "stream", "num_retries": 0} ) - _assert_internal_server_error(response) + _assert_openai_overloaded(response) _one_post(wire, "/v1/messages") diff --git a/tests/integration/observability/test_anthropic_overloaded_failure_signals.py b/tests/integration/observability/test_anthropic_overloaded_failure_signals.py index 044c58e44cc..e86ed98fae7 100644 --- a/tests/integration/observability/test_anthropic_overloaded_failure_signals.py +++ b/tests/integration/observability/test_anthropic_overloaded_failure_signals.py @@ -215,7 +215,7 @@ def _until(rig: _Rig, name: str, model: str) -> tuple[Sample, ...]: ( pytest.param("/v1/messages", _MESSAGES_ANTHROPIC, 529, "500", "InternalServerError", id="messages-anthropic"), pytest.param("/v1/messages", _MESSAGES_BEDROCK, 503, "503", "ServiceUnavailableError", id="messages-bedrock"), - pytest.param("/v1/chat/completions", _CHAT_ANTHROPIC, 500, "500", "InternalServerError", id="chat-anthropic"), + pytest.param("/v1/chat/completions", _CHAT_ANTHROPIC, 503, "500", "InternalServerError", id="chat-anthropic"), ), ) def test_the_failed_request_metric_counts_the_answered_status_and_the_deployment_metric_the_deployment_status( @@ -245,7 +245,7 @@ def _server_spans(rig: _Rig, since: int, trace_id: str) -> tuple[Span, ...]: ("route", "model", "answered"), ( pytest.param("/v1/messages", _TRACED_MESSAGES, 529, id="messages"), - pytest.param("/v1/chat/completions", _TRACED_CHAT, 500, id="chat"), + pytest.param("/v1/chat/completions", _TRACED_CHAT, 503, id="chat"), ), ) def test_the_server_span_carries_the_status_the_route_answered( @@ -313,7 +313,7 @@ def _datadog_status_codes(rig: _Rig, alias: str, expected: int) -> tuple[tuple[s ( pytest.param("/v1/messages", _DATADOG_MESSAGES_ANTHROPIC, 529, "500", id="messages-anthropic"), pytest.param("/v1/messages", _DATADOG_MESSAGES_BEDROCK, 503, "503", id="messages-bedrock"), - pytest.param("/v1/chat/completions", _DATADOG_CHAT_ANTHROPIC, 500, "500", id="chat-anthropic"), + pytest.param("/v1/chat/completions", _DATADOG_CHAT_ANTHROPIC, 503, "500", id="chat-anthropic"), ), ) def test_the_datadog_proxy_failure_log_carries_the_answered_status_and_the_llm_failure_log_the_deployment_status( @@ -361,7 +361,7 @@ def test_a_fail_closed_guardrail_outage_is_answered_and_recorded_with_the_routes ( pytest.param("/v1/messages", _TRACED_MESSAGES, 529, id="messages-anthropic"), pytest.param("/v1/messages", _MESSAGES_BEDROCK, 503, id="messages-bedrock"), - pytest.param("/v1/chat/completions", _TRACED_CHAT, 500, id="chat-anthropic"), + pytest.param("/v1/chat/completions", _TRACED_CHAT, 503, id="chat-anthropic"), ), ) def test_the_otel_v2_server_span_carries_the_status_the_route_answered( diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index 13f6f840cb7..8143636a435 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -22,6 +22,7 @@ from fastapi.responses import JSONResponse, StreamingResponse import litellm from litellm._uuid import uuid +from litellm.exceptions import APIError, InternalServerError, MidStreamFallbackError from litellm.litellm_core_utils.litellm_logging import Logging from litellm.anthropic_interface.exceptions import AnthropicErrorSseFrame, anthropic_error_sse_frame from litellm.caching.caching import DualCache @@ -5127,6 +5128,138 @@ class TestHandleLLMApiExceptionRetryAfter: assert "All deployments for selected model are in cooldown" not in body["message"] +class TestHandleLLMApiExceptionProviderOverload: + class _RecordingFailureHook(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.original_exception: Exception | None = None + + async def async_post_call_failure_hook( + self, + request_data: dict, + original_exception: Exception, + user_api_key_dict: UserAPIKeyAuth, + traceback_str: str | None = None, + ) -> None: + self.original_exception = original_exception + + async def _invoke( + self, + exc: Exception, + route_type: str, + monkeypatch: pytest.MonkeyPatch, + recording_hook: _RecordingFailureHook | None = None, + ) -> ProxyException: + processor: Final = ProxyBaseLLMRequestProcessing(data={}) + processor.route_type = route_type + callback: Final = recording_hook if recording_hook is not None else self._RecordingFailureHook() + monkeypatch.setattr(litellm, "callbacks", [callback]) + proxy_logging_obj: Final = ProxyLogging(user_api_key_cache=DualCache()) + + with pytest.raises(ProxyException) as raised: + await processor.handle_llm_api_exception( + e=exc, + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test"), + proxy_logging_obj=proxy_logging_obj, + ) + + return raised.value + + @pytest.mark.parametrize("route_type", ("acompletion", "atext_completion", "aresponses")) + async def test_openai_routes_answer_provider_overload_as_503( + self, route_type: str, monkeypatch: pytest.MonkeyPatch + ) -> None: + exc: Final = InternalServerError( + message="Overloaded", + llm_provider="anthropic", + model="claude", + provider_status_code=529, + ) + + proxy_exc: Final = await self._invoke(exc, route_type, monkeypatch) + + assert proxy_exc.code == "503" + assert proxy_exc.type == "service_unavailable_error" + assert proxy_exc.to_dict()["code"] == "server_is_overloaded" + + @pytest.mark.parametrize( + ("route_type", "provider_status_code"), + (("acompletion", 500), ("anthropic_messages", 529)), + ) + async def test_non_overload_and_non_openai_routes_keep_their_status( + self, route_type: str, provider_status_code: int, monkeypatch: pytest.MonkeyPatch + ) -> None: + exc: Final = InternalServerError( + message="Internal server error", + llm_provider="anthropic", + model="claude", + provider_status_code=provider_status_code, + ) + + proxy_exc: Final = await self._invoke(exc, route_type, monkeypatch) + + assert proxy_exc.code == "500" + assert proxy_exc.type == "internal_server_error" + assert proxy_exc.to_dict()["code"] == "500" + + async def test_openai_route_answers_wrapped_provider_overload_as_503( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + original: Final = InternalServerError( + message="Overloaded", + llm_provider="anthropic", + model="claude", + provider_status_code=529, + ) + exc: Final = MidStreamFallbackError( + message="Fallback failed", + llm_provider="anthropic", + model="claude", + original_exception=original, + ) + + proxy_exc: Final = await self._invoke(exc, "acompletion", monkeypatch) + + assert proxy_exc.code == "503" + assert proxy_exc.type == "service_unavailable_error" + assert proxy_exc.to_dict()["code"] == "server_is_overloaded" + + async def test_anthropic_text_api_overload_answers_503(self, monkeypatch: pytest.MonkeyPatch) -> None: + exc: Final = APIError( + status_code=529, + message="Overloaded", + llm_provider="anthropic_text", + model="claude", + ) + + proxy_exc: Final = await self._invoke(exc, "atext_completion", monkeypatch) + + assert proxy_exc.code == "503" + assert proxy_exc.type == "service_unavailable_error" + assert proxy_exc.to_dict()["code"] == "server_is_overloaded" + + @pytest.mark.parametrize( + ("route_type", "expected_status"), + (("acompletion", 503), ("anthropic_messages", 500)), + ) + async def test_failure_hook_receives_the_answered_overload_status( + self, route_type: str, expected_status: int, monkeypatch: pytest.MonkeyPatch + ) -> None: + exc: Final = InternalServerError( + message="Overloaded", + llm_provider="anthropic", + model="claude", + provider_status_code=529, + ) + recording_hook: Final = self._RecordingFailureHook() + + await self._invoke(exc, route_type, monkeypatch, recording_hook) + + original_exception: Final = recording_hook.original_exception + assert original_exception is not None + assert original_exception.status_code == expected_status + + class TestHandleLLMApiExceptionFramingHeaders: """HTTP-framing headers on the provider exception must be stripped before the proxy builds its own response, or they conflict with the framing the proxy @@ -7755,7 +7888,10 @@ class TestPreCallWithFallbacksOnLocalRateLimit: "litellm_params": {"model": "openai/text-embedding-3-small", "api_key": "sk-synthetic"}, "model_info": {"enforce_supported_endpoints": True, "supported_endpoints": ["/v1/embeddings"]}, }, - {"model_name": "chat-model", "litellm_params": {"model": "openai/gpt-5.4-nano", "api_key": "sk-synthetic"}}, + { + "model_name": "chat-model", + "litellm_params": {"model": "openai/gpt-5.4-nano", "api_key": "sk-synthetic"}, + }, ], fallbacks=[{"rate-limited": ["embed-alias", "chat-model"]}], ) @@ -9829,10 +9965,13 @@ def _hook_data(**kwargs: object) -> object: @pytest.mark.asyncio -@pytest.mark.parametrize("prompt, proxy_field, client_field", [ - ({"input": "hi"}, "litellm_metadata", "metadata"), - ({"messages": [{"role": "user", "content": "hi"}]}, "metadata", "litellm_metadata"), -]) +@pytest.mark.parametrize( + "prompt, proxy_field, client_field", + [ + ({"input": "hi"}, "litellm_metadata", "metadata"), + ({"messages": [{"role": "user", "content": "hi"}]}, "metadata", "litellm_metadata"), + ], +) async def test_cursor_route_writes_proxy_fields_only_to_the_field_its_body_shape_picks( prompt: dict[str, str | list[dict[str, str]]], proxy_field: str, @@ -9842,7 +9981,9 @@ async def test_cursor_route_writes_proxy_fields_only_to_the_field_its_body_shape logging: Final = MagicMock(spec=ProxyLogging) logging.pre_call_hook = AsyncMock(side_effect=_hook_data) - processor: Final = ProxyBaseLLMRequestProcessing(data={"model": "gpt-x", "metadata": {"client": "cursor"}, **prompt}) + processor: Final = ProxyBaseLLMRequestProcessing( + data={"model": "gpt-x", "metadata": {"client": "cursor"}, **prompt} + ) data, _ = await processor.common_processing_pre_call_logic( request=Request({"type": "http", "method": "POST", "path": "/cursor/chat/completions", "headers": []}), general_settings={}, @@ -9858,6 +9999,7 @@ async def test_cursor_route_writes_proxy_fields_only_to_the_field_its_body_shape assert "queue_time_seconds" not in data.get(client_field, {}) assert data["metadata"]["client"] == "cursor" + class TestPerRequestModelGroupAlias: """``router_settings.model_group_alias`` on a key or team has to be resolved by the proxy: the Router resolves aliases from its own shared instance @@ -12693,7 +12835,9 @@ class TestResponseCostFromLoggingObjForCacheHits: proxy_logging_obj=ProxyLogging(user_api_key_cache=DualCache()), ) - cost_headers: Final = {name: value for name, value in headers.items() if name.startswith("x-litellm-response-cost")} + cost_headers: Final = { + name: value for name, value in headers.items() if name.startswith("x-litellm-response-cost") + } assert cost_headers == {"x-litellm-response-cost-output": "0.0012"} def test_a_fresh_zero_cost_v1_messages_call_still_sends_no_cost_header(self): From 114ba4021a6482e580ab7cac242cff749cc06225 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 5 Nov 2026 00:51:36 +0000 Subject: [PATCH 06/39] fix(bedrock): add GovCloud Kimi, Haiku and Grok prices (#50128) * fix(bedrock): add GovCloud Kimi K3 runtime prices * fix(bedrock): omit retired max_tokens from Kimi rows * fix(bedrock): add GovCloud Haiku and Grok prices Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(bedrock): match GovCloud Grok 4.7 limits to the card and date GovCloud Claude 3 Haiku * fix(bedrock): add priced Grok Mantle base for regional routing Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: kerry --- ...odel_prices_and_context_window_backup.json | 187 ++++++++++++++++++ model_prices_and_context_window.json | 187 ++++++++++++++++++ 2 files changed, 374 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 1aa2ac27152..e805d631db0 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -85912,5 +85912,192 @@ ], "supports_pdf_input": true, "supports_vision": true + }, + "bedrock/us-gov-east-1/moonshotai.kimi-k3": { + "cache_creation_input_token_cost": 4.95e-06, + "cache_creation_input_token_cost_flex": 2.475e-06, + "cache_creation_input_token_cost_priority": 8.6625e-06, + "cache_read_input_token_cost": 3.96e-07, + "cache_read_input_token_cost_flex": 1.98e-07, + "cache_read_input_token_cost_priority": 6.93e-07, + "input_cost_per_token": 3.96e-06, + "input_cost_per_token_batches": 1.98e-06, + "input_cost_per_token_flex": 1.98e-06, + "input_cost_per_token_priority": 6.93e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 868928, + "max_output_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1.98e-05, + "output_cost_per_token_batches": 9.9e-06, + "output_cost_per_token_flex": 9.9e-06, + "output_cost_per_token_priority": 3.465e-05, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-east-1/index.json", + "supports_audio_input": false, + "supports_function_calling": true, + "supports_prompt_cache_breakpoint": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_regex_lookaround": false, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "token_count_framing": 64.0, + "token_count_tokenizer": "kimi_k3" + }, + "bedrock/us-gov-west-1/moonshotai.kimi-k3": { + "cache_creation_input_token_cost": 4.95e-06, + "cache_creation_input_token_cost_flex": 2.475e-06, + "cache_creation_input_token_cost_priority": 8.6625e-06, + "cache_read_input_token_cost": 3.96e-07, + "cache_read_input_token_cost_flex": 1.98e-07, + "cache_read_input_token_cost_priority": 6.93e-07, + "input_cost_per_token": 3.96e-06, + "input_cost_per_token_batches": 1.98e-06, + "input_cost_per_token_flex": 1.98e-06, + "input_cost_per_token_priority": 6.93e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 868928, + "max_output_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1.98e-05, + "output_cost_per_token_batches": 9.9e-06, + "output_cost_per_token_flex": 9.9e-06, + "output_cost_per_token_priority": 3.465e-05, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", + "supports_audio_input": false, + "supports_function_calling": true, + "supports_prompt_cache_breakpoint": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_regex_lookaround": false, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "token_count_framing": 64.0, + "token_count_tokenizer": "kimi_k3" + }, + "bedrock/us-gov-east-1/anthropic.claude-3-haiku-20240307-v1:0": { + "deprecation_date": "2026-09-10", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-3-haiku.md", + "input_cost_per_token": 3e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 195904, + "max_output_tokens": 4096, + "mode": "chat", + "output_cost_per_token": 1.5e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-gov-east-1/index.json", + "successor_model": "bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0", + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "bedrock_mantle/us-gov-east-1/xai.grok-4.7": { + "cache_read_input_token_cost": 6.6e-07, + "cache_read_input_token_cost_flex": 3.3e-07, + "cache_read_input_token_cost_priority": 1.155e-06, + "input_cost_per_token": 2.64e-06, + "input_cost_per_token_batches": 1.32e-06, + "input_cost_per_token_flex": 1.32e-06, + "input_cost_per_token_priority": 4.62e-06, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 524288, + "max_output_tokens": 524288, + "mode": "chat", + "output_cost_per_token": 7.92e-06, + "output_cost_per_token_batches": 3.96e-06, + "output_cost_per_token_flex": 3.96e-06, + "output_cost_per_token_priority": 1.386e-05, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-east-1/index.json", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "use_openai_responses_path": true + }, + "bedrock/us-gov-west-1/anthropic.claude-3-haiku-20240307-v1:0": { + "deprecation_date": "2026-09-10", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-3-haiku.md", + "input_cost_per_token": 3e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 195904, + "max_output_tokens": 4096, + "mode": "chat", + "output_cost_per_token": 1.5e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-gov-west-1/index.json", + "successor_model": "bedrock/us-gov-west-1/anthropic.claude-haiku-4-5-20251001-v1:0", + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "bedrock_mantle/us-gov-west-1/xai.grok-4.7": { + "cache_read_input_token_cost": 6.6e-07, + "cache_read_input_token_cost_flex": 3.3e-07, + "cache_read_input_token_cost_priority": 1.155e-06, + "input_cost_per_token": 2.64e-06, + "input_cost_per_token_batches": 1.32e-06, + "input_cost_per_token_flex": 1.32e-06, + "input_cost_per_token_priority": 4.62e-06, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 524288, + "max_output_tokens": 524288, + "mode": "chat", + "output_cost_per_token": 7.92e-06, + "output_cost_per_token_batches": 3.96e-06, + "output_cost_per_token_flex": 3.96e-06, + "output_cost_per_token_priority": 1.386e-05, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "use_openai_responses_path": true + }, + "bedrock_mantle/xai.grok-4.7": { + "cache_read_input_token_cost": 5.5e-07, + "cache_read_input_token_cost_flex": 2.75e-07, + "cache_read_input_token_cost_priority": 9.625e-07, + "input_cost_per_token": 2.2e-06, + "input_cost_per_token_batches": 1.1e-06, + "input_cost_per_token_flex": 1.1e-06, + "input_cost_per_token_priority": 3.85e-06, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 524288, + "max_output_tokens": 524288, + "mode": "chat", + "output_cost_per_token": 6.6e-06, + "output_cost_per_token_batches": 3.3e-06, + "output_cost_per_token_flex": 3.3e-06, + "output_cost_per_token_priority": 1.155e-05, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "use_openai_responses_path": true } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 1aa2ac27152..e805d631db0 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -85912,5 +85912,192 @@ ], "supports_pdf_input": true, "supports_vision": true + }, + "bedrock/us-gov-east-1/moonshotai.kimi-k3": { + "cache_creation_input_token_cost": 4.95e-06, + "cache_creation_input_token_cost_flex": 2.475e-06, + "cache_creation_input_token_cost_priority": 8.6625e-06, + "cache_read_input_token_cost": 3.96e-07, + "cache_read_input_token_cost_flex": 1.98e-07, + "cache_read_input_token_cost_priority": 6.93e-07, + "input_cost_per_token": 3.96e-06, + "input_cost_per_token_batches": 1.98e-06, + "input_cost_per_token_flex": 1.98e-06, + "input_cost_per_token_priority": 6.93e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 868928, + "max_output_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1.98e-05, + "output_cost_per_token_batches": 9.9e-06, + "output_cost_per_token_flex": 9.9e-06, + "output_cost_per_token_priority": 3.465e-05, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-east-1/index.json", + "supports_audio_input": false, + "supports_function_calling": true, + "supports_prompt_cache_breakpoint": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_regex_lookaround": false, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "token_count_framing": 64.0, + "token_count_tokenizer": "kimi_k3" + }, + "bedrock/us-gov-west-1/moonshotai.kimi-k3": { + "cache_creation_input_token_cost": 4.95e-06, + "cache_creation_input_token_cost_flex": 2.475e-06, + "cache_creation_input_token_cost_priority": 8.6625e-06, + "cache_read_input_token_cost": 3.96e-07, + "cache_read_input_token_cost_flex": 1.98e-07, + "cache_read_input_token_cost_priority": 6.93e-07, + "input_cost_per_token": 3.96e-06, + "input_cost_per_token_batches": 1.98e-06, + "input_cost_per_token_flex": 1.98e-06, + "input_cost_per_token_priority": 6.93e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 868928, + "max_output_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1.98e-05, + "output_cost_per_token_batches": 9.9e-06, + "output_cost_per_token_flex": 9.9e-06, + "output_cost_per_token_priority": 3.465e-05, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", + "supports_audio_input": false, + "supports_function_calling": true, + "supports_prompt_cache_breakpoint": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_regex_lookaround": false, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "token_count_framing": 64.0, + "token_count_tokenizer": "kimi_k3" + }, + "bedrock/us-gov-east-1/anthropic.claude-3-haiku-20240307-v1:0": { + "deprecation_date": "2026-09-10", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-3-haiku.md", + "input_cost_per_token": 3e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 195904, + "max_output_tokens": 4096, + "mode": "chat", + "output_cost_per_token": 1.5e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-gov-east-1/index.json", + "successor_model": "bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0", + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "bedrock_mantle/us-gov-east-1/xai.grok-4.7": { + "cache_read_input_token_cost": 6.6e-07, + "cache_read_input_token_cost_flex": 3.3e-07, + "cache_read_input_token_cost_priority": 1.155e-06, + "input_cost_per_token": 2.64e-06, + "input_cost_per_token_batches": 1.32e-06, + "input_cost_per_token_flex": 1.32e-06, + "input_cost_per_token_priority": 4.62e-06, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 524288, + "max_output_tokens": 524288, + "mode": "chat", + "output_cost_per_token": 7.92e-06, + "output_cost_per_token_batches": 3.96e-06, + "output_cost_per_token_flex": 3.96e-06, + "output_cost_per_token_priority": 1.386e-05, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-east-1/index.json", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "use_openai_responses_path": true + }, + "bedrock/us-gov-west-1/anthropic.claude-3-haiku-20240307-v1:0": { + "deprecation_date": "2026-09-10", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-3-haiku.md", + "input_cost_per_token": 3e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 195904, + "max_output_tokens": 4096, + "mode": "chat", + "output_cost_per_token": 1.5e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-gov-west-1/index.json", + "successor_model": "bedrock/us-gov-west-1/anthropic.claude-haiku-4-5-20251001-v1:0", + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "bedrock_mantle/us-gov-west-1/xai.grok-4.7": { + "cache_read_input_token_cost": 6.6e-07, + "cache_read_input_token_cost_flex": 3.3e-07, + "cache_read_input_token_cost_priority": 1.155e-06, + "input_cost_per_token": 2.64e-06, + "input_cost_per_token_batches": 1.32e-06, + "input_cost_per_token_flex": 1.32e-06, + "input_cost_per_token_priority": 4.62e-06, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 524288, + "max_output_tokens": 524288, + "mode": "chat", + "output_cost_per_token": 7.92e-06, + "output_cost_per_token_batches": 3.96e-06, + "output_cost_per_token_flex": 3.96e-06, + "output_cost_per_token_priority": 1.386e-05, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "use_openai_responses_path": true + }, + "bedrock_mantle/xai.grok-4.7": { + "cache_read_input_token_cost": 5.5e-07, + "cache_read_input_token_cost_flex": 2.75e-07, + "cache_read_input_token_cost_priority": 9.625e-07, + "input_cost_per_token": 2.2e-06, + "input_cost_per_token_batches": 1.1e-06, + "input_cost_per_token_flex": 1.1e-06, + "input_cost_per_token_priority": 3.85e-06, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 524288, + "max_output_tokens": 524288, + "mode": "chat", + "output_cost_per_token": 6.6e-06, + "output_cost_per_token_batches": 3.3e-06, + "output_cost_per_token_flex": 3.3e-06, + "output_cost_per_token_priority": 1.155e-05, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "use_openai_responses_path": true } } From a73e12de561d7196f0fbcb18960433a9d2f9a8cd Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 4 Nov 2026 16:59:42 -0800 Subject: [PATCH 07/39] refactor(ui): move shell, entry and public hub text onto i18n (#50110) Move batch B1 of the admin dashboard onto react-i18next: the dashboard shell (navbar, command palette, deployment health, header, layouts), the entry pages (login, onboarding, change password, home, API reference, UI theme, workflows), the public AI hub and the AI hub table, the shared components and helpers in src/components/shared, common_components, networking toasts, the toast titles and the shadcn primitive labels. Pure text move into t() and Trans with 1,056 new en.json keys. The English on screen is unchanged. --- .../_components/APIReferenceView.tsx | 9 +- .../api-reference/_components/DocLink.tsx | 8 +- .../app/(dashboard)/api-reference/page.tsx | 4 +- .../change-password/ChangePasswordForm.tsx | 44 +- .../(dashboard)/home/_components/HomePage.tsx | 18 +- .../home/_components/UpdatesFeed.tsx | 14 +- .../src/app/(dashboard)/layout.tsx | 18 +- .../_components/TelemetryEnvBanner.tsx | 12 +- .../(dashboard)/ui-theme/UIThemeSettings.tsx | 38 +- .../src/app/(dashboard)/workflows/page.tsx | 6 +- .../src/app/login/LoginPage.tsx | 87 +- .../src/app/model_hub/page.tsx | 8 +- .../src/app/model_hub_table/page.tsx | 10 +- .../app/onboarding/OnboardingErrorView.tsx | 8 +- .../src/app/onboarding/OnboardingForm.tsx | 6 +- .../src/app/onboarding/OnboardingFormBody.tsx | 31 +- .../app/onboarding/OnboardingLoadingView.tsx | 8 +- .../src/app/onboarding/page.tsx | 6 +- .../components/AIHub/AgentHubTableColumns.tsx | 60 +- .../components/AIHub/DecisionModelGarden.tsx | 46 +- .../components/AIHub/MCPHubTableColumns.tsx | 58 +- .../components/AIHub/ModelCustomMetadata.tsx | 7 +- .../components/AIHub/ModelHubProviders.tsx | 8 +- .../src/components/AIHub/ModelHubTable.tsx | 271 +-- .../components/AIHub/ModelHubTableColumns.tsx | 64 +- .../components/AIHub/SkillHubDashboard.tsx | 38 +- .../components/AIHub/SkillHubTableColumns.tsx | 39 +- .../AIHub/UsefulLinksManagement.tsx | 76 +- .../AIHub/forms/MakeAgentPublicForm.tsx | 58 +- .../AIHub/forms/MakeMCPPublicForm.tsx | 86 +- .../AIHub/forms/MakeModelPublicForm.tsx | 54 +- .../src/components/CodeBlock.tsx | 4 +- .../CommandPalette/CommandPalette.tsx | 79 +- .../CommandPalette/CommandPaletteHint.tsx | 11 +- .../CommandPalette/CommandPaletteTrigger.tsx | 6 +- .../CommandPalette/useShortcutLabel.ts | 7 +- .../src/components/CommandPalette/utils.ts | 5 +- .../DeploymentHealth/DeploymentHealth.tsx | 19 +- .../DeploymentHealthPanel.tsx | 132 +- .../DeploymentWarningContent.tsx | 48 +- .../src/components/DeprecationBanner.tsx | 32 +- .../Navbar/BlogDropdown/BlogDropdown.tsx | 14 +- .../CommunityEngagementButtons.tsx | 16 +- .../components/Navbar/DashboardResources.tsx | 40 +- .../components/Navbar/DocsLink/DocsLink.tsx | 16 +- .../NotificationsBell/NotificationsBell.tsx | 36 +- .../Navbar/UserDropdown/UserDropdown.tsx | 48 +- .../src/components/Navbar/ViewSwitcher.tsx | 19 +- .../Navbar/WorkerDropdown/WorkerDropdown.tsx | 16 +- .../src/components/Navbar/navDisplayName.ts | 6 +- .../components/PublicModelHubTableColumns.tsx | 177 +- .../DefaultProxyAdminTag.tsx | 5 +- .../common_components/DeleteResourceModal.tsx | 15 +- .../common_components/LoadingScreen.tsx | 4 +- .../check_openapi_schema.tsx | 66 +- .../fetch_available_models_team_key.tsx | 5 +- .../src/components/molecules/logo/Logo.tsx | 4 +- .../src/components/navbar.tsx | 14 +- .../src/components/networking.tsx | 17 +- .../publicModelHub/AgentInspector.tsx | 33 +- .../components/publicModelHub/FilterChip.tsx | 8 +- .../components/publicModelHub/HubEmpty.tsx | 30 +- .../components/publicModelHub/HubHeader.tsx | 24 +- .../publicModelHub/HubMasterDetail.tsx | 3 +- .../components/publicModelHub/HubParts.tsx | 25 +- .../components/publicModelHub/HubStatus.tsx | 10 +- .../publicModelHub/InspectorHeader.tsx | 11 +- .../publicModelHub/McpInspector.tsx | 17 +- .../publicModelHub/ModelInspector.tsx | 74 +- .../components/publicModelHub/ModelsPane.tsx | 61 +- .../components/publicModelHub/PublicAiHub.tsx | 12 +- .../publicModelHub/ResourcePanes.tsx | 31 +- .../publicModelHub/SkillInspector.tsx | 36 +- .../components/publicModelHub/hubFormat.ts | 77 +- .../components/publicModelHub/hubSnippets.ts | 5 +- .../src/components/public_model_hub.tsx | 353 ++-- .../src/components/shared/AdminOnlyNotice.tsx | 16 +- .../components/shared/BudgetPeriodTooltip.tsx | 26 +- .../src/components/shared/CopyButton.tsx | 3 +- .../components/shared/CreatedKeyDisplay.tsx | 35 +- .../components/shared/DataTable/DataTable.tsx | 9 +- .../DataTable/DataTableFilterDrawer.tsx | 7 +- .../shared/DataTable/DataTablePagination.tsx | 18 +- .../DataTable/DataTableSelectionColumn.tsx | 12 +- .../shared/DataTable/DataTableSortHeader.tsx | 33 +- .../shared/DataTable/DataTableToolbar.tsx | 17 +- .../shared/DataTable/DataTableViewOptions.tsx | 3 +- .../src/components/shared/ErrorState.tsx | 7 +- .../components/shared/InheritedBudgetHint.tsx | 11 +- .../src/components/shared/MultiSelect.tsx | 15 +- .../src/components/shared/PageHeader.tsx | 4 +- .../shared/PaginatedMultiSelect.tsx | 8 +- .../shared/PaginatedSearchSelect.tsx | 8 +- .../src/components/shared/PasswordInput.tsx | 4 +- .../src/components/shared/SavingsTiles.tsx | 28 +- .../components/shared/ScopedSavingsTab.tsx | 20 +- .../src/components/shared/SearchSelect.tsx | 5 +- .../src/components/shared/SummaryCard.tsx | 76 +- .../shared/advanced_date_picker.tsx | 48 +- .../shared/charts/StackedUsageChart.tsx | 3 +- .../components/shared/charts/area_chart.tsx | 4 +- .../components/shared/charts/bar_chart.tsx | 4 +- .../src/components/shared/errorUtils.tsx | 4 +- .../src/components/shared/numerical_input.tsx | 11 +- .../shared/table_cells/AutoRouterTag.tsx | 4 +- .../shared/table_cells/UserPopoverCell.tsx | 17 +- .../components/shared/table_cells/id_cell.tsx | 3 +- .../shared/table_cells/models_cell.tsx | 14 +- .../shared/table_cells/spend_budget_cell.tsx | 20 +- .../components/ui/ResizableSheetContent.tsx | 6 +- .../src/components/ui/breadcrumb.tsx | 8 +- .../src/components/ui/combobox.tsx | 7 +- .../src/components/ui/command.tsx | 5 +- .../src/components/ui/dialog.tsx | 11 +- .../src/components/ui/message-scroller.tsx | 6 +- .../src/components/ui/sheet.tsx | 4 +- .../src/components/ui/spinner.tsx | 4 +- ui/litellm-dashboard/src/i18n/locales/en.json | 1564 +++++++++++++++++ ui/litellm-dashboard/src/lib/toast.ts | 94 +- ui/litellm-dashboard/src/utils/dataUtils.ts | 7 +- ui/litellm-dashboard/src/utils/textUtils.ts | 4 +- 121 files changed, 3756 insertions(+), 1332 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/api-reference/_components/APIReferenceView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/api-reference/_components/APIReferenceView.tsx index 1339300b2ba..7f6f7ea1d17 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/api-reference/_components/APIReferenceView.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/api-reference/_components/APIReferenceView.tsx @@ -1,5 +1,6 @@ "use client"; import React from "react"; +import { useTranslation } from "react-i18next"; import CodeBlock from "@/components/CodeBlock"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import DocLink from "./DocLink"; @@ -12,6 +13,7 @@ interface ApiRefProps { } const APIReferenceView: React.FC = ({ proxySettings }) => { + const { t } = useTranslation(); let base_url = ""; const customDocBaseUrl = proxySettings?.LITELLM_UI_API_DOC_BASE_URL; if (customDocBaseUrl && customDocBaseUrl.trim()) { @@ -25,14 +27,11 @@ const APIReferenceView: React.FC = ({ proxySettings }) => {
{/* Header row with Docs link on the right */}
-

OpenAI Compatible Proxy: API Reference

+

{t("apiReference.view.title")}

-

- LiteLLM is OpenAI Compatible. This means your API Key works with the OpenAI SDK. Just replace the base_url to - point to your litellm proxy. Example Below{" "} -

+

{t("apiReference.view.description")}

diff --git a/ui/litellm-dashboard/src/app/(dashboard)/api-reference/_components/DocLink.tsx b/ui/litellm-dashboard/src/app/(dashboard)/api-reference/_components/DocLink.tsx index f5a9d5e3a15..cee174d7fdb 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/api-reference/_components/DocLink.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/api-reference/_components/DocLink.tsx @@ -1,5 +1,6 @@ import React from "react"; import { ExternalLink } from "lucide-react"; +import { useTranslation } from "react-i18next"; import { cn } from "@/lib/cva.config"; @@ -9,21 +10,22 @@ export type DocLinkProps = { }; const DocLink = ({ href, className }: DocLinkProps) => { + const { t } = useTranslation(); return ( - API Reference Docs + {t("apiReference.docLink.label")} - (opens in a new tab) + {t("apiReference.docLink.newTab")} ); }; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/api-reference/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/api-reference/page.tsx index d7c977b0870..4d8498a1be4 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/api-reference/page.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/api-reference/page.tsx @@ -1,17 +1,19 @@ "use client"; +import { useTranslation } from "react-i18next"; import APIReferenceView from "./_components/APIReferenceView"; import { DeprecationBanner } from "@/components/DeprecationBanner"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; import useProxySettings from "@/app/(dashboard)/hooks/proxySettings/useProxySettings"; const APIReferencePage = () => { + const { t } = useTranslation(); const { accessToken } = useAuthorized(); const proxySettings = useProxySettings(accessToken); return ( <> - + ); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/change-password/ChangePasswordForm.tsx b/ui/litellm-dashboard/src/app/(dashboard)/change-password/ChangePasswordForm.tsx index 9e9c2ad7fc8..6aee2bf7dfd 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/change-password/ChangePasswordForm.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/change-password/ChangePasswordForm.tsx @@ -1,6 +1,7 @@ "use client"; import React, { useState } from "react"; +import { useTranslation } from "react-i18next"; import { CircleAlert } from "lucide-react"; import { z } from "zod"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; @@ -17,21 +18,23 @@ import { useZodForm } from "@/lib/forms/useZodForm"; import { toast } from "@/lib/toast"; import { revokeSessionAndClearClientState } from "@/app/(dashboard)/hooks/useLogout"; import { getLoginUrl } from "@/utils/returnUrlUtils"; +import i18n from "@/i18n/i18n"; const changePasswordSchema = z .object({ - currentPassword: z.string().min(1, "Current password is required"), - newPassword: z.string().min(1, "New password is required"), - confirmNewPassword: z.string().min(1, "Confirm your new password"), + currentPassword: z.string().min(1, { error: () => i18n.t("changePassword.schema.currentRequired") }), + newPassword: z.string().min(1, { error: () => i18n.t("changePassword.schema.newRequired") }), + confirmNewPassword: z.string().min(1, { error: () => i18n.t("changePassword.schema.confirmRequired") }), }) .refine((values) => values.newPassword === values.confirmNewPassword, { - message: "New passwords do not match", + error: () => i18n.t("changePassword.schema.mismatch"), path: ["confirmNewPassword"], }); type ChangePasswordValues = z.infer; export function ChangePasswordForm() { + const { t } = useTranslation(); const { accessToken, passwordResetRequired } = useAuthorized(); const form = useZodForm(changePasswordSchema, { defaultValues: { currentPassword: "", newPassword: "", confirmNewPassword: "" }, @@ -49,12 +52,12 @@ export function ChangePasswordForm() { // The session key was minted restricted; only a fresh login lifts it. // Revoke it server-side too (best-effort) so it doesn't sit valid // until the expiry reaper gets to it. - toast.success("Password updated. Please log in with your new password."); + toast.success(t("changePassword.form.toasts.updatedSignIn")); await revokeSessionAndClearClientState(accessToken); window.location.replace(getUiPageUrl(getLoginUrl(getProxyBaseUrl()))); return; } - toast.success("Password updated"); + toast.success(t("changePassword.form.toasts.updated")); form.reset(); } catch (error) { setSubmitError(extractProxyErrorMessage(error)); @@ -67,34 +70,31 @@ export function ChangePasswordForm() {
-

Change Password

-

- Enter your current password and choose a new one. The new password must meet this proxy's password - policy. -

+

{t("changePassword.form.title")}

+

{t("changePassword.form.description")}

{passwordResetRequired && ( - - Your password must be changed before you can use the dashboard: it was either found in a known data - breach or set by an administrator as a temporary password. After updating it, you will be signed out to - log in again. - + {t("changePassword.form.resetRequired")} )}
- + {({ ref, ...field }) => } - + {({ ref, ...field }) => } - + {({ ref, ...field }) => } @@ -108,8 +108,10 @@ export function ChangePasswordForm() {
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/home/_components/HomePage.tsx b/ui/litellm-dashboard/src/app/(dashboard)/home/_components/HomePage.tsx index 1faa9c1d802..dfd461fb9ba 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/home/_components/HomePage.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/home/_components/HomePage.tsx @@ -1,6 +1,7 @@ "use client"; import { ArrowRight, Plus, Rocket, Sparkles } from "lucide-react"; +import { useTranslation } from "react-i18next"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; import { Button } from "@/components/ui/button"; import { Card } from "@/components/ui/card"; @@ -20,6 +21,7 @@ import { HOME_USAGE_DAYS, useHomeUsage } from "./useHomeUsage"; const LEADERBOARD = { metric: "tokens", dimension: "model_groups" } as const; export default function HomePage() { + const { t } = useTranslation(); const { isViewOnly } = useAuthorized(); const usage = useHomeUsage(); const blog = useBlogPosts(); @@ -29,23 +31,23 @@ export default function HomePage() { return ( - Home + {t("home.page.title")} {!isViewOnly && ( )} -
+
{usage.failed ? (
- Failed to load usage + {t("home.page.usageFailed")}
) : ( @@ -66,7 +68,7 @@ export default function HomePage() {

- What's new in LiteLLM? + {t("home.page.whatsNew")}

@@ -97,8 +99,8 @@ export default function HomePage() { )} ( /** Latest posts from the LiteLLM blog RSS feed, via /public/litellm_blog_posts, one row per post. */ export default function UpdatesFeed({ className }: { className?: string }) { + const { t } = useTranslation(); const disabled = useDisableBlogPosts(); const { data, isLoading, isError, refetch } = useBlogPosts(); if (disabled) return null; @@ -43,10 +45,10 @@ export default function UpdatesFeed({ className }: { className?: string }) {

- Updates + {t("home.updatesFeed.title")}

@@ -63,13 +65,15 @@ export default function UpdatesFeed({ className }: { className?: string }) { )} {isError && (
- Failed to load posts + {t("home.updatesFeed.failed")}
)} - {data && posts.length === 0 &&

No posts yet

} + {data && posts.length === 0 && ( +

{t("home.updatesFeed.empty")}

+ )} {posts.length > 0 && (
    {posts.map((post: BlogPost) => ( diff --git a/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx b/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx index 9a94e225932..5c93d214812 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx @@ -1,6 +1,7 @@ "use client"; import React, { Suspense, useState, useRef, useEffect } from "react"; +import { useTranslation } from "react-i18next"; import { Dialog as DialogPrimitive } from "@base-ui/react/dialog"; import { DashboardHeader } from "@/components/DashboardHeader"; import Navbar from "@/components/navbar"; @@ -44,6 +45,7 @@ function PluginModeProviderWithAuth({ children }: { children: React.ReactNode }) } export function AgentControlPlaneView() { + const { t } = useTranslation(); const { activePlugin } = usePluginMode(); const activePluginName = activePlugin?.name; const agentPlatformUrl = activePlugin?.url ?? ""; @@ -88,8 +90,8 @@ export function AgentControlPlaneView() { return (
    -

    Plugin

    -

    Configure the plugin URL in settings

    +

    {t("shell.agentControlPlane.plugin")}

    +

    {t("shell.agentControlPlane.configure")}

    ); @@ -107,7 +109,7 @@ export function AgentControlPlaneView() { flex: 1, minHeight: "calc(100vh - 56px)", }} - title={activePlugin?.display_name ?? "Plugin"} + title={activePlugin?.display_name ?? t("shell.agentControlPlane.plugin")} allow="clipboard-write" /> ); @@ -116,6 +118,7 @@ export function AgentControlPlaneView() { const FULL_BLEED_SEGMENTS = new Set(["logs"]); function DashboardShell({ children }: { children: React.ReactNode }) { + const { t } = useTranslation(); const { accessToken } = useAuth(); const { mode } = usePluginMode(); const pathname = usePathname(); @@ -186,7 +189,7 @@ function DashboardShell({ children }: { children: React.ReactNode }) { if (event.target instanceof Element && event.target.closest("a[href]")) setMobileNavigationKey(null); }} > - Navigation + {t("shell.dashboardShell.navigation")} setMobileNavigationKey(null)} /> @@ -198,7 +201,12 @@ function DashboardShell({ children }: { children: React.ReactNode }) { +
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/workflows/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/workflows/page.tsx index 51db7579f82..d3ce686d127 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/workflows/page.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/workflows/page.tsx @@ -1,5 +1,6 @@ "use client"; +import { useTranslation } from "react-i18next"; import WorkflowRuns from "./WorkflowRuns"; import { DeprecationBanner } from "@/components/DeprecationBanner"; import { AdminOnlyNotice } from "@/components/shared/AdminOnlyNotice"; @@ -7,16 +8,17 @@ import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; import useCan from "@/app/(dashboard)/hooks/useCan"; export default function Workflows() { + const { t } = useTranslation(); const { accessToken } = useAuthorized(); const canViewWorkflowRuns = useCan("viewWorkflowRuns"); if (!canViewWorkflowRuns) { - return ; + return ; } return ( <> - + ); diff --git a/ui/litellm-dashboard/src/app/login/LoginPage.tsx b/ui/litellm-dashboard/src/app/login/LoginPage.tsx index 10d8051232c..8c8a2d741dd 100644 --- a/ui/litellm-dashboard/src/app/login/LoginPage.tsx +++ b/ui/litellm-dashboard/src/app/login/LoginPage.tsx @@ -27,17 +27,20 @@ import { consumeReturnUrl, getLoginUrl, getReturnUrl, isValidReturnUrl } from "@ import { CircleAlert, Info, TriangleAlert, X } from "lucide-react"; import { useRouter } from "next/navigation"; import { useEffect, useId, useState } from "react"; +import { Trans, useTranslation } from "react-i18next"; import { z } from "zod"; import { useWorker } from "@/hooks/useWorker"; +import i18n from "@/i18n/i18n"; const loginSchema = z.object({ - username: z.string().min(1, "Please enter your username"), - password: z.string().min(1, "Please enter your password"), + username: z.string().min(1, { error: () => i18n.t("login.schema.usernameRequired") }), + password: z.string().min(1, { error: () => i18n.t("login.schema.passwordRequired") }), }); type LoginFormValues = z.infer; function SsoEnabledNotice() { + const { t } = useTranslation(); const [dismissed, setDismissed] = useState(false); if (dismissed) { @@ -48,13 +51,22 @@ function SsoEnabledNotice() { - Single Sign-On (SSO) is enabled. LiteLLM no longer automatically redirects to the SSO login flow upon loading - this page. To re-enable auto-redirect-to-SSO, set{" "} - AUTO_REDIRECT_UI_LOGIN_TO_SSO=true in your - environment configuration. + AUTO_REDIRECT_UI_LOGIN_TO_SSO=true + ), + }} + /> - @@ -63,6 +75,7 @@ function SsoEnabledNotice() { } function LoginPageContent() { + const { t } = useTranslation(); const [isLoading, setIsLoading] = useState(true); const { data: uiConfig, isLoading: isConfigLoading } = useUIConfig(); const loginMutation = useLogin(); @@ -203,12 +216,9 @@ function LoginPageContent() { - Admin UI Disabled + {t("login.adminDisabled.title")} -

- The Admin UI has been disabled by the administrator. To re-enable it, please update the following - environment variable: -

+

{t("login.adminDisabled.description")}

DISABLE_ADMIN_UI=False

@@ -232,26 +242,33 @@ function LoginPageContent() {
-

Login

-

Access your LiteLLM Admin UI.

+

{t("login.form.title")}

+

{t("login.form.subtitle")}

{!uiConfig?.hide_default_credentials_hint && ( - Default Credentials + {t("login.form.defaultCredentials.title")}

- By default, Username is admin and - Password is your set LiteLLM Proxy - MASTER_KEY. + admin, + masterkey: MASTER_KEY, + }} + />

- Need to set UI credentials or SSO?{" "} - - Check the documentation - - . + + ), + }} + />

@@ -268,14 +285,14 @@ function LoginPageContent() { {uiConfig?.is_control_plane && workers.length > 0 && ( - Worker + {t("login.form.worker.label")} - + {({ ref, ...field }) => ( {!uiConfig?.sso_configured ? ( }> - Please configure SSO to log in with SSO. + {t("login.form.ssoUnavailable")} ) : ( )} diff --git a/ui/litellm-dashboard/src/app/model_hub/page.tsx b/ui/litellm-dashboard/src/app/model_hub/page.tsx index 60a2f054dff..0e066dc2a34 100644 --- a/ui/litellm-dashboard/src/app/model_hub/page.tsx +++ b/ui/litellm-dashboard/src/app/model_hub/page.tsx @@ -2,6 +2,7 @@ import { useSearchParams } from "next/navigation"; import { Suspense } from "react"; +import { useTranslation } from "react-i18next"; import PublicAiHub from "@/components/publicModelHub/PublicAiHub"; @@ -11,8 +12,13 @@ function PublicAiHubPage() { } export default function PublicModelHub() { + const { t } = useTranslation(); return ( - Loading...
}> + {t("publicModelHub.modelHubPage.loading")} + } + > ); diff --git a/ui/litellm-dashboard/src/app/model_hub_table/page.tsx b/ui/litellm-dashboard/src/app/model_hub_table/page.tsx index 99df5ed8cfd..dd922636e9e 100644 --- a/ui/litellm-dashboard/src/app/model_hub_table/page.tsx +++ b/ui/litellm-dashboard/src/app/model_hub_table/page.tsx @@ -2,6 +2,7 @@ import { useSearchParams } from "next/navigation"; import { Suspense } from "react"; +import { useTranslation } from "react-i18next"; import PublicAiHub from "@/components/publicModelHub/PublicAiHub"; @@ -11,8 +12,15 @@ function PublicAiHubPage() { } export default function PublicModelHubTable() { + const { t } = useTranslation(); return ( - Loading...}> + + {t("publicModelHub.modelHubTablePage.loading")} + + } + > ); diff --git a/ui/litellm-dashboard/src/app/onboarding/OnboardingErrorView.tsx b/ui/litellm-dashboard/src/app/onboarding/OnboardingErrorView.tsx index 2fa35205a82..4831566a587 100644 --- a/ui/litellm-dashboard/src/app/onboarding/OnboardingErrorView.tsx +++ b/ui/litellm-dashboard/src/app/onboarding/OnboardingErrorView.tsx @@ -1,20 +1,22 @@ import React from "react"; +import { useTranslation } from "react-i18next"; import { CircleAlert } from "lucide-react"; import { Alert, AlertDescription, AlertTitle } from "@/components/shared/Alert"; import { buttonVariants } from "@/components/ui/button"; import { getLoginUrl } from "@/utils/returnUrlUtils"; export function OnboardingErrorView() { + const { t } = useTranslation(); return (
- Failed to load invitation - The invitation link may be invalid or expired. + {t("onboarding.errorView.title")} + {t("onboarding.errorView.description")}
diff --git a/ui/litellm-dashboard/src/app/onboarding/OnboardingForm.tsx b/ui/litellm-dashboard/src/app/onboarding/OnboardingForm.tsx index 304ee161a9a..6a2c4a28783 100644 --- a/ui/litellm-dashboard/src/app/onboarding/OnboardingForm.tsx +++ b/ui/litellm-dashboard/src/app/onboarding/OnboardingForm.tsx @@ -1,5 +1,6 @@ "use client"; import React from "react"; +import { useTranslation } from "react-i18next"; import { useSearchParams } from "next/navigation"; import { jwtDecode } from "jwt-decode"; import { useOnboardingCredentials, useClaimOnboardingToken } from "@/app/(dashboard)/hooks/onboarding/useOnboarding"; @@ -14,6 +15,7 @@ type OnboardingFormProps = { }; export function OnboardingForm({ variant }: OnboardingFormProps) { + const { t } = useTranslation(); const searchParams = useSearchParams()!; const inviteId = searchParams.get("invitation_id"); const [claimError, setClaimError] = React.useState(null); @@ -41,7 +43,7 @@ export function OnboardingForm({ variant }: OnboardingFormProps) { { onSuccess: (data: { token?: string }) => { if (!data?.token) { - setClaimError("Failed to start session. Please try again."); + setClaimError(t("onboarding.form.errors.sessionFailed")); return; } // Invite signup is a principal-change boundary — the prior admin's @@ -56,7 +58,7 @@ export function OnboardingForm({ variant }: OnboardingFormProps) { ); }, onError: (error: Error) => { - setClaimError(error.message || "Failed to submit. Please try again."); + setClaimError(error.message || t("onboarding.form.errors.submitFailed")); }, }, ); diff --git a/ui/litellm-dashboard/src/app/onboarding/OnboardingFormBody.tsx b/ui/litellm-dashboard/src/app/onboarding/OnboardingFormBody.tsx index bf6f1d09558..fa93dd22998 100644 --- a/ui/litellm-dashboard/src/app/onboarding/OnboardingFormBody.tsx +++ b/ui/litellm-dashboard/src/app/onboarding/OnboardingFormBody.tsx @@ -1,4 +1,5 @@ import React from "react"; +import { useTranslation } from "react-i18next"; import { CircleAlert, Info } from "lucide-react"; import { z } from "zod"; import { Alert, AlertDescription, AlertTitle } from "@/components/shared/Alert"; @@ -11,6 +12,7 @@ import { Input } from "@/components/ui/input"; import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner"; import { cn } from "@/lib/cva.config"; import { useZodForm } from "@/lib/forms/useZodForm"; +import i18n from "@/i18n/i18n"; type OnboardingFormBodyProps = { variant: "signup" | "reset_password"; @@ -21,16 +23,17 @@ type OnboardingFormBodyProps = { }; const onboardingSchema = z.object({ - password: z.string().min(1, "password required to sign up"), + password: z.string().min(1, { error: () => i18n.t("onboarding.formBody.schema.passwordRequired") }), }); type OnboardingFormValues = z.infer; export function OnboardingFormBody({ variant, userEmail, isPending, claimError, onSubmit }: OnboardingFormBodyProps) { + const { t } = useTranslation(); const form = useZodForm(onboardingSchema, { defaultValues: { password: "" } }); const emailFieldId = React.useId(); const isResetPassword = variant === "reset_password"; - const actionLabel = isResetPassword ? "Reset Password" : "Sign Up"; + const actionLabel = isResetPassword ? t("onboarding.formBody.resetPassword") : t("onboarding.formBody.signUp"); const handleSubmit = (values: OnboardingFormValues) => onSubmit({ password: values.password }); @@ -41,25 +44,23 @@ export function OnboardingFormBody({ variant, userEmail, isPending, claimError,
🚅 LiteLLM

{actionLabel}

- {isResetPassword - ? "Reset your password to access Admin UI." - : "Claim your user account to login to Admin UI."} + {isResetPassword ? t("onboarding.formBody.resetDescription") : t("onboarding.formBody.signUpDescription")}

{variant === "signup" && ( - SSO + {t("onboarding.formBody.sso.title")}
- SSO is under the Enterprise Tier. + {t("onboarding.formBody.sso.description")} - Get Free Trial + {t("onboarding.formBody.sso.trial")}
@@ -69,15 +70,19 @@ export function OnboardingFormBody({ variant, userEmail, isPending, claimError,
- Email Address + {t("onboarding.formBody.email")} {({ ref, ...field }) => } @@ -92,7 +97,9 @@ export function OnboardingFormBody({ variant, userEmail, isPending, claimError,
diff --git a/ui/litellm-dashboard/src/app/onboarding/OnboardingLoadingView.tsx b/ui/litellm-dashboard/src/app/onboarding/OnboardingLoadingView.tsx index 4d5d2a1371e..9b288ca542a 100644 --- a/ui/litellm-dashboard/src/app/onboarding/OnboardingLoadingView.tsx +++ b/ui/litellm-dashboard/src/app/onboarding/OnboardingLoadingView.tsx @@ -1,10 +1,16 @@ import React from "react"; +import { useTranslation } from "react-i18next"; import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner"; export function OnboardingLoadingView() { + const { t } = useTranslation(); return (
- +
); } diff --git a/ui/litellm-dashboard/src/app/onboarding/page.tsx b/ui/litellm-dashboard/src/app/onboarding/page.tsx index ad0bfc69299..168abf354b7 100644 --- a/ui/litellm-dashboard/src/app/onboarding/page.tsx +++ b/ui/litellm-dashboard/src/app/onboarding/page.tsx @@ -1,6 +1,7 @@ "use client"; import React, { Suspense } from "react"; import { useSearchParams } from "next/navigation"; +import { useTranslation } from "react-i18next"; import { OnboardingForm } from "./OnboardingForm"; function OnboardingContent() { @@ -11,8 +12,11 @@ function OnboardingContent() { } export default function Onboarding() { + const { t } = useTranslation(); return ( - Loading...}> + {t("onboarding.page.loading")}} + > ); diff --git a/ui/litellm-dashboard/src/components/AIHub/AgentHubTableColumns.tsx b/ui/litellm-dashboard/src/components/AIHub/AgentHubTableColumns.tsx index 643f2628e73..2cd01d56344 100644 --- a/ui/litellm-dashboard/src/components/AIHub/AgentHubTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/AgentHubTableColumns.tsx @@ -2,6 +2,7 @@ import { ColumnDef } from "@tanstack/react-table"; import { Copy, Info, MoreHorizontal } from "lucide-react"; +import { useTranslation } from "react-i18next"; import { DataTableSortHeader } from "@/components/shared/DataTable"; import { StatusBadge } from "@/components/shared/table_cells"; @@ -15,6 +16,7 @@ import { DropdownMenuTrigger, } from "@/components/ui/dropdown-menu"; import { cn } from "@/lib/cva.config"; +import i18n from "@/i18n/i18n"; import { copyToClipboard } from "@/utils/dataUtils"; export interface AgentHubData { @@ -48,10 +50,11 @@ interface AgentHubRowActionsProps { } function AgentHubRowActions({ agent, onAgentClick }: AgentHubRowActionsProps) { + const { t } = useTranslation(); return ( @@ -60,14 +63,14 @@ function AgentHubRowActions({ agent, onAgentClick }: AgentHubRowActionsProps) { onAgentClick(agent)}> - View details + {t("aiHub.agentHubColumns.rowActions.viewDetails")} void copyToClipboard(agent.name, "Agent name copied")} + onClick={() => void copyToClipboard(agent.name, t("aiHub.agentHubColumns.rowActions.nameCopied"))} > - Copy agent name + {t("aiHub.agentHubColumns.rowActions.copyName")} @@ -82,8 +85,8 @@ export const getAgentHubTableColumns = ({ onAgentClick }: AgentHubTableColumnsDe { id: "name", accessorKey: "name", - meta: { title: "Agent Name" }, - header: ({ column }) => , + meta: { title: i18n.t("aiHub.agentHubColumns.name") }, + header: ({ column }) => , size: 200, enableSorting: true, sortingFn: "alphanumeric", @@ -94,8 +97,8 @@ export const getAgentHubTableColumns = ({ onAgentClick }: AgentHubTableColumnsDe { id: "description", accessorKey: "description", - meta: { title: "Description", className: "hidden md:table-cell" }, - header: ({ column }) => , + meta: { title: i18n.t("aiHub.agentHubColumns.description"), className: "hidden md:table-cell" }, + header: ({ column }) => , size: 240, enableSorting: true, sortingFn: "alphanumeric", @@ -108,8 +111,8 @@ export const getAgentHubTableColumns = ({ onAgentClick }: AgentHubTableColumnsDe { id: "version", accessorKey: "version", - meta: { title: "Version", skeleton: "badge", className: "hidden lg:table-cell" }, - header: ({ column }) => , + meta: { title: i18n.t("aiHub.agentHubColumns.version"), skeleton: "badge", className: "hidden lg:table-cell" }, + header: ({ column }) => , size: 100, enableSorting: true, sortingFn: "alphanumeric", @@ -122,8 +125,8 @@ export const getAgentHubTableColumns = ({ onAgentClick }: AgentHubTableColumnsDe { id: "protocolVersion", accessorKey: "protocolVersion", - meta: { title: "Protocol", className: "hidden lg:table-cell" }, - header: ({ column }) => , + meta: { title: i18n.t("aiHub.agentHubColumns.protocol"), className: "hidden lg:table-cell" }, + header: ({ column }) => , size: 100, enableSorting: true, sortingFn: "alphanumeric", @@ -131,8 +134,8 @@ export const getAgentHubTableColumns = ({ onAgentClick }: AgentHubTableColumnsDe }, { id: "skills", - meta: { title: "Skills", skeleton: "chips" }, - header: "Skills", + meta: { title: i18n.t("aiHub.agentHubColumns.skills"), skeleton: "chips" }, + header: i18n.t("aiHub.agentHubColumns.skills"), size: 180, enableSorting: false, cell: ({ row }) => { @@ -140,7 +143,7 @@ export const getAgentHubTableColumns = ({ onAgentClick }: AgentHubTableColumnsDe return (
- {skills.length} skill{skills.length !== 1 ? "s" : ""} + {i18n.t("aiHub.agentHubColumns.skillCount", { count: skills.length })} {skills.length > 0 && (
@@ -158,8 +161,8 @@ export const getAgentHubTableColumns = ({ onAgentClick }: AgentHubTableColumnsDe }, { id: "capabilities", - meta: { title: "Capabilities", skeleton: "chips" }, - header: "Capabilities", + meta: { title: i18n.t("aiHub.agentHubColumns.capabilities"), skeleton: "chips" }, + header: i18n.t("aiHub.agentHubColumns.capabilities"), size: 160, enableSorting: false, cell: ({ row }) => { @@ -182,8 +185,8 @@ export const getAgentHubTableColumns = ({ onAgentClick }: AgentHubTableColumnsDe }, { id: "io_modes", - meta: { title: "I/O Modes", skeleton: "twoLine", className: "hidden xl:table-cell" }, - header: "I/O Modes", + meta: { title: i18n.t("aiHub.agentHubColumns.ioModes"), skeleton: "twoLine", className: "hidden xl:table-cell" }, + header: i18n.t("aiHub.agentHubColumns.ioModes"), size: 150, enableSorting: false, cell: ({ row }) => { @@ -192,10 +195,12 @@ export const getAgentHubTableColumns = ({ onAgentClick }: AgentHubTableColumnsDe return (
- In: {inputModes.join(", ") || "-"} + {i18n.t("aiHub.agentHubColumns.inputLabel")}{" "} + {inputModes.join(", ") || "-"} - Out: {outputModes.join(", ") || "-"} + {i18n.t("aiHub.agentHubColumns.outputLabel")}{" "} + {outputModes.join(", ") || "-"}
); @@ -204,8 +209,8 @@ export const getAgentHubTableColumns = ({ onAgentClick }: AgentHubTableColumnsDe { id: "is_public", accessorKey: "is_public", - meta: { title: "Public", skeleton: "badge", className: "hidden md:table-cell" }, - header: ({ column }) => , + meta: { title: i18n.t("aiHub.agentHubColumns.public"), skeleton: "badge", className: "hidden md:table-cell" }, + header: ({ column }) => , size: 100, enableSorting: true, sortingFn: (rowA, rowB) => { @@ -215,13 +220,18 @@ export const getAgentHubTableColumns = ({ onAgentClick }: AgentHubTableColumnsDe }, cell: ({ row }) => { const isPublic = row.original.is_public === true; - return ; + return ( + + ); }, }, { id: "actions", meta: { className: "text-right", headerClassName: "text-right" }, - header: () => Actions, + header: () => {i18n.t("aiHub.agentHubColumns.actions")}, size: 64, enableSorting: false, enableHiding: false, diff --git a/ui/litellm-dashboard/src/components/AIHub/DecisionModelGarden.tsx b/ui/litellm-dashboard/src/components/AIHub/DecisionModelGarden.tsx index dcd2528c761..116b539c250 100644 --- a/ui/litellm-dashboard/src/components/AIHub/DecisionModelGarden.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/DecisionModelGarden.tsx @@ -1,5 +1,7 @@ import React, { useMemo, useState } from "react"; +import type { ParseKeys } from "i18next"; import { BookOpen, Plus, Sparkles } from "lucide-react"; +import { Trans, useTranslation } from "react-i18next"; import { useModelCostMap } from "@/app/(dashboard)/hooks/models/useModelCostMap"; import { useProviderFields } from "@/app/(dashboard)/hooks/providers/useProviderFields"; @@ -20,17 +22,18 @@ import { uiHref } from "@/utils/uiHref"; const PREVIEW_COUNT = 8; -const plural = (count: number, one: string, many: string): string => (count === 1 ? one : many); - -const ROUTE_LINKS: Readonly> = { - decisions: { href: DECISIONS_DOCS_URL, label: "/v1/decisions docs" }, - "self-hosted": { href: DECISIONS_SELF_HOSTED_DOCS_URL, label: "Self-hosted, own route" }, - "needs-upgrade": { href: DECISIONS_PROVIDERS_DOCS_URL, label: "Needs a newer LiteLLM" }, +const ROUTE_LINKS: Readonly> = { + decisions: { href: DECISIONS_DOCS_URL, label: "aiHub.decisionModelGarden.routes.decisions" }, + "self-hosted": { href: DECISIONS_SELF_HOSTED_DOCS_URL, label: "aiHub.decisionModelGarden.routes.selfHosted" }, + "needs-upgrade": { href: DECISIONS_PROVIDERS_DOCS_URL, label: "aiHub.decisionModelGarden.routes.needsUpgrade" }, }; const DEPLOYMENT_NAMED_PROVIDERS: ReadonlySet = new Set(["azure", "azure_ai"]); -const OWN_DEPLOYMENT_LINK = { href: DECISIONS_PROVIDERS_DOCS_URL, label: "Deploy under your own name" } as const; +const OWN_DEPLOYMENT_LINK = { + href: DECISIONS_PROVIDERS_DOCS_URL, + label: "aiHub.decisionModelGarden.routes.ownDeployment", +} as const; const needsOwnDeploymentName = (row: DecisionGardenModel): boolean => row.route === "decisions" && DEPLOYMENT_NAMED_PROVIDERS.has(row.provider); @@ -49,6 +52,7 @@ interface DecisionModelCardProps { } function DecisionModelCard({ row, canAddModels }: DecisionModelCardProps) { + const { t } = useTranslation(); const { displayName } = getProviderLogoAndName(row.provider); const link = cardLink(row); return ( @@ -70,12 +74,12 @@ function DecisionModelCard({ row, canAddModels }: DecisionModelCardProps) { rel="noopener noreferrer" className="text-xs text-muted-foreground underline" > - {link.label} + {t(link.label)} {canAddModels && ( )}
@@ -89,6 +93,7 @@ interface DecisionModelGardenProps { } const DecisionModelGarden: React.FC = ({ accessToken, canAddModels }) => { + const { t } = useTranslation(); const { data: costMap } = useModelCostMap(true, true); const { data: providerFields } = useProviderFields(); const { data: proxyProviders, isLoading: proxyProvidersLoading } = useProxyDecisionProviders(accessToken); @@ -118,15 +123,16 @@ const DecisionModelGarden: React.FC = ({ accessToken,
-

Decision models LiteLLM supports

+

{t("aiHub.decisionModelGarden.title")}

- {routedRows.length} decision {plural(routedRows.length, "model", "models")} across {routedProviderCount}{" "} - {plural(routedProviderCount, "provider", "providers")} {plural(routedRows.length, "answers", "answer")} on{" "} - /v1/decisions. - {selfHostedCount > 0 && - ` ${selfHostedCount} more ${plural(selfHostedCount, "is self-hosted and answers on its own route", "are self-hosted and answer on their own routes")}.`} - {needsUpgradeCount > 0 && - ` ${needsUpgradeCount} more ${plural(needsUpgradeCount, "needs", "need")} a newer LiteLLM version.`} + }} + /> + {selfHostedCount > 0 && ` ${t("aiHub.decisionModelGarden.selfHosted", { count: selfHostedCount })}`} + {needsUpgradeCount > 0 && ` ${t("aiHub.decisionModelGarden.needsUpgrade", { count: needsUpgradeCount })}`}

@@ -148,7 +154,9 @@ const DecisionModelGarden: React.FC = ({ accessToken, {rows.length > PREVIEW_COUNT && (
)} diff --git a/ui/litellm-dashboard/src/components/AIHub/MCPHubTableColumns.tsx b/ui/litellm-dashboard/src/components/AIHub/MCPHubTableColumns.tsx index db53e97569b..2807a651648 100644 --- a/ui/litellm-dashboard/src/components/AIHub/MCPHubTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/MCPHubTableColumns.tsx @@ -2,6 +2,7 @@ import { ColumnDef } from "@tanstack/react-table"; import { Copy, Info, MoreHorizontal } from "lucide-react"; +import { useTranslation } from "react-i18next"; import { DataTableSortHeader } from "@/components/shared/DataTable"; import { MCP_REACHABLE_DESCRIPTION } from "@/components/mcp_tools/types"; @@ -15,6 +16,7 @@ import { DropdownMenuTrigger, } from "@/components/ui/dropdown-menu"; import { cn } from "@/lib/cva.config"; +import i18n from "@/i18n/i18n"; import { copyToClipboard } from "@/utils/dataUtils"; export interface MCPServerData { @@ -60,10 +62,11 @@ interface MCPHubRowActionsProps { } function MCPHubRowActions({ server, onServerClick }: MCPHubRowActionsProps) { + const { t } = useTranslation(); return ( @@ -72,14 +75,14 @@ function MCPHubRowActions({ server, onServerClick }: MCPHubRowActionsProps) { onServerClick(server)}> - View details + {t("aiHub.mcpHubColumns.rowActions.viewDetails")} void copyToClipboard(server.server_name, "Server name copied")} + onClick={() => void copyToClipboard(server.server_name, t("aiHub.mcpHubColumns.rowActions.nameCopied"))} > - Copy server name + {t("aiHub.mcpHubColumns.rowActions.copyName")} @@ -94,8 +97,8 @@ export const getMCPHubTableColumns = ({ onServerClick }: MCPHubTableColumnsDeps) { id: "server_name", accessorKey: "server_name", - meta: { title: "Server Name" }, - header: ({ column }) => , + meta: { title: i18n.t("aiHub.mcpHubColumns.serverName") }, + header: ({ column }) => , size: 200, enableSorting: true, sortingFn: "alphanumeric", @@ -106,8 +109,8 @@ export const getMCPHubTableColumns = ({ onServerClick }: MCPHubTableColumnsDeps) { id: "description", accessorKey: "description", - meta: { title: "Description", className: "hidden md:table-cell" }, - header: ({ column }) => , + meta: { title: i18n.t("aiHub.mcpHubColumns.description"), className: "hidden md:table-cell" }, + header: ({ column }) => , size: 240, enableSorting: true, sortingFn: "alphanumeric", @@ -120,8 +123,8 @@ export const getMCPHubTableColumns = ({ onServerClick }: MCPHubTableColumnsDeps) { id: "transport", accessorKey: "transport", - meta: { title: "Transport", skeleton: "badge", className: "hidden md:table-cell" }, - header: ({ column }) => , + meta: { title: i18n.t("aiHub.mcpHubColumns.transport"), skeleton: "badge", className: "hidden md:table-cell" }, + header: ({ column }) => , size: 110, enableSorting: true, sortingFn: "alphanumeric", @@ -134,8 +137,8 @@ export const getMCPHubTableColumns = ({ onServerClick }: MCPHubTableColumnsDeps) { id: "auth_type", accessorKey: "auth_type", - meta: { title: "Auth Type", skeleton: "badge", className: "hidden md:table-cell" }, - header: ({ column }) => , + meta: { title: i18n.t("aiHub.mcpHubColumns.authType"), skeleton: "badge", className: "hidden md:table-cell" }, + header: ({ column }) => , size: 110, enableSorting: true, sortingFn: "alphanumeric", @@ -146,23 +149,23 @@ export const getMCPHubTableColumns = ({ onServerClick }: MCPHubTableColumnsDeps) { id: "status", accessorKey: "status", - meta: { title: "Status", skeleton: "badge" }, - header: ({ column }) => , + meta: { title: i18n.t("aiHub.mcpHubColumns.status"), skeleton: "badge" }, + header: ({ column }) => , size: 110, enableSorting: true, sortingFn: "alphanumeric", cell: ({ row }) => ( ), }, { id: "allowed_tools", - meta: { title: "Tools", skeleton: "chips", className: "hidden lg:table-cell" }, - header: "Tools", + meta: { title: i18n.t("aiHub.mcpHubColumns.tools"), skeleton: "chips", className: "hidden lg:table-cell" }, + header: i18n.t("aiHub.mcpHubColumns.tools"), size: 180, enableSorting: false, cell: ({ row }) => { @@ -170,7 +173,9 @@ export const getMCPHubTableColumns = ({ onServerClick }: MCPHubTableColumnsDeps) return (
- {tools.length > 0 ? `${tools.length} tool${tools.length !== 1 ? "s" : ""}` : "All tools"} + {tools.length > 0 + ? i18n.t("aiHub.mcpHubColumns.toolCount", { count: tools.length }) + : i18n.t("aiHub.mcpHubColumns.allTools")} {tools.length > 0 && (
@@ -189,8 +194,8 @@ export const getMCPHubTableColumns = ({ onServerClick }: MCPHubTableColumnsDeps) { id: "created_by", accessorKey: "created_by", - meta: { title: "Created By", className: "hidden xl:table-cell" }, - header: ({ column }) => , + meta: { title: i18n.t("aiHub.mcpHubColumns.createdBy"), className: "hidden xl:table-cell" }, + header: ({ column }) => , size: 140, enableSorting: true, sortingFn: "alphanumeric", @@ -203,8 +208,8 @@ export const getMCPHubTableColumns = ({ onServerClick }: MCPHubTableColumnsDeps) { id: "is_public", accessorFn: (row) => row.mcp_info?.is_public === true, - meta: { title: "Hub listing", skeleton: "badge", className: "hidden md:table-cell" }, - header: ({ column }) => , + meta: { title: i18n.t("aiHub.mcpHubColumns.hubListing"), skeleton: "badge", className: "hidden md:table-cell" }, + header: ({ column }) => , size: 100, enableSorting: true, sortingFn: (rowA, rowB) => { @@ -214,13 +219,18 @@ export const getMCPHubTableColumns = ({ onServerClick }: MCPHubTableColumnsDeps) }, cell: ({ row }) => { const isPublic = row.original.mcp_info?.is_public === true; - return ; + return ( + + ); }, }, { id: "actions", meta: { className: "text-right", headerClassName: "text-right" }, - header: () => Actions, + header: () => {i18n.t("aiHub.mcpHubColumns.actions")}, size: 64, enableSorting: false, enableHiding: false, diff --git a/ui/litellm-dashboard/src/components/AIHub/ModelCustomMetadata.tsx b/ui/litellm-dashboard/src/components/AIHub/ModelCustomMetadata.tsx index be2f509b646..2278faaf88e 100644 --- a/ui/litellm-dashboard/src/components/AIHub/ModelCustomMetadata.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/ModelCustomMetadata.tsx @@ -1,3 +1,5 @@ +import { useTranslation } from "react-i18next"; + import { Badge } from "@/components/ui/badge"; export type ModelCustomMetadataValue = string | number | boolean; @@ -20,19 +22,20 @@ export function ModelCustomTags({ tags, className = "" }: { tags: string[]; clas } export function ModelCustomMetadata({ customTags, customMetadata }: ModelCustomMetadataProps) { + const { t } = useTranslation(); const tags = customTags ?? []; const entries = Object.entries(customMetadata ?? {}); return ( <> {tags.length > 0 && (
-

Tags:

+

{t("aiHub.modelCustomMetadata.tags")}

)} {entries.length > 0 && (
-

Custom Metadata:

+

{t("aiHub.modelCustomMetadata.customMetadata")}

{entries.map(([key, value]) => (
diff --git a/ui/litellm-dashboard/src/components/AIHub/ModelHubProviders.tsx b/ui/litellm-dashboard/src/components/AIHub/ModelHubProviders.tsx index aca783e56d1..d15f32ca6a8 100644 --- a/ui/litellm-dashboard/src/components/AIHub/ModelHubProviders.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/ModelHubProviders.tsx @@ -2,6 +2,7 @@ import Image from "next/image"; import { useState } from "react"; +import { useTranslation } from "react-i18next"; import { getProviderLogoAndName } from "@/components/provider_info_helpers"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; @@ -55,12 +56,13 @@ export default function ModelHubProviders({ onSelect, loading = false, }: ModelHubProvidersProps) { + const { t } = useTranslation(); const availableProviders = Array.from(new Set(providers)).filter(Boolean); const selectedProvider = selectedProviders[0] ?? ""; const initialLoading = loading && availableProviders.length === 0; - const emptyLabel = initialLoading ? "Loading providers" : "All providers"; + const emptyLabel = initialLoading ? t("aiHub.modelHubProviders.loading") : t("aiHub.modelHubProviders.allProviders"); const items = [ - { value: "", label: "All providers" }, + { value: "", label: t("aiHub.modelHubProviders.allProviders") }, ...availableProviders.map((provider) => ({ value: provider, label: getProviderLogoAndName(provider).displayName, @@ -75,7 +77,7 @@ export default function ModelHubProviders({ disabled={initialLoading} > diff --git a/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx b/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx index 59f26b72481..4d44d0bb809 100644 --- a/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx @@ -1,4 +1,5 @@ import { useTerminology } from "@/contexts/TerminologyContext"; +import i18n from "@/i18n/i18n"; import { AgentHubData, getAgentHubTableColumns } from "@/components/AIHub/AgentHubTableColumns"; import MakeAgentPublicForm from "@/components/AIHub/forms/MakeAgentPublicForm"; import MakeMCPPublicForm from "@/components/AIHub/forms/MakeMCPPublicForm"; @@ -38,6 +39,7 @@ import { InputGroup, InputGroupAddon, InputGroupButton, InputGroupInput } from " import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { Copy, Inbox, Search as SearchIcon, X } from "lucide-react"; import React, { useCallback, useEffect, useMemo, useState } from "react"; +import { Trans, useTranslation } from "react-i18next"; import { Prism as SyntaxHighlighter } from "react-syntax-highlighter"; import { prism } from "react-syntax-highlighter/dist/esm/styles/prism"; @@ -78,13 +80,19 @@ function modelTableEmptyState( ): { title: string; body: string } { if (showsDecisionGarden && selectedMode === DECISIONS_MODE && !models.some((model) => isDecisionMode(model.mode))) { return { - title: "No model on this proxy uses the decisions mode", - body: "The decision models LiteLLM supports are listed below.", + title: i18n.t("aiHub.modelHubTable.models.empty.decisionsTitle"), + body: i18n.t("aiHub.modelHubTable.models.empty.decisionsBody"), }; } return models.length > 0 - ? { title: "No matching models", body: "Adjust the filters to see more models." } - : { title: "No models yet", body: "Models added to this proxy will appear here." }; + ? { + title: i18n.t("aiHub.modelHubTable.models.empty.noMatchTitle"), + body: i18n.t("aiHub.modelHubTable.models.empty.noMatchBody"), + } + : { + title: i18n.t("aiHub.modelHubTable.models.empty.noneTitle"), + body: i18n.t("aiHub.modelHubTable.models.empty.noneBody"), + }; } function HubEmptyState({ title, body }: { title: string; body: string }) { @@ -109,6 +117,7 @@ const ModelHubTable: React.FC = ({ variant = "dashboard", }) => { const term = useTerminology(); + const { t } = useTranslation(); const syntaxTheme = useSyntaxTheme(prism); const isChat = variant === "chat"; // Admin Viewer follows the read-parity rule: see the AI Hub catalog, but @@ -459,25 +468,21 @@ const ModelHubTable: React.FC = ({ {!isChat && (
-

{term("AI Hub")}

+

{term(t("aiHub.modelHubTable.title"))}

{isAdminRole(userRole || "") ? ( -

- Make models, agents, and MCP servers public for developers to know what's available. -

+

{t("aiHub.modelHubTable.adminDescription")}

) : ( -

- A list of all public model names personally available to you. -

+

{t("aiHub.modelHubTable.userDescription")}

)}
-

Model Hub URL:

+

{t("aiHub.modelHubTable.hubUrlLabel")}

{getUiPageUrl(`${getProxyBaseUrl()}/ui/model_hub_table`)}

@@ -498,16 +503,16 @@ const ModelHubTable: React.FC = ({ {!isChat && ( - Model Hub + {t("aiHub.modelHubTable.tabs.models")} - Agent Hub + {t("aiHub.modelHubTable.tabs.agents")} - MCP Hub + {t("aiHub.modelHubTable.tabs.mcp")} - Skill Hub + {t("aiHub.modelHubTable.tabs.skills")} )} @@ -519,8 +524,11 @@ const ModelHubTable: React.FC = ({ {isChat && (
-

Models

- +

{t("aiHub.modelHubTable.models.chatTitle")}

+ {filteredData.length}
@@ -528,7 +536,9 @@ const ModelHubTable: React.FC = ({ {/* Header with Make Public Button */} {publicPage == false && canModify && (
- +
)} @@ -555,7 +565,7 @@ const ModelHubTable: React.FC = ({ sorting={modelSorting} onSortingChange={setModelSorting} isLoading={loading} - loadingMessage="Loading models…" + loadingMessage={t("aiHub.modelHubTable.models.loading")} noDataMessage={ } @@ -566,7 +576,10 @@ const ModelHubTable: React.FC = ({ {!isChat && (

- Showing {filteredData.length} of {modelHubData?.length || 0} models + {t("aiHub.modelHubTable.models.showing", { + shown: filteredData.length, + total: modelHubData?.length || 0, + })}

)} @@ -580,18 +593,20 @@ const ModelHubTable: React.FC = ({ {/* Header with Make Public Button */} {publicPage == false && canModify && (
- +
)}
-

Search Agents:

+

{t("aiHub.modelHubTable.agents.searchLabel")}

setAgentSearchTerm(e.target.value)} /> @@ -599,7 +614,7 @@ const ModelHubTable: React.FC = ({ setAgentSearchTerm("")} > @@ -619,14 +634,18 @@ const ModelHubTable: React.FC = ({ sorting={agentSorting} onSortingChange={setAgentSorting} isLoading={agentLoading} - loadingMessage="Loading agents…" + loadingMessage={t("aiHub.modelHubTable.agents.loading")} noDataMessage={ } @@ -636,7 +655,10 @@ const ModelHubTable: React.FC = ({

- Showing {filteredAgentData.length} of {agentHubData?.length || 0} agents + {t("aiHub.modelHubTable.agents.showing", { + shown: filteredAgentData.length, + total: agentHubData?.length || 0, + })}

@@ -651,7 +673,7 @@ const ModelHubTable: React.FC = ({ onClick={() => handleMakeMcpPublicPage()} disabled={isMCPHubVisibilityDisabled(mcpLoading, mcpHubData)} > - Manage MCP Hub Visibility + {t("aiHub.modelHubTable.mcp.manageVisibility")}
)} @@ -666,11 +688,11 @@ const ModelHubTable: React.FC = ({ sorting={mcpSorting} onSortingChange={setMcpSorting} isLoading={mcpLoading} - loadingMessage="Loading MCP servers…" + loadingMessage={t("aiHub.modelHubTable.mcp.loading")} noDataMessage={ } size="compact" @@ -679,7 +701,7 @@ const ModelHubTable: React.FC = ({

- Showing {mcpHubData?.length || 0} MCP server{mcpHubData?.length !== 1 ? "s" : ""} + {t("aiHub.modelHubTable.mcp.showing", { count: mcpHubData?.length || 0 })}

@@ -688,7 +710,9 @@ const ModelHubTable: React.FC = ({ {publicPage == false && canModify && (
- +
)} = ({
) : ( -

Public Model Hub not enabled.

-

- Ask your proxy admin to enable this on their Admin UI. -

+

{t("aiHub.modelHubTable.publicNotEnabled.title")}

+

{t("aiHub.modelHubTable.publicNotEnabled.body")}

 )} @@ -721,24 +743,26 @@ const ModelHubTable: React.FC = ({ className={`${isChat ? "chat-theme chat-model-hub-dialog " : ""}max-h-[calc(100dvh-2rem)] overflow-y-auto sm:max-w-[1000px]`} > - {selectedModel?.model_group || "Model Details"} + + {selectedModel?.model_group || t("aiHub.modelHubTable.modelDialog.fallbackTitle")} + {selectedModel && (
{/* Model Overview */}
-

Model Overview

+

{t("aiHub.modelHubTable.modelDialog.overview")}

-

Model Group:

+

{t("aiHub.modelHubTable.modelDialog.modelGroup")}

{selectedModel.model_group}

-

Mode:

-

{displayMode(selectedModel.mode) ?? "Not specified"}

+

{t("aiHub.modelHubTable.modelDialog.mode")}

+

{displayMode(selectedModel.mode) ?? t("aiHub.modelHubTable.modelDialog.notSpecified")}

-

Providers:

+

{t("aiHub.modelHubTable.modelDialog.providers")}

{selectedModel.providers.map((provider) => ( @@ -749,7 +773,7 @@ const ModelHubTable: React.FC = ({
{selectedModel.description && (
-

Description:

+

{t("aiHub.modelHubTable.modelDialog.description")}

{selectedModel.description}

)} @@ -762,30 +786,36 @@ const ModelHubTable: React.FC = ({ {/* Token and Cost Information */}
-

Token & Cost Information

+

{t("aiHub.modelHubTable.modelDialog.tokenCost")}

-

Max Input Tokens:

-

{selectedModel.max_input_tokens?.toLocaleString() || "Not specified"}

-
-
-

Max Output Tokens:

-

{selectedModel.max_output_tokens?.toLocaleString() || "Not specified"}

-
-
-

Input Cost per 1M Tokens:

+

{t("aiHub.modelHubTable.modelDialog.maxInputTokens")}

- {selectedModel.input_cost_per_token - ? formatCost(selectedModel.input_cost_per_token) - : "Not specified"} + {selectedModel.max_input_tokens?.toLocaleString() || + t("aiHub.modelHubTable.modelDialog.notSpecified")}

-

Output Cost per 1M Tokens:

+

{t("aiHub.modelHubTable.modelDialog.maxOutputTokens")}

+

+ {selectedModel.max_output_tokens?.toLocaleString() || + t("aiHub.modelHubTable.modelDialog.notSpecified")} +

+
+
+

{t("aiHub.modelHubTable.modelDialog.inputCost")}

+

+ {selectedModel.input_cost_per_token + ? formatCost(selectedModel.input_cost_per_token) + : t("aiHub.modelHubTable.modelDialog.notSpecified")} +

+
+
+

{t("aiHub.modelHubTable.modelDialog.outputCost")}

{selectedModel.output_cost_per_token ? formatCost(selectedModel.output_cost_per_token) - : "Not specified"} + : t("aiHub.modelHubTable.modelDialog.notSpecified")}

@@ -793,14 +823,16 @@ const ModelHubTable: React.FC = ({ {/* Capabilities */}
-

Capabilities

+

{t("aiHub.modelHubTable.modelDialog.capabilities")}

{(() => { const capabilities = getModelCapabilities(selectedModel); const colors = ["green", "blue", "purple", "orange", "red", "yellow"]; if (capabilities.length === 0) { - return

No special capabilities listed

; + return ( +

{t("aiHub.modelHubTable.modelDialog.noCapabilities")}

+ ); } return capabilities.map((capability, index) => ( @@ -815,17 +847,17 @@ const ModelHubTable: React.FC = ({ {/* Rate Limits */} {(selectedModel.tpm || selectedModel.rpm) && (
-

Rate Limits

+

{t("aiHub.modelHubTable.modelDialog.rateLimits")}

{selectedModel.tpm && (
-

Tokens per Minute:

+

{t("aiHub.modelHubTable.modelDialog.tpm")}

{selectedModel.tpm.toLocaleString()}

)} {selectedModel.rpm && (
-

Requests per Minute:

+

{t("aiHub.modelHubTable.modelDialog.rpm")}

{selectedModel.rpm.toLocaleString()}

)} @@ -836,7 +868,7 @@ const ModelHubTable: React.FC = ({ {/* Supported OpenAI Parameters */} {selectedModel.supported_openai_params && (
-

Supported OpenAI Parameters

+

{t("aiHub.modelHubTable.modelDialog.openaiParams")}

{selectedModel.supported_openai_params.map((param) => ( @@ -848,19 +880,18 @@ const ModelHubTable: React.FC = ({ )}
-

Usage Example

+

{t("aiHub.modelHubTable.modelDialog.usageExample")}

{isDecisionMode(selectedModel.mode) && (

- Decision models answer on /v1/decisions and /v1/systemone, not{" "} - /chat/completions.{" "} + }} />{" "} - How to call decision models + {t("aiHub.modelHubTable.modelDialog.howToCall")} {canOpenPlayground && ( <> {" · "} - Try it in the Playground + {t("aiHub.modelHubTable.modelDialog.tryPlayground")} )} @@ -884,28 +915,28 @@ const ModelHubTable: React.FC = ({

!open && handleCancel()}> - {selectedAgent?.name || "Agent Details"} + {selectedAgent?.name || t("aiHub.modelHubTable.agentDialog.fallbackTitle")} {selectedAgent && (
{/* Agent Overview */}
-

Agent Overview

+

{t("aiHub.modelHubTable.agentDialog.overview")}

-

Name:

+

{t("aiHub.modelHubTable.agentDialog.name")}

{selectedAgent.name}

-

Version:

+

{t("aiHub.modelHubTable.agentDialog.version")}

v{selectedAgent.version}
-

Protocol Version:

+

{t("aiHub.modelHubTable.agentDialog.protocolVersion")}

{selectedAgent.protocolVersion}

-

URL:

+

{t("aiHub.modelHubTable.agentDialog.url")}

{selectedAgent.url}

= ({
-

Description:

+

{t("aiHub.modelHubTable.agentDialog.description")}

{selectedAgent.description}

@@ -924,7 +955,7 @@ const ModelHubTable: React.FC = ({ {/* Capabilities */} {selectedAgent.capabilities && Object.keys(selectedAgent.capabilities).length > 0 && (
-

Capabilities

+

{t("aiHub.modelHubTable.agentDialog.capabilities")}

{Object.entries(selectedAgent.capabilities) .filter(([_, value]) => value === true) @@ -939,26 +970,26 @@ const ModelHubTable: React.FC = ({ {/* Input/Output Modes */}
-

Input/Output Modes

+

{t("aiHub.modelHubTable.agentDialog.ioModes")}

-

Input Modes:

+

{t("aiHub.modelHubTable.agentDialog.inputModes")}

{selectedAgent.defaultInputModes?.map((mode) => ( {mode} - )) ||

Not specified

} + )) ||

{t("aiHub.modelHubTable.agentDialog.notSpecified")}

}
-

Output Modes:

+

{t("aiHub.modelHubTable.agentDialog.outputModes")}

{selectedAgent.defaultOutputModes?.map((mode) => ( {mode} - )) ||

Not specified

} + )) ||

{t("aiHub.modelHubTable.agentDialog.notSpecified")}

}
@@ -967,14 +998,16 @@ const ModelHubTable: React.FC = ({ {/* Skills */} {selectedAgent.skills && selectedAgent.skills.length > 0 && (
-

Skills

+

{t("aiHub.modelHubTable.agentDialog.skills")}

{selectedAgent.skills.map((skill) => (

{skill.name}

-

ID: {skill.id}

+

+ {t("aiHub.modelHubTable.agentDialog.skillId", { id: skill.id })} +

{skill.tags && skill.tags.length > 0 && (
@@ -989,7 +1022,9 @@ const ModelHubTable: React.FC = ({

{skill.description}

{skill.examples && skill.examples.length > 0 && (
-

Examples:

+

+ {t("aiHub.modelHubTable.agentDialog.examples")} +

{skill.examples.map((example, idx) => ( @@ -1008,8 +1043,10 @@ const ModelHubTable: React.FC = ({ {/* Additional Properties */} {selectedAgent.supportsAuthenticatedExtendedCard && (
-

Additional Features

- Supports Authenticated Extended Card +

+ {t("aiHub.modelHubTable.agentDialog.additionalFeatures")} +

+ {t("aiHub.modelHubTable.agentDialog.extendedCard")}
)}
@@ -1021,20 +1058,22 @@ const ModelHubTable: React.FC = ({ !open && handleCancel()}> - {selectedMcpServer?.server_name || "MCP Server Details"} + + {selectedMcpServer?.server_name || t("aiHub.modelHubTable.mcpDialog.fallbackTitle")} + {selectedMcpServer && (
{/* Server Overview */}
-

Server Overview

+

{t("aiHub.modelHubTable.mcpDialog.overview")}

-

Server Name:

+

{t("aiHub.modelHubTable.mcpDialog.serverName")}

{selectedMcpServer.server_name}

-

Server ID:

+

{t("aiHub.modelHubTable.mcpDialog.serverId")}

{selectedMcpServer.server_id}

= ({
{selectedMcpServer.alias && (
-

Alias:

+

{t("aiHub.modelHubTable.mcpDialog.alias")}

{selectedMcpServer.alias}

)}
-

Transport:

+

{t("aiHub.modelHubTable.mcpDialog.transport")}

{selectedMcpServer.transport}
-

Auth Type:

+

{t("aiHub.modelHubTable.mcpDialog.authType")}

{selectedMcpServer.auth_type}
-

Status:

+

{t("aiHub.modelHubTable.mcpDialog.status")}

= ({ : "outline" } > - {selectedMcpServer.status || "unknown"} + {selectedMcpServer.status || t("aiHub.modelHubTable.mcpDialog.statusUnknown")}
{selectedMcpServer.description && (
-

Description:

+

{t("aiHub.modelHubTable.mcpDialog.description")}

{selectedMcpServer.description}

)} @@ -1084,11 +1123,11 @@ const ModelHubTable: React.FC = ({ {/* Connection Details */}
-

Connection Details

+

{t("aiHub.modelHubTable.mcpDialog.connectionDetails")}

{selectedMcpServer.command && (
-

Command:

+

{t("aiHub.modelHubTable.mcpDialog.command")}

{selectedMcpServer.command}

)} @@ -1098,7 +1137,7 @@ const ModelHubTable: React.FC = ({ {/* Tools */} {selectedMcpServer.allowed_tools && selectedMcpServer.allowed_tools.length > 0 && (
-

Allowed Tools

+

{t("aiHub.modelHubTable.mcpDialog.allowedTools")}

{selectedMcpServer.allowed_tools.map((tool, idx) => ( @@ -1112,7 +1151,7 @@ const ModelHubTable: React.FC = ({ {/* Teams */} {selectedMcpServer.teams && selectedMcpServer.teams.length > 0 && (
-

Teams

+

{t("aiHub.modelHubTable.mcpDialog.teams")}

{selectedMcpServer.teams.map((team, idx) => ( @@ -1126,7 +1165,7 @@ const ModelHubTable: React.FC = ({ {/* Access Groups */} {selectedMcpServer.mcp_access_groups && selectedMcpServer.mcp_access_groups.length > 0 && (
-

Access Groups

+

{t("aiHub.modelHubTable.mcpDialog.accessGroups")}

{selectedMcpServer.mcp_access_groups.map((group, idx) => ( @@ -1139,34 +1178,36 @@ const ModelHubTable: React.FC = ({ {/* Metadata */}
-

Metadata

+

{t("aiHub.modelHubTable.mcpDialog.metadata")}

-

Created By:

+

{t("aiHub.modelHubTable.mcpDialog.createdBy")}

{selectedMcpServer.created_by}

-

Updated By:

+

{t("aiHub.modelHubTable.mcpDialog.updatedBy")}

{selectedMcpServer.updated_by}

-

Created At:

+

{t("aiHub.modelHubTable.mcpDialog.createdAt")}

{new Date(selectedMcpServer.created_at).toLocaleString()}

-

Updated At:

+

{t("aiHub.modelHubTable.mcpDialog.updatedAt")}

{new Date(selectedMcpServer.updated_at).toLocaleString()}

{selectedMcpServer.last_health_check && (
-

Last Health Check:

+

{t("aiHub.modelHubTable.mcpDialog.lastHealthCheck")}

{new Date(selectedMcpServer.last_health_check).toLocaleString()}

)}
{selectedMcpServer.health_check_error && (
-

Health Check Error:

+

+ {t("aiHub.modelHubTable.mcpDialog.healthCheckError")} +

{selectedMcpServer.health_check_error}

)} @@ -1174,7 +1215,7 @@ const ModelHubTable: React.FC = ({ {/* Usage Example */}
-

Usage Example

+

{t("aiHub.modelHubTable.mcpDialog.usageExample")}

{`from fastmcp import Client import asyncio diff --git a/ui/litellm-dashboard/src/components/AIHub/ModelHubTableColumns.tsx b/ui/litellm-dashboard/src/components/AIHub/ModelHubTableColumns.tsx index 188bfc90263..2e4af18455f 100644 --- a/ui/litellm-dashboard/src/components/AIHub/ModelHubTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/ModelHubTableColumns.tsx @@ -2,6 +2,7 @@ import { ColumnDef } from "@tanstack/react-table"; import { Copy, Info, MoreHorizontal } from "lucide-react"; +import { useTranslation } from "react-i18next"; import { DataTableSortHeader } from "@/components/shared/DataTable"; import { IdentityCell, StatusBadge } from "@/components/shared/table_cells"; @@ -14,6 +15,7 @@ import { DropdownMenuTrigger, } from "@/components/ui/dropdown-menu"; import { cn } from "@/lib/cva.config"; +import i18n from "@/i18n/i18n"; import { displayMode } from "@/lib/decisionModels"; import { copyToClipboard } from "@/utils/dataUtils"; @@ -68,10 +70,11 @@ interface ModelHubRowActionsProps { } function ModelHubRowActions({ model, onModelClick }: ModelHubRowActionsProps) { + const { t } = useTranslation(); return ( @@ -80,14 +83,14 @@ function ModelHubRowActions({ model, onModelClick }: ModelHubRowActionsProps) { onModelClick(model)}> - View details + {t("aiHub.modelHubColumns.rowActions.viewDetails")} void copyToClipboard(model.model_group, "Model name copied")} + onClick={() => void copyToClipboard(model.model_group, t("aiHub.modelHubColumns.rowActions.nameCopied"))} > - Copy model name + {t("aiHub.modelHubColumns.rowActions.copyName")} @@ -107,9 +110,20 @@ export const getModelHubTableColumns = ({ { id: "model_group", accessorKey: "model_group", - meta: { title: showProviderLogos ? "Model" : "Public Model Name" }, + meta: { + title: showProviderLogos + ? i18n.t("aiHub.modelHubColumns.modelChat") + : i18n.t("aiHub.modelHubColumns.publicModelName"), + }, header: ({ column }) => ( - + ), size: 220, enableSorting: true, @@ -141,8 +155,8 @@ export const getModelHubTableColumns = ({ { id: "providers", accessorKey: "providers", - meta: { title: "Provider", skeleton: "chips", className: "hidden md:table-cell" }, - header: ({ column }) => , + meta: { title: i18n.t("aiHub.modelHubColumns.provider"), skeleton: "chips", className: "hidden md:table-cell" }, + header: ({ column }) => , size: 150, enableSorting: true, sortingFn: (rowA, rowB) => rowA.original.providers.join(", ").localeCompare(rowB.original.providers.join(", ")), @@ -163,8 +177,8 @@ export const getModelHubTableColumns = ({ { id: "mode", accessorFn: (row) => (row.mode ? displayMode(row.mode) : ""), - meta: { title: "Mode", className: "hidden lg:table-cell" }, - header: ({ column }) => , + meta: { title: i18n.t("aiHub.modelHubColumns.mode"), className: "hidden lg:table-cell" }, + header: ({ column }) => , size: 110, enableSorting: true, sortingFn: "alphanumeric", @@ -180,8 +194,8 @@ export const getModelHubTableColumns = ({ { id: "max_input_tokens", accessorKey: "max_input_tokens", - meta: { title: "Tokens", className: "hidden lg:table-cell", numeric: true }, - header: ({ column }) => , + meta: { title: i18n.t("aiHub.modelHubColumns.tokens"), className: "hidden lg:table-cell", numeric: true }, + header: ({ column }) => , size: 110, enableSorting: true, sortingFn: (rowA, rowB) => { @@ -202,8 +216,8 @@ export const getModelHubTableColumns = ({ { id: "input_cost_per_token", accessorKey: "input_cost_per_token", - meta: { title: "Cost/1M", skeleton: "twoLine", numeric: true }, - header: ({ column }) => , + meta: { title: i18n.t("aiHub.modelHubColumns.cost"), skeleton: "twoLine", numeric: true }, + header: ({ column }) => , size: 110, enableSorting: true, sortingFn: (rowA, rowB) => { @@ -225,8 +239,12 @@ export const getModelHubTableColumns = ({ }, { id: "capabilities", - meta: { title: "Features", skeleton: "chips", className: showProviderLogos ? "hidden md:table-cell" : undefined }, - header: "Features", + meta: { + title: i18n.t("aiHub.modelHubColumns.features"), + skeleton: "chips", + className: showProviderLogos ? "hidden md:table-cell" : undefined, + }, + header: i18n.t("aiHub.modelHubColumns.features"), size: 220, enableSorting: false, cell: ({ row }) => { @@ -247,8 +265,8 @@ export const getModelHubTableColumns = ({ }, { id: "custom_tags", - meta: { title: "Tags", skeleton: "chips" }, - header: "Tags", + meta: { title: i18n.t("aiHub.modelHubColumns.tags"), skeleton: "chips" }, + header: i18n.t("aiHub.modelHubColumns.tags"), size: 160, enableSorting: false, cell: ({ row }) => { @@ -262,8 +280,8 @@ export const getModelHubTableColumns = ({ { id: "is_public_model_group", accessorKey: "is_public_model_group", - meta: { title: "Public", skeleton: "badge", className: "hidden md:table-cell" }, - header: ({ column }) => , + meta: { title: i18n.t("aiHub.modelHubColumns.public"), skeleton: "badge", className: "hidden md:table-cell" }, + header: ({ column }) => , size: 100, enableSorting: true, sortingFn: (rowA, rowB) => { @@ -273,15 +291,15 @@ export const getModelHubTableColumns = ({ }, cell: ({ row }) => row.original.is_public_model_group === true ? ( - + ) : ( - + ), }, { id: "actions", meta: { className: "text-right", headerClassName: "text-right" }, - header: () => Actions, + header: () => {i18n.t("aiHub.modelHubColumns.actions")}, size: 64, enableSorting: false, enableHiding: false, diff --git a/ui/litellm-dashboard/src/components/AIHub/SkillHubDashboard.tsx b/ui/litellm-dashboard/src/components/AIHub/SkillHubDashboard.tsx index 9cede3b4497..4b73cafa9ff 100644 --- a/ui/litellm-dashboard/src/components/AIHub/SkillHubDashboard.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/SkillHubDashboard.tsx @@ -1,6 +1,7 @@ import React, { useMemo, useState } from "react"; import { SortingState } from "@tanstack/react-table"; import { Inbox, Search, X } from "lucide-react"; +import { useTranslation } from "react-i18next"; import { Plugin } from "@/components/claude_code_plugins/types"; import { DataTable } from "@/components/shared/DataTable"; import { getSkillHubTableColumns } from "@/components/AIHub/SkillHubTableColumns"; @@ -20,16 +21,21 @@ interface SkillHubDashboardProps { } function SkillsEmptyState({ filtered }: { filtered: boolean }) { + const { t } = useTranslation(); return (
-
{filtered ? "No matching skills" : "No skills yet"}
+
+ {filtered + ? t("aiHub.skillHubDashboard.emptyState.noMatchTitle") + : t("aiHub.skillHubDashboard.emptyState.noneTitle")} +
{filtered - ? "Adjust the search or domain filter to see more skills." - : "Skills added here will appear for developers."} + ? t("aiHub.skillHubDashboard.emptyState.noMatchBody") + : t("aiHub.skillHubDashboard.emptyState.noneBody")}
); @@ -43,6 +49,7 @@ const SkillHubDashboard: React.FC = ({ publicPage = false, onPublishSuccess, }) => { + const { t } = useTranslation(); const [search, setSearch] = useState(""); const [domainFilter, setDomainFilter] = useState(undefined); const [selectedSkill, setSelectedSkill] = useState(null); @@ -79,8 +86,11 @@ const SkillHubDashboard: React.FC = ({ const columns = useMemo(() => getSkillHubTableColumns({ onSkillClick: setSelectedSkill }), []); const domainItems = useMemo( - () => [{ value: ALL_DOMAINS, label: "All Domains" }, ...domains.map((d) => ({ value: d, label: d }))], - [domains], + () => [ + { value: ALL_DOMAINS, label: t("aiHub.skillHubDashboard.allDomains") }, + ...domains.map((d) => ({ value: d, label: d })), + ], + [domains, t], ); const hasActiveFilter = search.trim().length > 0 || domainFilter != null; @@ -102,15 +112,15 @@ const SkillHubDashboard: React.FC = ({ {/* Stats row */}
-
Total Skills
+
{t("aiHub.skillHubDashboard.totalSkills")}
{totalSkills}
-
Namespaces
+
{t("aiHub.skillHubDashboard.namespaces")}
{namespaces.length}
-
Domains
+
{t("aiHub.skillHubDashboard.domains")}
{domains.length}
@@ -118,7 +128,9 @@ const SkillHubDashboard: React.FC = ({ {/* Search + filters + table */}
-

All {publicPage ? "Public " : ""}Skills

+

+ {publicPage ? t("aiHub.skillHubDashboard.allPublicSkills") : t("aiHub.skillHubDashboard.allSkills")} +

= ({ accessTok displayName: e.target.value, }) } - placeholder="Friendly name" + placeholder={t("aiHub.usefulLinks.addForm.displayNamePlaceholder")} className="w-full px-3 py-2 border border-border rounded-md text-sm" />
- + = ({ accessTok className={`flex items-center px-4 py-2 rounded-md text-sm ${!newLink.url || !newLink.displayName ? "bg-border text-muted-foreground cursor-not-allowed" : "bg-success text-success-foreground hover:bg-success/80"}`} > - Add Link + {t("aiHub.usefulLinks.addForm.submit")}
-

Manage Existing Links

+

{t("aiHub.usefulLinks.manageExisting")}

- Public Model Hub + {t("aiHub.usefulLinks.publicHub")} {!isRearranging ? ( @@ -306,7 +308,7 @@ const UsefulLinksManagement: React.FC = ({ accessTok onClick={handleStartRearranging} className="text-xs bg-purple-50 text-purple-600 px-3 py-1.5 rounded-sm hover:bg-purple-100 flex items-center dark:bg-purple-950 dark:text-purple-300 dark:hover:bg-purple-900" > - Rearrange Order + {t("aiHub.usefulLinks.rearrange")} ) : (
@@ -314,13 +316,13 @@ const UsefulLinksManagement: React.FC = ({ accessTok onClick={handleSaveRearranging} className="text-xs bg-success text-success-foreground px-3 py-1.5 rounded-sm hover:bg-success/80" > - Save Order + {t("aiHub.usefulLinks.saveOrder")}
)} @@ -331,9 +333,9 @@ const UsefulLinksManagement: React.FC = ({ accessTok - Display Name - URL - Actions + {t("aiHub.usefulLinks.table.displayName")} + {t("aiHub.usefulLinks.table.url")} + {t("aiHub.usefulLinks.table.actions")} @@ -373,13 +375,13 @@ const UsefulLinksManagement: React.FC = ({ accessTok onClick={handleUpdateLink} className="text-xs bg-info/10 text-info px-2 py-1 rounded-sm hover:bg-info/15" > - Save + {t("aiHub.usefulLinks.save")} @@ -394,17 +396,17 @@ const UsefulLinksManagement: React.FC = ({ accessTok handleMoveUp(index)} - tooltipText="Move up" + tooltipText={t("aiHub.usefulLinks.moveUp")} disabled={index === 0} - disabledTooltipText="Already at the top" + disabledTooltipText={t("aiHub.usefulLinks.alreadyTop")} dataTestId={`move-up-${link.id}`} /> handleMoveDown(index)} - tooltipText="Move down" + tooltipText={t("aiHub.usefulLinks.moveDown")} disabled={index === links.length - 1} - disabledTooltipText="Already at the bottom" + disabledTooltipText={t("aiHub.usefulLinks.alreadyBottom")} dataTestId={`move-down-${link.id}`} /> @@ -413,19 +415,19 @@ const UsefulLinksManagement: React.FC = ({ accessTok setCurrentLink(link.url)} - tooltipText="Open link" + tooltipText={t("aiHub.usefulLinks.openLink")} dataTestId={`open-link-${link.id}`} /> handleEditLink(link)} - tooltipText="Edit link" + tooltipText={t("aiHub.usefulLinks.editLink")} dataTestId={`edit-link-${link.id}`} /> deleteLink(link.id)} - tooltipText="Delete link" + tooltipText={t("aiHub.usefulLinks.deleteLink")} dataTestId={`delete-link-${link.id}`} /> @@ -438,7 +440,7 @@ const UsefulLinksManagement: React.FC = ({ accessTok {links.length === 0 && ( - No links added yet. Add a new link above. + {t("aiHub.usefulLinks.empty")} )} diff --git a/ui/litellm-dashboard/src/components/AIHub/forms/MakeAgentPublicForm.tsx b/ui/litellm-dashboard/src/components/AIHub/forms/MakeAgentPublicForm.tsx index ef6d72f9c41..331ed40d4a6 100644 --- a/ui/litellm-dashboard/src/components/AIHub/forms/MakeAgentPublicForm.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/forms/MakeAgentPublicForm.tsx @@ -1,5 +1,7 @@ import React, { useState, useEffect } from "react"; +import type { ParseKeys } from "i18next"; import { Loader2 } from "lucide-react"; +import { Trans, useTranslation } from "react-i18next"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; import { Checkbox } from "@/components/ui/checkbox"; @@ -9,7 +11,7 @@ import { makeAgentsPublicCall } from "../../networking"; import { toast } from "@/lib/toast"; import { AgentHubData } from "@/components/AIHub/AgentHubTableColumns"; -const STEP_TITLES = ["Select Agents", "Confirm"]; +const STEP_TITLES: ParseKeys[] = ["aiHub.makeAgentPublic.steps.selectAgents", "aiHub.makeAgentPublic.steps.confirm"]; interface MakeAgentPublicFormProps { visible: boolean; @@ -26,6 +28,7 @@ const MakeAgentPublicForm: React.FC = ({ agentHubData, onSuccess, }) => { + const { t } = useTranslation(); const [currentStep, setCurrentStep] = useState(0); const [selectedAgents, setSelectedAgents] = useState>(new Set()); const [loading, setLoading] = useState(false); @@ -39,7 +42,7 @@ const MakeAgentPublicForm: React.FC = ({ const handleNext = () => { if (currentStep === 0) { if (selectedAgents.size === 0) { - toast.fromError("Please select at least one agent to make public"); + toast.fromError(t("aiHub.makeAgentPublic.selectAtLeastOne")); return; } setCurrentStep(1); @@ -85,7 +88,7 @@ const MakeAgentPublicForm: React.FC = ({ const handleSubmit = async () => { if (selectedAgents.size === 0) { - toast.fromError("Please select at least one agent to make public"); + toast.fromError(t("aiHub.makeAgentPublic.selectAtLeastOne")); return; } @@ -96,12 +99,12 @@ const MakeAgentPublicForm: React.FC = ({ // Make batch API call for all agents await makeAgentsPublicCall(accessToken, agentIdsToMakePublic); - toast.success(`Successfully made ${agentIdsToMakePublic.length} agent(s) public!`); + toast.success(t("aiHub.makeAgentPublic.success", { total: agentIdsToMakePublic.length })); handleClose(); onSuccess(); } catch (error) { console.error("Error making agents public:", error); - toast.fromError("Failed to make agents public. Please try again."); + toast.fromError(t("aiHub.makeAgentPublic.failed")); } finally { setLoading(false); } @@ -115,7 +118,7 @@ const MakeAgentPublicForm: React.FC = ({ return (
-

Select Agents to Make Public

+

{t("aiHub.makeAgentPublic.step1Title")}

-

- Select the agents you want to be visible on the public model hub. Users will still require a valid Virtual Key - to use these agents. -

+

{t("aiHub.makeAgentPublic.description")}

{agentHubData.length === 0 ? (
-

No agents available.

+

{t("aiHub.makeAgentPublic.noAgents")}

) : ( agentHubData.map((agent) => { @@ -163,7 +163,9 @@ const MakeAgentPublicForm: React.FC = ({ ))} {agent.skills.length > 3 && ( -

+{agent.skills.length - 3} more

+

+ {t("aiHub.makeAgentPublic.moreSkills", { extra: agent.skills.length - 3 })} +

)}
)} @@ -178,7 +180,11 @@ const MakeAgentPublicForm: React.FC = ({ {selectedAgents.size > 0 && (

- {selectedAgents.size} agent{selectedAgents.size !== 1 ? "s" : ""} selected + }} + />

)} @@ -189,17 +195,16 @@ const MakeAgentPublicForm: React.FC = ({ const renderStep2Content = () => { return (
-

Confirm Making Agents Public

+

{t("aiHub.makeAgentPublic.step2Title")}

- Warning: Once you make these agents public, anyone who can go to the{" "} - /ui/model_hub_table will be able to know they exist on the proxy. + , code: }} />

-

Agents to be made public:

+

{t("aiHub.makeAgentPublic.toBeMadePublic")}

{Array.from(selectedAgents).map((agentId) => { @@ -224,8 +229,11 @@ const MakeAgentPublicForm: React.FC = ({

- Total: {selectedAgents.size} agent{selectedAgents.size !== 1 ? "s" : ""} will be made - public + }} + />

@@ -247,20 +255,20 @@ const MakeAgentPublicForm: React.FC = ({ return (
{currentStep === 0 && ( )} {currentStep === 1 && ( )}
@@ -272,7 +280,7 @@ const MakeAgentPublicForm: React.FC = ({ !open && handleClose()} disablePointerDismissal> - Make Agents Public + {t("aiHub.makeAgentPublic.title")}
@@ -294,7 +302,7 @@ const MakeAgentPublicForm: React.FC = ({ {index + 1} - {title} + {t(title)} ))} diff --git a/ui/litellm-dashboard/src/components/AIHub/forms/MakeMCPPublicForm.tsx b/ui/litellm-dashboard/src/components/AIHub/forms/MakeMCPPublicForm.tsx index 2448732a236..2914610e445 100644 --- a/ui/litellm-dashboard/src/components/AIHub/forms/MakeMCPPublicForm.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/forms/MakeMCPPublicForm.tsx @@ -1,5 +1,7 @@ import React, { useState, useEffect } from "react"; +import type { ParseKeys } from "i18next"; import { Loader2 } from "lucide-react"; +import { Trans, useTranslation } from "react-i18next"; import CodeBlock from "@/components/CodeBlock"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; @@ -10,7 +12,7 @@ import { makeMCPPublicCall } from "../../networking"; import { toast } from "@/lib/toast"; import { MCPServerData } from "@/components/AIHub/MCPHubTableColumns"; -const STEP_TITLES = ["Select Servers", "Confirm"]; +const STEP_TITLES: ParseKeys[] = ["aiHub.makeMcpPublic.steps.selectServers", "aiHub.makeMcpPublic.steps.confirm"]; const statusVariant = (status?: string) => { if (status === "active" || status === "healthy") { @@ -42,6 +44,7 @@ const MakeMCPPublicForm: React.FC = ({ mcpHubData, onSuccess, }) => { + const { t } = useTranslation(); const [currentStep, setCurrentStep] = useState(0); const [selection, setSelection] = useState(null); const [loading, setLoading] = useState(false); @@ -115,7 +118,7 @@ const MakeMCPPublicForm: React.FC = ({ // Make batch API call for all servers await makeMCPPublicCall(accessToken, serverIdsToMakePublic); - toast.success("MCP Hub publication list updated"); + toast.success(t("aiHub.makeMcpPublic.updated")); handleClose(); onSuccess(); } catch (error) { @@ -134,7 +137,7 @@ const MakeMCPPublicForm: React.FC = ({ return (
-

Select MCP Servers for the Hub

+

{t("aiHub.makeMcpPublic.step1Title")}

-

- Select the complete list of MCP servers to publish on the public hub. Uncheck a server to remove it from this - list, or uncheck all to clear it. Authentication and access permissions still apply -

+

{t("aiHub.makeMcpPublic.description")}

-

- Legacy mode also lists servers with public IP access enabled. Set public_mcp_hub_strict_whitelist to true in - your configuration to use only the publication list -

+

{t("aiHub.makeMcpPublic.legacyNote")}

{mcpHubData.length === 0 ? (
-

No MCP servers available.

+

{t("aiHub.makeMcpPublic.noServers")}

) : ( mcpHubData.map((server) => { @@ -173,7 +170,7 @@ const MakeMCPPublicForm: React.FC = ({ className="flex items-center space-x-3 p-3 border rounded-lg hover:bg-accent" > handleServerSelection(server.server_id, checked === true)} /> @@ -182,11 +179,15 @@ const MakeMCPPublicForm: React.FC = ({

{server.server_name}

{isPublic && ( - {server.mcp_info?.is_public_explicit === false ? "Listed by legacy mode" : "Listed"} + {server.mcp_info?.is_public_explicit === false + ? t("aiHub.makeMcpPublic.listedLegacy") + : t("aiHub.makeMcpPublic.listed")} )} {server.transport} - {server.status || "unknown"} + + {server.status || t("aiHub.makeMcpPublic.statusUnknown")} +

{server.server_id}

@@ -200,7 +201,9 @@ const MakeMCPPublicForm: React.FC = ({ ))} {server.allowed_tools.length > 3 && ( -

+{server.allowed_tools.length - 3} more

+

+ {t("aiHub.makeMcpPublic.moreTools", { extra: server.allowed_tools.length - 3 })} +

)}
)} @@ -213,13 +216,9 @@ const MakeMCPPublicForm: React.FC = ({
- Configure in YAML + {t("aiHub.makeMcpPublic.configureYaml")}
-

- Merge these settings into your proxy configuration and reload it. Entries use the server IDs shown above, - not names or aliases. For servers defined in YAML, pin server_id in each existing mcp_servers entry so the - publication list stays stable -

+

{t("aiHub.makeMcpPublic.yamlHelp")}

@@ -227,7 +226,11 @@ const MakeMCPPublicForm: React.FC = ({ {selectedServers.size > 0 && (

- {selectedServers.size} MCP server{selectedServers.size !== 1 ? "s" : ""} selected + }} + />

)} @@ -238,20 +241,19 @@ const MakeMCPPublicForm: React.FC = ({ const renderStep2Content = () => { return (
-

Confirm MCP Hub Publication

+

{t("aiHub.makeMcpPublic.step2Title")}

- Anyone who can open /ui/model_hub_table can discover published servers. Explicitly published - server IDs also allow requests from public IPs. Authentication and access permissions still apply + }} />

-

MCP servers in the publication list:

+

{t("aiHub.makeMcpPublic.publicationList")}

- {selectedServers.size === 0 &&

No explicitly published servers

} + {selectedServers.size === 0 &&

{t("aiHub.makeMcpPublic.noPublished")}

} {Array.from(selectedServers).map((serverId) => { const server = mcpHubData.find((s) => s.server_id === serverId); return ( @@ -262,7 +264,9 @@ const MakeMCPPublicForm: React.FC = ({ {server && ( <> {server.transport} - {server.status || "unknown"} + + {server.status || t("aiHub.makeMcpPublic.statusUnknown")} + )}
@@ -280,8 +284,11 @@ const MakeMCPPublicForm: React.FC = ({

- Saving replaces the publication list with {selectedServers.size} MCP server - {selectedServers.size !== 1 ? "s" : ""}. Legacy mode may still list servers with public IP access enabled + }} + />

@@ -292,12 +299,11 @@ const MakeMCPPublicForm: React.FC = ({ if (!hasPublicationMetadata) { return (
- This proxy does not provide explicit publication status for every MCP server. Update the proxy to manage - visibility here, or edit litellm_settings.public_mcp_servers in its existing configuration + {t("aiHub.makeMcpPublic.noMetadata")}
); } - if (!canManagePublication) return

Loading publication settings

; + if (!canManagePublication) return

{t("aiHub.makeMcpPublic.loadingSettings")}

; switch (currentStep) { case 0: return renderStep1Content(); @@ -312,20 +318,20 @@ const MakeMCPPublicForm: React.FC = ({ return (
{currentStep === 0 && ( )} {currentStep === 1 && ( )}
@@ -337,7 +343,7 @@ const MakeMCPPublicForm: React.FC = ({ !open && handleClose()} disablePointerDismissal> - Manage MCP Hub Visibility + {t("aiHub.makeMcpPublic.title")}
@@ -359,7 +365,7 @@ const MakeMCPPublicForm: React.FC = ({ {index + 1} - {title} + {t(title)} ))} diff --git a/ui/litellm-dashboard/src/components/AIHub/forms/MakeModelPublicForm.tsx b/ui/litellm-dashboard/src/components/AIHub/forms/MakeModelPublicForm.tsx index 20fa0295fed..1cc5d016082 100644 --- a/ui/litellm-dashboard/src/components/AIHub/forms/MakeModelPublicForm.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/forms/MakeModelPublicForm.tsx @@ -1,5 +1,7 @@ import React, { useState, useCallback, useEffect } from "react"; +import type { ParseKeys } from "i18next"; import { Loader2 } from "lucide-react"; +import { Trans, useTranslation } from "react-i18next"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; import { Checkbox } from "@/components/ui/checkbox"; @@ -10,7 +12,7 @@ import ModelFilters from "../../model_filters"; import { toast } from "@/lib/toast"; import { displayMode } from "@/lib/decisionModels"; -const STEP_TITLES = ["Select Models", "Confirm"]; +const STEP_TITLES: ParseKeys[] = ["aiHub.makeModelPublic.steps.selectModels", "aiHub.makeModelPublic.steps.confirm"]; interface ModelGroupInfo { model_group: string; @@ -45,6 +47,7 @@ const MakeModelPublicForm: React.FC = ({ modelHubData, onSuccess, }) => { + const { t } = useTranslation(); const [currentStep, setCurrentStep] = useState(0); const [selectedModels, setSelectedModels] = useState>(new Set()); const [filteredData, setFilteredData] = useState([]); @@ -60,7 +63,7 @@ const MakeModelPublicForm: React.FC = ({ const handleNext = () => { if (currentStep === 0) { if (selectedModels.size === 0) { - toast.fromError("Please select at least one model to make public"); + toast.fromError(t("aiHub.makeModelPublic.selectAtLeastOne")); return; } setCurrentStep(1); @@ -113,7 +116,7 @@ const MakeModelPublicForm: React.FC = ({ const handleSubmit = async () => { if (selectedModels.size === 0) { - toast.fromError("Please select at least one model to make public"); + toast.fromError(t("aiHub.makeModelPublic.selectAtLeastOne")); return; } @@ -122,12 +125,12 @@ const MakeModelPublicForm: React.FC = ({ const modelGroupsToMakePublic = Array.from(selectedModels); await makeModelGroupPublic(accessToken, modelGroupsToMakePublic); - toast.success(`Successfully made ${modelGroupsToMakePublic.length} model group(s) public!`); + toast.success(t("aiHub.makeModelPublic.success", { total: modelGroupsToMakePublic.length })); handleClose(); onSuccess(); } catch (error) { console.error("Error making model groups public:", error); - toast.fromError("Failed to make model groups public. Please try again."); + toast.fromError(t("aiHub.makeModelPublic.failed")); } finally { setLoading(false); } @@ -141,7 +144,7 @@ const MakeModelPublicForm: React.FC = ({ return (
-

Select Models to Make Public

+

{t("aiHub.makeModelPublic.step1Title")}

-

- Select the models you want to be visible on the public model hub. Users will still require a valid Virtual Key - to use these models. -

+

{t("aiHub.makeModelPublic.description")}

{/* Filters */} = ({
{filteredData.length === 0 ? (
-

No models match the current filters.

+

{t("aiHub.makeModelPublic.noMatch")}

) : ( filteredData.map((model) => ( @@ -206,7 +206,11 @@ const MakeModelPublicForm: React.FC = ({ {selectedModels.size > 0 && (

- {selectedModels.size} model{selectedModels.size !== 1 ? "s" : ""} selected + }} + />

)} @@ -217,17 +221,16 @@ const MakeModelPublicForm: React.FC = ({ const renderStep2Content = () => { return (
-

Confirm Making Models Public

+

{t("aiHub.makeModelPublic.step2Title")}

- Warning: Once you make these models public, anyone who can go to the{" "} - /ui/model_hub_table will be able to know they exist on the proxy. + , code: }} />

-

Models to be made public:

+

{t("aiHub.makeModelPublic.toBeMadePublic")}

{Array.from(selectedModels).map((modelGroup) => { @@ -255,8 +258,11 @@ const MakeModelPublicForm: React.FC = ({

- Total: {selectedModels.size} model{selectedModels.size !== 1 ? "s" : ""} will be made - public + }} + />

@@ -278,20 +284,20 @@ const MakeModelPublicForm: React.FC = ({ return (
{currentStep === 0 && ( )} {currentStep === 1 && ( )}
@@ -303,7 +309,7 @@ const MakeModelPublicForm: React.FC = ({ !open && handleClose()} disablePointerDismissal> - Make Models Public + {t("aiHub.makeModelPublic.title")}
@@ -325,7 +331,7 @@ const MakeModelPublicForm: React.FC = ({ {index + 1} - {title} + {t(title)} ))} diff --git a/ui/litellm-dashboard/src/components/CodeBlock.tsx b/ui/litellm-dashboard/src/components/CodeBlock.tsx index a39d7417302..c40ba19e380 100644 --- a/ui/litellm-dashboard/src/components/CodeBlock.tsx +++ b/ui/litellm-dashboard/src/components/CodeBlock.tsx @@ -1,5 +1,6 @@ import { useState } from "react"; import { CheckIcon, ClipboardIcon } from "lucide-react"; +import { useTranslation } from "react-i18next"; import { Prism as SyntaxHighlighter } from "react-syntax-highlighter"; import { oneLight } from "react-syntax-highlighter/dist/esm/styles/prism"; @@ -11,6 +12,7 @@ interface CodeBlockProps { } const CodeBlock = ({ code, language }: CodeBlockProps) => { + const { t } = useTranslation(); const syntaxTheme = useSyntaxTheme(oneLight); const [copied, setCopied] = useState(false); const copyToClipboard = () => { @@ -24,7 +26,7 @@ const CodeBlock = ({ code, language }: CodeBlockProps) => { diff --git a/ui/litellm-dashboard/src/components/CommandPalette/CommandPalette.tsx b/ui/litellm-dashboard/src/components/CommandPalette/CommandPalette.tsx index 8b3d2a02837..9323cce85af 100644 --- a/ui/litellm-dashboard/src/components/CommandPalette/CommandPalette.tsx +++ b/ui/litellm-dashboard/src/components/CommandPalette/CommandPalette.tsx @@ -5,6 +5,7 @@ import { Command as CommandPrimitive } from "cmdk"; import { useDebouncedValue } from "@tanstack/react-pacer/debouncer"; import { FileText, KeyRound, LoaderCircle, Search } from "lucide-react"; import { usePathname, useRouter } from "next/navigation"; +import { useTranslation } from "react-i18next"; import { useKeys } from "@/app/(dashboard)/hooks/keys/useKeys"; import { useTeams } from "@/app/(dashboard)/hooks/teams/useTeams"; import { Badge } from "@/components/ui/badge"; @@ -20,6 +21,7 @@ import { } from "@/components/ui/command"; import { InputGroupAddon } from "@/components/ui/input-group"; import { useTerminology } from "@/contexts/TerminologyContext"; +import i18n from "@/i18n/i18n"; import { DEBOUNCE_WAIT_MS } from "@/utils/debounceConstants"; import { keyDetailHref } from "@/utils/entityLinks"; import { routeSegmentForPathname, uiHref } from "@/utils/uiHref"; @@ -32,7 +34,7 @@ type PaletteEntry = { title: string; subtitle?: ReactNode; icon: ReactNode; - typeLabel: "Key" | "Page" | "Action"; + typeLabel: "shell.commandPalette.types.key" | "shell.commandPalette.types.page" | "shell.commandPalette.types.action"; } & ( | { kind: "key"; key: KeyResponse } | { kind: "filter" } @@ -46,9 +48,9 @@ interface PaletteGroup { } const emptyMessageFor = (scope: PaletteScope, query: string, hasKeyError: boolean): string => { - if (hasKeyError) return "Unable to search keys"; - if (scope === "keys" && !query) return "No recent keys"; - return `No results for “${query}”`; + if (hasKeyError) return i18n.t("shell.commandPalette.empty.keyError"); + if (scope === "keys" && !query) return i18n.t("shell.commandPalette.empty.noRecentKeys"); + return i18n.t("shell.commandPalette.empty.noResults", { query }); }; // TODO(https://github.com/dip/cmdk/issues/413): remove once cmdk keeps aria-activedescendant on the selected row @@ -75,12 +77,13 @@ function useActiveDescendantOnSelectedRow(inputRef: RefObject void }) { + const { t } = useTranslation(); return ( @@ -88,6 +91,7 @@ export function CommandPalette({ open, setOpen }: { open: boolean; setOpen: (ope } function CommandPaletteContent({ setOpen }: { setOpen: (open: boolean) => void }) { + const { t } = useTranslation(); const pathname = usePathname(); const router = useRouter(); const inputRef = useRef(null); @@ -130,22 +134,27 @@ function CommandPaletteContent({ setOpen }: { setOpen: (open: boolean) => void } title: page.label, subtitle: page.section, icon: page.icon ?? , - typeLabel: "Page", + typeLabel: "shell.commandPalette.types.page", page, })); - const pagesGroups = pages.length > 0 ? [{ heading: "Pages", entries: pages }] : []; + const pagesGroups = pages.length > 0 ? [{ heading: t("shell.commandPalette.groups.pages"), entries: pages }] : []; if (scope === "global") { const showSearchKeysAction = !trimmedQuery || "search virtual keys".includes(trimmedQuery.toLowerCase()); const searchKeysAction: PaletteEntry = { id: "action:search-keys", kind: "search-keys", - title: "Search virtual keys", - subtitle: "Find a key by alias or ID", + title: t("shell.commandPalette.searchKeys.title"), + subtitle: t("shell.commandPalette.searchKeys.subtitle"), icon: , - typeLabel: "Action", + typeLabel: "shell.commandPalette.types.action", }; - return [...(showSearchKeysAction ? [{ heading: "Actions", entries: [searchKeysAction] }] : []), ...pagesGroups]; + return [ + ...(showSearchKeysAction + ? [{ heading: t("shell.commandPalette.groups.actions"), entries: [searchKeysAction] }] + : []), + ...pagesGroups, + ]; } const keyEntries: PaletteEntry[] = (isLoadingKeys ? [] : keyResults.data?.keys ?? []).map((key) => { @@ -155,7 +164,7 @@ function CommandPaletteContent({ setOpen }: { setOpen: (open: boolean) => void } return { id: `key:${key.token}`, kind: "key", - title: key.key_alias || "Unnamed key", + title: key.key_alias || t("shell.commandPalette.unnamedKey"), subtitle: ( <> {key.key_name} @@ -163,25 +172,38 @@ function CommandPaletteContent({ setOpen }: { setOpen: (open: boolean) => void } ), icon: , - typeLabel: "Key", + typeLabel: "shell.commandPalette.types.key", key, }; }); const keysGroups = - keyEntries.length > 0 ? [{ heading: trimmedQuery ? "Keys" : "Recent keys", entries: keyEntries }] : []; + keyEntries.length > 0 + ? [ + { + heading: trimmedQuery + ? t("shell.commandPalette.groups.keys") + : t("shell.commandPalette.groups.recentKeys"), + entries: keyEntries, + }, + ] + : []; if (!trimmedQuery) return keysGroups; const filterAction: PaletteEntry = { id: "action:filter-keys", kind: "filter", - title: `Filter the keys table for “${trimmedQuery}”`, - subtitle: "Open Virtual Keys with this search applied", + title: t("shell.commandPalette.filterKeys.title", { query: trimmedQuery }), + subtitle: t("shell.commandPalette.filterKeys.subtitle"), icon: , - typeLabel: "Action", + typeLabel: "shell.commandPalette.types.action", }; - return [...keysGroups, { heading: "Actions", entries: [filterAction] }, ...pagesGroups]; - }, [hasKeyError, isLoadingKeys, keyResults.data?.keys, navMatches, scope, teamAliases, trimmedQuery]); + return [ + ...keysGroups, + { heading: t("shell.commandPalette.groups.actions"), entries: [filterAction] }, + ...pagesGroups, + ]; + }, [hasKeyError, isLoadingKeys, keyResults.data?.keys, navMatches, scope, t, teamAliases, trimmedQuery]); const resultSet = [scope, query, isLoadingKeys].join("\n"); const [highlight, setHighlight] = useState<{ value: string; resultSet: string } | null>(null); @@ -216,7 +238,7 @@ function CommandPaletteContent({ setOpen }: { setOpen: (open: boolean) => void } return ( void } - Virtual Keys + {t("shell.commandPalette.virtualKeys")} )}
event.preventDefault()}> {isLoadingKeys && !hasKeyError ? ( - + - Searching… + {t("shell.commandPalette.searching")} ) : ( @@ -259,7 +286,7 @@ function CommandPaletteContent({ setOpen }: { setOpen: (open: boolean) => void } {entry.title} {entry.subtitle && {entry.subtitle}} - {entry.typeLabel} + {t(entry.typeLabel)} ))} diff --git a/ui/litellm-dashboard/src/components/CommandPalette/CommandPaletteHint.tsx b/ui/litellm-dashboard/src/components/CommandPalette/CommandPaletteHint.tsx index 6fa362f4509..f7c75571344 100644 --- a/ui/litellm-dashboard/src/components/CommandPalette/CommandPaletteHint.tsx +++ b/ui/litellm-dashboard/src/components/CommandPalette/CommandPaletteHint.tsx @@ -1,6 +1,7 @@ "use client"; import { Search, X } from "lucide-react"; +import { Trans, useTranslation } from "react-i18next"; import { z } from "zod"; import { Kbd } from "@/components/ui/kbd"; import { storageKey } from "@/lib/storage"; @@ -16,6 +17,7 @@ interface CommandPaletteHintProps { } export function CommandPaletteHint({ scope, onDismiss }: CommandPaletteHintProps) { + const { t } = useTranslation(); const { setOpen } = useCommandPalette(); const shortcut = useShortcutLabel(); const mounted = useClientMounted(); @@ -26,14 +28,15 @@ export function CommandPaletteHint({ scope, onDismiss }: CommandPaletteHintProps
); }; @@ -128,10 +131,14 @@ const ItemAction = ({ }; const IgnoreMeta = ({ ignore }: { ignore: IgnoreState }) => { + const { t } = useTranslation(); if (ignore.source === "env_var") { return ( - By env var {ignore.env_var} + {ignore.env_var} }} + /> ); } @@ -140,28 +147,34 @@ const IgnoreMeta = ({ ignore }: { ignore: IgnoreState }) => { - Until {formatLocalMinute(ignore.ignored_until)} + {t("shell.deploymentHealthPanel.ignoreMeta.until", { time: formatLocalMinute(ignore.ignored_until) })} {" "} - · by {ignore.ignored_by} + {t("shell.deploymentHealthPanel.ignoreMeta.byUser", { user: ignore.ignored_by })} ); } return ( - Since {formatLocalDay(ignore.ignored_at)} · by {ignore.ignored_by} + {t("shell.deploymentHealthPanel.ignoreMeta.since", { + date: formatLocalDay(ignore.ignored_at), + user: ignore.ignored_by, + })} ); }; const IgnoreStatus = ({ ignore }: { ignore: IgnoreState }) => { + const { t } = useTranslation(); const snoozed = ignore.source === "admin" && Boolean(ignore.ignored_until); const Icon = snoozed ? Clock : BellOff; return (
- {snoozed ? "Snoozed" : "Ignored"} + {snoozed + ? t("shell.deploymentHealthPanel.ignoreStatus.snoozed") + : t("shell.deploymentHealthPanel.ignoreStatus.ignored")}
@@ -169,6 +182,7 @@ const IgnoreStatus = ({ ignore }: { ignore: IgnoreState }) => { }; const WarningItem = ({ warning, ...permissions }: ItemPermissions & { warning: DeploymentWarning }) => { + const { t } = useTranslation(); const content = deploymentWarningContent(warning.id); const critical = warning.severity === "critical"; const SeverityIcon = critical ? CircleAlert : TriangleAlert; @@ -196,7 +210,7 @@ const WarningItem = ({ warning, ...permissions }: ItemPermissions & { warning: D "h-auto gap-1 p-0 text-[13px] leading-5 font-medium text-foreground no-underline hover:underline", )} > - Read the docs + {t("shell.deploymentHealthPanel.item.docs")} ) : null} @@ -251,36 +265,42 @@ const PanelSkeleton = () => (
); -const NoActiveWarnings = ({ hasIgnored }: { hasIgnored: boolean }) => ( - - - - - - No active warnings - - {hasIgnored - ? "Snoozed and ignored warnings are listed below" - : "Deployment warnings show up here when this proxy\u2019s setup needs attention"} - - - -); +const NoActiveWarnings = ({ hasIgnored }: { hasIgnored: boolean }) => { + const { t } = useTranslation(); + return ( + + + + + + + {t("shell.deploymentHealthPanel.empty.title")} + + + {hasIgnored + ? t("shell.deploymentHealthPanel.empty.ignoredBelow") + : t("shell.deploymentHealthPanel.empty.description")} + + + + ); +}; const PanelBody = () => { + const { t } = useTranslation(); const { canManage } = useDeploymentHealthAccess(); const { data, isError } = useDeploymentWarnings({ refetchOnOpen: true }); if (!data) { return (
{isError ? ( -

Could not load deployment warnings

+

{t("shell.deploymentHealthPanel.body.loadError")}

) : ( <> - Loading deployment warnings + {t("shell.deploymentHealthPanel.body.loading")} )} @@ -298,24 +318,36 @@ const PanelBody = () => { {SEVERITY_GROUPS.map(({ severity, label }) => { const group = active.filter((warning) => warning.severity === severity); return group.length === 0 ? null : ( - + ); })} {ignored.length > 0 ? ( - + ) : null}
); }; -export const DeploymentHealthPanel = ({ handle }: { handle: DeploymentHealthHandle }) => ( - - - - Deployment health - Warnings about how this proxy is deployed - - - - -); +export const DeploymentHealthPanel = ({ handle }: { handle: DeploymentHealthHandle }) => { + const { t } = useTranslation(); + return ( + + + + + {t("shell.deploymentHealthPanel.sheet.title")} + + + {t("shell.deploymentHealthPanel.sheet.description")} + + + + + + ); +}; diff --git a/ui/litellm-dashboard/src/components/DeploymentHealth/DeploymentWarningContent.tsx b/ui/litellm-dashboard/src/components/DeploymentHealth/DeploymentWarningContent.tsx index fa1b979e2fd..da2f1b680dc 100644 --- a/ui/litellm-dashboard/src/components/DeploymentHealth/DeploymentWarningContent.tsx +++ b/ui/litellm-dashboard/src/components/DeploymentHealth/DeploymentWarningContent.tsx @@ -1,4 +1,6 @@ import type { ReactNode } from "react"; +import { Trans } from "react-i18next"; +import i18n from "@/i18n/i18n"; import { formatExpiryDate, getDaysUntilExpiration } from "@/utils/licenseUtils"; const LICENSE_EXPIRY_PREFIX = "license_expiry:"; @@ -14,41 +16,38 @@ const STATIC_CONTENT: Readonly> = { env_credential_login: { title: "Environment-credential login is enabled", fix: ( - <> - Anyone with UI_USERNAME/UI_PASSWORD (or the master key, when UI_PASSWORD{" "} - is unset) can sign in as a proxy admin with a shared static secret. Create a regular admin account with its own - password, then set general_settings.disable_env_credential_login: true - + UI_USERNAME, + password: UI_PASSWORD, + setting: general_settings.disable_env_credential_login: true, + }} + /> ), docsUrl: "https://docs.litellm.ai/docs/proxy/ui", }, no_redis: { title: "No Redis configured", - fix: ( - <> - This proxy runs more than one worker (or the worker count could not be verified). Without Redis, rate limits, - budgets, router state and cache invalidation are per worker, so limits are enforced once per worker and spend - can overshoot - - ), + fix: , docsUrl: "https://docs.litellm.ai/docs/proxy/redis_requirements", }, detailed_debug: { title: "Detailed debug logging is on", fix: ( - <> - LITELLM_LOG=DEBUG logs extensive diagnostic information and significantly slows the proxy. Use it - only while troubleshooting and turn it off in production - + LITELLM_LOG=DEBUG }} + /> ), docsUrl: "https://docs.litellm.ai/docs/proxy/prod", }, }; const describeCountdown = (days: number): string => { - if (days <= 0) return "expires today"; - if (days === 1) return "expires in 1 day"; - return `expires in ${days} days`; + if (days <= 0) return i18n.t("shell.deploymentWarnings.license.expiresToday"); + if (days === 1) return i18n.t("shell.deploymentWarnings.license.expiresIn", { count: 1 }); + return i18n.t("shell.deploymentWarnings.license.expiresIn", { count: days }); }; const licenseExpiryContent = (expirationDate: string, now: Date): DeploymentWarningContent => { @@ -57,13 +56,16 @@ const licenseExpiryContent = (expirationDate: string, now: Date): DeploymentWarn const salesLink = {SALES_EMAIL}; return days < 0 ? { - title: `Your LiteLLM Enterprise license expired on ${formattedDate}`, - fix: <>Enterprise features are now disabled. Reach out to {salesLink} to restore access, + title: i18n.t("shell.deploymentWarnings.license.expiredTitle", { date: formattedDate }), + fix: , docsUrl: "https://docs.litellm.ai/docs/enterprise", } : { - title: `Your LiteLLM Enterprise license ${describeCountdown(days)} (${formattedDate})`, - fix: <>Renew before it lapses to keep enterprise features. Reach out to {salesLink}, + title: i18n.t("shell.deploymentWarnings.license.expiringTitle", { + countdown: describeCountdown(days), + date: formattedDate, + }), + fix: , docsUrl: "https://docs.litellm.ai/docs/enterprise", }; }; diff --git a/ui/litellm-dashboard/src/components/DeprecationBanner.tsx b/ui/litellm-dashboard/src/components/DeprecationBanner.tsx index 75541e4087c..3307db65a7b 100644 --- a/ui/litellm-dashboard/src/components/DeprecationBanner.tsx +++ b/ui/litellm-dashboard/src/components/DeprecationBanner.tsx @@ -1,6 +1,7 @@ "use client"; import React from "react"; +import { Trans, useTranslation } from "react-i18next"; import Link from "next/link"; import { Info } from "lucide-react"; import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; @@ -11,16 +12,21 @@ interface DeprecationBannerProps { featureName: string; } -export const DeprecationBanner: React.FC = ({ featureName }) => ( - - - {`${featureName} is deprecated`} - - {`${featureName} will be removed in the near future. See the `} - - deprecation discussion - - {" for details."} - - -); +export const DeprecationBanner: React.FC = ({ featureName }) => { + const { t } = useTranslation(); + return ( + + + {t("shell.deprecationBanner.title", { featureName })} + + {featureName}, + discussion: , + }} + /> + + + ); +}; diff --git a/ui/litellm-dashboard/src/components/Navbar/BlogDropdown/BlogDropdown.tsx b/ui/litellm-dashboard/src/components/Navbar/BlogDropdown/BlogDropdown.tsx index eaaf3301a8b..a98b35e7937 100644 --- a/ui/litellm-dashboard/src/components/Navbar/BlogDropdown/BlogDropdown.tsx +++ b/ui/litellm-dashboard/src/components/Navbar/BlogDropdown/BlogDropdown.tsx @@ -11,6 +11,7 @@ import { } from "@/components/ui/dropdown-menu"; import { ChevronDown, LoaderCircle } from "lucide-react"; import React from "react"; +import { useTranslation } from "react-i18next"; function formatDate(dateStr: string): string { const date = new Date(dateStr + "T00:00:00"); @@ -22,6 +23,7 @@ function formatDate(dateStr: string): string { } export const BlogDropdown: React.FC = () => { + const { t } = useTranslation(); const disableBlogPosts = useDisableBlogPosts(); const { data, isLoading, isError, refetch } = useBlogPosts(); @@ -34,7 +36,7 @@ export const BlogDropdown: React.FC = () => { if (isLoading) { return (
- +
); } @@ -42,16 +44,16 @@ export const BlogDropdown: React.FC = () => { if (isError) { return (
- Failed to load posts + {t("shell.blogDropdown.failed")}
); } if (!data || data.posts.length === 0) { - return
No posts available
; + return
{t("shell.blogDropdown.empty")}
; } return ( @@ -72,7 +74,7 @@ export const BlogDropdown: React.FC = () => { - View all posts + {t("shell.blogDropdown.viewAll")} @@ -87,7 +89,7 @@ export const BlogDropdown: React.FC = () => { closeDelay={100} render={ )} {data?.posts.slice(0, 3).map((post) => ( @@ -91,7 +111,7 @@ export function DashboardResources({ children }: { children?: ReactNode }) { )}
-
@@ -109,6 +112,7 @@ export const NotificationsBellView: React.FC = ({ latestRelease, deploymentHealthLink, }) => { + const { t } = useTranslation(); const upgrade = upgradeAnnouncement(currentVersion, latestRelease); const announcements = upgrade ? [upgrade, AUTO_ROUTER_ANNOUNCEMENT] : [AUTO_ROUTER_ANNOUNCEMENT]; const unreadCount = useUnreadAnnouncementCount(announcements.map((announcement) => announcement.readKey)); @@ -117,7 +121,11 @@ export const NotificationsBellView: React.FC = ({ } - aria-label={unreadCount > 0 ? `Notifications, ${unreadCount} unread` : "Notifications"} + aria-label={ + unreadCount > 0 + ? t("shell.notifications.bell.triggerUnread", { unread: unreadCount }) + : t("shell.notifications.bell.trigger") + } > @@ -126,10 +134,12 @@ export const NotificationsBellView: React.FC = ({ - What's new + + {t("shell.notifications.bell.title")} +
{announcements.map((announcement, index) => ( diff --git a/ui/litellm-dashboard/src/components/Navbar/UserDropdown/UserDropdown.tsx b/ui/litellm-dashboard/src/components/Navbar/UserDropdown/UserDropdown.tsx index 69117073ced..48d8a1102cb 100644 --- a/ui/litellm-dashboard/src/components/Navbar/UserDropdown/UserDropdown.tsx +++ b/ui/litellm-dashboard/src/components/Navbar/UserDropdown/UserDropdown.tsx @@ -24,6 +24,7 @@ import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/comp import CopyButton from "@/components/shared/CopyButton"; import { cn } from "@/lib/cva.config"; import React, { useEffect, useState } from "react"; +import { useTranslation } from "react-i18next"; function hueFromString(seed: string): number { let h = 0; @@ -68,6 +69,7 @@ interface UserDropdownProps { } const UserDropdown: React.FC = ({ onLogout, variant = "navbar", collapsed = false }) => { + const { t } = useTranslation(); const { userId, userEmail, @@ -101,16 +103,16 @@ const UserDropdown: React.FC = ({ onLogout, variant = "navbar {premiumUser ? ( - Premium + {t("shell.userMenu.premium")} ) : ( }> - Standard + {t("shell.userMenu.standard")} - Upgrade to Premium for advanced features + {t("shell.userMenu.upgradeHint")} )} @@ -119,25 +121,25 @@ const UserDropdown: React.FC = ({ onLogout, variant = "navbar
- User ID + {t("shell.userMenu.userId")}
{userId || "-"} - +
- Role + {t("shell.userMenu.role")}
{userRole}
- Hide New Feature Indicators + {t("shell.userMenu.hideNewBadges")} = ({ onLogout, variant = "navbar emitLocalStorageChange("disableShowNewBadge"); } }} - aria-label="Toggle hide new feature indicators" + aria-label={t("shell.userMenu.toggleHideNewBadges")} />
- Hide All Prompts + {t("shell.userMenu.hidePrompts")} = ({ onLogout, variant = "navbar emitLocalStorageChange("disableShowPrompts"); } }} - aria-label="Toggle hide all prompts" + aria-label={t("shell.userMenu.toggleHidePrompts")} />
- Hide Blog Posts + {t("shell.userMenu.hideBlogPosts")} = ({ onLogout, variant = "navbar emitLocalStorageChange("disableBlogPosts"); } }} - aria-label="Toggle hide blog posts" + aria-label={t("shell.userMenu.toggleHideBlogPosts")} />
- Hide Bouncing Icon + {t("shell.userMenu.hideBouncingIcon")} = ({ onLogout, variant = "navbar emitLocalStorageChange("disableBouncingIcon"); } }} - aria-label="Toggle hide bouncing icon" + aria-label={t("shell.userMenu.toggleHideBouncingIcon")} />
{premiumUser === true && canUseLiteAdmin && (
- Hide {ARIA_AGENT_LABEL} + {t("shell.userMenu.hideAgent", { agent: ARIA_AGENT_LABEL })}
)} @@ -235,7 +237,10 @@ const UserDropdown: React.FC = ({ onLogout, variant = "navbar "flex w-full items-center rounded-lg border border-transparent transition-colors hover:bg-sidebar-accent", collapsed ? "justify-center px-0 py-1" : "gap-2.5 px-2 py-1.5 text-left", )} - aria-label={`Account menu — ${userRole ?? "Unknown role"} — signed in as ${userEmail || userId || "unknown"}`} + aria-label={t("shell.userMenu.triggerAria", { + role: userRole ?? t("shell.userMenu.unknownRole"), + identity: userEmail || userId || t("shell.userMenu.unknownIdentity"), + })} aria-haspopup="dialog" title={collapsed ? displayName : undefined} /> @@ -262,7 +267,10 @@ const UserDropdown: React.FC = ({ onLogout, variant = "navbar )} diff --git a/ui/litellm-dashboard/src/components/Navbar/ViewSwitcher.tsx b/ui/litellm-dashboard/src/components/Navbar/ViewSwitcher.tsx index aa860226e3b..4d13522aee5 100644 --- a/ui/litellm-dashboard/src/components/Navbar/ViewSwitcher.tsx +++ b/ui/litellm-dashboard/src/components/Navbar/ViewSwitcher.tsx @@ -1,5 +1,6 @@ import { useTerminology } from "@/contexts/TerminologyContext"; import React from "react"; +import { useTranslation } from "react-i18next"; import { usePathname } from "next/navigation"; import { Button } from "@/components/ui/button"; import { @@ -13,6 +14,7 @@ import { usePluginMode } from "@/contexts/PluginModeContext"; import { useUISettings } from "@/app/(dashboard)/hooks/uiSettings/useUISettings"; import { uiHref } from "@/utils/uiHref"; import { cn } from "@/lib/cva.config"; +import i18n from "@/i18n/i18n"; const GATEWAY = "ai-gateway"; const CHAT = "chat"; @@ -30,7 +32,7 @@ function activeWorkspaceLabel( pluginLabel: string | undefined, gatewayLabel: string, ): string { - if (isChatRoute) return "Chat"; + if (isChatRoute) return i18n.t("shell.viewSwitcher.chat"); if (isMoyaiRoute) return "Moyai"; return pluginLabel ?? gatewayLabel; } @@ -48,6 +50,7 @@ interface ViewSwitcherProps { } export default function ViewSwitcher({ className, contentClassName }: ViewSwitcherProps) { + const { t } = useTranslation(); const term = useTerminology(); const { mode, setMode, plugins } = usePluginMode(); const { data: uiSettings } = useUISettings(); @@ -61,7 +64,7 @@ export default function ViewSwitcher({ className, contentClassName }: ViewSwitch const isMoyaiRoute = isWorkspaceRoute(normalizedPathname, MOYAI); const isStandaloneRoute = isChatRoute || isMoyaiRoute; - const gatewayLabel = term("AI Gateway"); + const gatewayLabel = term(t("shell.viewSwitcher.aiGateway")); const activeLabel = activeWorkspaceLabel( isChatRoute, isMoyaiRoute, @@ -87,11 +90,11 @@ export default function ViewSwitcher({ className, contentClassName }: ViewSwitch label: (
- Chat{" "} - Chat UI for using LLMs, MCPs, Agents{" "} + {t("shell.viewSwitcher.chat")}{" "} + {t("shell.viewSwitcher.chatDescription")}{" "} {!chatEnabled && ( - Admins can enable in Settings + {t("shell.viewSwitcher.chatDisabled")} )} @@ -119,7 +122,9 @@ export default function ViewSwitcher({ className, contentClassName }: ViewSwitch
Moyai{" "} - Cloud Coding Agent + + {t("shell.viewSwitcher.moyaiDescription")} + {isMoyaiRoute && }
@@ -131,7 +136,7 @@ export default function ViewSwitcher({ className, contentClassName }: ViewSwitch return ( = ({ onWorkerSwitch, compact = false }) => { + const { t } = useTranslation(); const { isControlPlane, selectedWorker, workers } = useWorker(); if (!isControlPlane || !selectedWorker) return null; @@ -51,21 +53,27 @@ const WorkerDropdown: React.FC = ({ onWorkerSwitch, compact {compact ? ( } - aria-label={`Switch worker, current: ${selectedWorker.name}`} + aria-label={t("shell.workerDropdown.trigger", { worker: selectedWorker.name })} title={selectedWorker.name} > ) : ( - + )} - {compact && } - No matching workers + {compact && ( + + )} + {t("shell.workerDropdown.empty")} {(option: WorkerOption) => ( diff --git a/ui/litellm-dashboard/src/components/Navbar/navDisplayName.ts b/ui/litellm-dashboard/src/components/Navbar/navDisplayName.ts index d6f51dc37a8..d979798eaf5 100644 --- a/ui/litellm-dashboard/src/components/Navbar/navDisplayName.ts +++ b/ui/litellm-dashboard/src/components/Navbar/navDisplayName.ts @@ -1,3 +1,5 @@ +import i18n from "@/i18n/i18n"; + /** Primary label for the navbar account control — avoids raw placeholder JWT/user IDs in the UI. */ export function navAccountDisplayName(userEmail: string | null, userId: string | null): string { const email = userEmail?.trim(); @@ -6,10 +8,10 @@ export function navAccountDisplayName(userEmail: string | null, userId: string | } const id = userId?.trim(); if (!id) { - return "Account"; + return i18n.t("shell.navDisplayName.account"); } if (/^default[_\s-]?user[_\s-]?id$/i.test(id)) { - return "Account"; + return i18n.t("shell.navDisplayName.account"); } return id; } diff --git a/ui/litellm-dashboard/src/components/PublicModelHubTableColumns.tsx b/ui/litellm-dashboard/src/components/PublicModelHubTableColumns.tsx index e70ce5756eb..08d6b973b87 100644 --- a/ui/litellm-dashboard/src/components/PublicModelHubTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/PublicModelHubTableColumns.tsx @@ -7,6 +7,7 @@ import { CellTooltip, IdentityCell, StatusBadge, type StatusTone } from "@/compo import { Badge } from "@/components/ui/badge"; import { getProviderLogoAndName } from "@/components/provider_info_helpers"; import { PUBLIC_MODEL_HUB_SORTABLE_FIELDS } from "@/components/publicModelHub/publicModelHubFilters"; +import i18n from "@/i18n/i18n"; import { displayMode } from "@/lib/decisionModels"; import type { ModelCustomMetadataValue } from "@/components/AIHub/ModelCustomMetadata"; import { formatModelModeLabel, ProviderLogo } from "@/components/AIHub/ModelHubProviders"; @@ -88,14 +89,17 @@ const formatCapabilityName = (key: string) => const formatCost = (cost: number) => `$${(cost * 1_000_000).toFixed(4)}`; const formatTokens = (tokens: number | undefined) => { - if (!tokens) return "N/A"; + if (!tokens) return i18n.t("aiHub.publicModelHubColumns.notAvailable"); if (tokens >= 1000) return `${(tokens / 1000).toFixed(0)}K`; return tokens.toString(); }; const formatLimits = (rpm?: number, tpm?: number) => { - const limits = [...(rpm ? [`RPM: ${rpm.toLocaleString()}`] : []), ...(tpm ? [`TPM: ${tpm.toLocaleString()}`] : [])]; - return limits.length > 0 ? limits.join(", ") : "N/A"; + const limits = [ + ...(rpm ? [i18n.t("aiHub.publicModelHubColumns.rpmLimit", { value: rpm.toLocaleString() })] : []), + ...(tpm ? [i18n.t("aiHub.publicModelHubColumns.tpmLimit", { value: tpm.toLocaleString() })] : []), + ]; + return limits.length > 0 ? limits.join(", ") : i18n.t("aiHub.publicModelHubColumns.notAvailable"); }; const getModeIcon = (mode: string) => { @@ -179,9 +183,20 @@ export const getPublicModelHubColumns = ({ { id: "model_group", accessorKey: "model_group", - meta: { title: showProviderLogos ? "Model" : "Model Name" }, + meta: { + title: showProviderLogos + ? i18n.t("aiHub.publicModelHubColumns.models.modelChat") + : i18n.t("aiHub.publicModelHubColumns.models.modelName"), + }, header: ({ column }) => ( - + ), size: 200, sortingFn: "alphanumeric", @@ -213,8 +228,10 @@ export const getPublicModelHubColumns = ({ { id: "providers", accessorKey: "providers", - meta: { title: "Providers", skeleton: "chips" }, - header: ({ column }) => , + meta: { title: i18n.t("aiHub.publicModelHubColumns.models.providers"), skeleton: "chips" }, + header: ({ column }) => ( + + ), size: 150, sortingFn: (rowA, rowB) => (rowA.original.providers ?? []).join(", ").localeCompare((rowB.original.providers ?? []).join(", ")), @@ -223,15 +240,19 @@ export const getPublicModelHubColumns = ({ { id: "mode", accessorKey: "mode", - meta: { title: "Mode" }, - header: ({ column }) => , + meta: { title: i18n.t("aiHub.publicModelHubColumns.models.mode") }, + header: ({ column }) => ( + + ), size: 110, sortingFn: "alphanumeric", cell: ({ row }) => ( {!showProviderLogos && {getModeIcon(row.original.mode || "")}} - {showProviderLogos ? formatModelModeLabel(row.original.mode) : displayMode(row.original.mode) ?? "Chat"} + {showProviderLogos + ? formatModelModeLabel(row.original.mode) + : displayMode(row.original.mode) ?? i18n.t("aiHub.publicModelHubColumns.chatMode")} ), @@ -239,59 +260,89 @@ export const getPublicModelHubColumns = ({ { id: "max_input_tokens", accessorKey: "max_input_tokens", - meta: { title: "Max Input", numeric: true }, - header: ({ column }) => , + meta: { title: i18n.t("aiHub.publicModelHubColumns.models.maxInput"), numeric: true }, + header: ({ column }) => ( + + ), size: 100, cell: ({ row }) => {formatTokens(row.original.max_input_tokens)}, }, { id: "max_output_tokens", accessorKey: "max_output_tokens", - meta: { title: "Max Output", numeric: true }, - header: ({ column }) => , + meta: { title: i18n.t("aiHub.publicModelHubColumns.models.maxOutput"), numeric: true }, + header: ({ column }) => ( + + ), size: 100, cell: ({ row }) => {formatTokens(row.original.max_output_tokens)}, }, { id: "input_cost_per_token", accessorKey: "input_cost_per_token", - meta: { title: showProviderLogos ? "Cost / 1M tokens" : "Input $/1M", numeric: true }, + meta: { + title: showProviderLogos + ? i18n.t("aiHub.publicModelHubColumns.models.costChat") + : i18n.t("aiHub.publicModelHubColumns.models.inputCost"), + numeric: true, + }, header: ({ column }) => ( - + ), size: 110, cell: ({ row }) => showProviderLogos ? (
- Input {row.original.input_cost_per_token ? formatCost(row.original.input_cost_per_token) : "Free"} + {i18n.t("aiHub.publicModelHubColumns.inputCost", { + cost: row.original.input_cost_per_token + ? formatCost(row.original.input_cost_per_token) + : i18n.t("aiHub.publicModelHubColumns.free"), + })} - Output {row.original.output_cost_per_token ? formatCost(row.original.output_cost_per_token) : "Free"} + {i18n.t("aiHub.publicModelHubColumns.outputCost", { + cost: row.original.output_cost_per_token + ? formatCost(row.original.output_cost_per_token) + : i18n.t("aiHub.publicModelHubColumns.free"), + })}
) : ( - {row.original.input_cost_per_token ? formatCost(row.original.input_cost_per_token) : "Free"} + {row.original.input_cost_per_token + ? formatCost(row.original.input_cost_per_token) + : i18n.t("aiHub.publicModelHubColumns.free")} ), }, { id: "output_cost_per_token", accessorKey: "output_cost_per_token", - meta: { title: "Output $/1M", numeric: true }, - header: ({ column }) => , + meta: { title: i18n.t("aiHub.publicModelHubColumns.models.outputCost"), numeric: true }, + header: ({ column }) => ( + + ), size: 110, cell: ({ row }) => ( - {row.original.output_cost_per_token ? formatCost(row.original.output_cost_per_token) : "Free"} + {row.original.output_cost_per_token + ? formatCost(row.original.output_cost_per_token) + : i18n.t("aiHub.publicModelHubColumns.free")} ), }, { id: "features", - meta: { title: "Features", skeleton: "chips" }, - header: "Features", + meta: { title: i18n.t("aiHub.publicModelHubColumns.models.features"), skeleton: "chips" }, + header: i18n.t("aiHub.publicModelHubColumns.models.features"), size: 140, cell: ({ row }) => { const features = Object.entries(row.original) @@ -302,25 +353,29 @@ export const getPublicModelHubColumns = ({ }, { id: "custom_tags", - meta: { title: "Tags", skeleton: "chips" }, - header: "Tags", + meta: { title: i18n.t("aiHub.publicModelHubColumns.models.tags"), skeleton: "chips" }, + header: i18n.t("aiHub.publicModelHubColumns.models.tags"), size: 140, cell: ({ row }) => , }, { id: "health_status", accessorKey: "health_status", - meta: { title: "Health Status", skeleton: "badge" }, - header: ({ column }) => , + meta: { title: i18n.t("aiHub.publicModelHubColumns.models.healthStatus"), skeleton: "badge" }, + header: ({ column }) => ( + + ), size: 130, cell: ({ row }) => { const model = row.original; const responseTimeLabel = model.health_response_time - ? `Response Time: ${Number(model.health_response_time).toFixed(2)}ms` - : "N/A"; + ? i18n.t("aiHub.publicModelHubColumns.responseTime", { ms: Number(model.health_response_time).toFixed(2) }) + : i18n.t("aiHub.publicModelHubColumns.notAvailable"); const lastCheckedLabel = model.health_checked_at - ? `Last Checked: ${new Date(model.health_checked_at).toLocaleString()}` - : "N/A"; + ? i18n.t("aiHub.publicModelHubColumns.lastChecked", { + date: new Date(model.health_checked_at).toLocaleString(), + }) + : i18n.t("aiHub.publicModelHubColumns.notAvailable"); return ( } @@ -344,8 +399,10 @@ export const getPublicModelHubColumns = ({ { id: "rpm", accessorKey: "rpm", - meta: { title: "Limits" }, - header: ({ column }) => , + meta: { title: i18n.t("aiHub.publicModelHubColumns.models.limits") }, + header: ({ column }) => ( + + ), size: 150, cell: ({ row }) => ( {formatLimits(row.original.rpm, row.original.tpm)} @@ -378,8 +435,10 @@ export const getPublicAgentHubColumns = ({ onAgentClick }: PublicAgentHubColumns { id: "name", accessorKey: "name", - meta: { title: "Agent Name" }, - header: ({ column }) => , + meta: { title: i18n.t("aiHub.publicModelHubColumns.agents.name") }, + header: ({ column }) => ( + + ), size: 200, enableSorting: true, sortingFn: "alphanumeric", @@ -395,8 +454,8 @@ export const getPublicAgentHubColumns = ({ onAgentClick }: PublicAgentHubColumns { id: "description", accessorKey: "description", - meta: { title: "Description" }, - header: "Description", + meta: { title: i18n.t("aiHub.publicModelHubColumns.agents.description") }, + header: i18n.t("aiHub.publicModelHubColumns.agents.description"), size: 260, enableSorting: false, cell: ({ row }) => ( @@ -408,8 +467,10 @@ export const getPublicAgentHubColumns = ({ onAgentClick }: PublicAgentHubColumns { id: "version", accessorKey: "version", - meta: { title: "Version" }, - header: ({ column }) => , + meta: { title: i18n.t("aiHub.publicModelHubColumns.agents.version") }, + header: ({ column }) => ( + + ), size: 90, enableSorting: true, sortingFn: "alphanumeric", @@ -417,8 +478,8 @@ export const getPublicAgentHubColumns = ({ onAgentClick }: PublicAgentHubColumns }, { id: "provider", - meta: { title: "Provider" }, - header: "Provider", + meta: { title: i18n.t("aiHub.publicModelHubColumns.agents.provider") }, + header: i18n.t("aiHub.publicModelHubColumns.agents.provider"), size: 130, enableSorting: false, cell: ({ row }) => @@ -430,16 +491,16 @@ export const getPublicAgentHubColumns = ({ onAgentClick }: PublicAgentHubColumns }, { id: "skills", - meta: { title: "Skills", skeleton: "chips" }, - header: "Skills", + meta: { title: i18n.t("aiHub.publicModelHubColumns.agents.skills"), skeleton: "chips" }, + header: i18n.t("aiHub.publicModelHubColumns.agents.skills"), size: 160, enableSorting: false, cell: ({ row }) => skill.name)} />, }, { id: "capabilities", - meta: { title: "Capabilities", skeleton: "chips" }, - header: "Capabilities", + meta: { title: i18n.t("aiHub.publicModelHubColumns.agents.capabilities"), skeleton: "chips" }, + header: i18n.t("aiHub.publicModelHubColumns.agents.capabilities"), size: 160, enableSorting: false, cell: ({ row }) => { @@ -470,8 +531,10 @@ export const getPublicMCPHubColumns = ({ onServerClick }: PublicMCPHubColumnsDep { id: "server_name", accessorKey: "server_name", - meta: { title: "Server Name" }, - header: ({ column }) => , + meta: { title: i18n.t("aiHub.publicModelHubColumns.mcp.serverName") }, + header: ({ column }) => ( + + ), size: 180, enableSorting: true, sortingFn: "alphanumeric", @@ -486,8 +549,8 @@ export const getPublicMCPHubColumns = ({ onServerClick }: PublicMCPHubColumnsDep }, { id: "description", - meta: { title: "Description" }, - header: "Description", + meta: { title: i18n.t("aiHub.publicModelHubColumns.mcp.description") }, + header: i18n.t("aiHub.publicModelHubColumns.mcp.description"), size: 260, enableSorting: false, cell: ({ row }) => { @@ -502,8 +565,10 @@ export const getPublicMCPHubColumns = ({ onServerClick }: PublicMCPHubColumnsDep { id: "transport", accessorKey: "transport", - meta: { title: "Transport", skeleton: "badge" }, - header: ({ column }) => , + meta: { title: i18n.t("aiHub.publicModelHubColumns.mcp.transport"), skeleton: "badge" }, + header: ({ column }) => ( + + ), size: 110, enableSorting: true, sortingFn: "alphanumeric", @@ -516,8 +581,10 @@ export const getPublicMCPHubColumns = ({ onServerClick }: PublicMCPHubColumnsDep { id: "auth_type", accessorKey: "auth_type", - meta: { title: "Auth Type", skeleton: "badge" }, - header: ({ column }) => , + meta: { title: i18n.t("aiHub.publicModelHubColumns.mcp.authType"), skeleton: "badge" }, + header: ({ column }) => ( + + ), size: 110, enableSorting: true, sortingFn: "alphanumeric", diff --git a/ui/litellm-dashboard/src/components/common_components/DefaultProxyAdminTag.tsx b/ui/litellm-dashboard/src/components/common_components/DefaultProxyAdminTag.tsx index 308f722a8f2..6a6619e278c 100644 --- a/ui/litellm-dashboard/src/components/common_components/DefaultProxyAdminTag.tsx +++ b/ui/litellm-dashboard/src/components/common_components/DefaultProxyAdminTag.tsx @@ -1,3 +1,5 @@ +import { useTranslation } from "react-i18next"; + import { Badge } from "@/components/ui/badge"; import { DEFAULT_PROXY_ADMIN_USER_ID } from "@/utils/sentinels"; @@ -6,8 +8,9 @@ interface DefaultProxyAdminTagProps { } export default function DefaultProxyAdminTag({ userId }: DefaultProxyAdminTagProps) { + const { t } = useTranslation(); if (userId === DEFAULT_PROXY_ADMIN_USER_ID) { - return Default Proxy Admin; + return {t("commonComponents.defaultProxyAdminTag.label")}; } return {userId}; diff --git a/ui/litellm-dashboard/src/components/common_components/DeleteResourceModal.tsx b/ui/litellm-dashboard/src/components/common_components/DeleteResourceModal.tsx index 2d8a523397a..22842f8c358 100644 --- a/ui/litellm-dashboard/src/components/common_components/DeleteResourceModal.tsx +++ b/ui/litellm-dashboard/src/components/common_components/DeleteResourceModal.tsx @@ -1,5 +1,6 @@ import { CircleAlert } from "lucide-react"; import React, { useState, useEffect } from "react"; +import { Trans, useTranslation } from "react-i18next"; import { Alert, AlertTitle } from "@/components/shared/Alert"; import { Button } from "@/components/ui/button"; import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; @@ -35,6 +36,7 @@ export default function DeleteResourceModal({ confirmLoading, requiredConfirmation, }: DeleteResourceModalProps) { + const { t } = useTranslation(); const [requiredConfirmationInput, setRequiredConfirmationInput] = useState(""); useEffect(() => { @@ -78,7 +80,12 @@ export default function DeleteResourceModal({ {requiredConfirmation && (

- Type {requiredConfirmation} to confirm deletion: + {requiredConfirmation}, + }} + />

@@ -96,14 +103,16 @@ export default function DeleteResourceModal({
diff --git a/ui/litellm-dashboard/src/components/common_components/LoadingScreen.tsx b/ui/litellm-dashboard/src/components/common_components/LoadingScreen.tsx index 2d46efa3a2d..2e486f6607d 100644 --- a/ui/litellm-dashboard/src/components/common_components/LoadingScreen.tsx +++ b/ui/litellm-dashboard/src/components/common_components/LoadingScreen.tsx @@ -1,14 +1,16 @@ +import { useTranslation } from "react-i18next"; import { cx } from "@/lib/cva.config"; import { UiLoadingSpinner } from "../ui/ui-loading-spinner"; export default function LoadingScreen() { + const { t } = useTranslation(); return (
🚅 LiteLLM
- Loading... + {t("shell.loadingScreen.loading")}
); diff --git a/ui/litellm-dashboard/src/components/common_components/check_openapi_schema.tsx b/ui/litellm-dashboard/src/components/common_components/check_openapi_schema.tsx index cdeef9acbf7..c5ed733d05d 100644 --- a/ui/litellm-dashboard/src/components/common_components/check_openapi_schema.tsx +++ b/ui/litellm-dashboard/src/components/common_components/check_openapi_schema.tsx @@ -5,6 +5,8 @@ import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@ import { Info } from "lucide-react"; import { SimpleTooltip } from "@/components/ui/tooltip"; import type { UseFormSetValue } from "react-hook-form"; +import { useTranslation } from "react-i18next"; +import i18n from "@/i18n/i18n"; import { getOpenAPISchema } from "../networking"; import { formatLabel } from "@/utils/textUtils"; import { MountedFormField, type MountedFormValues } from "./MountedFormField"; @@ -71,28 +73,28 @@ const getFieldHelp = (key: string, property: SchemaProperty, type: string): stri // Default help text based on type const defaultHelp = { - string: "Text input", - number: "Numeric input", - integer: "Whole number input", - boolean: "True/False value", - }[type] || "Text input"; + string: i18n.t("commonComponents.checkOpenapiSchema.typeHelp.string"), + number: i18n.t("commonComponents.checkOpenapiSchema.typeHelp.number"), + integer: i18n.t("commonComponents.checkOpenapiSchema.typeHelp.integer"), + boolean: i18n.t("commonComponents.checkOpenapiSchema.typeHelp.boolean"), + }[type] || i18n.t("commonComponents.checkOpenapiSchema.typeHelp.fallback"); // Specific field help text const specificHelp: { [key: string]: string } = { - max_budget: "Enter maximum budget in USD (e.g., 100.50)", - budget_duration: "Select a time period for budget reset", - tpm_limit: "Enter maximum tokens per minute (whole number)", - rpm_limit: "Enter maximum requests per minute (whole number)", - duration: "Enter duration (e.g., 30s, 24h, 7d)", - metadata: 'Enter JSON object with key-value pairs\nExample: {"team": "research", "project": "nlp"}', - config: 'Enter configuration as JSON object\nExample: {"setting": "value"}', - permissions: "Enter comma-separated permission strings", - enforced_params: 'Enter parameters as JSON object\nExample: {"param": "value"}', - blocked: "Enter true/false or specific block conditions", - aliases: 'Enter aliases as JSON object\nExample: {"alias1": "value1", "alias2": "value2"}', - models: "Select one or more model names", - key_alias: "Enter a unique identifier for this key", - tags: "Enter comma-separated tag strings", + max_budget: i18n.t("commonComponents.checkOpenapiSchema.fieldHelp.maxBudget"), + budget_duration: i18n.t("commonComponents.checkOpenapiSchema.fieldHelp.budgetDuration"), + tpm_limit: i18n.t("commonComponents.checkOpenapiSchema.fieldHelp.tpmLimit"), + rpm_limit: i18n.t("commonComponents.checkOpenapiSchema.fieldHelp.rpmLimit"), + duration: i18n.t("commonComponents.checkOpenapiSchema.fieldHelp.duration"), + metadata: i18n.t("commonComponents.checkOpenapiSchema.fieldHelp.metadata"), + config: i18n.t("commonComponents.checkOpenapiSchema.fieldHelp.config"), + permissions: i18n.t("commonComponents.checkOpenapiSchema.fieldHelp.permissions"), + enforced_params: i18n.t("commonComponents.checkOpenapiSchema.fieldHelp.enforcedParams"), + blocked: i18n.t("commonComponents.checkOpenapiSchema.fieldHelp.blocked"), + aliases: i18n.t("commonComponents.checkOpenapiSchema.fieldHelp.aliases"), + models: i18n.t("commonComponents.checkOpenapiSchema.fieldHelp.models"), + key_alias: i18n.t("commonComponents.checkOpenapiSchema.fieldHelp.keyAlias"), + tags: i18n.t("commonComponents.checkOpenapiSchema.fieldHelp.tags"), }; // Get specific help text or use default based on type @@ -100,11 +102,11 @@ const getFieldHelp = (key: string, property: SchemaProperty, type: string): stri // Add format requirements for special cases if (isJSONField(key, property)) { - return `${helpText}\nMust be valid JSON format`; + return i18n.t("commonComponents.checkOpenapiSchema.jsonHelp", { helpText }); } if (property.enum) { - return `Select from available options\nAllowed values: ${property.enum.join(", ")}`; + return i18n.t("commonComponents.checkOpenapiSchema.enumHelp", { values: property.enum.join(", ") }); } return helpText; @@ -119,6 +121,7 @@ const SchemaFormFields: React.FC = ({ customValidation = {}, defaultValues = {}, }) => { + const { t } = useTranslation(); const [schemaProperties, setSchemaProperties] = useState(null); const [error, setError] = useState(null); @@ -141,12 +144,12 @@ const SchemaFormFields: React.FC = ({ }); } catch (error) { console.error("Schema fetch error:", error); - setError(error instanceof Error ? error.message : "Failed to fetch schema"); + setError(error instanceof Error ? error.message : t("commonComponents.checkOpenapiSchema.fetchFailed")); } }; fetchOpenAPISchema(); - }, [schemaComponent, setValue, excludedFields]); + }, [schemaComponent, setValue, excludedFields, t]); const getPropertyType = (property: SchemaProperty): string => { if (property.type) { @@ -169,7 +172,8 @@ const SchemaFormFields: React.FC = ({ const validate = { ...(isRequired && { - required: (value: unknown) => (isBlank(value) ? `${label} is required` : true), + required: (value: unknown) => + isBlank(value) ? t("commonComponents.checkOpenapiSchema.required", { label }) : true, }), ...(customValidation[key] && { custom: async (value: unknown) => { @@ -183,7 +187,9 @@ const SchemaFormFields: React.FC = ({ }), ...(isJSONField(key, property) && { json: (value: unknown) => - value && !validateJSON(value as string) ? "Please enter valid JSON" : (true as const), + value && !validateJSON(value as string) + ? t("commonComponents.checkOpenapiSchema.invalidJson") + : (true as const), }), }; @@ -216,7 +222,7 @@ const SchemaFormFields: React.FC = ({ {...control} value={control.value as string | undefined} rows={4} - placeholder="Enter as JSON" + placeholder={t("commonComponents.checkOpenapiSchema.jsonPlaceholder")} className="font-mono" /> ); @@ -256,7 +262,11 @@ const SchemaFormFields: React.FC = ({ } if (key === "duration") { return ( - + ); } return ; @@ -266,7 +276,7 @@ const SchemaFormFields: React.FC = ({ }; if (error) { - return
Error: {error}
; + return
{t("commonComponents.checkOpenapiSchema.error", { error })}
; } if (!schemaProperties?.properties) { diff --git a/ui/litellm-dashboard/src/components/key_team_helpers/fetch_available_models_team_key.tsx b/ui/litellm-dashboard/src/components/key_team_helpers/fetch_available_models_team_key.tsx index 3a3226336b2..d270017332c 100644 --- a/ui/litellm-dashboard/src/components/key_team_helpers/fetch_available_models_team_key.tsx +++ b/ui/litellm-dashboard/src/components/key_team_helpers/fetch_available_models_team_key.tsx @@ -1,3 +1,4 @@ +import i18n from "@/i18n/i18n"; import { modelAvailableCall } from "../networking"; export const fetchAvailableModelsForTeamOrKey = async ( @@ -43,11 +44,11 @@ export const hasAllModelsSentinel = (models: string[]): boolean => export const getModelDisplayName = (model: string) => { if (model === "all-proxy-models") { - return "All Proxy Models"; + return i18n.t("keyTeamHelpers.fetchAvailableModelsTeamKey.allProxyModels"); } if (model.endsWith("/*")) { const provider = model.replace("/*", ""); - return `All ${provider} models`; + return i18n.t("keyTeamHelpers.fetchAvailableModelsTeamKey.allProviderModels", { provider }); } return model; }; diff --git a/ui/litellm-dashboard/src/components/molecules/logo/Logo.tsx b/ui/litellm-dashboard/src/components/molecules/logo/Logo.tsx index 66c10f1e222..d87caa74aec 100644 --- a/ui/litellm-dashboard/src/components/molecules/logo/Logo.tsx +++ b/ui/litellm-dashboard/src/components/molecules/logo/Logo.tsx @@ -1,4 +1,5 @@ import React, { useState } from "react"; +import { useTranslation } from "react-i18next"; import { getProviderLogoAndName } from "@/components/provider_info_helpers"; import { resolveLogoSrc } from "@/lib/assetPaths"; import { cn } from "@/lib/cva.config"; @@ -15,6 +16,7 @@ const DARK_TREATMENT_CLASS: Readonly> = { }; export const Logo: React.FC = ({ provider, src, label, className = "w-4 h-4" }) => { + const { t } = useTranslation(); const [erroredSrc, setErroredSrc] = useState(null); const resolvedSrc = provider !== undefined ? getProviderLogoAndName(provider).logo : resolveLogoSrc(src) ?? ""; const name = label ?? provider ?? ""; @@ -32,7 +34,7 @@ export const Logo: React.FC = ({ provider, src, label, className = "w return ( {`${name { console.warn(`Logo failed to load: ${resolvedSrc}`); diff --git a/ui/litellm-dashboard/src/components/navbar.tsx b/ui/litellm-dashboard/src/components/navbar.tsx index c475c73c40a..7ddbd77bb7f 100644 --- a/ui/litellm-dashboard/src/components/navbar.tsx +++ b/ui/litellm-dashboard/src/components/navbar.tsx @@ -11,6 +11,7 @@ import { Badge } from "@/components/ui/badge"; import { PanelLeftClose, PanelLeftOpen } from "lucide-react"; import Link from "next/link"; import React from "react"; +import { useTranslation } from "react-i18next"; import { BlogDropdown } from "./Navbar/BlogDropdown/BlogDropdown"; import { DocsLink } from "./Navbar/DocsLink/DocsLink"; import { CommunityEngagementButtons } from "./Navbar/CommunityEngagementButtons/CommunityEngagementButtons"; @@ -40,6 +41,7 @@ const Navbar: React.FC = ({ deploymentHealth, deploymentHealthLink, }) => { + const { t } = useTranslation(); const baseUrl = getProxyBaseUrl(); const { logoUrl, logoUrlDark } = useTheme(); const { data: healthData } = useHealthReadinessDetails(accessToken); @@ -69,7 +71,7 @@ const Navbar: React.FC = ({ )} {(agent.skills ?? []).length > 0 && ( - + {agent.skills.map((skill) => ( @@ -65,11 +78,11 @@ export function AgentInspector({ agent }: { agent: AgentCard }) { )} - + {agentSnippets(agent).map((snippet, index) => ( - Step {index + 1}: {snippet.title} + {t("publicModelHub.agentInspector.step", { step: index + 1, title: snippet.title })} diff --git a/ui/litellm-dashboard/src/components/publicModelHub/FilterChip.tsx b/ui/litellm-dashboard/src/components/publicModelHub/FilterChip.tsx index c25a1619845..a5aeff6f068 100644 --- a/ui/litellm-dashboard/src/components/publicModelHub/FilterChip.tsx +++ b/ui/litellm-dashboard/src/components/publicModelHub/FilterChip.tsx @@ -1,6 +1,7 @@ "use client"; import { CirclePlus, X } from "lucide-react"; +import { useTranslation } from "react-i18next"; import { Button } from "@/components/ui/button"; import { ButtonGroup } from "@/components/ui/button-group"; @@ -37,6 +38,7 @@ export function FilterChip({ value: string[]; onChange: (value: string[]) => void; }) { + const { t } = useTranslation(); const byValue = new Map(options.map((option) => [option.value, option])); const selected = value.map((v) => byValue.get(v)?.label ?? v); const active = value.length > 0; @@ -64,8 +66,8 @@ export function FilterChip({ {active && {firstPlus(selected)}} - - No {noun} found + + {t("publicModelHub.filterChip.empty", { noun })} {(option: FilterOption) => ( @@ -80,7 +82,7 @@ export function FilterChip({ )} @@ -45,32 +50,34 @@ export function NoResults({ } export function HubNotEnabled() { + const { t } = useTranslation(); return ( - This AI Hub isn't available - Ask your proxy admin to make it public from their Admin UI + {t("publicModelHub.notEnabled.title")} + {t("publicModelHub.notEnabled.description")} ); } export function LoadFailed({ noun, onRetry }: { noun: string; onRetry: () => void }) { + const { t } = useTranslation(); return ( - Couldn't load {noun} - The gateway returned an error. Try again in a moment + {t("publicModelHub.loadFailed.title", { noun })} + {t("publicModelHub.loadFailed.description")} @@ -78,14 +85,15 @@ export function LoadFailed({ noun, onRetry }: { noun: string; onRetry: () => voi } export function NotListed({ name }: { name: string }) { + const { t } = useTranslation(); return ( - “{name}” isn't listed here - It may have been renamed or unpublished + {t("publicModelHub.notListed.title", { name })} + {t("publicModelHub.notListed.description")} ); diff --git a/ui/litellm-dashboard/src/components/publicModelHub/HubHeader.tsx b/ui/litellm-dashboard/src/components/publicModelHub/HubHeader.tsx index ca8df489cfa..532df227ea2 100644 --- a/ui/litellm-dashboard/src/components/publicModelHub/HubHeader.tsx +++ b/ui/litellm-dashboard/src/components/publicModelHub/HubHeader.tsx @@ -1,6 +1,7 @@ "use client"; import { ChevronDown, ExternalLink, LayoutGrid } from "lucide-react"; +import { useTranslation } from "react-i18next"; import { Button } from "@/components/ui/button"; import { @@ -23,12 +24,13 @@ import type { HubInfo } from "./useHubData"; export type HubStatus = "loading" | "operational" | "degraded" | "unavailable"; function LinksMenu({ info }: { info: HubInfo | undefined }) { + const { t } = useTranslation(); const links = usefulLinkList(info?.useful_links); if (links.length === 0) return null; return ( }> - Links + {t("publicModelHub.linksMenu.trigger")} @@ -42,7 +44,9 @@ function LinksMenu({ info }: { info: HubInfo | undefined }) { <> - LiteLLM v{info.litellm_version} + + {t("publicModelHub.linksMenu.version", { version: info.litellm_version })} + )} @@ -52,44 +56,46 @@ function LinksMenu({ info }: { info: HubInfo | undefined }) { } function BaseUrlField() { + const { t } = useTranslation(); const url = gatewayUrl(); return ( - Base URL + {t("publicModelHub.baseUrlField.label")} - + - Use your LiteLLM key as the API key in any OpenAI-compatible SDK + {t("publicModelHub.baseUrlField.hint")} ); } export function HubHeader({ info, status }: { info: HubInfo | undefined; status: HubStatus }) { + const { t } = useTranslation(); return (

- AI Hub + {t("publicModelHub.header.title")}

- {status === "operational" && } + {status === "operational" && }

- {info?.custom_docs_description || "Models, agents, MCP servers and skills you can call through this gateway"} + {info?.custom_docs_description || t("publicModelHub.header.description")}

{status !== "unavailable" && ( diff --git a/ui/litellm-dashboard/src/components/publicModelHub/HubMasterDetail.tsx b/ui/litellm-dashboard/src/components/publicModelHub/HubMasterDetail.tsx index 63153c63c76..715e1cad07f 100644 --- a/ui/litellm-dashboard/src/components/publicModelHub/HubMasterDetail.tsx +++ b/ui/litellm-dashboard/src/components/publicModelHub/HubMasterDetail.tsx @@ -7,6 +7,7 @@ import { ScrollArea } from "@/components/ui/scroll-area"; import { Sheet, SheetContent, SheetTitle } from "@/components/ui/sheet"; import { Skeleton } from "@/components/ui/skeleton"; import { cn } from "@/lib/cva.config"; +import i18n from "@/i18n/i18n"; import { NotListed } from "./HubEmpty"; import { InSheet } from "./InspectorHeader"; @@ -121,7 +122,7 @@ function useSheet( const { row } = content; return { open: chosen !== undefined || missingKey !== undefined, - title: row ? getKey(row) : content.missingKey ?? "Details", + title: row ? getKey(row) : content.missingKey ?? i18n.t("publicModelHub.masterDetail.details"), body: row ? renderDetail(row) : content.missingKey && , }; } diff --git a/ui/litellm-dashboard/src/components/publicModelHub/HubParts.tsx b/ui/litellm-dashboard/src/components/publicModelHub/HubParts.tsx index e325bd42eed..834a82c57f4 100644 --- a/ui/litellm-dashboard/src/components/publicModelHub/HubParts.tsx +++ b/ui/litellm-dashboard/src/components/publicModelHub/HubParts.tsx @@ -2,6 +2,7 @@ import { Copy, Search, X } from "lucide-react"; import * as React from "react"; +import { useTranslation } from "react-i18next"; import { Avatar, AvatarFallback, AvatarImage } from "@/components/ui/avatar"; import { Badge } from "@/components/ui/badge"; @@ -11,11 +12,13 @@ import { InputGroup, InputGroupAddon, InputGroupButton, InputGroupInput } from " import { ItemTitle } from "@/components/ui/item"; import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; import { cn } from "@/lib/cva.config"; +import i18n from "@/i18n/i18n"; import { copyToClipboard } from "@/utils/dataUtils"; import { providerLogo, providerName, SECTION_TITLE } from "./hubFormat"; export function CopyIconButton({ value, label, className }: { value: string; label: string; className?: string }) { + const { t } = useTranslation(); return ( void copyToClipboard(value, "Copied")} + onClick={() => void copyToClipboard(value, t("publicModelHub.copyIconButton.copied"))} /> } > @@ -36,7 +39,13 @@ export function CopyIconButton({ value, label, className }: { value: string; lab ); } -export function CodeBlock({ code, label = "Copy code" }: { code: string; label?: string }) { +export function CodeBlock({ + code, + label = i18n.t("publicModelHub.codeBlock.copyCode"), +}: { + code: string; + label?: string; +}) { return (
@@ -125,13 +134,14 @@ export function Facts({ groups }: { groups: Fact[][] }) { } export function EndpointField({ id, value }: { id: string; value: string }) { + const { t } = useTranslation(); return ( - Endpoint + {t("publicModelHub.endpointField.label")} - + @@ -147,6 +157,7 @@ export function SearchInput({ onChange: (value: string) => void; placeholder: string; }) { + const { t } = useTranslation(); return ( @@ -160,7 +171,11 @@ export function SearchInput({ /> {value && ( - onChange("")}> + onChange("")} + > diff --git a/ui/litellm-dashboard/src/components/publicModelHub/HubStatus.tsx b/ui/litellm-dashboard/src/components/publicModelHub/HubStatus.tsx index cae5211f3ab..9eb9729469f 100644 --- a/ui/litellm-dashboard/src/components/publicModelHub/HubStatus.tsx +++ b/ui/litellm-dashboard/src/components/publicModelHub/HubStatus.tsx @@ -1,6 +1,7 @@ "use client"; import { CircleCheck, CircleDashed, CircleX } from "lucide-react"; +import { useTranslation } from "react-i18next"; import type { ModelGroupInfo } from "@/components/PublicModelHubTableColumns"; import { Badge } from "@/components/ui/badge"; @@ -27,6 +28,7 @@ export function StatusBadge({ tone, label }: { tone: HealthTone; label: string } } export function HealthBadge({ model }: { model: ModelGroupInfo }) { + const { t } = useTranslation(); const badge = ; if (!model.health_status) return badge; return ( @@ -36,11 +38,13 @@ export function HealthBadge({ model }: { model: ModelGroupInfo }) {
-
Response time
+
{t("publicModelHub.healthBadge.responseTime")}
- {model.health_response_time ? `${Math.round(model.health_response_time)} ms` : "-"} + {model.health_response_time + ? t("publicModelHub.healthBadge.milliseconds", { value: Math.round(model.health_response_time) }) + : "-"}
-
Last checked
+
{t("publicModelHub.healthBadge.lastChecked")}
{relativeTime(model.health_checked_at)}
 diff --git a/ui/litellm-dashboard/src/components/publicModelHub/InspectorHeader.tsx b/ui/litellm-dashboard/src/components/publicModelHub/InspectorHeader.tsx index 4400b18ce1f..bf76bd49239 100644 --- a/ui/litellm-dashboard/src/components/publicModelHub/InspectorHeader.tsx +++ b/ui/litellm-dashboard/src/components/publicModelHub/InspectorHeader.tsx @@ -2,6 +2,7 @@ import { X } from "lucide-react"; import * as React from "react"; +import { useTranslation } from "react-i18next"; import { Button } from "@/components/ui/button"; import { SheetClose } from "@/components/ui/sheet"; @@ -25,6 +26,7 @@ export function InspectorHeader({ status?: React.ReactNode; description?: React.ReactNode; }) { + const { t } = useTranslation(); const inSheet = React.useContext(InSheet); return (
@@ -36,7 +38,14 @@ export function InspectorHeader({ {status && {status}} {inSheet && ( } + render={ +
@@ -969,7 +1042,9 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded > - {selectedAgent?.name || "Agent Details"} + + {selectedAgent?.name || t("aiHub.publicModelHub.agentDialog.fallbackTitle")} + {selectedAgent && ( = ({ accessToken, isEmbedded /> } /> - Copy agent name + + {t("aiHub.publicModelHub.agentDialog.copyName")} + )} @@ -989,23 +1066,23 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded
{/* Agent Overview */}
-

Agent Overview

+

{t("aiHub.publicModelHub.agentDialog.overview")}

-

Name:

+

{t("aiHub.publicModelHub.agentDialog.name")}

{selectedAgent.name}

-

Version:

+

{t("aiHub.publicModelHub.agentDialog.version")}

{selectedAgent.version}

-

Description:

+

{t("aiHub.publicModelHub.agentDialog.description")}

{selectedAgent.description}

{selectedAgent.url && (
-

URL:

+

{t("aiHub.publicModelHub.agentDialog.url")}

 = ({ accessToken, isEmbedded {/* Capabilities */} {selectedAgent.capabilities && (
-

Capabilities

+

{t("aiHub.publicModelHub.agentDialog.capabilities")}

{Object.entries(selectedAgent.capabilities) .filter(([_, value]) => value === true) @@ -1038,7 +1115,7 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded {/* Skills */} {selectedAgent.skills && selectedAgent.skills.length > 0 && (
-

Skills

+

{t("aiHub.publicModelHub.agentDialog.skills")}

{selectedAgent.skills.map((skill, index) => (
@@ -1065,10 +1142,10 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded {/* Input/Output Modes */}
-

Input/Output Modes

+

{t("aiHub.publicModelHub.agentDialog.ioModes")}

-

Input Modes:

+

{t("aiHub.publicModelHub.agentDialog.inputModes")}

{(selectedAgent.defaultInputModes ?? []).map((mode) => ( @@ -1078,7 +1155,7 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded
-

Output Modes:

+

{t("aiHub.publicModelHub.agentDialog.outputModes")}

{(selectedAgent.defaultOutputModes ?? []).map((mode) => ( @@ -1093,7 +1170,9 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded {/* Documentation */} {selectedAgent.documentationUrl && ( )} {/* A2A Usage Example */}
-

Usage Example (A2A Protocol)

+

{t("aiHub.publicModelHub.agentDialog.usageExample")}

{/* Step 1: Retrieve Agent Card */}
-

Step 1: Retrieve Agent Card

+

+ {t("aiHub.publicModelHub.agentDialog.stepRetrieveCard")} + 

                           {`base_url = '${selectedAgent.url}'
@@ -1200,14 +1281,16 @@ if _public_card.supports_authenticated_extended_card:
                           }}
                           className="text-sm text-info hover:text-info/80 cursor-pointer"
                         >
-                          Copy to clipboard
+                          {t("aiHub.publicModelHub.agentDialog.copyToClipboard")}
{/* Step 2: Call the Agent */}
-

Step 2: Call the Agent

+

+ {t("aiHub.publicModelHub.agentDialog.stepCallAgent")} + 

                           {`client = A2AClient(
@@ -1257,7 +1340,7 @@ print(response.model_dump(mode='json', exclude_none=True))`;
                           }}
                           className="text-sm text-info hover:text-info/80 cursor-pointer"
                         >
-                          Copy to clipboard
+                          {t("aiHub.publicModelHub.agentDialog.copyToClipboard")}
@@ -1274,7 +1357,9 @@ print(response.model_dump(mode='json', exclude_none=True))`; > - {selectedMcpServer?.server_name || "MCP Server Details"} + + {selectedMcpServer?.server_name || t("aiHub.publicModelHub.mcpDialog.fallbackTitle")} + {selectedMcpServer && ( } /> - Copy server name + + {t("aiHub.publicModelHub.mcpDialog.copyName")} + )} @@ -1294,30 +1381,30 @@ print(response.model_dump(mode='json', exclude_none=True))`;
{/* Server Overview */}
-

Server Overview

+

{t("aiHub.publicModelHub.mcpDialog.overview")}

-

Server Name:

+

{t("aiHub.publicModelHub.mcpDialog.serverName")}

{selectedMcpServer.server_name}

-

Transport:

+

{t("aiHub.publicModelHub.mcpDialog.transport")}

{selectedMcpServer.transport}
{selectedMcpServer.alias && (
-

Alias:

+

{t("aiHub.publicModelHub.mcpDialog.alias")}

{selectedMcpServer.alias}

)}
-

Auth Type:

+

{t("aiHub.publicModelHub.mcpDialog.authType")}

{selectedMcpServer.auth_type}
-

Description:

+

{t("aiHub.publicModelHub.mcpDialog.description")}

{selectedMcpServer.mcp_info?.description || "-"}

@@ -1326,7 +1413,7 @@ print(response.model_dump(mode='json', exclude_none=True))`; {/* Additional Info */} {selectedMcpServer.mcp_info && Object.keys(selectedMcpServer.mcp_info).length > 0 && (
-

Additional Information

+

{t("aiHub.publicModelHub.mcpDialog.additionalInfo")}

                           {JSON.stringify(selectedMcpServer.mcp_info, null, 2)}
@@ -1337,7 +1424,7 @@ print(response.model_dump(mode='json', exclude_none=True))`;
 
                   {/* Usage Example */}
                   

-                    
Usage Example

+                    
{t("aiHub.publicModelHub.mcpDialog.usageExample")}

                     

                       
                         {`# Using MCP Server with Python FastMCP
@@ -1419,7 +1506,7 @@ if __name__ == "__main__":
                         }}
                         className="text-sm text-info hover:text-info/80 cursor-pointer"
                       >
-                        Copy to clipboard
+                        {t("aiHub.publicModelHub.mcpDialog.copyToClipboard")}
                       
                     

                   

diff --git a/ui/litellm-dashboard/src/components/shared/AdminOnlyNotice.tsx b/ui/litellm-dashboard/src/components/shared/AdminOnlyNotice.tsx
index f618ff899fb..6d4dd79b251 100644
--- a/ui/litellm-dashboard/src/components/shared/AdminOnlyNotice.tsx
+++ b/ui/litellm-dashboard/src/components/shared/AdminOnlyNotice.tsx
@@ -1,14 +1,18 @@
 "use client";
 
 import React from "react";
+import { useTranslation } from "react-i18next";
 
 interface AdminOnlyNoticeProps {
   pageTitle: string;
 }
 
-export const AdminOnlyNotice: React.FC = ({ pageTitle }) => (
-  

-    
{pageTitle}

-    
{pageTitle} is only available to admin users.

-  

-);
+export const AdminOnlyNotice: React.FC = ({ pageTitle }) => {
+  const { t } = useTranslation();
+  return (
+    

+      
{pageTitle}

+      
{t("shell.adminOnlyNotice.description", { pageTitle })}

+    

+  );
+};
diff --git a/ui/litellm-dashboard/src/components/shared/BudgetPeriodTooltip.tsx b/ui/litellm-dashboard/src/components/shared/BudgetPeriodTooltip.tsx
index 56c0667b2a5..39f2e2e4e11 100644
--- a/ui/litellm-dashboard/src/components/shared/BudgetPeriodTooltip.tsx
+++ b/ui/litellm-dashboard/src/components/shared/BudgetPeriodTooltip.tsx
@@ -4,6 +4,7 @@ import dayjs from "dayjs";
 import { CircleHelp } from "lucide-react";
 
 import { getBudgetDurationLabel } from "@/components/common_components/budget_duration_dropdown";
+import i18n from "@/i18n/i18n";
 import { SimpleTooltip } from "@/components/ui/tooltip";
 import { budgetPeriodRange } from "@/utils/budgetUtils";
 import { formatNumberWithCommas } from "@/utils/dataUtils";
@@ -21,10 +22,14 @@ const describePeriod = (budgetDuration: string, budgetResetAt: string | null | u
   const timeZone = Intl.DateTimeFormat().resolvedOptions().timeZone;
   const period = budgetPeriodRange(budgetDuration, budgetResetAt);
   if (period) {
-    return `Budget period: ${formatPeriodDate(period.start)} to ${formatPeriodDate(period.end)} (${timeZone})`;
+    return i18n.t("shared.budgetPeriodTooltip.budgetPeriod", {
+      start: formatPeriodDate(period.start),
+      end: formatPeriodDate(period.end),
+      timeZone,
+    });
   }
   if (budgetResetAt && dayjs(budgetResetAt).isValid()) {
-    return `Next reset: ${formatPeriodDate(budgetResetAt)} (${timeZone})`;
+    return i18n.t("shared.budgetPeriodTooltip.nextReset", { date: formatPeriodDate(budgetResetAt), timeZone });
   }
   return null;
 };
@@ -35,15 +40,22 @@ export const budgetPeriodLines = ({
   budgetDuration,
   budgetResetAt,
 }: BudgetPeriodTooltipProps): readonly string[] => {
-  const spendLine = `${budgetDuration ? "Spend this period" : "Spend so far"}: $${formatNumberWithCommas(spend ?? 0, 4)}`;
-  if (maxBudget == null) return ["No budget set", spendLine];
-  const maxBudgetLine = `Max budget: $${formatNumberWithCommas(maxBudget, 2)}`;
-  if (!budgetDuration) return ["Lifetime budget, never resets", maxBudgetLine, spendLine];
+  const spendLine = i18n.t("shared.budgetPeriodTooltip.spendLine", {
+    label: budgetDuration
+      ? i18n.t("shared.budgetPeriodTooltip.spendThisPeriod")
+      : i18n.t("shared.budgetPeriodTooltip.spendSoFar"),
+    amount: formatNumberWithCommas(spend ?? 0, 4),
+  });
+  if (maxBudget == null) return [i18n.t("shared.budgetPeriodTooltip.noBudgetSet"), spendLine];
+  const maxBudgetLine = i18n.t("shared.budgetPeriodTooltip.maxBudget", {
+    amount: formatNumberWithCommas(maxBudget, 2),
+  });
+  if (!budgetDuration) return [i18n.t("shared.budgetPeriodTooltip.lifetimeBudget"), maxBudgetLine, spendLine];
   const periodLine = describePeriod(budgetDuration, budgetResetAt);
   return [
     ...(periodLine ? [periodLine] : []),
     maxBudgetLine,
-    `Reset interval: ${getBudgetDurationLabel(budgetDuration)}`,
+    i18n.t("shared.budgetPeriodTooltip.resetInterval", { interval: getBudgetDurationLabel(budgetDuration) }),
     spendLine,
   ];
 };
diff --git a/ui/litellm-dashboard/src/components/shared/CopyButton.tsx b/ui/litellm-dashboard/src/components/shared/CopyButton.tsx
index 225154e4bcf..eba571a37f3 100644
--- a/ui/litellm-dashboard/src/components/shared/CopyButton.tsx
+++ b/ui/litellm-dashboard/src/components/shared/CopyButton.tsx
@@ -2,6 +2,7 @@
 
 import { Button } from "@/components/ui/button";
 import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip";
+import i18n from "@/i18n/i18n";
 import { cn } from "@/lib/cva.config";
 import { copyToClipboard } from "@/utils/dataUtils";
 import { Check, Copy } from "lucide-react";
@@ -37,7 +38,7 @@ const CopyButton: React.FC = ({
   iconClassName,
   variant = "icon",
   iconOnly = false,
-  copiedLabel = "Copied",
+  copiedLabel = i18n.t("shared.copyButton.copied"),
 }) => {
   const [copied, setCopied] = useState(false);
   const isAction = variant === "action";
diff --git a/ui/litellm-dashboard/src/components/shared/CreatedKeyDisplay.tsx b/ui/litellm-dashboard/src/components/shared/CreatedKeyDisplay.tsx
index d58a1864575..cb0607cb885 100644
--- a/ui/litellm-dashboard/src/components/shared/CreatedKeyDisplay.tsx
+++ b/ui/litellm-dashboard/src/components/shared/CreatedKeyDisplay.tsx
@@ -1,6 +1,7 @@
 import React, { useState } from "react";
 import { CopyToClipboard } from "react-copy-to-clipboard";
 import { ChevronDown, ChevronUp } from "lucide-react";
+import { Trans, useTranslation } from "react-i18next";
 import anthropicLogo from "../../../public/assets/logos/anthropic.svg";
 import openaiLogo from "../../../public/assets/logos/openai_small.svg";
 import { getProxyBaseUrl } from "@/components/networking";
@@ -41,6 +42,7 @@ export const buildAgentInstructions = (baseUrl: string, apiKey: string): string
  * Used on the Virtual Keys page and in the Add Agent wizard.
  */
 const CreatedKeyDisplay: React.FC = ({ apiKey }) => {
+  const { t } = useTranslation();
   const [copied, setCopied] = useState(false);
   const [instructionsCopied, setInstructionsCopied] = useState(false);
   const [showInstructions, setShowInstructions] = useState(false);
@@ -49,39 +51,42 @@ const CreatedKeyDisplay: React.FC = ({ apiKey }) => {
 
   const handleCopy = () => {
     setCopied(true);
-    toast.success("Key copied to clipboard");
+    toast.success(t("shared.createdKeyDisplay.toasts.keyCopied"));
     setTimeout(() => setCopied(false), 2000);
   };
 
   const handleCopyInstructions = () => {
     setInstructionsCopied(true);
-    toast.success("Instructions copied");
+    toast.success(t("shared.createdKeyDisplay.toasts.instructionsCopied"));
     setTimeout(() => setInstructionsCopied(false), 2000);
   };
 
   return (
     

       

-        Please save this secret key somewhere safe and accessible. For security reasons,{" "}
-        you will not be able to view it again through your LiteLLM account. If you lose this secret key, you will
-        need to generate a new one.
+         }} />
       

 
-      
Virtual Key:

+      
{t("shared.createdKeyDisplay.virtualKey")}

       

         
{apiKey}

       

 
       
-        
+        
       
 
-      
Gateway URL:

+      
{t("shared.createdKeyDisplay.gatewayUrl")}

       

         
{baseUrl}

-         toast.success("Gateway URL copied to clipboard")}>
+         toast.success(t("shared.createdKeyDisplay.toasts.gatewayUrlCopied"))}
+        >
           
         
       

@@ -89,13 +94,17 @@ const CreatedKeyDisplay: React.FC = ({ apiKey }) => {
       

         

           

-            
Agent instructions

+            
{t("shared.createdKeyDisplay.agentInstructions")}

             Claude Code
             Codex
           

           

             
-              
+              
             
             
                     
                   

                 

diff --git a/ui/litellm-dashboard/src/components/shared/charts/StackedUsageChart.tsx b/ui/litellm-dashboard/src/components/shared/charts/StackedUsageChart.tsx
index 1dce03d23fc..5d58a0b0c08 100644
--- a/ui/litellm-dashboard/src/components/shared/charts/StackedUsageChart.tsx
+++ b/ui/litellm-dashboard/src/components/shared/charts/StackedUsageChart.tsx
@@ -3,6 +3,7 @@
 import * as React from "react";
 import { Bar, BarChart, CartesianGrid, XAxis, YAxis } from "recharts";
 import { type ChartConfig, ChartContainer, ChartTooltip, ChartTooltipContent } from "@/components/ui/chart";
+import i18n from "@/i18n/i18n";
 import { cn } from "@/lib/cva.config";
 
 /** The Model Leaderboard palette; shared so the usage page stacks in the same colors. */
@@ -77,7 +78,7 @@ export function StackedUsageChart({
   xLabel = String,
   format,
   totalFor,
-  totalLabel = "Total",
+  totalLabel = i18n.t("shared.charts.stackedUsageChart.total"),
   scale = "linear",
   className,
 }: StackedUsageChartProps) {
diff --git a/ui/litellm-dashboard/src/components/shared/charts/area_chart.tsx b/ui/litellm-dashboard/src/components/shared/charts/area_chart.tsx
index eba9f6b1d82..d3943b8273b 100644
--- a/ui/litellm-dashboard/src/components/shared/charts/area_chart.tsx
+++ b/ui/litellm-dashboard/src/components/shared/charts/area_chart.tsx
@@ -1,6 +1,7 @@
 "use client";
 
 import * as React from "react";
+import { useTranslation } from "react-i18next";
 import { Area, AreaChart as RechartsAreaChart, CartesianGrid, XAxis, YAxis } from "recharts";
 import { ChartContainer, ChartLegend, ChartLegendContent, ChartTooltip, type ChartConfig } from "@/components/ui/chart";
 import { cn } from "@/lib/cva.config";
@@ -38,6 +39,7 @@ export function AreaChart>({
   className,
   style,
 }: AreaChartProps) {
+  const { t } = useTranslation();
   const gradientId = React.useId().replace(/:/g, "");
 
   if (data.length === 0) {
@@ -46,7 +48,7 @@ export function AreaChart>({
         className={cn("flex h-80 w-full items-center justify-center rounded-lg border border-dashed", className)}
         style={style}
       >
-        
No data

+        
{t("shared.charts.areaChart.noData")}

       

     );
   }
diff --git a/ui/litellm-dashboard/src/components/shared/charts/bar_chart.tsx b/ui/litellm-dashboard/src/components/shared/charts/bar_chart.tsx
index c959bf0bae0..357536cb421 100644
--- a/ui/litellm-dashboard/src/components/shared/charts/bar_chart.tsx
+++ b/ui/litellm-dashboard/src/components/shared/charts/bar_chart.tsx
@@ -1,6 +1,7 @@
 "use client";
 
 import * as React from "react";
+import { useTranslation } from "react-i18next";
 import { Bar, BarChart as RechartsBarChart, CartesianGrid, Cell, XAxis, YAxis } from "recharts";
 import { ChartContainer, ChartLegend, ChartLegendContent, ChartTooltip, type ChartConfig } from "@/components/ui/chart";
 import { cn } from "@/lib/cva.config";
@@ -52,13 +53,14 @@ export function BarChart>({
   className,
   style,
 }: BarChartProps) {
+  const { t } = useTranslation();
   if (data.length === 0) {
     return (
       

-        
No data

+        
{t("shared.charts.barChart.noData")}

       

     );
   }
diff --git a/ui/litellm-dashboard/src/components/shared/errorUtils.tsx b/ui/litellm-dashboard/src/components/shared/errorUtils.tsx
index 054642e3e72..21c49225fd2 100644
--- a/ui/litellm-dashboard/src/components/shared/errorUtils.tsx
+++ b/ui/litellm-dashboard/src/components/shared/errorUtils.tsx
@@ -1,10 +1,12 @@
+import i18n from "@/i18n/i18n";
+
 /**
  * Extracts a user-friendly error message from various error formats
  * @param {any} error - The error object or message
  * @returns {string} - A clean error message
  */
 export const parseErrorMessage = (error: any): string => {
-  if (!error) return "An unknown error occurred";
+  if (!error) return i18n.t("shared.errorUtils.unknownError");
 
   // If error is already a string, return it
   if (typeof error === "string") return error;
diff --git a/ui/litellm-dashboard/src/components/shared/numerical_input.tsx b/ui/litellm-dashboard/src/components/shared/numerical_input.tsx
index 604651f3d15..1a25f6b5ea8 100644
--- a/ui/litellm-dashboard/src/components/shared/numerical_input.tsx
+++ b/ui/litellm-dashboard/src/components/shared/numerical_input.tsx
@@ -1,5 +1,6 @@
 import React from "react";
 import { Input } from "@/components/ui/input";
+import i18n from "@/i18n/i18n";
 
 interface NumericalInputProps {
   step?: number;
@@ -24,7 +25,15 @@ interface NumericalInputProps {
  */
 const NumericalInput = React.forwardRef(
   (
-    { step = 0.01, style = { width: "100%" }, placeholder = "Enter a numerical value", min, max, onChange, ...rest },
+    {
+      step = 0.01,
+      style = { width: "100%" },
+      placeholder = i18n.t("shared.numericalInput.placeholder"),
+      min,
+      max,
+      onChange,
+      ...rest
+    },
     ref,
   ) => (
     
       
diff --git a/ui/litellm-dashboard/src/components/shared/table_cells/UserPopoverCell.tsx b/ui/litellm-dashboard/src/components/shared/table_cells/UserPopoverCell.tsx
index eb786e45541..5aea68818d2 100644
--- a/ui/litellm-dashboard/src/components/shared/table_cells/UserPopoverCell.tsx
+++ b/ui/litellm-dashboard/src/components/shared/table_cells/UserPopoverCell.tsx
@@ -1,5 +1,7 @@
 "use client";
 
+import { useTranslation } from "react-i18next";
+
 import DefaultProxyAdminTag from "@/components/common_components/DefaultProxyAdminTag";
 import { HoverCard, HoverCardContent, HoverCardTrigger } from "@/components/ui/hover-card";
 import { userDetailHref } from "@/utils/entityLinks";
@@ -18,20 +20,27 @@ interface UserPopoverCellProps {
 }
 
 export function UserPopoverCell({ userAlias, userEmail, userId, width }: UserPopoverCellProps) {
+  const { t } = useTranslation();
   const displayValue = userAlias || userEmail || userId;
   const isDefaultAdmin = userId === DEFAULT_PROXY_ADMIN_USER_ID;
 
   const popoverContent = (
     

       {[
-        { label: "User Alias", value: userAlias },
-        { label: "User Email", value: userEmail },
-        { label: "User ID", value: userId },
+        { label: t("shared.tableCells.userPopoverCell.userAlias"), value: userAlias },
+        { label: t("shared.tableCells.userPopoverCell.userEmail"), value: userEmail },
+        { label: t("shared.tableCells.userPopoverCell.userId"), value: userId },
       ].map(({ label, value }) => (
         

           {label}
           {value ? (
-            
+            
           ) : (
             -
           )}
diff --git a/ui/litellm-dashboard/src/components/shared/table_cells/id_cell.tsx b/ui/litellm-dashboard/src/components/shared/table_cells/id_cell.tsx
index 47d0d750223..7781b382651 100644
--- a/ui/litellm-dashboard/src/components/shared/table_cells/id_cell.tsx
+++ b/ui/litellm-dashboard/src/components/shared/table_cells/id_cell.tsx
@@ -4,6 +4,7 @@ import { Copy } from "lucide-react";
 import * as React from "react";
 
 import { useEntityLinkClick } from "@/components/shared/EntityLink";
+import i18n from "@/i18n/i18n";
 import { cn } from "@/lib/cva.config";
 import { copyToClipboard } from "@/utils/dataUtils";
 
@@ -43,7 +44,7 @@ export function IdCell({
   href,
   onClick,
   copyable = false,
-  copyLabel = "Copy ID",
+  copyLabel = i18n.t("shared.tableCells.idCell.copyId"),
   truncate = true,
   fallback = "-",
   tooltip,
diff --git a/ui/litellm-dashboard/src/components/shared/table_cells/models_cell.tsx b/ui/litellm-dashboard/src/components/shared/table_cells/models_cell.tsx
index 81928d6fe87..e1f5efb66b6 100644
--- a/ui/litellm-dashboard/src/components/shared/table_cells/models_cell.tsx
+++ b/ui/litellm-dashboard/src/components/shared/table_cells/models_cell.tsx
@@ -1,8 +1,11 @@
 "use client";
 
+import { useTranslation } from "react-i18next";
+
 import { deriveKeyModelScope } from "@/components/key_scope";
 import { getModelDisplayName } from "@/components/key_team_helpers/fetch_available_models_team_key";
 import { Badge } from "@/components/ui/badge";
+import i18n from "@/i18n/i18n";
 
 import { CellTooltip } from "./cell_tooltip";
 
@@ -17,28 +20,29 @@ const WILDCARD_MODEL = "all-proxy-models";
 
 const formatModel = (model: string): string => {
   if (model === WILDCARD_MODEL) {
-    return "All Proxy Models";
+    return i18n.t("shared.tableCells.modelsCell.allProxyModels");
   }
   const name = getModelDisplayName(model);
   return name.length > 30 ? `${name.slice(0, 30)}...` : name;
 };
 
 export function ModelsCell({ models, maxVisible = 3, allowedRoutes, keyType }: ModelsCellProps) {
+  const { t } = useTranslation();
   if (!Array.isArray(models) || models.length === 0) {
     const scope = deriveKeyModelScope(allowedRoutes, keyType);
     if (!scope.hasModelAccess) {
       return (
         
-              No model access
+              {t("shared.tableCells.modelsCell.noModelAccess")}
             
           }
         />
       );
     }
-    return All Proxy Models;
+    return {t("shared.tableCells.modelsCell.allProxyModels")};
   }
 
   const visible = models.slice(0, maxVisible);
@@ -62,7 +66,7 @@ export function ModelsCell({ models, maxVisible = 3, allowedRoutes, keyType }: M
           }
           trigger={
             
-              +{overflow.length} more
+              {t("shared.tableCells.modelsCell.more", { number: overflow.length })}
             
           }
         />
diff --git a/ui/litellm-dashboard/src/components/shared/table_cells/spend_budget_cell.tsx b/ui/litellm-dashboard/src/components/shared/table_cells/spend_budget_cell.tsx
index 6090b42e93d..c3714c668ef 100644
--- a/ui/litellm-dashboard/src/components/shared/table_cells/spend_budget_cell.tsx
+++ b/ui/litellm-dashboard/src/components/shared/table_cells/spend_budget_cell.tsx
@@ -1,7 +1,10 @@
 "use client";
 
+import { useTranslation } from "react-i18next";
+
 import { InheritedBudgetHint, type InheritedBudgetGate } from "@/components/shared/InheritedBudgetHint";
 import { Meter, MeterIndicator, MeterTrack } from "@/components/shared/Meter";
+import i18n from "@/i18n/i18n";
 import { formatNumberWithCommas, getSpendString } from "@/utils/dataUtils";
 
 interface SpendBudgetCellProps {
@@ -25,10 +28,14 @@ const getBudgetLabel = (
   memberSpendText: string,
   budgetDecimals: number,
 ): string => {
-  if (budget === null) return "· Unlimited";
+  if (budget === null) return i18n.t("shared.tableCells.spendBudgetCell.unlimited");
   const formattedBudget = formatNumberWithCommas(budget, budgetDecimals);
-  if (usesTeamMemberBudget) return `· ${memberSpendText} of $${formattedBudget} (team member)`;
-  return `of $${formattedBudget}`;
+  if (usesTeamMemberBudget)
+    return i18n.t("shared.tableCells.spendBudgetCell.memberBudget", {
+      spend: memberSpendText,
+      budget: formattedBudget,
+    });
+  return i18n.t("shared.tableCells.spendBudgetCell.ofBudget", { budget: formattedBudget });
 };
 
 const getMeterValueText = (
@@ -39,7 +46,7 @@ const getMeterValueText = (
 ): string => {
   if (!hasBudget || budget === null) return "";
   const formattedBudget = formatNumberWithCommas(budget, budgetDecimals);
-  return `${meterSpendText} of $${formattedBudget}`;
+  return i18n.t("shared.tableCells.spendBudgetCell.meterValue", { spend: meterSpendText, budget: formattedBudget });
 };
 
 export function SpendBudgetCell({
@@ -50,6 +57,7 @@ export function SpendBudgetCell({
   spendDecimals = 4,
   budgetDecimals = 0,
 }: SpendBudgetCellProps) {
+  const { t } = useTranslation();
   const spendValue = typeof spend === "number" && !Number.isNaN(spend) ? spend : 0;
   const ownBudget = maxBudget ?? null;
   const budget = ownBudget ?? teamMemberGate?.maxBudget ?? null;
@@ -63,7 +71,9 @@ export function SpendBudgetCell({
   const spendText = spendValue > 0 ? getSpendString(spendValue, spendDecimals) : "$0.00";
   const memberSpendText = memberSpendValue > 0 ? getSpendString(memberSpendValue, spendDecimals) : "$0.00";
   const budgetLabel = getBudgetLabel(budget, usesTeamMemberBudget, memberSpendText, budgetDecimals);
-  const meterSpendText = usesTeamMemberBudget ? `team member spend ${memberSpendText}` : spendText;
+  const meterSpendText = usesTeamMemberBudget
+    ? t("shared.tableCells.spendBudgetCell.memberSpend", { spend: memberSpendText })
+    : spendText;
   const meterValueText = getMeterValueText(hasBudget, budget, meterSpendText, budgetDecimals);
 
   return (
diff --git a/ui/litellm-dashboard/src/components/ui/ResizableSheetContent.tsx b/ui/litellm-dashboard/src/components/ui/ResizableSheetContent.tsx
index b21a5b2c56e..cdfdc3bba43 100644
--- a/ui/litellm-dashboard/src/components/ui/ResizableSheetContent.tsx
+++ b/ui/litellm-dashboard/src/components/ui/ResizableSheetContent.tsx
@@ -2,6 +2,7 @@
 
 import * as React from "react";
 import { Maximize2Icon, Minimize2Icon } from "lucide-react";
+import { useTranslation } from "react-i18next";
 
 import { cn } from "@/lib/cva.config";
 import { Button } from "@/components/ui/button";
@@ -44,6 +45,7 @@ function ResizableSheetContent({
   minWidthPercent = 40,
   ...props
 }: ResizableSheetContentProps) {
+  const { t } = useTranslation();
   const [width, setWidth] = React.useState(() => readStoredWidth(storageKey, defaultWidthPercent, minWidthPercent));
   const widthRef = React.useRef(width);
   const lastNonFullWidthRef = React.useRef(defaultWidthPercent);
@@ -134,7 +136,7 @@ function ResizableSheetContent({
       

         {isFull ?  : }
diff --git a/ui/litellm-dashboard/src/components/ui/breadcrumb.tsx b/ui/litellm-dashboard/src/components/ui/breadcrumb.tsx
index 731a014362d..69a2dfcce7e 100644
--- a/ui/litellm-dashboard/src/components/ui/breadcrumb.tsx
+++ b/ui/litellm-dashboard/src/components/ui/breadcrumb.tsx
@@ -1,11 +1,13 @@
 import * as React from "react";
 import { ChevronRight } from "lucide-react";
+import { useTranslation } from "react-i18next";
 
 import { cn } from "@/lib/cva.config";
 
-const Breadcrumb = React.forwardRef>(({ ...props }, ref) => (
-  

           
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.tsx
index 92890031ddd..ff027ac8b3c 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.tsx
@@ -6,6 +6,7 @@ import { useCredentials } from "@/app/(dashboard)/hooks/credentials/useCredentia
 import { credentialLabelsByName } from "@/components/shared/credentialOptions";
 import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized";
 import DeleteResourceModal from "@/components/common_components/DeleteResourceModal";
+import i18n from "@/i18n/i18n";
 import ModelSettingsModal from "@/components/model_dashboard/ModelSettingsModal/ModelSettingsModal";
 import { ModelData } from "@/components/model_dashboard/types";
 import { toast } from "@/lib/toast";
@@ -18,6 +19,7 @@ import { ColumnFiltersState, functionalUpdate, OnChangeFn, PaginationState, Sort
 import { Info } from "lucide-react";
 import { createParser, parseAsInteger, parseAsString, parseAsStringLiteral, useQueryStates } from "nuqs";
 import { useCallback, useMemo, useState } from "react";
+import { Trans, useTranslation } from "react-i18next";
 
 import { useModelsInfo } from "../../hooks/models/useModels";
 import { ROUTING_STATUS_FILTER_ID, routingStatusToBlocked } from "../utils/routingStatus";
@@ -82,6 +84,7 @@ const AllModelsTab = ({
   setSelectedModelId,
   setSelectedTeamId,
 }: AllModelsTabProps) => {
+  const { t } = useTranslation();
   const { data: modelCostMapData, isLoading: isLoadingModelCostMap } = useModelCostMap();
   const { accessToken, userId, userRole, isViewOnly } = useAuthorized();
   const { data: teams, isLoading: isLoadingTeams } = useTeams();
@@ -240,7 +243,7 @@ const AllModelsTab = ({
 
   const teamOptions = useMemo(
     () => [
-      { value: PERSONAL_TEAM_VALUE, label: "Personal" },
+      { value: PERSONAL_TEAM_VALUE, label: i18n.t("modelsAndEndpoints.allModelsTab.personalTeam") },
       ...(teams ?? [])
         .filter((team) => team.team_id)
         .map((team) => ({ value: team.team_id, label: team.team_alias ? team.team_alias : team.team_id })),
@@ -263,7 +266,7 @@ const AllModelsTab = ({
     try {
       setDeleteLoading(true);
       await modelDeleteCall(accessToken, deleteModalModelId);
-      toast.success("Model deleted successfully");
+      toast.success(t("modelsAndEndpoints.allModelsTab.toasts.deleted"));
       queryClient.invalidateQueries({ queryKey: ["models", "list"] });
       refetchModels();
     } catch (error) {
@@ -281,7 +284,11 @@ const AllModelsTab = ({
       try {
         setPausingModelId(modelId);
         await modelPatchUpdateCall(accessToken, { blocked }, modelId);
-        toast.success(blocked ? "Model paused" : "Model resumed");
+        toast.success(
+          blocked
+            ? i18n.t("modelsAndEndpoints.allModelsTab.toasts.paused")
+            : i18n.t("modelsAndEndpoints.allModelsTab.toasts.resumed"),
+        );
         // invalidateQueries already schedules a refetch for active observers
         // on this key — no need to also call refetchModels() (would double-fetch).
         queryClient.invalidateQueries({ queryKey: ["models", "list"] });
@@ -352,19 +359,22 @@ const AllModelsTab = ({
             
             {selectedTeamValue === PERSONAL_TEAM_VALUE ? (
               
-                To access these models, create a Virtual Key without selecting a team on the{" "}
-                
-                  Virtual Keys page
-                
-                .
+                ,
+                  }}
+                />
               
             ) : (
               
-                To access these models, create a Virtual Key and select Team as "{teamAccessLabel}" on the{" "}
-                
-                  Virtual Keys page
-                
-                .
+                {teamAccessLabel},
+                    keys: ,
+                  }}
+                />
               
             )}
           

@@ -373,28 +383,29 @@ const AllModelsTab = ({
 
        = {
   [MODEL_NAME_COLUMN_ID]: "Public Model Name",
@@ -42,13 +45,13 @@ const FILTER_LABELS: Record = {
 };
 
 const ROUTING_STATUS_FILTER_VALUES = {
-  all: "All",
-  active: "Active",
-  paused: "Paused",
+  all: "modelsAndEndpoints.allModelsTable.status.all",
+  active: "modelsAndEndpoints.allModelsTable.status.active",
+  paused: "modelsAndEndpoints.allModelsTable.status.paused",
 } as const;
 
-const VIEW_MODE_LABELS: Record = {
-  current_team: "Current Team Models",
+const VIEW_MODE_LABELS: Record = {
+  current_team: "modelsAndEndpoints.allModelsTable.viewModes.currentTeam",
   all: ALL_PROXY_MODELS_LABEL,
 };
 
@@ -93,14 +96,17 @@ interface AllModelsTableProps {
 }
 
 function EmptyState() {
+  const { t } = useTranslation();
   return (
     

       

         
       

-      
No models found

+      

+        {t("modelsAndEndpoints.allModelsTable.empty.title")}
+      

       

-        No models match your search or filters. Try resetting them.
+        {t("modelsAndEndpoints.allModelsTable.empty.description")}
       

     

   );
@@ -140,6 +146,7 @@ export function AllModelsTable({
   pausingModelId,
   credentialLabels,
 }: AllModelsTableProps) {
+  const { t } = useTranslation();
   const [filtersOpen, setFiltersOpen] = useState(false);
 
   const columns = useMemo(() => {
@@ -169,8 +176,8 @@ export function AllModelsTable({
 
   const modelGroupOptions = useMemo(
     () => [
-      { label: ALL_PROXY_MODELS_LABEL, value: ALL_MODEL_GROUPS_VALUE },
-      { label: "Wildcard Models (*)", value: WILDCARD_MODEL_GROUP_VALUE },
+      { label: i18n.t(ALL_PROXY_MODELS_LABEL), value: ALL_MODEL_GROUPS_VALUE },
+      { label: i18n.t("modelsAndEndpoints.allModelsTable.wildcardModels"), value: WILDCARD_MODEL_GROUP_VALUE },
       ...availableModelGroups.map((group) => ({ label: group, value: group })),
     ],
     [availableModelGroups],
@@ -178,7 +185,7 @@ export function AllModelsTable({
 
   const accessGroupOptions = useMemo(
     () => [
-      { label: "All Model Access Groups", value: ALL_MODEL_GROUPS_VALUE },
+      { label: i18n.t("modelsAndEndpoints.allModelsTable.allAccessGroups"), value: ALL_MODEL_GROUPS_VALUE },
       ...availableModelAccessGroups.map((accessGroup) => ({ label: accessGroup, value: accessGroup })),
     ],
     [availableModelAccessGroups],
@@ -187,10 +194,12 @@ export function AllModelsTable({
   const formatFilterValue = (columnId: string, value: unknown): string => {
     const raw = String(value);
     if (columnId === MODEL_NAME_COLUMN_ID && raw === WILDCARD_MODEL_GROUP_VALUE) {
-      return "Wildcard Models (*)";
+      return t("modelsAndEndpoints.allModelsTable.wildcardModels");
     }
     if (columnId === ROUTING_STATUS_FILTER_ID) {
-      return ROUTING_STATUS_FILTER_VALUES[raw as keyof typeof ROUTING_STATUS_FILTER_VALUES] ?? raw;
+      return Object.hasOwn(ROUTING_STATUS_FILTER_VALUES, raw)
+        ? t(ROUTING_STATUS_FILTER_VALUES[raw as keyof typeof ROUTING_STATUS_FILTER_VALUES])
+        : raw;
     }
     return raw;
   };
@@ -219,7 +228,7 @@ export function AllModelsTable({
       enableColumnResizing
       maxBodyHeight={MODEL_TABLE_BODY_HEIGHT}
       isLoading={isLoading}
-      loadingMessage="Loading models…"
+      loadingMessage={t("modelsAndEndpoints.allModelsTable.loading")}
       noDataMessage={}
       size="compact"
       toolbar={(table) => (
@@ -228,7 +237,7 @@ export function AllModelsTable({
             table={table}
             searchValue={searchValue}
             onSearchChange={onSearchChange}
-            searchPlaceholder="Search model names…"
+            searchPlaceholder={t("modelsAndEndpoints.allModelsTable.searchPlaceholder")}
             onOpenFilters={() => setFiltersOpen(true)}
             onRefresh={onRefresh}
             isRefreshing={isRefreshing}
@@ -238,7 +247,7 @@ export function AllModelsTable({
             
 
             
 
@@ -283,8 +297,8 @@ export function AllModelsTable({
             
         )}
       

@@ -105,8 +104,8 @@ export function AutoRoutersPanel({
           className="max-h-[90vh] overflow-y-auto sm:max-w-4xl"
         >
           
-            Add Auto Router
-            Choose models for your auto router.
+            {t("modelsAndEndpoints.autoRoutersPanel.addDialogTitle")}
+            {t("modelsAndEndpoints.autoRoutersPanel.addDialogDescription")}
           
            setDeletingRouter(null)}
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersTable.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersTable.tsx
index 2102f5e55d9..bb09da7073c 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersTable.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersTable.tsx
@@ -2,6 +2,7 @@
 
 import { SortingState } from "@tanstack/react-table";
 import { useMemo } from "react";
+import { useTranslation } from "react-i18next";
 
 import { DataTable } from "@/components/shared/DataTable";
 import { AutoRouterIcon } from "@/components/shared/table_cells";
@@ -25,16 +26,17 @@ const DEFAULT_SORTING: SortingState = [
 ];
 
 function EmptyState({ canModify }: { canModify: boolean }) {
+  const { t } = useTranslation();
   return (
     

       

         
       

-      
No auto routers yet

+      
{t("modelsAndEndpoints.autoRoutersTable.emptyTitle")}

       

         {canModify
-          ? "Create an auto router to pick the right model per request instead of pinning one."
-          : "An auto router picks the right model per request instead of pinning one."}
+          ? t("modelsAndEndpoints.autoRoutersTable.emptyCreate")
+          : t("modelsAndEndpoints.autoRoutersTable.emptyReadOnly")}
       

     

   );
@@ -47,9 +49,10 @@ export function AutoRoutersTable({
   onRouterClick,
   onDeleteClick,
 }: AutoRoutersTableProps) {
+  const { t } = useTranslation();
   const columns = useMemo(
-    () => getAutoRoutersTableColumns({ canModify, onRouterClick, onDeleteClick }),
-    [canModify, onRouterClick, onDeleteClick],
+    () => getAutoRoutersTableColumns({ canModify, onRouterClick, onDeleteClick }, t),
+    [canModify, onRouterClick, onDeleteClick, t],
   );
 
   return (
@@ -62,7 +65,7 @@ export function AutoRoutersTable({
       paginationMode="client"
       pageSizeOptions={PAGE_SIZE_OPTIONS}
       isLoading={isLoading}
-      loadingMessage="Loading auto routers…"
+      loadingMessage={t("modelsAndEndpoints.autoRoutersTable.loading")}
       noDataMessage={}
       size="compact"
     />
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersTableColumns.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersTableColumns.tsx
index 4a99062988f..4878ebff6bc 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersTableColumns.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersTableColumns.tsx
@@ -1,6 +1,8 @@
 "use client";
 
 import { ColumnDef } from "@tanstack/react-table";
+import type { TFunction } from "i18next";
+import { useTranslation } from "react-i18next";
 import { useEffect, useMemo, useRef, useState } from "react";
 import { MoreHorizontal, Trash2 } from "lucide-react";
 
@@ -14,6 +16,7 @@ import {
   DropdownMenuItem,
   DropdownMenuTrigger,
 } from "@/components/ui/dropdown-menu";
+import i18n from "@/i18n/i18n";
 import { cn } from "@/lib/cva.config";
 
 import { AutoRouterRow } from "./autoRouterRows";
@@ -71,10 +74,11 @@ function AutoRouterRowActions({
   row: AutoRouterRow;
   onDeleteClick: (row: AutoRouterRow) => void;
 }) {
+  const { t } = useTranslation();
   return (
     
       
@@ -87,7 +91,7 @@ function AutoRouterRowActions({
           onClick={() => onDeleteClick(row)}
         >
           
-          Delete auto router
+          {t("modelsAndEndpoints.autoRoutersTableColumns.deleteAutoRouter")}
         
       
     
@@ -100,16 +104,17 @@ interface AutoRoutersTableColumnsDeps {
   onDeleteClick: (row: AutoRouterRow) => void;
 }
 
-export const getAutoRoutersTableColumns = ({
-  canModify,
-  onRouterClick,
-  onDeleteClick,
-}: AutoRoutersTableColumnsDeps): ColumnDef[] => [
+export const getAutoRoutersTableColumns = (
+  { canModify, onRouterClick, onDeleteClick }: AutoRoutersTableColumnsDeps,
+  t: TFunction = i18n.t.bind(i18n),
+): ColumnDef[] => [
   {
     id: "name",
     accessorKey: "name",
-    meta: { title: "Name" },
-    header: ({ column }) => ,
+    meta: { title: t("modelsAndEndpoints.autoRoutersTableColumns.name") },
+    header: ({ column }) => (
+      
+    ),
     size: 260,
     enableSorting: true,
     cell: ({ row }) =>  onRouterClick(row.original)} />,
@@ -117,16 +122,16 @@ export const getAutoRoutersTableColumns = ({
   {
     id: "kind",
     accessorKey: "kind",
-    meta: { title: "Type" },
-    header: "Type",
+    meta: { title: t("modelsAndEndpoints.autoRoutersTableColumns.type") },
+    header: t("modelsAndEndpoints.autoRoutersTableColumns.type"),
     size: 180,
     enableSorting: false,
     cell: ({ row }) => ,
   },
   {
     id: "targets",
-    meta: { title: "Routes to" },
-    header: "Routes to",
+    meta: { title: t("modelsAndEndpoints.autoRoutersTableColumns.routesTo") },
+    header: t("modelsAndEndpoints.autoRoutersTableColumns.routesTo"),
     size: 320,
     enableSorting: false,
     cell: ({ row }) => ,
@@ -134,8 +139,8 @@ export const getAutoRoutersTableColumns = ({
   {
     id: "defaultModel",
     accessorKey: "defaultModel",
-    meta: { title: "Default model" },
-    header: "Default model",
+    meta: { title: t("modelsAndEndpoints.autoRoutersTableColumns.defaultModel") },
+    header: t("modelsAndEndpoints.autoRoutersTableColumns.defaultModel"),
     size: 200,
     enableSorting: false,
     cell: ({ row }) =>
@@ -150,8 +155,10 @@ export const getAutoRoutersTableColumns = ({
   {
     id: "createdAt",
     accessorKey: "createdAt",
-    meta: { title: "Created" },
-    header: ({ column }) => ,
+    meta: { title: t("modelsAndEndpoints.autoRoutersTableColumns.created") },
+    header: ({ column }) => (
+      
+    ),
     size: 150,
     enableSorting: true,
     sortingFn: "datetime",
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelRetrySettingsTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelRetrySettingsTab.tsx
index a61eccbb5a2..4d6d542a0b4 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelRetrySettingsTab.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelRetrySettingsTab.tsx
@@ -1,5 +1,6 @@
 import { LoaderCircle } from "lucide-react";
 import React from "react";
+import { useTranslation } from "react-i18next";
 
 import { Button } from "@/components/ui/button";
 import { Input } from "@/components/ui/input";
@@ -53,9 +54,10 @@ const ModelRetrySettingsTab = ({
   handleSaveRetrySettings,
   isSaving = false,
 }: ModelRetrySettingsTabProps) => {
+  const { t } = useTranslation();
   const isGlobalScope = selectedModelGroup === "global";
   const scopeItems = [
-    { value: "global", label: "Global Default" },
+    { value: "global", label: t("modelsAndEndpoints.modelRetrySettingsTab.globalDefault") },
     ...availableModelGroups.map((group) => ({ value: group, label: group })),
   ];
 
@@ -86,7 +88,7 @@ const ModelRetrySettingsTab = ({
   return (
     

       

-        
+
@@ -161,7 +167,7 @@ const ModelRetrySettingsTab = ({
{exceptionType} {!isGlobalScope && ( - (Global: {inheritedValue}) + + {t("modelsAndEndpoints.modelRetrySettingsTab.inheritedValue", { value: inheritedValue })} + )} {!isGlobalScope && hasOverride && ( )}
); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelsTableColumns.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelsTableColumns.tsx index 1b7369c9520..0e286418355 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelsTableColumns.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelsTableColumns.tsx @@ -1,6 +1,8 @@ "use client"; import { ColumnDef } from "@tanstack/react-table"; +import type { TFunction } from "i18next"; +import { useTranslation } from "react-i18next"; import { CredentialIdentity } from "@/components/shared/CredentialIdentity"; import { Copy, Info, Loader2, Pencil, RefreshCw, Trash2 } from "lucide-react"; @@ -14,6 +16,7 @@ import { HoverCard, HoverCardContent, HoverCardTrigger } from "@/components/ui/h import { Switch } from "@/components/ui/switch"; import { getDisplayModelName } from "@/components/view_model/model_name_display"; import { copyToClipboard, formatPerSecondCost } from "@/utils/dataUtils"; +import i18n from "@/i18n/i18n"; export const MODEL_ID_COLUMN_ID = "model_info_id"; export const MODEL_NAME_COLUMN_ID = "model_name"; @@ -56,6 +59,7 @@ const formatShortDate = (value: string | null | undefined): string | null => { }; function ModelInformationCell({ model, displayName }: { model: ModelData; displayName: string }) { + const { t } = useTranslation(); const litellmModelName = model.litellm_model_name || "-"; return ( @@ -85,26 +89,37 @@ function ModelInformationCell({ model, displayName }: { model: ModelData; displa
{model.provider ? : null} - {model.provider || "Unknown provider"} + + {model.provider || t("modelsAndEndpoints.modelsTableColumns.unknownProvider")} +
- Public Model Name + + {t("modelsAndEndpoints.modelsTableColumns.information.publicModelName")} + {displayName}
- LiteLLM Model Name + + {t("modelsAndEndpoints.modelsTableColumns.information.litellmModelName")} + {litellmModelName} @@ -117,15 +132,16 @@ function ModelInformationCell({ model, displayName }: { model: ModelData; displa } function CredentialsHeader() { + const { t } = useTranslation(); return ( - Credentials + {t("modelsAndEndpoints.modelsTableColumns.columns.credentials")} @@ -135,23 +151,25 @@ function CredentialsHeader() {
- Credential types + + {t("modelsAndEndpoints.modelsTableColumns.credentialTypes.title")} +
- Reusable + {t("modelsAndEndpoints.modelsTableColumns.credentialTypes.reusable")} - Credentials saved in LiteLLM that can be added to models repeatedly. + {t("modelsAndEndpoints.modelsTableColumns.credentialTypes.reusableDescription")}
- Manual + {t("modelsAndEndpoints.modelsTableColumns.credentialTypes.manual")} - Credentials added directly during model creation or defined in the config file. + {t("modelsAndEndpoints.modelsTableColumns.credentialTypes.manualDescription")}
@@ -162,11 +180,12 @@ function CredentialsHeader() { } function CredentialsCell({ credentialName, label }: { credentialName: string | undefined; label: string | undefined }) { + const { t } = useTranslation(); if (!credentialName) { return ( - Manual + {t("modelsAndEndpoints.modelsTableColumns.credentialTypes.manual")} ); } @@ -182,10 +201,13 @@ function CredentialsCell({ credentialName, label }: { credentialName: string | u } function CreatedByCell({ model }: { model: ModelData }) { + const { t } = useTranslation(); const isConfigModel = !model.model_info?.db_model; const createdAt = formatShortDate(model.model_info.created_at); - const primary = isConfigModel ? "Defined in config" : model.model_info.created_by || "Unknown"; - const secondaryForDbModel = createdAt ?? "Unknown date"; + const primary = isConfigModel + ? t("modelsAndEndpoints.modelsTableColumns.createdBy.definedInConfig") + : model.model_info.created_by || t("modelsAndEndpoints.modelsTableColumns.createdBy.unknown"); + const secondaryForDbModel = createdAt ?? t("modelsAndEndpoints.modelsTableColumns.createdBy.unknownDate"); return (
@@ -207,6 +229,7 @@ function CostRow({ label, value }: { label: string; value: string }) { } function CostsCell({ model }: { model: ModelData }) { + const { t } = useTranslation(); const { input_cost: inputCost, output_cost: outputCost, output_cost_per_second: perSecond } = model; const hasPerSecond = perSecond != null; const showInput = inputCost != null && (!hasPerSecond || Number(inputCost) > 0); @@ -218,12 +241,25 @@ function CostsCell({ model }: { model: ModelData }) { return ( - {showInput && } - {showOutput && } - {hasPerSecond && } + {showInput && ( + + )} + {showOutput && ( + + )} + {hasPerSecond && ( + + )}
} /> @@ -231,6 +267,7 @@ function CostsCell({ model }: { model: ModelData }) { } function AccessGroupsCell({ accessGroups }: { accessGroups: string[] | null }) { + const { t } = useTranslation(); if (!accessGroups || accessGroups.length === 0) { return -; } @@ -253,7 +290,7 @@ function AccessGroupsCell({ accessGroups }: { accessGroups: string[] | null }) { } trigger={ - +{overflow.length} more + {t("modelsAndEndpoints.modelsTableColumns.accessGroups.more", { hiddenCount: overflow.length })} } /> @@ -281,6 +318,7 @@ function ModelRowActions({ onDeleteClick, onTogglePauseClick, }: ModelRowActionsProps) { + const { t } = useTranslation(); const modelId = model.model_info?.id; const isConfigModel = !model.model_info?.db_model; const isAdmin = userRole === "Admin" && !isViewOnly; @@ -290,17 +328,19 @@ function ModelRowActions({ const resolvePauseTooltip = (): string => { if (isConfigModel) { - return "Config models cannot be paused from the dashboard. Pause is DB-backed."; + return t("modelsAndEndpoints.modelsTableColumns.actions.configPauseTooltip"); } if (!isAdmin) { - return "Only proxy admins can pause or resume a model."; + return t("modelsAndEndpoints.modelsTableColumns.actions.adminOnlyPauseTooltip"); } - return isBlocked ? "Resume model — restore normal routing." : "Pause model — stop routing requests until resumed."; + return isBlocked + ? t("modelsAndEndpoints.modelsTableColumns.actions.resumeTooltip") + : t("modelsAndEndpoints.modelsTableColumns.actions.pauseTooltip"); }; const deleteTooltip = isConfigModel - ? "Config model cannot be deleted on the dashboard. Please delete it from the config file." - : "Delete model"; + ? t("modelsAndEndpoints.modelsTableColumns.actions.configDeleteTooltip") + : t("modelsAndEndpoints.modelsTableColumns.actions.delete"); return (
@@ -319,7 +359,11 @@ function ModelRowActions({ size="sm" checked={!isBlocked} disabled={!isPauseToggleable} - aria-label={isBlocked ? "Resume model" : "Pause model"} + aria-label={ + isBlocked + ? t("modelsAndEndpoints.modelsTableColumns.actions.resume") + : t("modelsAndEndpoints.modelsTableColumns.actions.pause") + } data-testid={`model-pause-toggle-${modelId}`} onCheckedChange={(nextChecked) => { if (isPauseToggleable && onTogglePauseClick && modelId) { @@ -339,7 +383,7 @@ function ModelRowActions({
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AccessGroupBudgetsPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AccessGroupBudgetsPanel.tsx index 20eb7b2e64c..d632408c85a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AccessGroupBudgetsPanel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AccessGroupBudgetsPanel.tsx @@ -3,6 +3,7 @@ import { SortingState } from "@tanstack/react-table"; import { Inbox } from "lucide-react"; import React, { useMemo, useState } from "react"; +import { useTranslation } from "react-i18next"; import DeleteResourceModal from "@/components/common_components/DeleteResourceModal"; import { DataTable } from "@/components/shared/DataTable"; @@ -21,20 +22,24 @@ import { getAccessGroupBudgetColumns } from "@/app/(dashboard)/models-and-endpoi const DEFAULT_SORTING: SortingState = [{ id: "access_group", desc: false }]; function EmptyState() { + const { t } = useTranslation(); return (
-
No model access groups yet
+
+ {t("modelsAndEndpoints.accessGroupBudgetsPanel.empty.title")} +
- Put a deployment in an access group from its model settings, then give the group a shared budget here. + {t("modelsAndEndpoints.accessGroupBudgetsPanel.empty.description")}
); } export default function AccessGroupBudgetsPanel() { + const { t } = useTranslation(); const { userRole } = useAuthorized(); const { data: accessGroups, isLoading } = useModelAccessGroups(); const setBudget = useSetModelAccessGroupBudget(); @@ -57,7 +62,7 @@ export default function AccessGroupBudgetsPanel() { { accessGroup, params }, { onSuccess: () => { - toast.success(`Budget saved for "${accessGroup}"`); + toast.success(t("modelsAndEndpoints.accessGroupBudgetsPanel.toasts.saved", { accessGroup })); setEditing(null); }, }, @@ -69,7 +74,7 @@ export default function AccessGroupBudgetsPanel() { const accessGroup = clearing.access_group; clearBudget.mutate(accessGroup, { onSuccess: () => { - toast.success(`Budget cleared for "${accessGroup}"`); + toast.success(t("modelsAndEndpoints.accessGroupBudgetsPanel.toasts.cleared", { accessGroup })); setClearing(null); }, }); @@ -77,10 +82,7 @@ export default function AccessGroupBudgetsPanel() { return (
-

- A model access group can carry one budget that every key granted the group by name draws from together. Keys - that reach the group's models through a wildcard or all-proxy-models are not charged against it. -

+

{t("modelsAndEndpoints.accessGroupBudgetsPanel.description")}

} size="compact" /> @@ -105,12 +107,19 @@ export default function AccessGroupBudgetsPanel() { setClearing(null)} onOk={handleConfirmClear} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/ModelRetrySettingsPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/ModelRetrySettingsPanel.tsx index a5c10ca8a8b..50dc4908865 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/ModelRetrySettingsPanel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/ModelRetrySettingsPanel.tsx @@ -1,6 +1,7 @@ "use client"; import { useCallback, useEffect, useState } from "react"; +import { useTranslation } from "react-i18next"; import ModelRetrySettingsTab from "@/app/(dashboard)/models-and-endpoints/components/ModelRetrySettingsTab"; import { getCallbacksCall } from "@/components/networking"; import { useUpdateRetryPolicy } from "@/app/(dashboard)/hooks/routerSettings/useUpdateRetryPolicy"; @@ -23,6 +24,7 @@ interface RouterSettings { } export default function ModelRetrySettingsPanel() { + const { t } = useTranslation(); const { accessToken, userId: userID, userRole } = useAuthorized(); const { availableModelGroups } = useModelDashboardData(); const updateRetryPolicy = useUpdateRetryPolicy(accessToken); @@ -69,7 +71,7 @@ export default function ModelRetrySettingsPanel() { { retry_policy: globalRetryPolicy, model_group_retry_policy: modelGroupRetryPolicy }, { onSuccess: () => { - toast.success("Retry settings saved successfully"); + toast.success(t("modelsAndEndpoints.modelRetrySettingsPanel.toasts.saved")); void fetchRetrySettings().then((routerSettings) => { if (routerSettings) { applyRetrySettings(routerSettings); @@ -77,7 +79,7 @@ export default function ModelRetrySettingsPanel() { }); }, onError: () => { - toast.fromError("Failed to save retry settings"); + toast.fromError(t("modelsAndEndpoints.modelRetrySettingsPanel.toasts.saveFailed")); }, }, ); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/transform-request/TransformRequestPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/transform-request/TransformRequestPanel.tsx index cf1eecdb03f..8fe92b51943 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/transform-request/TransformRequestPanel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/transform-request/TransformRequestPanel.tsx @@ -1,4 +1,5 @@ import React, { useState } from "react"; +import { Trans, useTranslation } from "react-i18next"; import { ArrowRight, Copy } from "lucide-react"; import { Button } from "@/components/ui/button"; import { Card, CardContent, CardDescription, CardFooter, CardHeader, CardTitle } from "@/components/ui/card"; @@ -12,6 +13,7 @@ interface TransformRequestPanelProps { } const TransformRequestPanel: React.FC = ({ accessToken }) => { + const { t } = useTranslation(); const [originalRequestJSON, setOriginalRequestJSON] = useState(`{ "model": "openai/gpt-4o", "messages": [ @@ -69,7 +71,7 @@ ${formattedBody} try { requestBody = JSON.parse(originalRequestJSON); } catch (e) { - toast.fromError("Invalid JSON in request body"); + toast.fromError(t("transformRequest.transformRequestPanel.toasts.invalidJson")); setIsLoading(false); return; } @@ -82,7 +84,7 @@ ${formattedBody} // Make the API call using fetch if (!accessToken) { - toast.fromError("No access token found"); + toast.fromError(t("transformRequest.transformRequestPanel.toasts.noAccessToken")); setIsLoading(false); return; } @@ -100,17 +102,17 @@ ${formattedBody} // Update state with the formatted curl command setTransformedResponse(formattedCurl); - toast.success("Request transformed successfully"); + toast.success(t("transformRequest.transformRequestPanel.toasts.transformed")); } else { // Handle the case where the API returns a different format // Try to extract the parts from a string response if needed const rawText = typeof data === "string" ? data : JSON.stringify(data); setTransformedResponse(rawText); - toast.info("Transformed request received in unexpected format"); + toast.info(t("transformRequest.transformRequestPanel.toasts.unexpectedFormat")); } } catch (err) { console.error("Error transforming request:", err); - toast.fromError("Failed to transform request"); + toast.fromError(t("transformRequest.transformRequestPanel.toasts.failed")); } finally { setIsLoading(false); } @@ -126,16 +128,16 @@ ${formattedBody} return (
-

Playground

-

- See how LiteLLM transforms your request for the specified provider. -

+

{t("transformRequest.transformRequestPanel.title")}

+

{t("transformRequest.transformRequestPanel.description")}

{/* Original Request Panel */} - Original Request - The request you would send to LiteLLM /chat/completions endpoint. + + {t("transformRequest.transformRequestPanel.original.title")} + + {t("transformRequest.transformRequestPanel.original.description")} @@ -144,13 +146,13 @@ ${formattedBody} value={originalRequestJSON} onChange={(e) => setOriginalRequestJSON(e.target.value)} onKeyDown={handleKeyDown} - placeholder="Press Cmd/Ctrl + Enter to transform" + placeholder={t("transformRequest.transformRequestPanel.original.placeholder")} /> @@ -159,9 +161,13 @@ ${formattedBody} {/* Transformed Request Panel */} - Transformed Request - How LiteLLM transforms your request for the specified provider. -

Note: Sensitive headers are not shown.

+ + {t("transformRequest.transformRequestPanel.transformed.title")} + + {t("transformRequest.transformRequestPanel.transformed.description")} +

+ {t("transformRequest.transformRequestPanel.transformed.sensitiveHeaders")} +

@@ -187,11 +193,11 @@ ${formattedBody}
diff --git a/ui/litellm-dashboard/src/components/GuardrailSettingsView.tsx b/ui/litellm-dashboard/src/components/GuardrailSettingsView.tsx index 5510eb62bdb..fa810521757 100644 --- a/ui/litellm-dashboard/src/components/GuardrailSettingsView.tsx +++ b/ui/litellm-dashboard/src/components/GuardrailSettingsView.tsx @@ -1,5 +1,6 @@ import React from "react"; import { Globe2 } from "lucide-react"; +import { useTranslation } from "react-i18next"; import { Badge } from "@/components/ui/badge"; import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; import { cn } from "@/lib/cva.config"; @@ -21,6 +22,7 @@ export function GuardrailSettingsView({ variant = "card", className = "", }: GuardrailSettingsViewProps) { + const { t } = useTranslation(); const optedOutSet = new Set(optedOutGlobalGuardrails); const globalsRunning = Array.from(globalGuardrailNames).filter((n) => !optedOutSet.has(n)); const nonGlobalOptIns = teamGuardrails.filter((n) => !globalGuardrailNames.has(n)); @@ -28,16 +30,16 @@ export function GuardrailSettingsView({ const isEmpty = !killSwitchOn && globalsRunning.length === 0 && nonGlobalOptIns.length === 0; const content = isEmpty ? ( - No guardrails configured + {t("guardrailSettingsView.empty")} ) : (
- - Global + + {t("guardrailSettingsView.global")} {killSwitchOn ? ( - Bypassed for this team + {t("guardrailSettingsView.bypassed")} ) : globalsRunning.length > 0 ? (
{globalsRunning.map((name) => ( @@ -45,11 +47,13 @@ export function GuardrailSettingsView({ ))}
) : ( - None configured + {t("guardrailSettingsView.noneConfigured")} )}
- Team-specific + + {t("guardrailSettingsView.teamSpecific")} + {nonGlobalOptIns.length > 0 ? (
{nonGlobalOptIns.map((name) => ( @@ -57,7 +61,7 @@ export function GuardrailSettingsView({ ))}
) : ( - None configured + {t("guardrailSettingsView.noneConfigured")} )}
@@ -67,8 +71,8 @@ export function GuardrailSettingsView({ return ( - Guardrails Settings - Global and team-specific guardrails applied to this team + {t("guardrailSettingsView.title")} + {t("guardrailSettingsView.description")} {content} @@ -77,7 +81,7 @@ export function GuardrailSettingsView({ return (
- Guardrails Settings + {t("guardrailSettingsView.title")} {content}
); diff --git a/ui/litellm-dashboard/src/components/ModelInfoEditForm.tsx b/ui/litellm-dashboard/src/components/ModelInfoEditForm.tsx index 085334567ed..08af899cf7b 100644 --- a/ui/litellm-dashboard/src/components/ModelInfoEditForm.tsx +++ b/ui/litellm-dashboard/src/components/ModelInfoEditForm.tsx @@ -3,8 +3,10 @@ import { zodResolver } from "@hookform/resolvers/zod"; import { CircleHelp } from "lucide-react"; import type { Dayjs } from "dayjs"; +import type { ParseKeys, TFunction } from "i18next"; import * as React from "react"; import { useForm, type Resolver } from "react-hook-form"; +import { useTranslation } from "react-i18next"; import { z } from "zod"; import { TagsInput } from "@/app/(dashboard)/guardrails/_components/content_filter/TagsInput"; @@ -21,6 +23,7 @@ import { Switch } from "@/components/ui/switch"; import { Textarea } from "@/components/ui/textarea"; import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"; import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner"; +import i18n from "@/i18n/i18n"; import CacheControlInjectionPoints, { CACHE_CONTROL_LABEL, @@ -50,17 +53,28 @@ import { interface PtuEditField { name: string; - label: string; + label: ParseKeys; input: "number" | "datetime"; - placeholder?: string; + placeholder?: ParseKeys; isCount?: boolean; } const PTU_EDIT_FIELDS: PtuEditField[] = [ - { name: PTU_COUNT_FIELD, label: "PTU Count", input: "number", placeholder: "e.g. 15", isCount: true }, - { name: PTU_RATE_FIELD, label: "Cost per PTU / Hour (USD)", input: "number", placeholder: "e.g. 2.00" }, - { name: PTU_START_FIELD, label: "PTU Effective From (UTC)", input: "datetime" }, - { name: PTU_END_FIELD, label: "PTU Effective To (UTC)", input: "datetime" }, + { + name: PTU_COUNT_FIELD, + label: "modelInfoEditForm.fields.ptuCount.label", + input: "number", + placeholder: "modelInfoEditForm.fields.ptuCount.placeholder", + isCount: true, + }, + { + name: PTU_RATE_FIELD, + label: "modelInfoEditForm.fields.ptuRate.label", + input: "number", + placeholder: "modelInfoEditForm.fields.ptuRate.placeholder", + }, + { name: PTU_START_FIELD, label: "modelInfoEditForm.fields.ptuEffectiveFrom.label", input: "datetime" }, + { name: PTU_END_FIELD, label: "modelInfoEditForm.fields.ptuEffectiveTo.label", input: "datetime" }, ]; const PRICING_FIELDS = [ @@ -86,10 +100,9 @@ const textish = z.string().optional(); const isBlank = (value: string | number | null | undefined): boolean => value == null || String(value).trim() === ""; export const perMillionToPerToken = scalar - .refine( - (perMillion) => isBlank(perMillion) || (Number.isFinite(Number(perMillion)) && Number(perMillion) >= 0), - "Enter a price of 0 or more", - ) + .refine((perMillion) => isBlank(perMillion) || (Number.isFinite(Number(perMillion)) && Number(perMillion) >= 0), { + error: () => i18n.t("modelInfoEditForm.validation.priceNonNegative"), + }) .transform((perMillion) => (isBlank(perMillion) ? null : Number(perMillion) / 1_000_000)); const modelEditShape = { @@ -127,13 +140,17 @@ const modelEditShape = { type ModelEditFieldName = keyof typeof modelEditShape; -const buildSchema = (ptuEnabled: boolean, isFieldTouched: (field: TouchedPricingField) => boolean) => +const buildSchema = ( + ptuEnabled: boolean, + isFieldTouched: (field: TouchedPricingField) => boolean, + t: TFunction = i18n.t.bind(i18n), +) => z.object(modelEditShape).superRefine((values, ctx) => { const reject = (path: ModelEditFieldName, message: string) => ctx.addIssue({ code: "custom", path: [path], message }); if (parseModelExtraParams(values.litellm_extra_params) === null) { - reject("litellm_extra_params", "Please enter a valid JSON object"); + reject("litellm_extra_params", t("modelInfoEditForm.validation.invalidJson")); } // antd validates only mounted fields, and the PTU block does not render when the flag is off. @@ -142,24 +159,24 @@ const buildSchema = (ptuEnabled: boolean, isFieldTouched: (field: TouchedPricing } if (!isPositiveWholePtuCount(values.ptu_count)) { - reject("ptu_count", `PTU Count must be a whole number between 1 and ${MAX_PTU_COUNT.toLocaleString()}`); + reject("ptu_count", t("modelInfoEditForm.validation.ptuCountRange", { max: MAX_PTU_COUNT.toLocaleString() })); } if (!isNonNegativePtuRate(values.cost_per_ptu_per_hour)) { reject( "cost_per_ptu_per_hour", - `Cost per PTU / Hour must be between 0 and ${MAX_COST_PER_PTU_PER_HOUR.toLocaleString()}`, + t("modelInfoEditForm.validation.ptuRateRange", { max: MAX_COST_PER_PTU_PER_HOUR.toLocaleString() }), ); } if (isFilledPtuValue(values.ptu_count) !== isFilledPtuValue(values.cost_per_ptu_per_hour)) { - const message = "PTU Count and Cost per PTU / Hour must be set together"; + const message = t("modelInfoEditForm.validation.ptuPair"); reject("ptu_count", message); reject("cost_per_ptu_per_hour", message); } if (isFilledPtuValue(values.ptu_count) && !isFilledPtuValue(values.ptu_effective_from)) { - reject("ptu_effective_from", "PTU Effective From is required when PTU Count is set"); + reject("ptu_effective_from", t("modelInfoEditForm.validation.ptuStartRequired")); } if (!ptuWindowIsOrdered(values.ptu_effective_from, values.ptu_effective_to)) { - const message = "PTU Effective To must be after PTU Effective From"; + const message = t("modelInfoEditForm.validation.ptuWindowOrder"); reject("ptu_effective_from", message); reject("ptu_effective_to", message); } @@ -172,7 +189,7 @@ const buildSchema = (ptuEnabled: boolean, isFieldTouched: (field: TouchedPricing isFilledPtuValue(value) && Number(value) !== 0 ) { - reject(field, "A PTU deployment bills by reserved capacity, so this cost must be 0 or blank"); + reject(field, t("modelInfoEditForm.validation.ptuPricing")); } } }); @@ -240,10 +257,10 @@ export const toModelEditFormValues = (localModelData: any, isWildcardModel: bool team_id: localModelData.model_info?.team_id ?? undefined, }); -const displayCost = (localModelData: any, field: TouchedPricingField): string => { +const displayCost = (localModelData: any, field: TouchedPricingField, t: TFunction = i18n.t.bind(i18n)): string => { const key = COST_SOURCES[field]; const rate = localModelData?.litellm_params?.[key] ?? localModelData?.model_info?.[key]; - return rate != null ? (Number(rate) * 1_000_000).toFixed(4) : "Not Set"; + return rate != null ? (Number(rate) * 1_000_000).toFixed(4) : t("modelInfoEditForm.notSet"); }; interface ModelInfoEditFormProps { @@ -298,8 +315,9 @@ const DocsHint: React.FC<{ text: string; href: string }> = ({ text, href }) => ( ); const ChipList: React.FC<{ values: unknown; emptyLabel: string }> = ({ values, emptyLabel }) => { + const { t } = useTranslation(); if (!values) { - return <>Not Set; + return <>{t("modelInfoEditForm.notSet")}; } if (!Array.isArray(values)) { return <>{String(values)}; @@ -338,6 +356,7 @@ const ModelInfoEditForm: React.FC = ({ healthCheckModelOptions, teams, }) => { + const { t } = useTranslation(); // Neither RHF's blur-based touchedFields nor its resettable dirtyFields matches antd's touched-on-change. const touchedRef = React.useRef>(new Set()); const isFieldTouched = React.useCallback((field: TouchedPricingField) => touchedRef.current.has(field), []); @@ -347,7 +366,7 @@ const ModelInfoEditForm: React.FC = ({ // react-hook-form refreshes control._options every render, so this rebuild is what the next submit runs. const resolver: Resolver = (values, context, options) => - zodResolver(buildSchema(ptuCostAttributionEnabled, isFieldTouched))(values, context, options); + zodResolver(buildSchema(ptuCostAttributionEnabled, isFieldTouched, t))(values, context, options); const form = useForm({ resolver, @@ -373,7 +392,7 @@ const ModelInfoEditForm: React.FC = ({ {({ value, ...control }) => } ) : ( - {(stored as string) || "Not Set"} + {(stored as string) || t("modelInfoEditForm.notSet")} )}
); @@ -386,7 +405,7 @@ const ModelInfoEditForm: React.FC = ({ {({ value, ...control }) => } ) : ( - {(stored as string) || "Not Set"} + {(stored as string) || t("modelInfoEditForm.notSet")} )}
); @@ -409,7 +428,7 @@ const ModelInfoEditForm: React.FC = ({ ) : (
{label} - {displayCost(localModelData, name)} + {displayCost(localModelData, name, t)}
); @@ -437,21 +456,34 @@ const ModelInfoEditForm: React.FC = ({
- {textField("model_name", "Model Name", "Enter model name", localModelData.model_name)} + {textField( + "model_name", + t("modelInfoEditForm.fields.modelName.label"), + t("modelInfoEditForm.fields.modelName.placeholder"), + localModelData.model_name, + )} {textField( "litellm_model_name", - "LiteLLM Model Name", - "Enter LiteLLM model name", + t("modelInfoEditForm.fields.litellmModelName.label"), + t("modelInfoEditForm.fields.litellmModelName.placeholder"), localModelData.litellm_model_name, )} - {pricingField("input_cost", "Input Cost (per 1M tokens)", "Enter input cost")} - {pricingField("output_cost", "Output Cost (per 1M tokens)", "Enter output cost")} + {pricingField( + "input_cost", + t("modelInfoEditForm.fields.inputCost.label"), + t("modelInfoEditForm.fields.inputCost.placeholder"), + )} + {pricingField( + "output_cost", + t("modelInfoEditForm.fields.outputCost.label"), + t("modelInfoEditForm.fields.outputCost.placeholder"), + )} {ptuCostAttributionEnabled && PTU_EDIT_FIELDS.map((ptuField) => (
- {ptuField.label} + {t(ptuField.label)} {isEditing ? ( {({ value, onChange, ...control }) => @@ -461,7 +493,7 @@ const ModelInfoEditForm: React.FC = ({ id={ptuField.name} onChange={onChange} value={value ?? ""} - placeholder={ptuField.placeholder} + placeholder={ptuField.placeholder && t(ptuField.placeholder)} step={ptuField.isCount ? 1 : undefined} min={ptuField.isCount ? 1 : 0} /> @@ -479,7 +511,7 @@ const ModelInfoEditForm: React.FC = ({ {(ptuField.input === "datetime" ? formatPtuUtcDisplay(localModelData?.model_info?.[ptuField.name]) - : localModelData?.model_info?.[ptuField.name]) ?? "Not Set"} + : localModelData?.model_info?.[ptuField.name]) ?? t("modelInfoEditForm.notSet")} )}
@@ -487,68 +519,96 @@ const ModelInfoEditForm: React.FC = ({ {pricingField( "cache_read_cost", - "Cache Read Cost (per 1M tokens)", - "Defaults to Input Cost if blank", - "If left blank on save, defaults to Input Cost.", + t("modelInfoEditForm.fields.cacheReadCost.label"), + t("modelInfoEditForm.fields.cacheReadCost.placeholder"), + t("modelInfoEditForm.fields.cacheReadCost.description"), )} {pricingField( "cache_write_cost", - "Cache Write Cost (per 1M tokens)", - "Defaults to Input Cost if blank", - "If left blank on save, defaults to Input Cost (backend falls back to input_cost_per_token).", + t("modelInfoEditForm.fields.cacheWriteCost.label"), + t("modelInfoEditForm.fields.cacheWriteCost.placeholder"), + t("modelInfoEditForm.fields.cacheWriteCost.description"), )} {pricingField( "cache_write_cost_1h", - "Cache Write Cost (1 hour, per 1M tokens)", - "Defaults to the cost map 1 hour rate, else Cache Write Cost", - "Rate for prompt cache writes with a 1 hour TTL. If left blank on save, known models use their cost map 1 hour rate, otherwise Cache Write Cost.", + t("modelInfoEditForm.fields.cacheWriteCost1h.label"), + t("modelInfoEditForm.fields.cacheWriteCost1h.placeholder"), + t("modelInfoEditForm.fields.cacheWriteCost1h.description"), )} - {textField("api_base", "API Base", "Enter API base", localModelData.litellm_params?.api_base)} + {textField( + "api_base", + t("modelInfoEditForm.fields.apiBase.label"), + t("modelInfoEditForm.fields.apiBase.placeholder"), + localModelData.litellm_params?.api_base, + )} {textField( "custom_llm_provider", - "Custom LLM Provider", - "Enter custom LLM provider", + t("modelInfoEditForm.fields.customLlmProvider.label"), + t("modelInfoEditForm.fields.customLlmProvider.placeholder"), localModelData.litellm_params?.custom_llm_provider, )} {textField( "organization", - "Organization", - "Enter organization", + t("modelInfoEditForm.fields.organization.label"), + t("modelInfoEditForm.fields.organization.placeholder"), localModelData.litellm_params?.organization, )} - {numberField("tpm", "TPM (Tokens per Minute)", "Enter TPM", localModelData.litellm_params?.tpm)} - {numberField("rpm", "RPM (Requests per Minute)", "Enter RPM", localModelData.litellm_params?.rpm)} - {numberField("max_retries", "Max Retries", "Enter max retries", localModelData.litellm_params?.max_retries)} - {numberField("timeout", "Timeout (seconds)", "Enter timeout", localModelData.litellm_params?.timeout)} + {numberField( + "tpm", + t("modelInfoEditForm.fields.tpm.label"), + t("modelInfoEditForm.fields.tpm.placeholder"), + localModelData.litellm_params?.tpm, + )} + {numberField( + "rpm", + t("modelInfoEditForm.fields.rpm.label"), + t("modelInfoEditForm.fields.rpm.placeholder"), + localModelData.litellm_params?.rpm, + )} + {numberField( + "max_retries", + t("modelInfoEditForm.fields.maxRetries.label"), + t("modelInfoEditForm.fields.maxRetries.placeholder"), + localModelData.litellm_params?.max_retries, + )} + {numberField( + "timeout", + t("modelInfoEditForm.fields.timeout.label"), + t("modelInfoEditForm.fields.timeout.placeholder"), + localModelData.litellm_params?.timeout, + )} {numberField( "stream_timeout", - "Stream Timeout (seconds)", - "Enter stream timeout", + t("modelInfoEditForm.fields.streamTimeout.label"), + t("modelInfoEditForm.fields.streamTimeout.placeholder"), localModelData.litellm_params?.stream_timeout, )}
- Model Access Groups + {t("modelInfoEditForm.fields.modelAccessGroups.label")} {isEditing ? ( tagsField( "model_access_group", (modelAccessGroups ?? []).map((group) => ({ value: group, label: group })), - "Select existing groups or type to create new ones", + t("modelInfoEditForm.fields.modelAccessGroups.placeholder"), ) ) : ( - + )}
- Guardrails + {t("modelInfoEditForm.fields.guardrails.label")} @@ -556,20 +616,23 @@ const ModelInfoEditForm: React.FC = ({ tagsField( "guardrails", guardrailsList.map((name) => ({ value: name, label: name })), - "Select existing guardrails or type to create new ones", + t("modelInfoEditForm.fields.guardrails.placeholder"), ) ) : ( - + )}
- Attached Knowledge Bases (RAG) + {t("modelInfoEditForm.fields.knowledgeBases.label")} @@ -580,7 +643,7 @@ const ModelInfoEditForm: React.FC = ({ value={value as string[] | undefined} onChange={onChange} accessToken={accessToken || ""} - placeholder="Select knowledge bases (optional)" + placeholder={t("modelInfoEditForm.fields.knowledgeBases.placeholder")} /> )} @@ -588,36 +651,39 @@ const ModelInfoEditForm: React.FC = ({ )}
- Tags + {t("modelInfoEditForm.fields.tags.label")} {isEditing ? ( tagsField( "tags", Object.values(tagsList).map((tag: Tag) => ({ value: tag.name, label: tag.name })), - "Select existing tags or type to create new ones", + t("modelInfoEditForm.fields.tags.placeholder"), ) ) : ( - + )}
- Existing Credentials + {t("modelInfoEditForm.fields.existingCredentials.label")} {isEditing ? ( {({ id, value, onChange }) => ( <> = ({ {isWildcardModel && (
- Health Check Model + {t("modelInfoEditForm.fields.healthCheckModel.label")} {isEditing ? ( {({ id, value, onChange, onBlur }) => ( @@ -654,10 +720,10 @@ const ModelInfoEditForm: React.FC = ({ onValueChange={onChange} > - + - None + {t("modelInfoEditForm.fields.healthCheckModel.none")} {healthCheckModelOptions.map((option) => ( {option.label} @@ -668,7 +734,7 @@ const ModelInfoEditForm: React.FC = ({ )} ) : ( - {localModelData.model_info?.health_check_model || "Not Set"} + {localModelData.model_info?.health_check_model || t("modelInfoEditForm.notSet")} )}
)} @@ -711,29 +777,34 @@ const ModelInfoEditForm: React.FC = ({ ) : (
- Cache Control + {t("modelInfoEditForm.fields.cacheControl.label")} {localModelData.litellm_params?.cache_control_injection_points ? (
-

Enabled

+

{t("modelInfoEditForm.fields.cacheControl.enabled")}

{localModelData.litellm_params.cache_control_injection_points.map((point: any, i: number) => (
- Location: {point.location},{point.role && Role: {point.role}} - {point.index !== undefined && Index: {point.index}} + {t("modelInfoEditForm.fields.cacheControl.location", { location: point.location })} + {point.role && ( + {t("modelInfoEditForm.fields.cacheControl.role", { role: point.role })} + )} + {point.index !== undefined && ( + {t("modelInfoEditForm.fields.cacheControl.index", { index: point.index })} + )}
))}
) : ( - "Disabled" + t("modelInfoEditForm.fields.cacheControl.disabled") )}
)}
- Model Info + {t("modelInfoEditForm.fields.modelInfo.label")} {isEditing ? ( {({ value, ...control }) => ( @@ -756,9 +827,9 @@ const ModelInfoEditForm: React.FC = ({
- LiteLLM Params + {t("modelInfoEditForm.fields.litellmParams.label")} @@ -767,7 +838,7 @@ const ModelInfoEditForm: React.FC = ({ {({ value, ...control }) => (