From df544fcc532179e3ff388a41032a514ce41c5020 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Sat, 5 Sep 2026 10:46:05 -0700 Subject: [PATCH 1/8] test(e2e): cover key spend reset, regenerate grace period, and the llm_api_routes grant Three deterministic proxy-only cells from the coverage registry that had no e2e test. A key over its max_budget is reset to 0 through /key/{key}/reset_spend and must both read back 0 on /key/info and serve traffic again. /key/regenerate with grace_period keeps the old key valid until the period elapses and rejects it 401 afterwards. A key whose allowed_routes is the llm_api_routes group must reach /chat/completions and /embeddings while /model/new stays 403. KeyRegenerateBody gains grace_period and the management client gains reset_key_spend so the tests stay on the shared typed transport. --- .../access_control/test_access_control_e2e.py | 28 ++++++++- tests/e2e/management/management_client.py | 16 ++++- .../e2e/management/test_key_management_e2e.py | 59 ++++++++++++++++++- tests/e2e/management/test_management_e2e.py | 34 +++++++++++ tests/e2e/models.py | 10 ++++ 5 files changed, 143 insertions(+), 4 deletions(-) diff --git a/tests/e2e/access_control/test_access_control_e2e.py b/tests/e2e/access_control/test_access_control_e2e.py index af7e9a099fd..c30dadc49ae 100644 --- a/tests/e2e/access_control/test_access_control_e2e.py +++ b/tests/e2e/access_control/test_access_control_e2e.py @@ -24,7 +24,7 @@ from access_control_client import ( from e2e_config import unique_marker from e2e_http import Success, UnauthorizedError, UnknownApiError, unwrap from lifecycle import ResourceManager -from models import ChatBody, ChatMessage, ChatResponse, LiteLLMParamsBody +from models import ChatBody, ChatMessage, ChatResponse, EmbedBody, LiteLLMParamsBody from proxy_client import ProxyClient pytestmark = pytest.mark.e2e @@ -32,6 +32,7 @@ pytestmark = pytest.mark.e2e ALLOWED_MODEL = "gemini-2.5-flash" DISALLOWED_MODEL = "gpt-5.5" VIRTUAL_KEY_BACKEND = "anthropic/claude-haiku-4-5-20251001" +EMBEDDING_MODEL = "openai-text-embedding-3-small" class TestAccessControl: @@ -71,6 +72,31 @@ class TestAccessControl: f"403 body must be a model-access denial, got: {result.body[:300]}" ) + @pytest.mark.covers("other.auth.virtual_key.route_group_allowed") + def test_llm_api_routes_group_grants_every_llm_endpoint( + self, client: AccessControlClient, resources: ResourceManager + ) -> None: + """allowed_routes=["llm_api_routes"] names a route group, not a path: one + entry must open every LLM endpoint while the management routes stay shut.""" + key = client.llm_only_key() + resources.defer(lambda: client.delete_key(key)) + + chat = client.chat_status(key, ALLOWED_MODEL, f"capital of France? {unique_marker()}") + assert chat.status_code == 200, ( + f"llm_api_routes key must reach /chat/completions, got {chat.status_code}: {chat.body[:300]}" + ) + assert ChatResponse.model_validate_json(chat.body).choices, ( + f"200 must carry a real completion, not an error envelope: {chat.body[:300]}" + ) + + embedding = unwrap(client.proxy.embed(key, EmbedBody(model=EMBEDDING_MODEL, input=f"route group {unique_marker()}"))) + assert embedding.model, f"llm_api_routes key reached /embeddings but got no model back: {embedding}" + + denied = client.create_model_status(key, f"e2e-route-group-{unique_marker()}") + assert denied.status_code == 403 and ROUTE_NOT_ALLOWED_MARKER in denied.body, ( + f"the same key must still be shut out of /model/new, got {denied.status_code}: {denied.body[:300]}" + ) + def test_llm_only_key_forbidden_from_management_route_403( self, client: AccessControlClient, resources: ResourceManager ) -> None: diff --git a/tests/e2e/management/management_client.py b/tests/e2e/management/management_client.py index 387280c8023..2b897f5f07f 100644 --- a/tests/e2e/management/management_client.py +++ b/tests/e2e/management/management_client.py @@ -40,6 +40,8 @@ from models import ( KeyListParams, KeyListResponse, KeyRegenerateBody, + KeyResetSpendBody, + KeyResetSpendResponse, KeyUpdateBody, ModelDeleteBody, OrgDeleteBody, @@ -191,16 +193,26 @@ class ManagementClient: response_type=NoBody, ) ) - def regenerate_key(self, key: str) -> str: + def regenerate_key(self, key: str, *, grace_period: str | None = None) -> str: return unwrap( self.proxy.transport.post( "/key/regenerate", headers=self.proxy.transport.master, - json=KeyRegenerateBody(key=key), + json=KeyRegenerateBody(key=key, grace_period=grace_period), response_type=KeyGenerateResponse, ) ).key + def reset_key_spend(self, key: str, reset_to: float) -> KeyResetSpendResponse: + return unwrap( + self.proxy.transport.post( + f"/key/{key}/reset_spend", + headers=self.proxy.transport.master, + json=KeyResetSpendBody(reset_to=reset_to), + response_type=KeyResetSpendResponse, + ) + ) + def key_list(self, key_alias: str, *, caller_key: str | None = None) -> Result[KeyListResponse]: """GET /key/list, the Virtual Keys page's own inventory call. `caller_key` is who is asking: the master key by default, or a virtual key.""" diff --git a/tests/e2e/management/test_key_management_e2e.py b/tests/e2e/management/test_key_management_e2e.py index 711175abb0d..d4347b8c0e7 100644 --- a/tests/e2e/management/test_key_management_e2e.py +++ b/tests/e2e/management/test_key_management_e2e.py @@ -18,7 +18,7 @@ from typing import Literal import pytest from e2e_config import unique_marker -from e2e_http import NoBody, unwrap +from e2e_http import NoBody, StreamingResponse, unwrap from lifecycle import ResourceManager from management_client import ManagementClient from models import KeyDeleteBody, KeyGenerateBody, KeyUpdateBody @@ -26,6 +26,9 @@ from pydantic import BaseModel pytestmark = pytest.mark.e2e +TINY_BUDGET = 3e-6 +SPEND_MODEL = "claude-haiku-4-5" + class KeyToggleBlockBody(BaseModel): key: str @@ -82,6 +85,34 @@ def _generate_key(client: ManagementClient, resources: ResourceManager, body: Ke return key +def _is_budget_block(outcome: StreamingResponse) -> bool: + return not outcome.ok and "budget_exceeded" in outcome.body + + +def _spend_until_budget_blocks(client: ManagementClient, key: str) -> None: + """Drive paid calls until the key's max_budget refuses one. The first call spends, + the reservation counter trips the cap, and the next call is the 429.""" + for _ in range(40): + outcome = client.chat_status(key, SPEND_MODEL, f"spend {unique_marker()}") + if _is_budget_block(outcome): + assert outcome.status_code == 429, ( + f"budget refusal must be 429, got {outcome.status_code}: {outcome.body[:200]}" + ) + return + assert outcome.ok, f"paid call failed before the budget tripped ({outcome.status_code}): {outcome.body[:300]}" + time.sleep(2) + pytest.fail(f"max_budget={TINY_BUDGET} never blocked a call on the key") + + +def _settled_spend(client: ManagementClient, key: str) -> float | None: + """The key's recorded spend once it is positive and unchanged across two reads a + poll interval apart, so no batched spend write is still in flight when we reset.""" + first = client.proxy.key_info(key).spend or 0.0 + time.sleep(client.proxy.poll_interval) + second = client.proxy.key_info(key).spend or 0.0 + return second if first > 0 and first == second else None + + def _block(client: ManagementClient, key: str) -> None: _ = unwrap( client.proxy.transport.post( @@ -197,6 +228,32 @@ class TestKeyManagementRoutes: "/key/info never reported max_budget 42.0 after /key/bulk_update before the deadline", ) + @pytest.mark.covers("other.key_mgmt.spend_reset.resets_to_value") + def test_reset_spend_zeroes_recorded_spend_and_lifts_the_budget_block( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + key = _generate_key(client, resources, KeyGenerateBody(models=[SPEND_MODEL], max_budget=TINY_BUDGET)) + _spend_until_budget_blocks(client, key) + recorded = _poll( + client, lambda: _settled_spend(client, key), "key spend never landed in /key/info before the deadline" + ) + + reset = client.reset_key_spend(key, reset_to=0.0) + assert reset.previous_spend == recorded, ( + f"reset_spend reported previous_spend {reset.previous_spend}, /key/info had recorded {recorded}" + ) + assert reset.spend == 0.0, f"reset_spend to 0 reported spend {reset.spend}" + assert client.proxy.key_info(key).spend == 0.0, "/key/info still reports spend after the reset to 0" + + def call_allowed_again() -> bool | None: + outcome = client.chat_status(key, SPEND_MODEL, f"after reset {unique_marker()}") + if _is_budget_block(outcome): + return None + assert outcome.ok, f"post-reset call failed ({outcome.status_code}): {outcome.body[:300]}" + return True + + _ = _poll(client, call_allowed_again, "the key stayed budget-blocked after its spend was reset to 0") + @pytest.mark.covers("mgmt.key.generate.admin_only") def test_generate_forbidden_for_non_admin_key( self, client: ManagementClient, resources: ResourceManager diff --git a/tests/e2e/management/test_management_e2e.py b/tests/e2e/management/test_management_e2e.py index a56eb853823..eace8f2e3b4 100644 --- a/tests/e2e/management/test_management_e2e.py +++ b/tests/e2e/management/test_management_e2e.py @@ -42,6 +42,10 @@ from models import ( pytestmark = pytest.mark.e2e +REGENERATE_GRACE_PERIOD = "15s" +REGENERATE_GRACE_SECONDS = 15.0 + + def _poll[T](client: ManagementClient, attempt: Callable[[], T | None], failure: str) -> T: deadline = time.monotonic() + client.proxy.poll_timeout while time.monotonic() < deadline: @@ -365,6 +369,36 @@ class TestKeyRegeneration: client, old_rejected, "old key was still accepted after regeneration (never rejected 401) at the deadline" ) + @pytest.mark.covers("other.key_mgmt.regenerate.grace_period_honored") + def test_regenerate_with_grace_period_keeps_old_key_until_revoked( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + old_key = _generate_key(client, resources, KeyGenerateBody(models=["gpt-5.5"])) + + new_key = client.regenerate_key(old_key, grace_period=REGENERATE_GRACE_PERIOD) + resources.defer(lambda: client.proxy.delete_key(new_key)) + revoke_at = time.monotonic() + REGENERATE_GRACE_SECONDS + assert new_key != old_key, "regenerate returned the same key string, so no rotation happened" + + def old_accepted() -> bool | None: + outcome = client.chat_status(old_key, "gpt-5.5", f"say hi {unique_marker()}") + return True if outcome.status_code != 401 else None + + _ = _poll(client, old_accepted, "old key was rejected 401 inside its grace period at the deadline") + assert time.monotonic() < revoke_at, ( + f"old key was only accepted after its {REGENERATE_GRACE_PERIOD} grace period had elapsed" + ) + + def old_rejected() -> bool | None: + outcome = client.chat_status(old_key, "gpt-5.5", f"say hi {unique_marker()}") + return True if outcome.status_code == 401 else None + + _ = _poll( + client, + old_rejected, + f"old key was still accepted past its {REGENERATE_GRACE_PERIOD} grace period (never 401) at the deadline", + ) + class TestTeamRoutes: @pytest.mark.covers("mgmt.team.new.persists") diff --git a/tests/e2e/models.py b/tests/e2e/models.py index b5229744d6f..5de49ead3ed 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -83,6 +83,16 @@ class KeyGenerateResponse(BaseModel): class KeyRegenerateBody(BaseModel): key: str + grace_period: str | None = None + + +class KeyResetSpendBody(BaseModel): + reset_to: float + + +class KeyResetSpendResponse(BaseModel): + spend: float + previous_spend: float class KeyDeleteBody(BaseModel): From 88c46fb1defa968348b9e780bd63453f5f8f3e94 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Sat, 5 Sep 2026 10:52:46 -0700 Subject: [PATCH 2/8] test(e2e): cover Anthropic and OpenAI prompt caching, Cohere embeddings, and costed /openai chat passthrough Four registry cells that had no e2e test. The cache_control suite gains a direct Anthropic case (the same cache_control prefix the Bedrock and Vertex rows send) and an OpenAI case, where caching is automatic so the prefix goes out as a plain system string with a prompt_cache_key; both assert the second identical call reports cache-read tokens. The shared second-call helper now takes the send callable so the OpenAI shape fits without a second copy of the retry loop. The embeddings suite gains a cohere/embed-v4.0 deployment that must return a non-zero vector, and the passthrough suite gains an OpenAI-format chat through the raw /openai/v1/chat/completions prefix that must relay a real completion and log a costed pass_through_endpoint row whose token counts match the usage the caller was served. --- .../e2e/llm_translation/test_cache_control.py | 76 +++++++++++++++++-- .../test_embeddings_endpoint_e2e.py | 22 +++++- .../llm_translation/test_passthrough_e2e.py | 34 ++++++++- 3 files changed, 124 insertions(+), 8 deletions(-) diff --git a/tests/e2e/llm_translation/test_cache_control.py b/tests/e2e/llm_translation/test_cache_control.py index a2c17b0fb66..0d224061381 100644 --- a/tests/e2e/llm_translation/test_cache_control.py +++ b/tests/e2e/llm_translation/test_cache_control.py @@ -10,6 +10,11 @@ Each case asserts the feature actually happened, not just a 200. Coverage matrix intentionally not covered here. - Vertex (gemini-2.5-flash): prompt caching via ``cache_control`` context caching; the second identical call must report cached prompt tokens > 0. +- Anthropic (claude-haiku-4-5, direct): the same ``cache_control`` prefix over + the OpenAI-compatible route; the second call must report cache-read tokens > 0. +- OpenAI (gpt-5.6): automatic prompt caching needs no request marker, so the + cacheable prefix goes out as a plain system string with a ``prompt_cache_key`` + and the second call must report ``prompt_tokens_details.cached_tokens`` > 0. service_tier lives in test_provider_features_e2e.py. @@ -21,6 +26,7 @@ built from the typed content blocks shared in ``endpoints_client.py``. from __future__ import annotations import time +from collections.abc import Callable import pytest from pydantic import BaseModel @@ -29,7 +35,7 @@ from e2e_config import unique_marker from e2e_http import Result, unwrap from endpoints_client import CacheControl, RichMessage, TextBlock from lifecycle import ResourceManager -from models import ChatResponse, LiteLLMParamsBody, Usage +from models import ChatBody, ChatMessage, ChatResponse, LiteLLMParamsBody, Usage from passthrough_client import PassthroughClient import os @@ -37,6 +43,8 @@ pytestmark = pytest.mark.e2e BEDROCK_MODEL = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" VERTEX_MODEL = "vertex_ai/gemini-2.5-flash" +ANTHROPIC_MODEL = "anthropic/claude-haiku-4-5-20251001" +OPENAI_MODEL = "openai/gpt-5.6" class CacheChatBody(BaseModel): @@ -89,17 +97,36 @@ def _cache_chat( ) +def _plain_cache_chat( + client: PassthroughClient, key: str, model: str, prefix: str, cache_key: str +) -> Result[ChatResponse]: + """The same cacheable prefix as a plain system string, for providers that cache + automatically and take no per-block marker (OpenAI).""" + return client.proxy.chat( + key, + ChatBody( + model=model, + messages=[ + ChatMessage(role="system", content=prefix), + ChatMessage(role="user", content="Reply with one word."), + ], + max_tokens=64, + prompt_cache_key=cache_key, + ), + ) + + def _assert_cache_read_on_second_call( - client: PassthroughClient, key: str, model: str + model: str, send: Callable[[str], Result[ChatResponse]] ) -> None: prefix = _cacheable_prefix() - first = unwrap(_cache_chat(client, key, model, prefix)) + first = unwrap(send(prefix)) assert first.choices, f"{model}: first cache-priming call returned no choices: {first}" deadline = time.monotonic() + 30.0 while True: - second = unwrap(_cache_chat(client, key, model, prefix)) + second = unwrap(send(prefix)) read_tokens = _cached_read_tokens(second.usage) if read_tokens > 0 or time.monotonic() >= deadline: break @@ -125,7 +152,8 @@ class TestCacheControl: LiteLLMParamsBody(model=BEDROCK_MODEL, aws_region_name="us-east-1"), ) resources.defer(lambda: client.proxy.delete_model(model_id)) - _assert_cache_read_on_second_call(client, resources.key(), model) + key = resources.key() + _assert_cache_read_on_second_call(model, lambda prefix: _cache_chat(client, key, model, prefix)) @pytest.mark.covers( "llm.chat_completions.vertex.prompt_cache_5m.nonstream.works", @@ -145,4 +173,40 @@ class TestCacheControl: ), ) resources.defer(lambda: client.proxy.delete_model(model_id)) - _assert_cache_read_on_second_call(client, resources.key(), model) + key = resources.key() + _assert_cache_read_on_second_call(model, lambda prefix: _cache_chat(client, key, model, prefix)) + + @pytest.mark.covers( + "llm.chat_completions.anthropic.prompt_cache_5m.nonstream.works", + exercised_on=[], + ) + def test_anthropic_prompt_caching_reads_cache( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + model = f"e2e-anthropic-cache-{unique_marker()}" + model_id = client.proxy.create_model( + model, + LiteLLMParamsBody(model=ANTHROPIC_MODEL, api_key="os.environ/ANTHROPIC_API_KEY"), + ) + resources.defer(lambda: client.proxy.delete_model(model_id)) + key = resources.key() + _assert_cache_read_on_second_call(model, lambda prefix: _cache_chat(client, key, model, prefix)) + + @pytest.mark.covers( + "llm.chat_completions.openai.prompt_cache_5m.nonstream.works", + exercised_on=[], + ) + def test_openai_prompt_caching_reads_cache( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + model = f"e2e-openai-cache-{unique_marker()}" + model_id = client.proxy.create_model( + model, + LiteLLMParamsBody(model=OPENAI_MODEL, api_key="os.environ/OPENAI_API_KEY"), + ) + resources.defer(lambda: client.proxy.delete_model(model_id)) + key = resources.key() + cache_key = f"e2e-openai-cache-{unique_marker()}" + _assert_cache_read_on_second_call( + model, lambda prefix: _plain_cache_chat(client, key, model, prefix, cache_key) + ) diff --git a/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py b/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py index f951eb328f5..5520ca0cee5 100644 --- a/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py +++ b/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py @@ -1,4 +1,4 @@ -"""Live e2e: POST /embeddings returns a real vector across OpenAI, Bedrock, Vertex. +"""Live e2e: POST /embeddings returns a real vector across OpenAI, Bedrock, Vertex, Cohere. Each test registers the deployment it needs at runtime (deleted on teardown) and asserts a non-empty, non-zero vector came back. The LIT-3167 guard in @@ -86,6 +86,26 @@ class TestEmbeddingsEndpoint: f"embedding vector is all zeros: {result.body[:300]}" ) + @pytest.mark.covers("llm.embeddings.cohere.basic.nonstream.works") + def test_cohere_embeddings_returns_vector( + self, endpoints_client: EndpointsClient, resources: ResourceManager + ) -> None: + model = f"e2e-embeddings-cohere-{unique_marker()}" + model_id = endpoints_client.create_model( + model, + LiteLLMParamsBody(model="cohere/embed-v4.0", api_key="os.environ/COHERE_API_KEY"), + ) + resources.defer(lambda: endpoints_client.delete_model(model_id)) + key = resources.key() + + result = endpoints_client.embeddings(key, model, "Say this is a test!") + require_successful_call(result) + parsed = EmbeddingsResult.model_validate_json(result.body) + assert parsed.first_vector, f"/embeddings returned no vector: {result.body[:300]}" + assert any(component != 0.0 for component in parsed.first_vector), ( + f"embedding vector is all zeros: {result.body[:300]}" + ) + @pytest.mark.covers("llm.embeddings.vertex.basic.nonstream.works") def test_vertex_embeddings_returns_vector( self, endpoints_client: EndpointsClient, resources: ResourceManager diff --git a/tests/e2e/llm_translation/test_passthrough_e2e.py b/tests/e2e/llm_translation/test_passthrough_e2e.py index 50ea8f4b4df..e50e83eaf77 100644 --- a/tests/e2e/llm_translation/test_passthrough_e2e.py +++ b/tests/e2e/llm_translation/test_passthrough_e2e.py @@ -17,7 +17,7 @@ import pytest from e2e_config import CHEAP_OPENAI_MODEL, unique_marker from e2e_http import require_successful_call, unwrap from lifecycle import ResourceManager -from models import KeyGenerateBody, SpendLogRow +from models import ChatResponse, KeyGenerateBody, SpendLogRow from passthrough_client import ( AnthropicTool, GeminiFunctionDeclaration, @@ -344,6 +344,38 @@ class TestOpenAIPassthroughSpend: ) +class TestOpenAIProviderPrefixChat: + """OpenAI-format chat through the raw `/openai/{endpoint}` passthrough (LIT-4752). + + The body goes to OpenAI untranslated with the proxy's own OPENAI_API_KEY swapped + in, so the customer gets OpenAI's real completion back, and the gateway must + still write a costed pass_through_endpoint row for it. + """ + + @pytest.mark.covers("llm.chat_completions.openai.passthrough.nonstream.cost_logged") + def test_openai_prefix_chat_returns_completion_and_logs_its_cost( + self, client: PassthroughClient, scoped_key: str + ) -> None: + result = client.openai_chat(scoped_key, CHEAP_OPENAI_MODEL, f"Say hi in one word. {unique_marker()}") + require_successful_call(result) + + completion = ChatResponse.model_validate_json(result.body) + assert completion.id, f"/openai/v1/chat/completions relayed no completion id: {result.body[:300]}" + content = completion.choices[0].message.content if completion.choices and completion.choices[0].message else None + assert content and content.strip(), f"/openai/v1/chat/completions relayed an empty completion: {result.body[:300]}" + assert completion.usage is not None, f"the completion carried no usage to price from: {completion}" + + row = _fetch_cost_breakdown(client, completion.id) + assert row.prompt_tokens == completion.usage.prompt_tokens, ( + f"logged {row.prompt_tokens} prompt tokens, the completion the customer read " + f"reported {completion.usage.prompt_tokens}" + ) + assert row.completion_tokens == completion.usage.completion_tokens, ( + f"logged {row.completion_tokens} completion tokens, the completion the customer read " + f"reported {completion.usage.completion_tokens}" + ) + + class TestOpenAIPassthroughWebsocket: """The OpenAI passthrough prefixes must answer a websocket upgrade, not only a POST. From 86038318ce408b4f63767a2fb753357ae7ed3cdf Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Sat, 5 Sep 2026 11:43:52 -0700 Subject: [PATCH 3/8] feat(ui): deep link guardrail detail with ?guardrail= on guardrails pages The Guardrails and Guardrails Monitor pages kept the selected guardrail in local React state, so the detail view could not be shared, reloaded, or reached with the browser back button. Both pages now read and write the selection through the nuqs `guardrail` query param, matching how the keys, teams, orgs, projects, users, models and logs pages deep link their detail views. Opening a guardrail pushes a history entry and closing it replaces the entry so back returns to the page the user came from --- .../GuardrailsMonitorView.test.tsx | 116 ++++++++++++++---- .../_components/GuardrailsMonitorView.tsx | 16 +-- .../_components/GuardrailsPanel.test.tsx | 77 +++++++++--- .../_components/GuardrailsPanel.tsx | 18 ++- 4 files changed, 179 insertions(+), 48 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsMonitorView.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsMonitorView.test.tsx index 9f27daab6b3..4e0590df72d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsMonitorView.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsMonitorView.test.tsx @@ -1,36 +1,61 @@ -import { render, screen, waitFor } from "@testing-library/react"; -import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; -import { describe, expect, it, vi } from "vitest"; +import { type UrlUpdateEvent } from "nuqs/adapters/testing"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import userEvent from "@testing-library/user-event"; import GuardrailsMonitorView from "./GuardrailsMonitorView"; import * as networking from "@/components/networking"; +import { renderWithProviders, screen, testQueryClient, waitFor } from "@/../tests/test-utils"; vi.mock("@/components/networking", () => ({ getGuardrailsUsageOverview: vi.fn(), + getGuardrailsUsageDetail: vi.fn(), + getGuardrailsUsageLogs: vi.fn(), formatDate: vi.fn((d: Date) => d.toISOString().slice(0, 10)), })); -const mockGetGuardrailsUsageOverview = vi.mocked(networking.getGuardrailsUsageOverview); +vi.mock("@/components/GuardrailsMonitor/LogViewer", () => ({ + LogViewer: ({ guardrailName }: { guardrailName: string }) =>
{guardrailName}
, +})); -function wrapper({ children }: { children: React.ReactNode }) { - const queryClient = new QueryClient({ - defaultOptions: { - queries: { retry: false }, - }, - }); - return {children}; -} +const mockGetGuardrailsUsageOverview = vi.mocked(networking.getGuardrailsUsageOverview); +const mockGetGuardrailsUsageDetail = vi.mocked(networking.getGuardrailsUsageDetail); +const mockGetGuardrailsUsageLogs = vi.mocked(networking.getGuardrailsUsageLogs); + +const emptyOverview = { rows: [], chart: [], totalRequests: 0, totalBlocked: 0, passRate: 100 }; + +const piiRow = { + id: "gr-pii", + name: "PII Guard", + type: "pii", + provider: "LiteLLM", + requestsEvaluated: 10, + failRate: 10, + status: "healthy" as const, + trend: "stable" as const, +}; + +const piiDetail = { + guardrail_name: "PII Guard", + description: "", + status: "healthy", + provider: "LiteLLM", + type: "pii", + requestsEvaluated: 10, + failRate: 10, + avgScore: 0.5, + avgLatency: 20, +}; describe("GuardrailsMonitorView", () => { - it("should render overview and fetch guardrails usage when accessToken is provided", async () => { - mockGetGuardrailsUsageOverview.mockResolvedValue({ - rows: [], - chart: [], - totalRequests: 0, - totalBlocked: 0, - passRate: 100, - }); + beforeEach(() => { + testQueryClient.clear(); + vi.clearAllMocks(); + mockGetGuardrailsUsageOverview.mockResolvedValue(emptyOverview); + mockGetGuardrailsUsageDetail.mockResolvedValue(piiDetail); + mockGetGuardrailsUsageLogs.mockResolvedValue({ logs: [], total: 0 }); + }); - render(, { wrapper }); + it("should render overview and fetch guardrails usage when accessToken is provided", async () => { + renderWithProviders(); expect(await screen.findByRole("heading", { name: /Guardrails Monitor/i })).toBeInTheDocument(); await waitFor(() => { @@ -39,7 +64,54 @@ describe("GuardrailsMonitorView", () => { }); it("should render without crashing when accessToken is null", async () => { - render(, { wrapper }); + renderWithProviders(); expect(await screen.findByRole("heading", { name: /Guardrails Monitor/i })).toBeInTheDocument(); }); + + describe("guardrail detail deep link (?guardrail=)", () => { + it("should open the detail view directly from a ?guardrail= deep link", async () => { + renderWithProviders(, { searchParams: "?guardrail=gr-pii" }); + + expect(await screen.findByRole("heading", { name: "PII Guard" })).toBeInTheDocument(); + expect(mockGetGuardrailsUsageDetail).toHaveBeenCalledWith( + "test-token", + "gr-pii", + expect.any(String), + expect.any(String), + ); + expect(screen.queryByRole("heading", { name: /Guardrails Monitor/i })).not.toBeInTheDocument(); + }); + + it("should push ?guardrail= as a new history entry when a guardrail is selected", async () => { + const user = userEvent.setup(); + const onUrlUpdate = vi.fn<(event: UrlUpdateEvent) => void>(); + mockGetGuardrailsUsageOverview.mockResolvedValue({ ...emptyOverview, rows: [piiRow] }); + renderWithProviders(, { onUrlUpdate }); + + await user.click(await screen.findByRole("button", { name: "PII Guard" })); + + await waitFor(() => expect(onUrlUpdate).toHaveBeenCalled()); + const lastUpdate = onUrlUpdate.mock.calls.at(-1)![0]; + expect(lastUpdate.searchParams.get("guardrail")).toBe("gr-pii"); + expect(lastUpdate.options.history).toBe("push"); + expect(await screen.findByRole("heading", { name: "PII Guard" })).toBeInTheDocument(); + }); + + it("should clear ?guardrail= by replacing history when going back to the overview", async () => { + const user = userEvent.setup(); + const onUrlUpdate = vi.fn<(event: UrlUpdateEvent) => void>(); + renderWithProviders(, { + searchParams: "?guardrail=gr-pii", + onUrlUpdate, + }); + + await user.click(await screen.findByRole("button", { name: /back to overview/i })); + + await waitFor(() => expect(onUrlUpdate).toHaveBeenCalled()); + const lastUpdate = onUrlUpdate.mock.calls.at(-1)![0]; + expect(lastUpdate.searchParams.has("guardrail")).toBe(false); + expect(lastUpdate.options.history).toBe("replace"); + expect(await screen.findByRole("heading", { name: /Guardrails Monitor/i })).toBeInTheDocument(); + }); + }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsMonitorView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsMonitorView.tsx index a9acf3e6377..f90a46e19e4 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsMonitorView.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsMonitorView.tsx @@ -1,12 +1,11 @@ import type { DateRangePickerValue } from "@/components/shared/date_picker_types"; +import { parseAsString, useQueryState } from "nuqs"; import React, { useCallback, useMemo, useState } from "react"; import { formatDate } from "@/components/networking"; import AdvancedDatePicker from "@/components/shared/advanced_date_picker"; import { GuardrailDetail } from "./GuardrailDetail"; import { GuardrailsOverview } from "./GuardrailsOverview"; -type View = { type: "overview" } | { type: "detail"; guardrailId: string }; - interface GuardrailsMonitorViewProps { accessToken?: string | null; } @@ -16,7 +15,10 @@ const defaultStart = new Date(); defaultStart.setDate(defaultStart.getDate() - 7); export default function GuardrailsMonitorView({ accessToken = null }: GuardrailsMonitorViewProps) { - const [view, setView] = useState({ type: "overview" }); + const [selectedGuardrailId, setSelectedGuardrailId] = useQueryState( + "guardrail", + parseAsString.withOptions({ history: "push" }), + ); const initialFrom = useMemo(() => new Date(defaultStart), []); const initialTo = useMemo(() => new Date(defaultEnd), []); @@ -34,11 +36,11 @@ export default function GuardrailsMonitorView({ accessToken = null }: Guardrails }, []); const handleSelectGuardrail = (id: string) => { - setView({ type: "detail", guardrailId: id }); + void setSelectedGuardrailId(id); }; const handleBack = () => { - setView({ type: "overview" }); + void setSelectedGuardrailId(null, { history: "replace" }); }; const dateRangeControl = ( @@ -47,7 +49,7 @@ export default function GuardrailsMonitorView({ accessToken = null }: Guardrails return (
- {view.type === "overview" ? ( + {!selectedGuardrailId ? (
{dateRangeControl}
({ getGuardrailsList: vi.fn(), @@ -15,16 +16,21 @@ vi.mock("./add_guardrail_form", () => ({ vi.mock("./guardrail_table", () => ({ __esModule: true, - default: ({ guardrailsList, onDeleteClick }: any) => ( + default: ({ guardrailsList, onDeleteClick, onGuardrailClick }: any) => (
Mock Guardrail Table
{guardrailsList.length > 0 && ( - + <> + + + )}
), @@ -32,7 +38,12 @@ vi.mock("./guardrail_table", () => ({ vi.mock("./guardrail_info", () => ({ __esModule: true, - default: () =>
Mock Guardrail Info View
, + default: ({ guardrailId, onClose }: { guardrailId: string; onClose: () => void }) => ( +
+
Mock Guardrail Info View {guardrailId}
+ +
+ ), })); vi.mock("./GuardrailTestPlayground", async () => { @@ -112,7 +123,7 @@ describe("GuardrailsPanel", () => { }); it("should render the component", async () => { - render(); + renderWithProviders(); expect(screen.getByText("Guardrails")).toBeInTheDocument(); // Activate the Guardrails tab so its content (including the Add button) is rendered fireEvent.click(screen.getByText("Guardrails")); @@ -120,7 +131,7 @@ describe("GuardrailsPanel", () => { }); it("should delete the clicked guardrail after confirming in the modal", async () => { - render(); + renderWithProviders(); fireEvent.click(screen.getByText("Guardrails")); fireEvent.click(await screen.findByTestId("delete-button")); @@ -139,14 +150,14 @@ describe("GuardrailsPanel", () => { }); it("should mount every tab panel up front so panel state survives tab switches", async () => { - render(); + renderWithProviders(); expect(await screen.findByLabelText("playground draft")).toBeInTheDocument(); expect(screen.getByText("Mock Team Guardrails Tab")).toBeInTheDocument(); }); it("should keep test playground state when switching tabs away and back", async () => { - render(); + renderWithProviders(); fireEvent.click(screen.getByText("Test Playground")); @@ -161,7 +172,7 @@ describe("GuardrailsPanel", () => { }); it("should not delete anything when the modal is cancelled", async () => { - render(); + renderWithProviders(); fireEvent.click(screen.getByText("Guardrails")); fireEvent.click(await screen.findByTestId("delete-button")); @@ -171,4 +182,42 @@ describe("GuardrailsPanel", () => { expect(mockDeleteGuardrailCall).not.toHaveBeenCalled(); }); + + describe("guardrail detail deep link (?guardrail=)", () => { + it("should open the guardrail info view directly from a ?guardrail= deep link", async () => { + renderWithProviders(, { searchParams: "?guardrail=test-guardrail-1" }); + + expect(await screen.findByTestId("guardrail-info-view")).toHaveTextContent("test-guardrail-1"); + expect(screen.queryByText("Mock Guardrail Table")).not.toBeInTheDocument(); + }); + + it("should push ?guardrail= as a new history entry when a guardrail row is clicked", async () => { + const onUrlUpdate = vi.fn<(event: UrlUpdateEvent) => void>(); + renderWithProviders(, { onUrlUpdate }); + + fireEvent.click(await screen.findByTestId("open-button")); + + await waitFor(() => expect(onUrlUpdate).toHaveBeenCalled()); + const lastUpdate = onUrlUpdate.mock.calls.at(-1)![0]; + expect(lastUpdate.searchParams.get("guardrail")).toBe("test-guardrail-1"); + expect(lastUpdate.options.history).toBe("push"); + expect(await screen.findByTestId("guardrail-info-view")).toHaveTextContent("test-guardrail-1"); + }); + + it("should clear ?guardrail= by replacing history when the info view is closed", async () => { + const onUrlUpdate = vi.fn<(event: UrlUpdateEvent) => void>(); + renderWithProviders(, { + searchParams: "?guardrail=test-guardrail-1", + onUrlUpdate, + }); + + fireEvent.click(await screen.findByRole("button", { name: "Close Guardrail Info" })); + + await waitFor(() => expect(onUrlUpdate).toHaveBeenCalled()); + const lastUpdate = onUrlUpdate.mock.calls.at(-1)![0]; + expect(lastUpdate.searchParams.has("guardrail")).toBe(false); + expect(lastUpdate.options.history).toBe("replace"); + expect(await screen.findByText("Mock Guardrail Table")).toBeInTheDocument(); + }); + }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailsPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailsPanel.tsx index 7e59abf8e3d..901e39004f3 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailsPanel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailsPanel.tsx @@ -1,3 +1,4 @@ +import { parseAsString, useQueryState } from "nuqs"; import React, { useState, useEffect } from "react"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { ChevronDown, Code, Plus } from "lucide-react"; @@ -40,7 +41,10 @@ const GuardrailsPanel: React.FC = ({ accessToken, userRole const [isDeleting, setIsDeleting] = useState(false); const [guardrailToDelete, setGuardrailToDelete] = useState(null); const [isDeleteModalOpen, setIsDeleteModalOpen] = useState(false); - const [selectedGuardrailId, setSelectedGuardrailId] = useState(null); + const [selectedGuardrailId, setSelectedGuardrailId] = useQueryState( + "guardrail", + parseAsString.withOptions({ history: "push" }), + ); const isAdmin = userRole ? isAdminRole(userRole) : false; const fetchGuardrails = async () => { @@ -63,16 +67,20 @@ const GuardrailsPanel: React.FC = ({ accessToken, userRole fetchGuardrails(); }, [accessToken]); + const closeGuardrailDetail = () => { + void setSelectedGuardrailId(null, { history: "replace" }); + }; + const handleAddGuardrail = () => { if (selectedGuardrailId) { - setSelectedGuardrailId(null); + closeGuardrailDetail(); } setIsAddModalVisible(true); }; const handleAddCustomCodeGuardrail = () => { if (selectedGuardrailId) { - setSelectedGuardrailId(null); + closeGuardrailDetail(); } setIsCustomCodeModalVisible(true); }; @@ -175,7 +183,7 @@ const GuardrailsPanel: React.FC = ({ accessToken, userRole {selectedGuardrailId ? ( setSelectedGuardrailId(null)} + onClose={closeGuardrailDetail} accessToken={accessToken} isAdmin={isAdmin} /> @@ -184,7 +192,7 @@ const GuardrailsPanel: React.FC = ({ accessToken, userRole guardrailsList={guardrailsList} isLoading={isLoading} onDeleteClick={handleDeleteClick} - onGuardrailClick={(id) => setSelectedGuardrailId(id)} + onGuardrailClick={(id) => void setSelectedGuardrailId(id)} /> )} From 9ac893df1562f35a56fbac17399481f1fb32f857 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Sat, 5 Sep 2026 11:47:15 -0700 Subject: [PATCH 4/8] style(e2e): wrap the openai passthrough content assertion under 120 columns --- tests/e2e/llm_translation/test_passthrough_e2e.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/e2e/llm_translation/test_passthrough_e2e.py b/tests/e2e/llm_translation/test_passthrough_e2e.py index e50e83eaf77..447fe7d30d9 100644 --- a/tests/e2e/llm_translation/test_passthrough_e2e.py +++ b/tests/e2e/llm_translation/test_passthrough_e2e.py @@ -361,8 +361,14 @@ class TestOpenAIProviderPrefixChat: completion = ChatResponse.model_validate_json(result.body) assert completion.id, f"/openai/v1/chat/completions relayed no completion id: {result.body[:300]}" - content = completion.choices[0].message.content if completion.choices and completion.choices[0].message else None - assert content and content.strip(), f"/openai/v1/chat/completions relayed an empty completion: {result.body[:300]}" + content = ( + completion.choices[0].message.content + if completion.choices and completion.choices[0].message + else None + ) + assert content and content.strip(), ( + f"/openai/v1/chat/completions relayed an empty completion: {result.body[:300]}" + ) assert completion.usage is not None, f"the completion carried no usage to price from: {completion}" row = _fetch_cost_breakdown(client, completion.id) From 45cc2ed08225300536894028624284e6f9eb8baf Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Sat, 5 Sep 2026 11:53:22 -0700 Subject: [PATCH 5/8] test(e2e): require a 200 inside the regenerate grace window and drop the helper docstrings --- tests/e2e/access_control/test_access_control_e2e.py | 6 +++--- tests/e2e/management/test_key_management_e2e.py | 4 ---- tests/e2e/management/test_management_e2e.py | 5 +++-- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/tests/e2e/access_control/test_access_control_e2e.py b/tests/e2e/access_control/test_access_control_e2e.py index c30dadc49ae..9d01f2915e7 100644 --- a/tests/e2e/access_control/test_access_control_e2e.py +++ b/tests/e2e/access_control/test_access_control_e2e.py @@ -76,8 +76,6 @@ class TestAccessControl: def test_llm_api_routes_group_grants_every_llm_endpoint( self, client: AccessControlClient, resources: ResourceManager ) -> None: - """allowed_routes=["llm_api_routes"] names a route group, not a path: one - entry must open every LLM endpoint while the management routes stay shut.""" key = client.llm_only_key() resources.defer(lambda: client.delete_key(key)) @@ -89,7 +87,9 @@ class TestAccessControl: f"200 must carry a real completion, not an error envelope: {chat.body[:300]}" ) - embedding = unwrap(client.proxy.embed(key, EmbedBody(model=EMBEDDING_MODEL, input=f"route group {unique_marker()}"))) + embedding = unwrap( + client.proxy.embed(key, EmbedBody(model=EMBEDDING_MODEL, input=f"route group {unique_marker()}")) + ) assert embedding.model, f"llm_api_routes key reached /embeddings but got no model back: {embedding}" denied = client.create_model_status(key, f"e2e-route-group-{unique_marker()}") diff --git a/tests/e2e/management/test_key_management_e2e.py b/tests/e2e/management/test_key_management_e2e.py index d4347b8c0e7..8b7d5f0eb6f 100644 --- a/tests/e2e/management/test_key_management_e2e.py +++ b/tests/e2e/management/test_key_management_e2e.py @@ -90,8 +90,6 @@ def _is_budget_block(outcome: StreamingResponse) -> bool: def _spend_until_budget_blocks(client: ManagementClient, key: str) -> None: - """Drive paid calls until the key's max_budget refuses one. The first call spends, - the reservation counter trips the cap, and the next call is the 429.""" for _ in range(40): outcome = client.chat_status(key, SPEND_MODEL, f"spend {unique_marker()}") if _is_budget_block(outcome): @@ -105,8 +103,6 @@ def _spend_until_budget_blocks(client: ManagementClient, key: str) -> None: def _settled_spend(client: ManagementClient, key: str) -> float | None: - """The key's recorded spend once it is positive and unchanged across two reads a - poll interval apart, so no batched spend write is still in flight when we reset.""" first = client.proxy.key_info(key).spend or 0.0 time.sleep(client.proxy.poll_interval) second = client.proxy.key_info(key).spend or 0.0 diff --git a/tests/e2e/management/test_management_e2e.py b/tests/e2e/management/test_management_e2e.py index eace8f2e3b4..476165b715d 100644 --- a/tests/e2e/management/test_management_e2e.py +++ b/tests/e2e/management/test_management_e2e.py @@ -12,6 +12,7 @@ from __future__ import annotations import math import time from collections.abc import Callable +from typing import Final import pytest @@ -377,12 +378,12 @@ class TestKeyRegeneration: new_key = client.regenerate_key(old_key, grace_period=REGENERATE_GRACE_PERIOD) resources.defer(lambda: client.proxy.delete_key(new_key)) - revoke_at = time.monotonic() + REGENERATE_GRACE_SECONDS + revoke_at: Final = time.monotonic() + REGENERATE_GRACE_SECONDS assert new_key != old_key, "regenerate returned the same key string, so no rotation happened" def old_accepted() -> bool | None: outcome = client.chat_status(old_key, "gpt-5.5", f"say hi {unique_marker()}") - return True if outcome.status_code != 401 else None + return True if outcome.ok else None _ = _poll(client, old_accepted, "old key was rejected 401 inside its grace period at the deadline") assert time.monotonic() < revoke_at, ( From ee5d66d030824c34e2bd15a1aecd62581b57dc61 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Sat, 5 Sep 2026 11:56:56 -0700 Subject: [PATCH 6/8] fix(ui): keep Back to Guardrails reachable when a ?guardrail= link is stale With the selection in the URL, a mistyped or deleted guardrail id lands on the info view's not-found branch, which rendered only the message and left no way back to the table short of editing the address bar. The not-found branch now shares the Back to Guardrails button with the loaded view --- .../_components/guardrail_info.test.tsx | 24 +++++++++++++++++++ .../guardrails/_components/guardrail_info.tsx | 21 ++++++++-------- 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.test.tsx index fcffc2122e7..836921104ff 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.test.tsx @@ -338,3 +338,27 @@ describe("Guardrail Info", () => { expect(screen.getByText("Guardrail Settings")).toBeInTheDocument(); }); }); + +describe("Guardrail Info when the guardrail cannot be loaded", () => { + afterEach(() => { + vi.clearAllMocks(); + }); + + it("should keep Back to Guardrails reachable so a stale ?guardrail= link is not a dead end", async () => { + vi.mocked(networking.getGuardrailInfo).mockRejectedValue(new Error("Guardrail stale-id not found")); + vi.mocked(networking.getGuardrailUISettings).mockResolvedValue({ + supported_entities: [], + supported_actions: [], + pii_entity_categories: [], + supported_modes: [], + }); + vi.mocked(networking.getGuardrailProviderSpecificParams).mockResolvedValue({}); + const onClose = vi.fn(); + + render(); + + expect(await screen.findByText("Guardrail not found")).toBeInTheDocument(); + fireEvent.click(screen.getByRole("button", { name: /back to guardrails/i })); + expect(onClose).toHaveBeenCalledTimes(1); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx index aaa656d15bb..c9162d99934 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx @@ -481,16 +481,18 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose, return
Loading...
; } + const backButton = ( + + ); + if (!guardrailData) { - return
Guardrail not found
; + return
{backButton}Guardrail not found
; } - // Format date helper function - const formatDate = (dateString?: string) => { - if (!dateString) return "-"; - const date = new Date(dateString); - return date.toLocaleString(); - }; + const formatDate = (dateString?: string) => (dateString ? new Date(dateString).toLocaleString() : "-"); // Format the provider display name and logo const { logo, displayName } = getGuardrailLogoAndName(guardrailData.litellm_params?.guardrail || ""); @@ -510,10 +512,7 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose, return (
- + {backButton}

{guardrailData.guardrail_name || "Unnamed Guardrail"}

{guardrailData.guardrail_id}

From da5af0cb27aa668527a0e5746e307ab3c1188a24 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Sat, 5 Sep 2026 12:02:48 -0700 Subject: [PATCH 7/8] test: repair two CI tests broken by intentional changes test_no_linear_scans_in_router: #39674 renamed heuristic_v2_router_limit_violation to auto_router_capability_violation, so the allowlist entry stopped matching and the same admin-only scan tripped the static check. Rename the entry to follow it. tableScrolling.spec.ts: 9ba6cab889 (LIT-4738) gave the Tags and Model Hub tables client-side pagination at 25 rows, so the 40 seeded rows no longer render on one page. Select 50 rows per page before counting, as the Logs case already does. --- tests/e2e/ui/tests/tables/tableScrolling.spec.ts | 2 ++ tests/router_unit_tests/test_router_index_management.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/e2e/ui/tests/tables/tableScrolling.spec.ts b/tests/e2e/ui/tests/tables/tableScrolling.spec.ts index 5c7438cfd11..0537ba193be 100644 --- a/tests/e2e/ui/tests/tables/tableScrolling.spec.ts +++ b/tests/e2e/ui/tests/tables/tableScrolling.spec.ts @@ -184,6 +184,7 @@ test.describe("Admin tables scroll inside the page", () => { ); try { await navigateToPage(page, Page.TagManagement); + await setRowsPerPage(page, "50"); await expectRowsAtLeast(page, SEED_ROWS); expect(await rowsPaintingPastAnAncestor(page)).toEqual([]); } finally { @@ -207,6 +208,7 @@ test.describe("Admin tables scroll inside the page", () => { ); try { await navigateToPage(page, Page.ModelHubTable); + await setRowsPerPage(page, "50"); await expectRowsAtLeast(page, SEED_ROWS); expect(await rowsPaintingPastAnAncestor(page)).toEqual([]); } finally { diff --git a/tests/router_unit_tests/test_router_index_management.py b/tests/router_unit_tests/test_router_index_management.py index 35d295d581a..291badd91b4 100644 --- a/tests/router_unit_tests/test_router_index_management.py +++ b/tests/router_unit_tests/test_router_index_management.py @@ -241,7 +241,7 @@ class TestRouterIndexManagement: "_get_deployment_by_litellm_model": "lookup by litellm_params.model, which is not indexed", "_finalize_adaptive_router_if_configured": 'init-time prefix scan for "auto_router/adaptive_router"; no index for prefix match', "config_deployments": "filters the whole list on model_info.db_model; admin path only (model add/upsert)", - "heuristic_v2_router_limit_violation": "counts heuristic_v2 routers across the whole list; admin path only (auto-router init/upsert)", + "auto_router_capability_violation": "counts gated auto-routers across the whole list; admin path only (auto-router init/upsert)", } # Get path to router.py From b290dd410e6bb9c59fc3ac7219a3bb197cf027d8 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 5 Sep 2026 12:22:52 -0700 Subject: [PATCH 8/8] feat(terraform/gcp): dependencies-only mode and bring-your-own-network for GKE (#39695) * feat(terraform/gcp): add dependencies-only mode Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(terraform/gcp): review fixes for dependencies-only mode Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/test-terraform-modules.yml | 31 ++++ terraform/litellm/gcp/README.md | 77 +++++++- terraform/litellm/gcp/bootstrap.tf | 6 +- terraform/litellm/gcp/cloudrun.tf | 109 +++++------ terraform/litellm/gcp/cloudsql.tf | 9 +- .../litellm/gcp/examples/default/main.tf | 5 + .../litellm/gcp/examples/default/outputs.tf | 30 +++ .../examples/default/terraform.tfvars.example | 8 + .../litellm/gcp/examples/default/variables.tf | 24 +++ terraform/litellm/gcp/iam.tf | 11 ++ terraform/litellm/gcp/load_balancer.tf | 58 ++++-- terraform/litellm/gcp/locals.tf | 5 +- terraform/litellm/gcp/network.tf | 20 +- terraform/litellm/gcp/outputs.tf | 58 ++++-- terraform/litellm/gcp/redis.tf | 4 +- .../litellm/gcp/tests/deps_only.tftest.hcl | 175 ++++++++++++++++++ terraform/litellm/gcp/variables.tf | 30 ++- 17 files changed, 540 insertions(+), 120 deletions(-) create mode 100644 terraform/litellm/gcp/tests/deps_only.tftest.hcl diff --git a/.github/workflows/test-terraform-modules.yml b/.github/workflows/test-terraform-modules.yml index 0e3e5330453..52006d9b578 100644 --- a/.github/workflows/test-terraform-modules.yml +++ b/.github/workflows/test-terraform-modules.yml @@ -4,6 +4,7 @@ on: push: paths: - "terraform/litellm/aws/**" + - "terraform/litellm/gcp/**" - ".github/workflows/test-terraform-modules.yml" pull_request: branches: @@ -13,6 +14,7 @@ on: - "litellm_**" paths: - "terraform/litellm/aws/**" + - "terraform/litellm/gcp/**" - ".github/workflows/test-terraform-modules.yml" permissions: @@ -52,3 +54,32 @@ jobs: # Plan-only, mock_provider-backed: no AWS credentials, no API calls. - name: test run: terraform test + + gcp-module: + name: fmt, validate, test (gcp) + runs-on: ubuntu-latest + timeout-minutes: 15 + defaults: + run: + working-directory: terraform/litellm/gcp + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3.1.2 + with: + terraform_version: 1.13.3 + terraform_wrapper: false + + - name: fmt + run: terraform fmt -recursive -check -diff + + - name: init + run: terraform init -backend=false -input=false + + - name: validate + run: terraform validate + + - name: test + run: terraform test diff --git a/terraform/litellm/gcp/README.md b/terraform/litellm/gcp/README.md index 88e9979148f..c93e5f6b303 100644 --- a/terraform/litellm/gcp/README.md +++ b/terraform/litellm/gcp/README.md @@ -392,6 +392,63 @@ with its own provider config (one `examples/default`-style root per project), or fork the module to add `configuration_aliases` and pass per-instance `providers = { ... }`. +## Dependencies only (run LiteLLM on GKE) + +Set `create_runtime = false` to provision Cloud SQL, Memorystore, GCS, +Secret Manager, and the runtime service account without Cloud Run or the +load balancer. For a Shared VPC, set the full host-project network ID and +skip PSA creation after the host project has configured it: + +```hcl +create_runtime = false +network_id = "projects//global/networks/" +create_psa_connection = false +``` + +The host project must already have Private Services Access configured on +that network and the Service Networking API enabled; the module cannot set +PSA up from a service project. GKE nodes must sit on the same Shared VPC so +the Cloud SQL and Memorystore private IPs are routable from the pods. Run +the root with its provider pointed at the project that should own the +dependencies. `create_runtime = true` with `network_id` set is also allowed, +but the Serverless VPC Access connector has to live in the same project as +the network, so that combination only works when the VPC is in the +deployment project + +Map the outputs into the Helm values as follows: + +```yaml +database: + writer: + host: + dbname: + passwordSecret: + name: + reader: + host: + dbname: + passwordSecret: + name: +redis: + host: + port: +masterKey: + secretName: +``` + +Create the database Secret with keys `username` (the `db_username` output) +and `password` (read it with `gcloud secrets versions access latest +--secret=`), and the master key Secret from +`master_key_secret_id` the same way. Memorystore only accepts TLS by +default, so store the `redis_server_ca_pem` output in a third Secret, +mount it into the gateway and backend pods via `volumes` / `volumeMounts`, +and add `REDIS_SSL=true` and `REDIS_SSL_CA_CERTS=` to each +component's `extraEnv`. Setting `redis_transit_encryption = false` removes +the CA plumbing at the cost of plaintext Redis traffic inside the VPC + +The chart's pre-install/pre-upgrade migration hook runs the Prisma +migration, so nothing replaces the Cloud Run migrations Job in this mode + ## Storage and database retention Two opt-in tripwires guard against accidental data loss on @@ -409,14 +466,15 @@ Flip `cloudsql_deletion_protection` to `false` or `gcs_force_destroy` to ## Redis encryption -Memorystore runs with `transit_encryption_mode = "SERVER_AUTHENTICATION"`, -so the proxy connects via `rediss://`. The instance's self-signed CA cert -(`server_ca_certs[0].cert`) is shipped to gateway + backend as -`REDIS_CA_PEM_B64`; their entrypoint shell decodes it to `/tmp/redis-ca.pem` -before uvicorn starts and points `REDIS_SSL_CA_CERTS` at that path. No -extra config needed — but if you ever swap Memorystore for an external -Redis, override `REDIS_HOST`/`REDIS_PORT` and either drop these env vars -or point them at your own CA. +By default, Memorystore runs with +`transit_encryption_mode = "SERVER_AUTHENTICATION"`, so Cloud Run connects +via `rediss://`. The instance's self-signed CA cert +(`server_ca_certs[0].cert`) is shipped to gateway and backend as +`REDIS_CA_PEM_B64`; their entrypoint shell decodes it to +`/tmp/redis-ca.pem` before uvicorn starts and points `REDIS_SSL_CA_CERTS` at +that path. Set `redis_transit_encryption = false` to use plaintext Redis. +For GKE, use `redis_server_ca_pem` as described in the dependencies-only +section, or accept the security tradeoff of disabling transit encryption ## Files @@ -434,4 +492,5 @@ or point them at your own CA. | `iam.tf` | Runtime SA + Cloud SQL client + Secret Manager accessor | | `cloudrun.tf` | 3 Cloud Run services + Cloud Run Job for migrations | | `load_balancer.tf`| External HTTPS LB, serverless NEGs, URL map for path routing | -| `outputs.tf` | LB IP, service URLs, secret IDs, migration `execute` command | +| `outputs.tf` | LB IP, service URLs, dependency endpoints, secret IDs, migration command | +| `tests/` | Plan-only mock-provider coverage for deployment modes and Redis encryption | diff --git a/terraform/litellm/gcp/bootstrap.tf b/terraform/litellm/gcp/bootstrap.tf index b929c4d76f3..dead5c41f6b 100644 --- a/terraform/litellm/gcp/bootstrap.tf +++ b/terraform/litellm/gcp/bootstrap.tf @@ -15,15 +15,17 @@ # enough to invoke Cloud Run admin APIs (`gcloud auth login`). resource "terraform_data" "migration" { + count = var.create_runtime ? 1 : 0 + triggers_replace = { - job_id = google_cloud_run_v2_job.migrations.id + job_id = google_cloud_run_v2_job.migrations[0].id job_image = local.migrations_image } provisioner "local-exec" { interpreter = ["bash", "-c"] environment = { - JOB = google_cloud_run_v2_job.migrations.name + JOB = google_cloud_run_v2_job.migrations[0].name REGION = var.region PROJECT = var.project_id } diff --git a/terraform/litellm/gcp/cloudrun.tf b/terraform/litellm/gcp/cloudrun.tf index 5a5c361b832..84ae8b9247f 100644 --- a/terraform/litellm/gcp/cloudrun.tf +++ b/terraform/litellm/gcp/cloudrun.tf @@ -6,25 +6,28 @@ locals { # Memorystore exposes a self-signed CA cert per instance; we ship it as # a base64 env var and decode it to a file at container startup so the # rediss:// connection can validate. Public cert, not sensitive. - redis_ca_pem_b64 = base64encode(google_redis_instance.this.server_ca_certs[0].cert) + redis_ca_pem_b64 = var.redis_transit_encryption ? base64encode(google_redis_instance.this.server_ca_certs[0].cert) : "" - shared_env_kv = [ - { name = "DATABASE_HOST", value = google_sql_database_instance.writer.private_ip_address }, - { name = "DATABASE_PORT", value = "5432" }, - { name = "DATABASE_USER", value = var.db_username }, - { name = "DATABASE_NAME", value = var.db_name }, - { name = "DATABASE_HOST_READ_REPLICA", value = google_sql_database_instance.reader.private_ip_address }, - { name = "DATABASE_PORT_READ_REPLICA", value = "5432" }, - { name = "REDIS_HOST", value = google_redis_instance.this.host }, - { name = "REDIS_PORT", value = tostring(google_redis_instance.this.port) }, - # _redis.get_redis_url_from_environment honors REDIS_SSL to flip the - # scheme to rediss://; REDIS_SSL_CA_CERTS is mapped via - # _get_redis_env_kwarg_mapping → ssl_ca_certs on the redis-py client. - { name = "REDIS_SSL", value = "true" }, - { name = "REDIS_SSL_CA_CERTS", value = "/tmp/redis-ca.pem" }, - { name = "REDIS_CA_PEM_B64", value = local.redis_ca_pem_b64 }, - { name = "GCS_BUCKET_NAME", value = google_storage_bucket.this.name }, - ] + shared_env_kv = concat( + [ + { name = "DATABASE_HOST", value = google_sql_database_instance.writer.private_ip_address }, + { name = "DATABASE_PORT", value = "5432" }, + { name = "DATABASE_USER", value = var.db_username }, + { name = "DATABASE_NAME", value = var.db_name }, + { name = "DATABASE_HOST_READ_REPLICA", value = google_sql_database_instance.reader.private_ip_address }, + { name = "DATABASE_PORT_READ_REPLICA", value = "5432" }, + { name = "REDIS_HOST", value = google_redis_instance.this.host }, + { name = "REDIS_PORT", value = tostring(google_redis_instance.this.port) }, + ], + var.redis_transit_encryption ? [ + { name = "REDIS_SSL", value = "true" }, + { name = "REDIS_SSL_CA_CERTS", value = "/tmp/redis-ca.pem" }, + { name = "REDIS_CA_PEM_B64", value = local.redis_ca_pem_b64 }, + ] : [], + [ + { name = "GCS_BUCKET_NAME", value = google_storage_bucket.this.name }, + ], + ) # OTel v2 is opt-in and gated on otel_endpoint, matching the AWS stack — # nothing OTel-related is added to the container env until an endpoint is @@ -126,9 +129,9 @@ locals { # Decode the Memorystore CA cert (passed as REDIS_CA_PEM_B64) to the # path REDIS_SSL_CA_CERTS points at, so the redis-py client can validate # the rediss:// handshake. - redis_ca_fragment = [ + redis_ca_fragment = var.redis_transit_encryption ? [ "python -c \"import os, base64, pathlib; pathlib.Path(os.environ['REDIS_SSL_CA_CERTS']).write_bytes(base64.b64decode(os.environ['REDIS_CA_PEM_B64']))\"" - ] + ] : [] database_url_fragment = [ "export DATABASE_URL=\"postgresql://$${DATABASE_USER}:$${DATABASE_PASSWORD}@$${DATABASE_HOST}:$${DATABASE_PORT}/$${DATABASE_NAME}\"", @@ -171,29 +174,7 @@ locals { # ---------- Gateway ---------- resource "google_cloud_run_v2_service" "gateway" { - # Metering needs a client certificate AND its key. Each secret is created only - # when its own PEM is supplied, so an endpoint set with a missing key would - # otherwise apply cleanly and leave the proxy logging "missing config" and - # never exporting. ca_cert_pem stays optional: empty means fall back to the - # system trust store. - # - # The guard lives here, on an unconditional resource, rather than on the cert - # secret: that secret is count-gated on the cert itself, so it has zero - # instances in exactly the case this must catch. Adding count or for_each to - # this resource would silently stop the guard from evaluating. - # - # endpoint cert key -> result - # "" any any -> metering off, no secrets created - # set set set -> metering on - # set any-missing -> plan fails here - lifecycle { - precondition { - condition = var.billing_metrics_endpoint == "" || ( - var.billing_metrics_client_cert_pem != "" && var.billing_metrics_client_key_pem != "" - ) - error_message = "billing_metrics_client_cert_pem and billing_metrics_client_key_pem are both required when billing_metrics_endpoint is set." - } - } + count = var.create_runtime ? 1 : 0 name = "${local.name}-gateway" location = var.region @@ -206,7 +187,7 @@ resource "google_cloud_run_v2_service" "gateway" { max_instance_request_concurrency = var.gateway_max_instance_request_concurrency vpc_access { - connector = google_vpc_access_connector.this.id + connector = google_vpc_access_connector.this[0].id egress = "PRIVATE_RANGES_ONLY" } @@ -312,17 +293,7 @@ resource "google_cloud_run_v2_service" "gateway" { # ---------- Backend ---------- resource "google_cloud_run_v2_service" "backend" { - # Same guard as the gateway: the backend meters too (it serves the named-server - # MCP transport), and a targeted apply of just this resource must not slip a - # billing endpoint through without the credentials to use it. - lifecycle { - precondition { - condition = var.billing_metrics_endpoint == "" || ( - var.billing_metrics_client_cert_pem != "" && var.billing_metrics_client_key_pem != "" - ) - error_message = "billing_metrics_client_cert_pem and billing_metrics_client_key_pem are both required when billing_metrics_endpoint is set." - } - } + count = var.create_runtime ? 1 : 0 name = "${local.name}-backend" location = var.region @@ -335,7 +306,7 @@ resource "google_cloud_run_v2_service" "backend" { max_instance_request_concurrency = var.backend_max_instance_request_concurrency vpc_access { - connector = google_vpc_access_connector.this.id + connector = google_vpc_access_connector.this[0].id egress = "PRIVATE_RANGES_ONLY" } @@ -443,6 +414,8 @@ resource "google_cloud_run_v2_service" "backend" { # with zero IAM bindings, so a compromised UI container can't pivot to # Secret Manager / Cloud SQL via the metadata service. resource "google_cloud_run_v2_service" "ui" { + count = var.create_runtime ? 1 : 0 + name = "${local.name}-ui" location = var.region ingress = "INGRESS_TRAFFIC_INTERNAL_LOAD_BALANCER" @@ -450,7 +423,7 @@ resource "google_cloud_run_v2_service" "ui" { deletion_protection = false template { - service_account = google_service_account.ui_runtime.email + service_account = google_service_account.ui_runtime[0].email max_instance_request_concurrency = var.ui_max_instance_request_concurrency scaling { @@ -491,25 +464,31 @@ resource "google_cloud_run_v2_service" "ui" { # (LITELLM_MASTER_KEY); these IAM bindings just open up Cloud Run's invoker # gate so the LB request makes it to the container. resource "google_cloud_run_v2_service_iam_member" "gateway_allusers" { + count = var.create_runtime ? 1 : 0 + project = var.project_id - location = google_cloud_run_v2_service.gateway.location - name = google_cloud_run_v2_service.gateway.name + location = google_cloud_run_v2_service.gateway[0].location + name = google_cloud_run_v2_service.gateway[0].name role = "roles/run.invoker" member = "allUsers" } resource "google_cloud_run_v2_service_iam_member" "backend_allusers" { + count = var.create_runtime ? 1 : 0 + project = var.project_id - location = google_cloud_run_v2_service.backend.location - name = google_cloud_run_v2_service.backend.name + location = google_cloud_run_v2_service.backend[0].location + name = google_cloud_run_v2_service.backend[0].name role = "roles/run.invoker" member = "allUsers" } resource "google_cloud_run_v2_service_iam_member" "ui_allusers" { + count = var.create_runtime ? 1 : 0 + project = var.project_id - location = google_cloud_run_v2_service.ui.location - name = google_cloud_run_v2_service.ui.name + location = google_cloud_run_v2_service.ui[0].location + name = google_cloud_run_v2_service.ui[0].name role = "roles/run.invoker" member = "allUsers" } @@ -519,6 +498,8 @@ resource "google_cloud_run_v2_service_iam_member" "ui_allusers" { # assembles DATABASE_URL from the DATABASE_* env vars and runs `prisma # migrate deploy`. No proxy_config, no master key, no shell wrapper. resource "google_cloud_run_v2_job" "migrations" { + count = var.create_runtime ? 1 : 0 + name = "${local.name}-migrations" location = var.region labels = local.labels @@ -529,7 +510,7 @@ resource "google_cloud_run_v2_job" "migrations" { service_account = google_service_account.runtime.email vpc_access { - connector = google_vpc_access_connector.this.id + connector = google_vpc_access_connector.this[0].id egress = "PRIVATE_RANGES_ONLY" } diff --git a/terraform/litellm/gcp/cloudsql.tf b/terraform/litellm/gcp/cloudsql.tf index c9c2d03b2de..777434b4727 100644 --- a/terraform/litellm/gcp/cloudsql.tf +++ b/terraform/litellm/gcp/cloudsql.tf @@ -36,7 +36,7 @@ resource "google_sql_database_instance" "writer" { ip_configuration { ipv4_enabled = false - private_network = google_compute_network.this.id + private_network = local.network_id } insights_config { @@ -55,6 +55,11 @@ resource "google_sql_database_instance" "writer" { # (full data loss). Set the initial size only; let Cloud SQL own it # thereafter. ignore_changes = [settings[0].disk_size] + + precondition { + condition = var.create_psa_connection || var.network_id != "" + error_message = "create_psa_connection must be true unless network_id references an existing VPC with Private Services Access configured." + } } } @@ -76,7 +81,7 @@ resource "google_sql_database_instance" "reader" { ip_configuration { ipv4_enabled = false - private_network = google_compute_network.this.id + private_network = local.network_id } } diff --git a/terraform/litellm/gcp/examples/default/main.tf b/terraform/litellm/gcp/examples/default/main.tf index 8760d445f0c..f44b2a9a001 100644 --- a/terraform/litellm/gcp/examples/default/main.tf +++ b/terraform/litellm/gcp/examples/default/main.tf @@ -31,6 +31,11 @@ module "litellm" { tenant = var.tenant env = var.env + create_runtime = var.create_runtime + network_id = var.network_id + create_psa_connection = var.create_psa_connection + redis_transit_encryption = var.redis_transit_encryption + litellm_master_key = var.litellm_master_key litellm_license = var.litellm_license ui_password = var.ui_password diff --git a/terraform/litellm/gcp/examples/default/outputs.tf b/terraform/litellm/gcp/examples/default/outputs.tf index 3a9343c4850..48cdc1af66e 100644 --- a/terraform/litellm/gcp/examples/default/outputs.tf +++ b/terraform/litellm/gcp/examples/default/outputs.tf @@ -38,6 +38,31 @@ output "redis_endpoint" { value = module.litellm.redis_endpoint } +output "redis_host" { + description = "Memorystore Redis host." + value = module.litellm.redis_host +} + +output "redis_port" { + description = "Memorystore Redis port." + value = module.litellm.redis_port +} + +output "redis_server_ca_pem" { + description = "Memorystore server CA PEM." + value = module.litellm.redis_server_ca_pem +} + +output "db_username" { + description = "Cloud SQL application username." + value = module.litellm.db_username +} + +output "db_name" { + description = "Cloud SQL database name." + value = module.litellm.db_name +} + output "gcs_bucket" { description = "GCS bucket name." value = module.litellm.gcs_bucket @@ -53,6 +78,11 @@ output "db_password_secret_id" { value = module.litellm.db_password_secret_id } +output "runtime_service_account_email" { + description = "Runtime service account email." + value = module.litellm.runtime_service_account_email +} + output "migration_run_command" { description = "Break-glass command to re-run the one-off migration job." value = module.litellm.migration_run_command diff --git a/terraform/litellm/gcp/examples/default/terraform.tfvars.example b/terraform/litellm/gcp/examples/default/terraform.tfvars.example index 4416cf0ee5d..c35206503bb 100644 --- a/terraform/litellm/gcp/examples/default/terraform.tfvars.example +++ b/terraform/litellm/gcp/examples/default/terraform.tfvars.example @@ -8,6 +8,14 @@ region = "us-central1" tenant = "acme" env = "stage" +# Deployment mode. For dependencies only on a Shared VPC, set +# create_runtime = false, network_id to the full host-project network ID, and +# create_psa_connection = false after configuring PSA on that network. +# create_runtime = true +# network_id = "" +# create_psa_connection = true +# redis_transit_encryption = true + # Tenant-supplied secrets. Prefer TF_VAR_litellm_master_key / # TF_VAR_litellm_license / TF_VAR_ui_password env vars so the values don't # end up in a committed tfvars file. All three are optional — when diff --git a/terraform/litellm/gcp/examples/default/variables.tf b/terraform/litellm/gcp/examples/default/variables.tf index 56e5ec88ef8..88b57ce27eb 100644 --- a/terraform/litellm/gcp/examples/default/variables.tf +++ b/terraform/litellm/gcp/examples/default/variables.tf @@ -26,6 +26,30 @@ variable "env" { type = string } +variable "create_runtime" { + description = "Create Cloud Run and load balancer resources." + type = bool + default = true +} + +variable "network_id" { + description = "Existing VPC network resource ID. Empty creates a VPC." + type = string + default = "" +} + +variable "create_psa_connection" { + description = "Create Private Services Access resources." + type = bool + default = true +} + +variable "redis_transit_encryption" { + description = "Enable Memorystore transit encryption." + type = bool + default = true +} + # Sensitive — prefer TF_VAR_litellm_master_key / TF_VAR_litellm_license / # TF_VAR_ui_password so values stay out of any committed tfvars file. variable "litellm_master_key" { diff --git a/terraform/litellm/gcp/iam.tf b/terraform/litellm/gcp/iam.tf index 09df5e7dff0..509e6d48ffd 100644 --- a/terraform/litellm/gcp/iam.tf +++ b/terraform/litellm/gcp/iam.tf @@ -6,6 +6,15 @@ resource "google_service_account" "runtime" { account_id = "${local.name}-runtime" display_name = "LiteLLM Cloud Run runtime" + + lifecycle { + precondition { + condition = !var.create_runtime || var.billing_metrics_endpoint == "" || ( + var.billing_metrics_client_cert_pem != "" && var.billing_metrics_client_key_pem != "" + ) + error_message = "billing_metrics_client_cert_pem and billing_metrics_client_key_pem are both required when billing_metrics_endpoint is set and create_runtime is true." + } + } } # UI runtime SA — no role bindings. The UI is static nginx with no DB, @@ -14,6 +23,8 @@ resource "google_service_account" "runtime" { # project's serverless service agent (not this SA), so it doesn't need # artifactregistry.reader either. resource "google_service_account" "ui_runtime" { + count = var.create_runtime ? 1 : 0 + account_id = "${local.name}-ui-runtime" display_name = "LiteLLM Cloud Run UI runtime (no data-plane access)" } diff --git a/terraform/litellm/gcp/load_balancer.tf b/terraform/litellm/gcp/load_balancer.tf index 11f30d0f944..57e8af8210f 100644 --- a/terraform/litellm/gcp/load_balancer.tf +++ b/terraform/litellm/gcp/load_balancer.tf @@ -14,77 +14,93 @@ locals { } resource "google_compute_global_address" "lb" { + count = var.create_runtime ? 1 : 0 + name = "${local.name}-lb-ip" labels = local.labels } # Serverless NEGs — one per Cloud Run service. resource "google_compute_region_network_endpoint_group" "gateway" { + count = var.create_runtime ? 1 : 0 + name = "${local.name}-gateway-neg" region = var.region network_endpoint_type = "SERVERLESS" cloud_run { - service = google_cloud_run_v2_service.gateway.name + service = google_cloud_run_v2_service.gateway[0].name } } resource "google_compute_region_network_endpoint_group" "backend" { + count = var.create_runtime ? 1 : 0 + name = "${local.name}-backend-neg" region = var.region network_endpoint_type = "SERVERLESS" cloud_run { - service = google_cloud_run_v2_service.backend.name + service = google_cloud_run_v2_service.backend[0].name } } resource "google_compute_region_network_endpoint_group" "ui" { + count = var.create_runtime ? 1 : 0 + name = "${local.name}-ui-neg" region = var.region network_endpoint_type = "SERVERLESS" cloud_run { - service = google_cloud_run_v2_service.ui.name + service = google_cloud_run_v2_service.ui[0].name } } # Backend services wrap each NEG. resource "google_compute_backend_service" "gateway" { + count = var.create_runtime ? 1 : 0 + name = "${local.name}-gateway-bs" protocol = "HTTP" load_balancing_scheme = "EXTERNAL_MANAGED" backend { - group = google_compute_region_network_endpoint_group.gateway.id + group = google_compute_region_network_endpoint_group.gateway[0].id } } resource "google_compute_backend_service" "backend" { + count = var.create_runtime ? 1 : 0 + name = "${local.name}-backend-bs" protocol = "HTTP" load_balancing_scheme = "EXTERNAL_MANAGED" backend { - group = google_compute_region_network_endpoint_group.backend.id + group = google_compute_region_network_endpoint_group.backend[0].id } } resource "google_compute_backend_service" "ui" { + count = var.create_runtime ? 1 : 0 + name = "${local.name}-ui-bs" protocol = "HTTP" load_balancing_scheme = "EXTERNAL_MANAGED" backend { - group = google_compute_region_network_endpoint_group.ui.id + group = google_compute_region_network_endpoint_group.ui[0].id } } # URL map. Default → backend (management API). Path matchers route the # gateway and UI prefixes elsewhere. resource "google_compute_url_map" "this" { + count = var.create_runtime ? 1 : 0 + name = local.name - default_service = google_compute_backend_service.backend.id + default_service = google_compute_backend_service.backend[0].id host_rule { hosts = ["*"] @@ -93,13 +109,13 @@ resource "google_compute_url_map" "this" { path_matcher { name = "main" - default_service = google_compute_backend_service.backend.id + default_service = google_compute_backend_service.backend[0].id # UI paths (catch them before any /v1/* gateway rules so /favicon.ico # and / take precedence). path_rule { paths = local.ui_path_prefixes - service = google_compute_backend_service.ui.id + service = google_compute_backend_service.ui[0].id } # Gateway path prefixes. GCP URL maps cap a path_rule at 10 path globs, @@ -108,7 +124,7 @@ resource "google_compute_url_map" "this" { for_each = { for idx, chunk in chunklist(local.gateway_path_prefixes, 10) : idx => chunk } content { paths = path_rule.value - service = google_compute_backend_service.gateway.id + service = google_compute_backend_service.gateway[0].id } } } @@ -118,7 +134,7 @@ resource "google_compute_url_map" "this" { # target proxy when TLS is enabled; otherwise the regular path-routing # URL map is attached to the HTTP proxy and everything stays plaintext. resource "google_compute_url_map" "https_redirect" { - count = local.tls_enabled ? 1 : 0 + count = var.create_runtime && local.tls_enabled ? 1 : 0 name = "${local.name}-redirect" default_url_redirect { @@ -129,8 +145,10 @@ resource "google_compute_url_map" "https_redirect" { } resource "google_compute_target_http_proxy" "this" { + count = var.create_runtime ? 1 : 0 + name = "${local.name}-http" - url_map = local.tls_enabled ? google_compute_url_map.https_redirect[0].id : google_compute_url_map.this.id + url_map = local.tls_enabled ? google_compute_url_map.https_redirect[0].id : google_compute_url_map.this[0].id # Default-deny on the HTTP-only path: TLS is the supported posture. # Operators must either supply DNS names or explicitly opt in. @@ -143,12 +161,14 @@ resource "google_compute_target_http_proxy" "this" { } resource "google_compute_global_forwarding_rule" "http" { + count = var.create_runtime ? 1 : 0 + name = "${local.name}-http" ip_protocol = "TCP" port_range = "80" load_balancing_scheme = "EXTERNAL_MANAGED" - ip_address = google_compute_global_address.lb.address - target = google_compute_target_http_proxy.this.id + ip_address = google_compute_global_address.lb[0].address + target = google_compute_target_http_proxy.this[0].id labels = local.labels } @@ -161,7 +181,7 @@ resource "google_compute_global_forwarding_rule" "http" { # transitions to ACTIVE. resource "google_compute_managed_ssl_certificate" "this" { - count = local.tls_enabled ? 1 : 0 + count = var.create_runtime && local.tls_enabled ? 1 : 0 # A managed cert's `domains` is immutable, so changing var.lb_domains # forces replacement, and the cert is referenced by the HTTPS target @@ -181,19 +201,19 @@ resource "google_compute_managed_ssl_certificate" "this" { } resource "google_compute_target_https_proxy" "this" { - count = local.tls_enabled ? 1 : 0 + count = var.create_runtime && local.tls_enabled ? 1 : 0 name = "${local.name}-https" - url_map = google_compute_url_map.this.id + url_map = google_compute_url_map.this[0].id ssl_certificates = [google_compute_managed_ssl_certificate.this[0].id] } resource "google_compute_global_forwarding_rule" "https" { - count = local.tls_enabled ? 1 : 0 + count = var.create_runtime && local.tls_enabled ? 1 : 0 name = "${local.name}-https" ip_protocol = "TCP" port_range = "443" load_balancing_scheme = "EXTERNAL_MANAGED" - ip_address = google_compute_global_address.lb.address + ip_address = google_compute_global_address.lb[0].address target = google_compute_target_https_proxy.this[0].id labels = local.labels } diff --git a/terraform/litellm/gcp/locals.tf b/terraform/litellm/gcp/locals.tf index 9a817eba605..3861413d496 100644 --- a/terraform/litellm/gcp/locals.tf +++ b/terraform/litellm/gcp/locals.tf @@ -21,6 +21,9 @@ locals { var.labels, ) + create_network = var.network_id == "" + network_id = local.create_network ? google_compute_network.this[0].id : var.network_id + gateway_path_prefixes = [ "/v1/chat/*", "/chat/*", "/v1/completions*", "/completions*", @@ -74,7 +77,7 @@ locals { "/ui/*", ] - proxy_config_enabled = length(keys(var.proxy_config)) > 0 + proxy_config_enabled = var.create_runtime && length(keys(var.proxy_config)) > 0 proxy_config_yaml = local.proxy_config_enabled ? yamlencode(var.proxy_config) : "" proxy_config_mount_path = "/etc/litellm" diff --git a/terraform/litellm/gcp/network.tf b/terraform/litellm/gcp/network.tf index a1ccaed02f9..47c7bf94a2b 100644 --- a/terraform/litellm/gcp/network.tf +++ b/terraform/litellm/gcp/network.tf @@ -1,13 +1,17 @@ resource "google_compute_network" "this" { + count = local.create_network ? 1 : 0 + name = local.name auto_create_subnetworks = false routing_mode = "REGIONAL" } resource "google_compute_subnetwork" "this" { + count = local.create_network ? 1 : 0 + name = "${local.name}-${var.region}" region = var.region - network = google_compute_network.this.id + network = google_compute_network.this[0].id ip_cidr_range = var.subnet_cidr private_ip_google_access = true } @@ -16,17 +20,21 @@ resource "google_compute_subnetwork" "this" { # managed services peer with the VPC over the connection below using # addresses from this range. resource "google_compute_global_address" "psa" { + count = var.create_psa_connection ? 1 : 0 + name = "${local.name}-psa" purpose = "VPC_PEERING" address_type = "INTERNAL" prefix_length = 16 - network = google_compute_network.this.id + network = local.network_id } resource "google_service_networking_connection" "psa" { - network = google_compute_network.this.id + count = var.create_psa_connection ? 1 : 0 + + network = local.network_id service = "servicenetworking.googleapis.com" - reserved_peering_ranges = [google_compute_global_address.psa.name] + reserved_peering_ranges = [google_compute_global_address.psa[0].name] } # Serverless VPC Access connector — required so Cloud Run can reach @@ -37,9 +45,11 @@ resource "google_service_networking_connection" "psa" { # for low-to-moderate Cloud Run egress; bump max if your services push # heavy private-network traffic. resource "google_vpc_access_connector" "this" { + count = var.create_runtime ? 1 : 0 + name = "${local.name}-conn" region = var.region - network = google_compute_network.this.name + network = local.network_id ip_cidr_range = var.vpc_connector_cidr min_instances = 2 max_instances = 3 diff --git a/terraform/litellm/gcp/outputs.tf b/terraform/litellm/gcp/outputs.tf index 6f1f1d5ccf4..2a4742f42cf 100644 --- a/terraform/litellm/gcp/outputs.tf +++ b/terraform/litellm/gcp/outputs.tf @@ -1,26 +1,26 @@ output "lb_ip" { - description = "Global anycast IP of the external HTTPS load balancer." - value = google_compute_global_address.lb.address + description = "Global anycast IP of the external HTTPS load balancer. Null when create_runtime is false." + value = var.create_runtime ? one(google_compute_global_address.lb[*].address) : null } output "lb_url" { - description = "Proxy URL. Switches scheme based on whether lb_domains is set; when TLS is enabled the URL points at the first listed domain (since managed certs are tied to the hostname, not the anycast IP). The dashboard is served at /, the API at /v1/*." - value = local.tls_enabled ? "https://${var.lb_domains[0]}" : "http://${google_compute_global_address.lb.address}" + description = "Proxy URL, or null when create_runtime is false. Switches scheme based on whether lb_domains is set." + value = var.create_runtime ? (local.tls_enabled ? "https://${var.lb_domains[0]}" : "http://${one(google_compute_global_address.lb[*].address)}") : null } output "gateway_service_url" { - description = "Default Cloud Run URL for the gateway (bypasses the LB)." - value = google_cloud_run_v2_service.gateway.uri + description = "Default Cloud Run URL for the gateway, or null when create_runtime is false." + value = var.create_runtime ? one(google_cloud_run_v2_service.gateway[*].uri) : null } output "backend_service_url" { - description = "Default Cloud Run URL for the backend (bypasses the LB)." - value = google_cloud_run_v2_service.backend.uri + description = "Default Cloud Run URL for the backend, or null when create_runtime is false." + value = var.create_runtime ? one(google_cloud_run_v2_service.backend[*].uri) : null } output "ui_service_url" { - description = "Default Cloud Run URL for the UI (bypasses the LB)." - value = google_cloud_run_v2_service.ui.uri + description = "Default Cloud Run URL for the UI, or null when create_runtime is false." + value = var.create_runtime ? one(google_cloud_run_v2_service.ui[*].uri) : null } output "cloudsql_writer_ip" { @@ -38,6 +38,36 @@ output "redis_endpoint" { value = "${google_redis_instance.this.host}:${google_redis_instance.this.port}" } +output "runtime_service_account_email" { + description = "Runtime service account email for Cloud Run or GKE Workload Identity." + value = google_service_account.runtime.email +} + +output "redis_host" { + description = "Memorystore Redis host." + value = google_redis_instance.this.host +} + +output "redis_port" { + description = "Memorystore Redis port." + value = google_redis_instance.this.port +} + +output "redis_server_ca_pem" { + description = "Memorystore server CA PEM. Mount it in the pod and set REDIS_SSL=true and REDIS_SSL_CA_CERTS= via extraEnv when transit encryption is enabled." + value = var.redis_transit_encryption ? google_redis_instance.this.server_ca_certs[0].cert : null +} + +output "db_username" { + description = "Cloud SQL application username." + value = var.db_username +} + +output "db_name" { + description = "Cloud SQL database name." + value = var.db_name +} + output "gcs_bucket" { description = "GCS bucket name. Exposed to gateway + backend as GCS_BUCKET_NAME. Reference from proxy_config via `os.environ/GCS_BUCKET_NAME`." value = google_storage_bucket.this.name @@ -54,11 +84,11 @@ output "db_password_secret_id" { } output "migration_run_command" { - description = "Shell command that executes the one-off migration job against Cloud SQL. Run this once after the first apply." - value = format( + description = "Shell command that executes the one-off migration job against Cloud SQL, or null when create_runtime is false." + value = var.create_runtime ? format( "gcloud run jobs execute %s --region %s --project %s --wait", - google_cloud_run_v2_job.migrations.name, + one(google_cloud_run_v2_job.migrations[*].name), var.region, var.project_id, - ) + ) : null } diff --git a/terraform/litellm/gcp/redis.tf b/terraform/litellm/gcp/redis.tf index 0e07c416e85..0602758f090 100644 --- a/terraform/litellm/gcp/redis.tf +++ b/terraform/litellm/gcp/redis.tf @@ -4,7 +4,7 @@ resource "google_redis_instance" "this" { memory_size_gb = var.redis_memory_size_gb region = var.region - authorized_network = google_compute_network.this.id + authorized_network = local.network_id connect_mode = "PRIVATE_SERVICE_ACCESS" redis_version = "REDIS_7_0" @@ -16,7 +16,7 @@ resource "google_redis_instance" "this" { # and passed to the proxy as REDIS_CA_PEM_B64); the proxy decodes it to # /tmp/redis-ca.pem at startup and uses it to validate the rediss:// # handshake. Mirrors `transit_encryption_enabled = true` on AWS. - transit_encryption_mode = "SERVER_AUTHENTICATION" + transit_encryption_mode = var.redis_transit_encryption ? "SERVER_AUTHENTICATION" : "DISABLED" depends_on = [google_service_networking_connection.psa] } diff --git a/terraform/litellm/gcp/tests/deps_only.tftest.hcl b/terraform/litellm/gcp/tests/deps_only.tftest.hcl new file mode 100644 index 00000000000..610c49d5b52 --- /dev/null +++ b/terraform/litellm/gcp/tests/deps_only.tftest.hcl @@ -0,0 +1,175 @@ +mock_provider "google" { + mock_resource "google_redis_instance" { + defaults = { + host = "10.0.0.4" + port = 6379 + server_ca_certs = [{ + cert = "-----BEGIN CERTIFICATE-----\nmock\n-----END CERTIFICATE-----" + }] + } + } +} + +mock_provider "google-beta" {} +mock_provider "random" {} + +variables { + project_id = "test-project" + tenant = "tenant" + env = "test" + allow_plaintext_lb = true + image_registry = "us-central1-docker.pkg.dev/test-project/litellm" +} + +run "default_creates_everything" { + command = plan + + assert { + condition = alltrue([ + length(google_compute_network.this) == 1, + length(google_compute_subnetwork.this) == 1, + length(google_compute_global_address.psa) == 1, + length(google_service_networking_connection.psa) == 1, + length(google_vpc_access_connector.this) == 1, + length(google_cloud_run_v2_service.gateway) == 1, + length(google_cloud_run_v2_service.backend) == 1, + length(google_cloud_run_v2_service.ui) == 1, + length(google_cloud_run_v2_job.migrations) == 1, + length(google_compute_global_address.lb) == 1, + length(terraform_data.migration) == 1, + ]) + error_message = "The default mode must create networking, runtime services, the load balancer, and migrations." + } + + assert { + condition = google_redis_instance.this.transit_encryption_mode == "SERVER_AUTHENTICATION" + error_message = "Redis transit encryption must remain enabled by default." + } + + assert { + condition = length(local.shared_env_kv) == 12 + error_message = "The default runtime environment must include GCS and the three Redis TLS entries." + } +} + +run "deps_only_creates_no_runtime" { + command = plan + + variables { + create_runtime = false + proxy_config = { + model_list = [] + } + } + + assert { + condition = alltrue([ + length(google_cloud_run_v2_service.gateway) == 0, + length(google_cloud_run_v2_service.backend) == 0, + length(google_cloud_run_v2_service.ui) == 0, + length(google_cloud_run_v2_job.migrations) == 0, + length(google_cloud_run_v2_service_iam_member.gateway_allusers) == 0, + length(google_cloud_run_v2_service_iam_member.backend_allusers) == 0, + length(google_cloud_run_v2_service_iam_member.ui_allusers) == 0, + length(google_compute_global_address.lb) == 0, + length(google_compute_region_network_endpoint_group.gateway) == 0, + length(google_compute_region_network_endpoint_group.backend) == 0, + length(google_compute_region_network_endpoint_group.ui) == 0, + length(google_compute_backend_service.gateway) == 0, + length(google_compute_backend_service.backend) == 0, + length(google_compute_backend_service.ui) == 0, + length(google_compute_url_map.this) == 0, + length(google_compute_url_map.https_redirect) == 0, + length(google_compute_target_http_proxy.this) == 0, + length(google_compute_global_forwarding_rule.http) == 0, + length(google_compute_managed_ssl_certificate.this) == 0, + length(google_compute_target_https_proxy.this) == 0, + length(google_compute_global_forwarding_rule.https) == 0, + length(terraform_data.migration) == 0, + length(google_vpc_access_connector.this) == 0, + length(google_service_account.ui_runtime) == 0, + length(google_storage_bucket.proxy_config) == 0, + ]) + error_message = "Dependencies-only mode must omit all runtime, load balancer, connector, UI identity, and proxy config resources." + } + + assert { + condition = alltrue([ + google_sql_database_instance.writer.name == "tenant-litellm-test", + google_sql_database_instance.reader.name == "tenant-litellm-test-reader", + google_redis_instance.this.name == "tenant-litellm-test", + google_storage_bucket.this.force_destroy == false, + google_secret_manager_secret.master_key.secret_id == "tenant-litellm-test-master-key", + google_secret_manager_secret.db_password.secret_id == "tenant-litellm-test-db-password", + google_service_account.runtime.account_id == "tenant-litellm-test-runtime", + ]) + error_message = "Dependencies-only mode must retain data stores, secrets, and the runtime service account." + } + + assert { + condition = output.lb_url == null && output.migration_run_command == null + error_message = "Runtime outputs must be null while dependency outputs remain available." + } +} + +run "existing_network_attaches_data_stores" { + command = plan + + variables { + network_id = "projects/host-proj/global/networks/shared" + create_psa_connection = false + create_runtime = false + } + + assert { + condition = alltrue([ + length(google_compute_network.this) == 0, + length(google_compute_subnetwork.this) == 0, + length(google_compute_global_address.psa) == 0, + length(google_service_networking_connection.psa) == 0, + google_sql_database_instance.writer.settings[0].ip_configuration[0].private_network == var.network_id, + google_redis_instance.this.authorized_network == var.network_id, + ]) + error_message = "An existing VPC must receive the Cloud SQL and Memorystore private-network attachments." + } +} + +run "psa_required_without_existing_network" { + command = plan + + variables { + create_psa_connection = false + } + + expect_failures = [ + google_sql_database_instance.writer, + ] +} + +run "redis_plaintext_drops_tls_env" { + command = plan + + variables { + redis_transit_encryption = false + } + + assert { + condition = google_redis_instance.this.transit_encryption_mode == "DISABLED" + error_message = "Redis transit encryption must be disabled when requested." + } + + assert { + condition = length(local.shared_env_kv) == 9 + error_message = "Plaintext Redis mode must include GCS and omit the three Redis TLS entries." + } + + assert { + condition = length([for env in local.shared_env_kv : env if env.name == "REDIS_SSL"]) == 0 + error_message = "Plaintext Redis mode must not set REDIS_SSL." + } + + assert { + condition = length(local.redis_ca_fragment) == 0 + error_message = "Plaintext Redis mode must not decode a Redis CA at startup." + } +} diff --git a/terraform/litellm/gcp/variables.tf b/terraform/litellm/gcp/variables.tf index 1162e100bb2..9c68ed3db76 100644 --- a/terraform/litellm/gcp/variables.tf +++ b/terraform/litellm/gcp/variables.tf @@ -79,16 +79,42 @@ variable "ui_password" { sensitive = true } +# ---------- Deployment mode ---------- + +variable "create_runtime" { + description = "Create Cloud Run, load balancer, VPC connector, runtime support resources, and the migration job. Set false for GKE or another external runtime." + type = bool + default = true +} + +variable "network_id" { + description = "Existing VPC network resource ID (`projects//global/networks/`). When set, no VPC or subnet is created. A VPC connector requires this network to be in the deployment project when create_runtime is true." + type = string + default = "" +} + +variable "create_psa_connection" { + description = "Create the Private Services Access range and connection for Cloud SQL and Memorystore. Set false when the existing network already has PSA configured." + type = bool + default = true +} + +variable "redis_transit_encryption" { + description = "Enable Memorystore transit encryption and inject Redis TLS settings into Cloud Run. Set false to use plaintext Redis." + type = bool + default = true +} + # ---------- Networking ---------- variable "subnet_cidr" { - description = "Primary CIDR block for the LiteLLM subnet." + description = "Primary CIDR block for the LiteLLM subnet. Unused when network_id is set." type = string default = "10.40.0.0/16" } variable "vpc_connector_cidr" { - description = "CIDR for the Serverless VPC Access connector. /28 required." + description = "CIDR for the Serverless VPC Access connector. /28 required. Unused when create_runtime is false." type = string default = "10.41.0.0/28" }