diff --git a/litellm/deploy/helm/litellm/templates/ingress.yaml b/litellm/deploy/helm/litellm/templates/ingress.yaml index 6c3670152ca..a7a977c10e3 100644 --- a/litellm/deploy/helm/litellm/templates/ingress.yaml +++ b/litellm/deploy/helm/litellm/templates/ingress.yaml @@ -96,25 +96,31 @@ /v2/user/info, /v2/team/list, /v2/model/info, /v2/login, /v2/guardrails/*, /v1/mcp/*) onto the gateway via a broad /v1 or /v2 prefix. */}} -{{- $gatewayPrefixes := list - "/v1/chat" "/chat" "/v1/completions" "/completions" "/v1/embeddings" "/embeddings" - "/v1/moderations" "/moderations" "/v1/audio" "/audio" "/v1/images" "/images" - "/v1/files" "/files" "/v1/batches" "/batches" "/v1/fine_tuning" "/fine_tuning" - "/v1/fine-tuning" "/fine-tuning" "/v1/responses" "/responses" "/v1/threads" "/threads" - "/v1/assistants" "/assistants" "/v1/vector_stores" "/vector_stores" "/v1/indexes" - "/v1/models" "/models" "/openai" "/engines" - "/v1/messages" "/messages" "/v1/skills" "/v1/a2a" "/a2a" "/api/event_logging" - "/v1/rerank" "/v2/rerank" "/rerank" "/v1/ocr" "/ocr" "/v1/rag" "/rag" - "/v1/video" "/v1/videos" "/video" "/videos" "/v1/search" "/search" - "/v1/containers" "/containers" "/v1/evals" "/v1/memory" "/queue/chat" - "/v1beta" "/interactions" - "/anthropic" "/azure" "/azure_ai" "/azure_speech" "/aws" "/bedrock" "/comprehendmedical" "/transcribe" "/cohere" "/gemini" "/google" - "/vertex_ai" "/vertex-ai" "/assemblyai" "/eu.assemblyai" "/langfuse" "/vllm" - "/mistral" "/groq" "/voyage" "/cursor" "/milvus" "/openai_passthrough" - "/toolset" - "/v1/realtime" "/realtime" - "/health" "/metrics" +{{- $gatewayPrefixGroups := list + (list "/chat" "/v1/chat") (list "/completions" "/v1/completions") (list "/embeddings" "/v1/embeddings") + (list "/moderations" "/v1/moderations") (list "/audio" "/v1/audio") (list "/images" "/v1/images") + (list "/files" "/v1/files") (list "/batches" "/v1/batches") (list "/fine_tuning" "/v1/fine_tuning") + (list "/fine-tuning" "/v1/fine-tuning") (list "/responses" "/v1/responses") (list "/threads" "/v1/threads") + (list "/assistants" "/v1/assistants") (list "/vector_stores" "/v1/vector_stores") (list "/models" "/v1/models") + (list "/messages" "/v1/messages") (list "/a2a" "/v1/a2a") (list "/rerank" "/v1/rerank") + (list "/decisions" "/v1/decisions") (list "/systemone" "/v1/systemone") (list "/extraction" "/v1/extraction") + (list "/ocr" "/v1/ocr") (list "/rag" "/v1/rag") (list "/video" "/v1/video") + (list "/videos" "/v1/videos") (list "/search" "/v1/search") (list "/containers" "/v1/containers") + (list "/realtime" "/v1/realtime") (list "/live/sessions" "/v1/live/sessions") (list "/v1/skills" "/v1/indexes") + (list "/v1/evals" "/v2/rerank") (list "/v1beta" "/v1/memory") (list "/openai" "/engines") + (list "/queue/chat" "/api/event_logging") (list "/toolset" "/interactions") (list "/azure" "/anthropic") + (list "/azure_ai" "/azure_speech") (list "/aws" "/bedrock") (list "/transcribe" "/comprehendmedical") + (list "/cohere" "/gemini") (list "/google" "/gigachat") (list "/vertex_ai" "/vertex-ai") + (list "/assemblyai" "/eu.assemblyai") (list "/fal_ai" "/deepgram") (list "/vllm" "/langfuse") + (list "/mistral" "/typesafe") (list "/openrouter" "/nvidia_nim") (list "/groq" "/voyage") + (list "/cursor" "/milvus") (list "/tinyfish" "/openai_passthrough") (list "/watsonx") + (list "/health" "/metrics") -}} +{{- $gatewayPrefixes := list -}} +{{- range $gatewayPrefixGroups -}} +{{- $gatewayPrefixes = concat $gatewayPrefixes . -}} +{{- end -}} +{{- $gatewayExactPaths := list "/test" "/debug/memory/summary" "/eu_jev/v1/systemone" -}} {{/* /test is gateway-only as an EXACT path (GATEWAY_EXACT_PATHS), but its children /test/connection and /test/tools/list are MCP-server management @@ -131,7 +137,16 @@ at "/" Prefix would swallow the whole backend management API) instead of adding to it. */}} -{{- $builtinPathKeys := ternary (list "/|Prefix") (list "/test|Exact" "/debug/memory/summary|Exact" "/|Prefix") $monolith -}} +{{- $builtinPathKeys := list "/|Prefix" -}} +{{/* + An ALB allows 100 listener rules and the AWS Load Balancer Controller makes + one per Ingress path, so on alb each $gatewayPrefixGroups entry is one rule. + Its first, shorter prefix is the Ingress path, which keeps any longer child + path ahead of it in the controller's longest-first order, and a conditions + annotation adds the other prefix to the same rule. +*/}} +{{- $albGrouping := and (eq $controller "alb") (not $monolith) -}} +{{- $gatewayConditions := get (.Values.ingress.annotations | default dict) (printf "alb.ingress.kubernetes.io/conditions.%s" $gatewayName) | default "[]" | trim | trimPrefix "[" | trimSuffix "]" | trim -}} {{- if $hasComponentHosts }} apiVersion: networking.k8s.io/v1 kind: Ingress @@ -198,9 +213,20 @@ metadata: name: {{ include "litellm.fullname" . }} labels: {{- include "litellm.commonLabels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} + {{- if or .Values.ingress.annotations $albGrouping }} annotations: + {{- with .Values.ingress.annotations }} {{- toYaml . | nindent 4 }} + {{- end }} + {{- if $albGrouping }} + {{- range $i, $group := $gatewayPrefixGroups }} + alb.ingress.kubernetes.io/actions.litellm-gateway-{{ $i }}: {{ printf `{"type":"forward","forwardConfig":{"targetGroups":[{"serviceName":"%s","servicePort":"http"}]}}` $gatewayName | quote }} + {{- $conditions := compact (list $gatewayConditions (ternary (printf `{"field":"path-pattern","pathPatternConfig":{"values":["%s","%s/*"]}}` (last $group) (last $group)) "" (eq (len $group) 2))) }} + {{- with $conditions }} + alb.ingress.kubernetes.io/conditions.litellm-gateway-{{ $i }}: {{ printf "[%s]" (join "," .) | quote }} + {{- end }} + {{- end }} + {{- end }} {{- end }} spec: {{- with .Values.ingress.className }} @@ -234,21 +260,31 @@ spec: # /test/* MCP management endpoints fall to the backend catch-all. # Exact /debug/memory/summary reads a serving worker's RSS (the e2e memory # gate); the rest of /debug/* stays on the backend. - - path: /test + {{- range $gatewayExactPaths }} + {{- $builtinPathKeys = append $builtinPathKeys (printf "%s|Exact" .) }} + - path: {{ . }} pathType: Exact backend: service: name: {{ $gatewayName }} port: number: {{ $gatewayPort }} - - path: /debug/memory/summary - pathType: Exact + {{- end }} + {{- if $albGrouping }} + {{- range $i, $group := $gatewayPrefixGroups }} + {{- range $group }} + {{- $builtinPathKeys = append $builtinPathKeys (printf "%s|Prefix" .) }} + {{- end }} + - path: {{ first $group }} + pathType: Prefix backend: service: - name: {{ $gatewayName }} + name: litellm-gateway-{{ $i }} port: - number: {{ $gatewayPort }} - {{- range $gatewayPrefixes }} + name: use-annotation + {{- end }} + {{- end }} + {{- range (ternary list $gatewayPrefixes $albGrouping) }} {{- $pathType := include "litellm.ingress.pathType" (dict "controller" $controller "path" . "pathType" "Prefix") }} {{- $builtinPathKeys = append $builtinPathKeys (printf "%s|%s" . $pathType) }} - path: {{ . }} diff --git a/litellm/deploy/helm/litellm/tests/ingress_controller_tests.yaml b/litellm/deploy/helm/litellm/tests/ingress_controller_tests.yaml index aa30db3c9c1..1cd39c91546 100644 --- a/litellm/deploy/helm/litellm/tests/ingress_controller_tests.yaml +++ b/litellm/deploy/helm/litellm/tests/ingress_controller_tests.yaml @@ -18,16 +18,8 @@ tests: name: RELEASE-NAME-litellm-ui port: number: 3000 - - contains: - path: spec.rules[0].http.paths - content: - path: /eu.assemblyai - pathType: Prefix - backend: - service: - name: RELEASE-NAME-litellm-gateway - port: - number: 4000 + - notExists: + path: spec.rules[0].http.paths[?(@.backend.service.port.name == "use-annotation" && @.pathType != "Prefix")] - contains: path: spec.rules[0].http.paths content: @@ -39,6 +31,39 @@ tests: port: number: 3000 + - it: fits the AWS Load Balancer Controller's one rule per path under the default 100 rules per ALB + set: + ingress.enabled: true + asserts: + - notExists: + path: spec.rules[0].http.paths[100] + + - it: routes each grouped gateway prefix to the gateway Service through a use-annotation action on alb + set: + ingress.enabled: true + asserts: + - equal: + path: metadata.annotations["alb.ingress.kubernetes.io/actions.litellm-gateway-0"] + value: '{"type":"forward","forwardConfig":{"targetGroups":[{"serviceName":"RELEASE-NAME-litellm-gateway","servicePort":"http"}]}}' + - matchRegex: + path: metadata.annotations["alb.ingress.kubernetes.io/conditions.litellm-gateway-0"] + pattern: '^\[\{"field":"path-pattern","pathPatternConfig":\{"values":\["(/[^"*]+)","/[^"*]+/\*"\]\}\}\]$' + - notExists: + path: spec.rules[0].http.paths[?(@.pathType == "Prefix" && @.backend.service.name == "RELEASE-NAME-litellm-gateway")] + + - it: keeps an operator's conditions for the gateway Service on every grouped alb rule + set: + ingress.enabled: true + ingress.annotations: + alb.ingress.kubernetes.io/conditions.RELEASE-NAME-litellm-gateway: '[{"field":"source-ip","sourceIpConfig":{"values":["10.0.0.0/8"]}}]' + asserts: + - matchRegex: + path: metadata.annotations["alb.ingress.kubernetes.io/conditions.litellm-gateway-0"] + pattern: '^\[\{"field":"source-ip","sourceIpConfig":\{"values":\["10\.0\.0\.0/8"\]\}\},\{"field":"path-pattern",' + - equal: + path: metadata.annotations["alb.ingress.kubernetes.io/conditions.litellm-gateway-50"] + value: '[{"field":"source-ip","sourceIpConfig":{"values":["10.0.0.0/8"]}}]' + - it: renders no dotted Exact or Prefix path for ingress-nginx, whose admission webhook rejects them set: ingress.enabled: true @@ -107,6 +132,26 @@ tests: name: RELEASE-NAME-litellm-gateway port: number: 4000 + - contains: + path: spec.rules[0].http.paths + content: + path: /eu_jev/v1/systemone + pathType: Exact + backend: + service: + name: RELEASE-NAME-litellm-gateway + port: + number: 4000 + - contains: + path: spec.rules[0].http.paths + content: + path: /openrouter + pathType: Prefix + backend: + service: + name: RELEASE-NAME-litellm-gateway + port: + number: 4000 - equal: path: spec.rules[0].http.paths[-1] value: diff --git a/litellm/deploy/helm/litellm/tests/ingress_extra_paths_tests.yaml b/litellm/deploy/helm/litellm/tests/ingress_extra_paths_tests.yaml index dee46528d04..dd2a853323d 100644 --- a/litellm/deploy/helm/litellm/tests/ingress_extra_paths_tests.yaml +++ b/litellm/deploy/helm/litellm/tests/ingress_extra_paths_tests.yaml @@ -7,6 +7,7 @@ tests: - it: renders nothing extra between the built-in gateway prefixes and the backend catch-all when unset set: ingress.enabled: true + ingress.controller: nginx asserts: - equal: path: spec.rules[0].http.paths[-1] @@ -33,12 +34,12 @@ tests: set: ingress.enabled: true ingress.extraPaths: - - path: /watsonx + - path: /hosted_vllm asserts: - equal: path: spec.rules[0].http.paths[-2] value: - path: /watsonx + path: /hosted_vllm pathType: Prefix backend: service: @@ -59,8 +60,9 @@ tests: - it: keeps every built-in path when extra paths are supplied set: ingress.enabled: true + ingress.controller: nginx ingress.extraPaths: - - path: /watsonx + - path: /hosted_vllm asserts: - contains: path: spec.rules[0].http.paths @@ -117,7 +119,7 @@ tests: set: ingress.enabled: true ingress.extraPaths: - - path: /watsonx + - path: /hosted_vllm service: gateway - path: /my-passthrough pathType: Exact @@ -129,7 +131,7 @@ tests: - equal: path: spec.rules[0].http.paths[-4] value: - path: /watsonx + path: /hosted_vllm pathType: Prefix backend: service: @@ -164,7 +166,7 @@ tests: backend.service.port: 8001 ui.service.port: 8080 ingress.extraPaths: - - path: /watsonx + - path: /hosted_vllm - path: /my-passthrough service: backend - path: /brand.txt @@ -184,21 +186,21 @@ tests: set: ingress.enabled: true ingress.extraPaths: - - path: /watsonx + - path: /hosted_vllm service: proxy asserts: - failedTemplate: - errorMessage: 'ingress.extraPaths[0] (path /watsonx): unknown service "proxy", expected one of backend, gateway, ui' + errorMessage: 'ingress.extraPaths[0] (path /hosted_vllm): unknown service "proxy", expected one of backend, gateway, ui' - it: rejects an entry whose pathType is not a kubernetes pathType set: ingress.enabled: true ingress.extraPaths: - - path: /watsonx + - path: /hosted_vllm pathType: prefix asserts: - failedTemplate: - errorMessage: 'ingress.extraPaths[0] (path /watsonx): unknown pathType "prefix", expected one of Exact, ImplementationSpecific, Prefix' + errorMessage: 'ingress.extraPaths[0] (path /hosted_vllm): unknown pathType "prefix", expected one of Exact, ImplementationSpecific, Prefix' - it: rejects an entry with no path set: @@ -322,7 +324,7 @@ tests: set: ingress.enabled: true ingress.extraPaths: - - /watsonx + - /hosted_vllm asserts: - failedTemplate: errorMessage: "ingress.extraPaths[0]: each entry must be a mapping with a 'path' key" @@ -372,7 +374,7 @@ tests: - path: /moyai serviceName: moyai-ingress servicePort: http - - path: /watsonx + - path: /hosted_vllm service: gateway asserts: - equal: diff --git a/litellm/deploy/helm/litellm/tests/ingress_monolith_tests.yaml b/litellm/deploy/helm/litellm/tests/ingress_monolith_tests.yaml index 08f7c598260..b3ace127ceb 100644 --- a/litellm/deploy/helm/litellm/tests/ingress_monolith_tests.yaml +++ b/litellm/deploy/helm/litellm/tests/ingress_monolith_tests.yaml @@ -41,6 +41,7 @@ tests: - it: componentized mode still splits traffic across ui, gateway and backend Services set: ingress.enabled: true + ingress.controller: nginx asserts: - equal: path: spec.rules[0].http.paths[0].backend.service.name diff --git a/litellm/deploy/helm/litellm/values.yaml b/litellm/deploy/helm/litellm/values.yaml index b4795cd9ca6..c5f63edf4d2 100644 --- a/litellm/deploy/helm/litellm/values.yaml +++ b/litellm/deploy/helm/litellm/values.yaml @@ -102,7 +102,7 @@ ingress: # The target component only answers paths its own route allowlist keeps, so # a path here still has to be one that component serves. extraPaths: [] - # - path: /watsonx + # - path: /hosted_vllm # pathType: Prefix # service: gateway diff --git a/tests/unit/gateway/routes/__init__.py b/tests/unit/gateway/routes/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/gateway/routes/test_allowlist.py b/tests/unit/gateway/routes/test_allowlist.py new file mode 100644 index 00000000000..c4b810093ea --- /dev/null +++ b/tests/unit/gateway/routes/test_allowlist.py @@ -0,0 +1,94 @@ +import re +from pathlib import Path +from typing import Final + +import pytest + +from backend.routes.allowlist import BACKEND_EXACT_PATHS, BACKEND_PATH_PREFIXES +from gateway.routes.allowlist import GATEWAY_EXACT_PATHS, GATEWAY_PATH_PREFIXES + +_INGRESS_TEMPLATE: Final = ( + Path(__file__).parents[4] / "litellm" / "deploy" / "helm" / "litellm" / "templates" / "ingress.yaml" +) + +# A route template whose first segment is any provider name (/{provider}/v1/files). An Ingress Prefix cannot +# express it, so those requests reach the backend unless the provider has its own prefix above. +_PREFIXES_AN_INGRESS_CANNOT_ROUTE: Final = frozenset({"/{provider}/"}) + + +def _template_block(name: str) -> str: + match: Final = re.search(rf"\${name} := list(.*?)-}}}}", _INGRESS_TEMPLATE.read_text(), re.S) + assert match is not None, f"${name} not found in {_INGRESS_TEMPLATE}" + return match.group(1) + + +def _quoted_paths(block: str) -> frozenset[str]: + return frozenset(re.findall(r'"(/[^"]*)"', block)) + + +_CHART_PREFIX_GROUPS: Final = tuple( + tuple(re.findall(r'"(/[^"]*)"', group)) + for group in re.findall(r"\(list ([^)]*)\)", _template_block("gatewayPrefixGroups")) +) +_CHART_PREFIXES: Final = _quoted_paths(_template_block("gatewayPrefixGroups")) +_CHART_EXACT_PATHS: Final = _quoted_paths(_template_block("gatewayExactPaths")) + + +def _backend_keeps(path: str) -> bool: + return path in BACKEND_EXACT_PATHS or any(path.startswith(prefix) for prefix in BACKEND_PATH_PREFIXES) + + +_GATEWAY_ONLY_PREFIXES: Final = sorted( + frozenset(prefix for prefix in GATEWAY_PATH_PREFIXES if not _backend_keeps(prefix)) + - _PREFIXES_AN_INGRESS_CANNOT_ROUTE +) +_GATEWAY_ONLY_EXACT_PATHS: Final = sorted(GATEWAY_EXACT_PATHS - BACKEND_EXACT_PATHS) + + +def _routed_to_gateway_by_chart_prefix(path: str) -> bool: + trimmed: Final = path.rstrip("/") + return any(trimmed == prefix or trimmed.startswith(f"{prefix}/") for prefix in _CHART_PREFIXES) + + +def _gateway_serves_something_under(chart_prefix: str) -> bool: + under: Final = f"{chart_prefix}/" + by_prefix: Final = any( + chart_prefix.startswith(prefix) or prefix.rstrip("/") == chart_prefix or prefix.startswith(under) + for prefix in GATEWAY_PATH_PREFIXES + ) + return by_prefix or any(path == chart_prefix or path.startswith(under) for path in GATEWAY_EXACT_PATHS) + + +@pytest.mark.parametrize("prefix", _GATEWAY_ONLY_PREFIXES) +def test_ingress_sends_every_gateway_only_prefix_to_the_gateway(prefix: str) -> None: + assert _routed_to_gateway_by_chart_prefix(prefix) + + +@pytest.mark.parametrize("path", _GATEWAY_ONLY_EXACT_PATHS) +def test_ingress_sends_every_gateway_only_exact_path_to_the_gateway(path: str) -> None: + assert path in _CHART_EXACT_PATHS or _routed_to_gateway_by_chart_prefix(path) + + +@pytest.mark.parametrize("chart_prefix", sorted(_CHART_PREFIXES)) +def test_ingress_sends_no_prefix_to_the_gateway_that_the_gateway_dropped(chart_prefix: str) -> None: + assert _gateway_serves_something_under(chart_prefix) + + +@pytest.mark.parametrize("chart_path", sorted(_CHART_EXACT_PATHS)) +def test_ingress_sends_no_exact_path_to_the_gateway_that_the_gateway_dropped(chart_path: str) -> None: + assert chart_path in GATEWAY_EXACT_PATHS + + +@pytest.mark.parametrize("group", _CHART_PREFIX_GROUPS, ids=lambda group: " ".join(group)) +def test_each_gateway_prefix_group_fits_one_alb_rule(group: tuple[str, ...]) -> None: + assert 1 <= len(group) <= 2, f"{group} needs {2 * len(group)} path-pattern values, an ALB rule takes 5" + + +def test_gateway_prefix_groups_list_each_prefix_once() -> None: + flattened: Final = [prefix for group in _CHART_PREFIX_GROUPS for prefix in group] + assert len(flattened) == len(set(flattened)), sorted(p for p in set(flattened) if flattened.count(p) > 1) + + +@pytest.mark.parametrize("group", _CHART_PREFIX_GROUPS, ids=lambda group: " ".join(group)) +def test_each_gateway_prefix_group_lists_its_shorter_prefix_first(group: tuple[str, ...]) -> None: + assert len(group[0]) <= len(group[-1]), f"{group} would order its ALB rule by the longer prefix"