diff --git a/enterprise/litellm_enterprise/enterprise_callbacks/pagerduty/pagerduty.py b/enterprise/litellm_enterprise/enterprise_callbacks/pagerduty/pagerduty.py index 12fdaeb6a81..f920aa7ac13 100644 --- a/enterprise/litellm_enterprise/enterprise_callbacks/pagerduty/pagerduty.py +++ b/enterprise/litellm_enterprise/enterprise_callbacks/pagerduty/pagerduty.py @@ -113,6 +113,10 @@ class PagerDutyAlerting(SlackAlerting): user_api_key_spend=_meta.get("user_api_key_spend"), user_api_key_max_budget=_meta.get("user_api_key_max_budget"), user_api_key_budget_reset_at=_meta.get("user_api_key_budget_reset_at"), + user_api_key_user_spend=_meta.get("user_api_key_user_spend"), + user_api_key_user_max_budget=_meta.get("user_api_key_user_max_budget"), + user_api_key_team_spend=_meta.get("user_api_key_team_spend"), + user_api_key_team_max_budget=_meta.get("user_api_key_team_max_budget"), user_api_key_org_id=_meta.get("user_api_key_org_id"), user_api_key_org_alias=_meta.get("user_api_key_org_alias"), user_api_key_team_id=_meta.get("user_api_key_team_id"), @@ -196,6 +200,10 @@ class PagerDutyAlerting(SlackAlerting): if user_api_key_dict.budget_reset_at else None ), + user_api_key_user_spend=user_api_key_dict.user_spend, + user_api_key_user_max_budget=user_api_key_dict.user_max_budget, + user_api_key_team_spend=user_api_key_dict.team_spend, + user_api_key_team_max_budget=user_api_key_dict.team_max_budget, user_api_key_org_id=user_api_key_dict.org_id, user_api_key_org_alias=user_api_key_dict.organization_alias, user_api_key_team_id=user_api_key_dict.team_id, diff --git a/helm/litellm/templates/NOTES.txt b/helm/litellm/templates/NOTES.txt index 5b939fe480a..468cf621b32 100644 --- a/helm/litellm/templates/NOTES.txt +++ b/helm/litellm/templates/NOTES.txt @@ -46,4 +46,9 @@ Reminders: - gateway.config.proxy_config (rendered into a ConfigMap and mounted at /app/config/config.yaml; gateway reads it via CONFIG_FILE_PATH) + - {component}.pdb.{enabled,minAvailable,maxUnavailable} (per-component PodDisruptionBudget; disabled by + default — with hpa.minReplicas of 1, minAvailable: 1 + would block node drains) + - {component}.topologySpreadConstraints (standard k8s list, e.g. spread replicas across + topology.kubernetes.io/zone) - Enable ingress.enabled=true to dispatch / → ui, gateway data-plane prefixes → gateway, and the catch-all → backend. diff --git a/helm/litellm/templates/_helpers.tpl b/helm/litellm/templates/_helpers.tpl index 043a0afc173..a0205c0a3a2 100644 --- a/helm/litellm/templates/_helpers.tpl +++ b/helm/litellm/templates/_helpers.tpl @@ -295,6 +295,52 @@ harmless no-op for the Job and authoritative for the app pods. {{- end }} {{- end -}} +{{/* +PodDisruptionBudget shared by gateway, backend, and ui. + +Invoke with a dict: + (dict "root" $ "component" .Values.gateway "componentName" "gateway" + "fullname" (include "litellm.gateway.fullname" .) + "selectorLabels" (include "litellm.gateway.selectorLabels" .)) + +Renders nothing unless both the component and its `pdb.enabled` are on. +Only one of minAvailable / maxUnavailable should be set; if both are, +minAvailable wins. If neither is set, falls back to `maxUnavailable: 1` so +an enabled-but-unconfigured PDB still permits node drains. + +"Set" means non-nil and non-empty-string, so an explicit 0 (e.g. +`maxUnavailable: 0` to forbid all voluntary disruptions) is honored rather +than silently replaced by the fallback. +*/}} +{{- define "litellm.pdb" -}} +{{- $root := .root -}} +{{- $component := .component -}} +{{- $min := $component.pdb.minAvailable -}} +{{- $max := $component.pdb.maxUnavailable -}} +{{- $minSet := not (or (kindIs "invalid" $min) (eq (printf "%v" $min) "")) -}} +{{- $maxSet := not (or (kindIs "invalid" $max) (eq (printf "%v" $max) "")) -}} +{{- if and $component.enabled $component.pdb $component.pdb.enabled }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ .fullname }} + labels: + {{- include "litellm.commonLabels" $root | nindent 4 }} + app.kubernetes.io/component: {{ .componentName }} +spec: + selector: + matchLabels: + {{- .selectorLabels | nindent 6 }} + {{- if $minSet }} + minAvailable: {{ $min }} + {{- else if $maxSet }} + maxUnavailable: {{ $max }} + {{- else }} + maxUnavailable: 1 + {{- end }} +{{- end }} +{{- end -}} + {{/* Renders `envFrom:` block for a component's `envConfigMaps` / `envSecrets` lists. Each entry is a resource name; the chart wires the whole ConfigMap / diff --git a/helm/litellm/templates/backend/deployment.yaml b/helm/litellm/templates/backend/deployment.yaml index 9d056167fe1..892b84ff7d5 100644 --- a/helm/litellm/templates/backend/deployment.yaml +++ b/helm/litellm/templates/backend/deployment.yaml @@ -98,4 +98,8 @@ spec: tolerations: {{- toYaml . | nindent 8 }} {{- end }} + {{- with .Values.backend.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml . | nindent 8 }} + {{- end }} {{- end }} diff --git a/helm/litellm/templates/backend/poddisruptionbudget.yaml b/helm/litellm/templates/backend/poddisruptionbudget.yaml new file mode 100644 index 00000000000..02853ac879c --- /dev/null +++ b/helm/litellm/templates/backend/poddisruptionbudget.yaml @@ -0,0 +1,6 @@ +{{- include "litellm.pdb" (dict + "root" $ + "component" .Values.backend + "componentName" "backend" + "fullname" (include "litellm.backend.fullname" .) + "selectorLabels" (include "litellm.backend.selectorLabels" .)) }} diff --git a/helm/litellm/templates/gateway/deployment.yaml b/helm/litellm/templates/gateway/deployment.yaml index 4c80d784156..b2e22612905 100644 --- a/helm/litellm/templates/gateway/deployment.yaml +++ b/helm/litellm/templates/gateway/deployment.yaml @@ -100,4 +100,8 @@ spec: tolerations: {{- toYaml . | nindent 8 }} {{- end }} + {{- with .Values.gateway.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml . | nindent 8 }} + {{- end }} {{- end }} diff --git a/helm/litellm/templates/gateway/poddisruptionbudget.yaml b/helm/litellm/templates/gateway/poddisruptionbudget.yaml new file mode 100644 index 00000000000..15e89af17d7 --- /dev/null +++ b/helm/litellm/templates/gateway/poddisruptionbudget.yaml @@ -0,0 +1,6 @@ +{{- include "litellm.pdb" (dict + "root" $ + "component" .Values.gateway + "componentName" "gateway" + "fullname" (include "litellm.gateway.fullname" .) + "selectorLabels" (include "litellm.gateway.selectorLabels" .)) }} diff --git a/helm/litellm/templates/ui/deployment.yaml b/helm/litellm/templates/ui/deployment.yaml index 79e9a3e43bb..cd1f8c08fd4 100644 --- a/helm/litellm/templates/ui/deployment.yaml +++ b/helm/litellm/templates/ui/deployment.yaml @@ -76,4 +76,8 @@ spec: tolerations: {{- toYaml . | nindent 8 }} {{- end }} + {{- with .Values.ui.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml . | nindent 8 }} + {{- end }} {{- end }} diff --git a/helm/litellm/templates/ui/poddisruptionbudget.yaml b/helm/litellm/templates/ui/poddisruptionbudget.yaml new file mode 100644 index 00000000000..f7a3a694e9c --- /dev/null +++ b/helm/litellm/templates/ui/poddisruptionbudget.yaml @@ -0,0 +1,6 @@ +{{- include "litellm.pdb" (dict + "root" $ + "component" .Values.ui + "componentName" "ui" + "fullname" (include "litellm.ui.fullname" .) + "selectorLabels" (include "litellm.ui.selectorLabels" .)) }} diff --git a/helm/litellm/tests/pdb_topology_spread_tests.yaml b/helm/litellm/tests/pdb_topology_spread_tests.yaml new file mode 100644 index 00000000000..8aa05f3a969 --- /dev/null +++ b/helm/litellm/tests/pdb_topology_spread_tests.yaml @@ -0,0 +1,188 @@ +suite: test pod disruption budgets and topology spread constraints +templates: + - gateway/poddisruptionbudget.yaml + - backend/poddisruptionbudget.yaml + - ui/poddisruptionbudget.yaml + - gateway/deployment.yaml + - gateway/configmap.yaml + - backend/deployment.yaml + - ui/deployment.yaml +values: + - ./values/required.yaml +tests: + - it: renders no PDB by default + templates: + - gateway/poddisruptionbudget.yaml + - backend/poddisruptionbudget.yaml + - ui/poddisruptionbudget.yaml + asserts: + - hasDocuments: + count: 0 + + - it: gateway PDB uses minAvailable and matches the gateway selector labels + template: gateway/poddisruptionbudget.yaml + set: + gateway.pdb.enabled: true + gateway.pdb.minAvailable: 1 + asserts: + - isKind: + of: PodDisruptionBudget + - equal: + path: apiVersion + value: policy/v1 + - equal: + path: metadata.name + value: RELEASE-NAME-litellm-gateway + - equal: + path: spec.minAvailable + value: 1 + - notExists: + path: spec.maxUnavailable + - equal: + path: spec.selector.matchLabels + value: + app.kubernetes.io/name: litellm + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: gateway + + - it: backend PDB uses maxUnavailable when minAvailable is unset + template: backend/poddisruptionbudget.yaml + set: + backend.pdb.enabled: true + backend.pdb.maxUnavailable: 25% + asserts: + - equal: + path: spec.maxUnavailable + value: 25% + - notExists: + path: spec.minAvailable + - equal: + path: spec.selector.matchLabels + value: + app.kubernetes.io/name: litellm + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: backend + + - it: minAvailable wins when both minAvailable and maxUnavailable are set + template: gateway/poddisruptionbudget.yaml + set: + gateway.pdb.enabled: true + gateway.pdb.minAvailable: 2 + gateway.pdb.maxUnavailable: 1 + asserts: + - equal: + path: spec.minAvailable + value: 2 + - notExists: + path: spec.maxUnavailable + + - it: an explicit maxUnavailable 0 is honored instead of the fallback + template: backend/poddisruptionbudget.yaml + set: + backend.pdb.enabled: true + backend.pdb.maxUnavailable: 0 + asserts: + - equal: + path: spec.maxUnavailable + value: 0 + - notExists: + path: spec.minAvailable + + - it: an explicit minAvailable 0 is honored and beats a set maxUnavailable + template: gateway/poddisruptionbudget.yaml + set: + gateway.pdb.enabled: true + gateway.pdb.minAvailable: 0 + gateway.pdb.maxUnavailable: 1 + asserts: + - equal: + path: spec.minAvailable + value: 0 + - notExists: + path: spec.maxUnavailable + + - it: enabled PDB with neither knob set falls back to maxUnavailable 1 + template: ui/poddisruptionbudget.yaml + set: + ui.pdb.enabled: true + asserts: + - equal: + path: spec.maxUnavailable + value: 1 + - notExists: + path: spec.minAvailable + - equal: + path: spec.selector.matchLabels + value: + app.kubernetes.io/name: litellm + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: ui + + - it: renders no PDB for a disabled component even when its pdb is enabled + template: gateway/poddisruptionbudget.yaml + set: + gateway.enabled: false + gateway.pdb.enabled: true + asserts: + - hasDocuments: + count: 0 + + - it: deployments omit topologySpreadConstraints by default + templates: + - gateway/deployment.yaml + - backend/deployment.yaml + - ui/deployment.yaml + asserts: + - notExists: + path: spec.template.spec.topologySpreadConstraints + + - it: gateway deployment renders configured topologySpreadConstraints + template: gateway/deployment.yaml + set: + gateway.topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app.kubernetes.io/component: gateway + asserts: + - equal: + path: spec.template.spec.topologySpreadConstraints + value: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app.kubernetes.io/component: gateway + + - it: backend deployment renders configured topologySpreadConstraints + template: backend/deployment.yaml + set: + backend.topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app.kubernetes.io/component: backend + asserts: + - equal: + path: spec.template.spec.topologySpreadConstraints[0].topologyKey + value: kubernetes.io/hostname + - equal: + path: spec.template.spec.topologySpreadConstraints[0].whenUnsatisfiable + value: DoNotSchedule + + - it: ui deployment renders configured topologySpreadConstraints + template: ui/deployment.yaml + set: + ui.topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: ScheduleAnyway + asserts: + - equal: + path: spec.template.spec.topologySpreadConstraints[0].topologyKey + value: topology.kubernetes.io/zone diff --git a/helm/litellm/values.yaml b/helm/litellm/values.yaml index 74d02a25b7a..461935b2f50 100644 --- a/helm/litellm/values.yaml +++ b/helm/litellm/values.yaml @@ -190,10 +190,28 @@ gateway: maxReplicas: 10 targetCPUUtilizationPercentage: 70 targetMemoryUtilizationPercentage: 80 + # PodDisruptionBudget for the gateway pods. Set exactly one of + # `minAvailable` / `maxUnavailable` (minAvailable wins if both are set; + # enabling without either falls back to `maxUnavailable: 1`). Disabled by + # default: with the default hpa.minReplicas of 1, a `minAvailable: 1` PDB + # would block node drains entirely. + pdb: + enabled: false + minAvailable: "" + maxUnavailable: "" podAnnotations: {} nodeSelector: {} tolerations: [] affinity: {} + # Standard k8s topologySpreadConstraints for the gateway pods, e.g. to + # spread replicas across zones: + # - maxSkew: 1 + # topologyKey: topology.kubernetes.io/zone + # whenUnsatisfiable: ScheduleAnyway + # labelSelector: + # matchLabels: + # app.kubernetes.io/component: gateway + topologySpreadConstraints: [] # ---------- backend (UI / management API) ---------- backend: @@ -233,10 +251,17 @@ backend: minReplicas: 1 maxReplicas: 4 targetCPUUtilizationPercentage: 70 + # Same shape as gateway.pdb. + pdb: + enabled: false + minAvailable: "" + maxUnavailable: "" podAnnotations: {} nodeSelector: {} tolerations: [] affinity: {} + # Same shape as gateway.topologySpreadConstraints. + topologySpreadConstraints: [] # ---------- ui (Next.js static dashboard) ---------- ui: @@ -279,7 +304,14 @@ ui: minReplicas: 1 maxReplicas: 3 targetCPUUtilizationPercentage: 80 + # Same shape as gateway.pdb. + pdb: + enabled: false + minAvailable: "" + maxUnavailable: "" podAnnotations: {} nodeSelector: {} tolerations: [] affinity: {} + # Same shape as gateway.topologySpreadConstraints. + topologySpreadConstraints: [] diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260715000000_add_issuer_to_mcp_server_table/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260715000000_add_issuer_to_mcp_server_table/migration.sql new file mode 100644 index 00000000000..f7f23e6a55e --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260715000000_add_issuer_to_mcp_server_table/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "LiteLLM_MCPServerTable" ADD COLUMN IF NOT EXISTS "issuer" TEXT; diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index a23cecc3911..f842bf13da9 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -325,6 +325,7 @@ model LiteLLM_MCPServerTable { command String? args String[] @default([]) env Json? @default("{}") + issuer String? authorization_url String? token_url String? registration_url String? diff --git a/litellm/exceptions.py b/litellm/exceptions.py index aca3fb551cc..fd0a2afb3e8 100644 --- a/litellm/exceptions.py +++ b/litellm/exceptions.py @@ -966,11 +966,15 @@ class BudgetExceededError(Exception): max_budget: float, message: Optional[str] = None, llm_provider: Optional[str] = None, + entity_type: Optional[str] = None, + entity_id: Optional[str] = None, ): self.current_cost = current_cost self.max_budget = max_budget self.status_code = 429 self.llm_provider = llm_provider or "" + self.entity_type = entity_type + self.entity_id = entity_id # Surface unified rate-limit fields without joining the RateLimitError # hierarchy so existing `except BudgetExceededError:` handlers keep # working; custom callbacks reading StandardLoggingPayload pick these diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 268dcc3df78..9a0b4937fdb 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -38,6 +38,7 @@ from litellm import ( ) from litellm._logging import _is_debugging_on, _redact_string, verbose_logger from litellm.exceptions import ( + BudgetExceededError, validate_rate_limit_category, validate_rate_limit_type, ) @@ -4597,6 +4598,10 @@ class StandardLoggingPayloadSetup: user_api_key_spend=None, user_api_key_max_budget=None, user_api_key_budget_reset_at=None, + user_api_key_user_spend=None, + user_api_key_user_max_budget=None, + user_api_key_team_spend=None, + user_api_key_team_max_budget=None, user_api_key_team_id=None, user_api_key_org_id=None, user_api_key_org_alias=None, @@ -4943,6 +4948,7 @@ class StandardLoggingPayloadSetup: rate_limit_category = validate_rate_limit_category(getattr(original_exception, "category", None)) rate_limit_type = validate_rate_limit_type(getattr(original_exception, "rate_limit_type", None)) + budget_error = original_exception if isinstance(original_exception, BudgetExceededError) else None return StandardLoggingPayloadErrorInformation( error_code=error_status, @@ -4952,6 +4958,10 @@ class StandardLoggingPayloadSetup: error_message=error_message, error_rate_limit_category=rate_limit_category, error_rate_limit_type=rate_limit_type, + error_budget_entity_type=budget_error.entity_type if budget_error else None, + error_budget_entity_id=budget_error.entity_id if budget_error else None, + error_budget_limit=budget_error.max_budget if budget_error else None, + error_budget_spend=budget_error.current_cost if budget_error else None, ) @staticmethod @@ -5428,6 +5438,10 @@ def get_standard_logging_metadata( user_api_key_spend=None, user_api_key_max_budget=None, user_api_key_budget_reset_at=None, + user_api_key_user_spend=None, + user_api_key_user_max_budget=None, + user_api_key_team_spend=None, + user_api_key_team_max_budget=None, user_api_key_team_id=None, user_api_key_org_id=None, user_api_key_org_alias=None, @@ -5527,6 +5541,10 @@ def create_dummy_standard_logging_payload() -> StandardLoggingPayload: user_api_key_team_id=str("test_team"), user_api_key_user_id=str("test_user"), user_api_key_team_alias=str("test_team_alias"), + user_api_key_user_spend=None, + user_api_key_user_max_budget=None, + user_api_key_team_spend=None, + user_api_key_team_max_budget=None, user_api_key_org_id=None, spend_logs_metadata=None, requester_ip_address=str("127.0.0.1"), diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 38bc68f2f78..d52d9849310 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -467,6 +467,7 @@ class ChunkProcessor: cache_read_input_tokens: Optional[int] = None completion_tokens_details: Optional[CompletionTokensDetails] = None prompt_tokens_details: Optional[PromptTokensDetailsWrapper] = None + cost: Optional[float] = None if "prompt_tokens" in usage_chunk: prompt_tokens = usage_chunk.get("prompt_tokens", 0) or 0 @@ -476,6 +477,8 @@ class ChunkProcessor: cache_creation_input_tokens = usage_chunk.get("cache_creation_input_tokens") if "cache_read_input_tokens" in usage_chunk: cache_read_input_tokens = usage_chunk.get("cache_read_input_tokens") + if "cost" in usage_chunk: + cost = usage_chunk.get("cost") if hasattr(usage_chunk, "completion_tokens_details"): if isinstance(usage_chunk.completion_tokens_details, dict): completion_tokens_details = CompletionTokensDetails(**usage_chunk.completion_tokens_details) @@ -494,6 +497,7 @@ class ChunkProcessor: "cache_read_input_tokens": cache_read_input_tokens, "completion_tokens_details": completion_tokens_details, "prompt_tokens_details": prompt_tokens_details, + "cost": cost, } def count_reasoning_tokens(self, response: ModelResponse) -> Optional[int]: @@ -512,6 +516,22 @@ class ChunkProcessor: return reasoning_tokens + @staticmethod + def _extract_usage_chunk(chunk: dict[str, Any] | ModelResponse | ModelResponseStream) -> Usage | None: + usage_chunk: Usage | dict[str, Any] | None = None + if hasattr(chunk, "usage") and chunk.usage is not None: + usage_chunk = chunk.usage + elif "usage" in chunk: + usage_chunk = chunk["usage"] + elif (isinstance(chunk, ModelResponse) or isinstance(chunk, ModelResponseStream)) and hasattr( + chunk, "_hidden_params" + ): + usage_chunk = chunk._hidden_params.get("usage", None) + + if isinstance(usage_chunk, dict): + return Usage(**usage_chunk) + return usage_chunk + def _calculate_usage_per_chunk( self, chunks: List[Union[Dict[str, Any], ModelResponse]], @@ -548,18 +568,12 @@ class ChunkProcessor: # is last-wins, so without preserving this separately the 1h breakdown is # lost and 1h cache writes get billed at the 5m rate. cache_creation_token_details: Optional[CacheCreationTokenDetails] = None + cost: Optional[float] = None + for chunk in chunks: - usage_chunk: Optional[Usage] = None - if "usage" in chunk: - usage_chunk = chunk["usage"] - elif (isinstance(chunk, ModelResponse) or isinstance(chunk, ModelResponseStream)) and hasattr( - chunk, "_hidden_params" - ): - usage_chunk = chunk._hidden_params.get("usage", None) + usage_chunk = self._extract_usage_chunk(chunk) if usage_chunk is not None: - if isinstance(usage_chunk, dict): - usage_chunk = Usage(**usage_chunk) usage_chunk_dict = self._usage_chunk_calculation_helper(usage_chunk) if usage_chunk_dict["prompt_tokens"] is not None and usage_chunk_dict["prompt_tokens"] > 0: prompt_tokens = usage_chunk_dict["prompt_tokens"] @@ -610,6 +624,9 @@ class ChunkProcessor: prompt_tokens_details, cache_creation_token_details ) + if usage_chunk_dict["cost"] is not None: + cost = usage_chunk_dict["cost"] + prompt_tokens_details = self._attach_cache_creation_token_details( prompt_tokens_details, cache_creation_token_details ) @@ -629,6 +646,7 @@ class ChunkProcessor: web_search_requests=web_search_requests, completion_tokens_details=completion_tokens_details, prompt_tokens_details=prompt_tokens_details, + cost=cost, ) @staticmethod @@ -727,6 +745,7 @@ class ChunkProcessor: prompt_tokens_details: Optional[PromptTokensDetailsWrapper] = calculated_usage_per_chunk[ "prompt_tokens_details" ] + cost: Optional[float] = calculated_usage_per_chunk["cost"] try: returned_usage.prompt_tokens = prompt_tokens or token_counter(model=model, messages=messages) @@ -784,6 +803,9 @@ class ChunkProcessor: else: returned_usage.prompt_tokens_details.web_search_requests = web_search_requests + if cost is not None: + setattr(returned_usage, "cost", cost) + # Return a new usage object with the new values returned_usage = Usage(**returned_usage.model_dump()) diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index b3288992854..fa5671d5a73 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -963,10 +963,11 @@ class CustomStreamWrapper: if self.custom_llm_provider == "bedrock" and "trace" in model_response: return model_response - # Default - return StopIteration - if hasattr(model_response, "usage"): - self.chunks.append(model_response) - raise StopIteration + # Don't raise StopIteration here - some providers (like OpenRouter) + # send usage/cost data in chunks after the finish_reason chunk + if hasattr(model_response, "usage") and model_response.usage is not None: + return model_response + return # flush any remaining holding chunk if len(self.holding_chunk) > 0: if model_response.choices[0].delta.content is None: @@ -1486,12 +1487,16 @@ class CustomStreamWrapper: self.tool_call = True + if hasattr(chunk, "usage") and chunk.usage is not None: + model_response.usage = chunk.usage + ## RETURN ARG - return self.return_processed_chunk_logic( + result = self.return_processed_chunk_logic( completion_obj=completion_obj, model_response=model_response, # type: ignore response_obj=response_obj, ) + return result except StopIteration: raise StopIteration @@ -1698,6 +1703,21 @@ class CustomStreamWrapper: model_response.choices[0].finish_reason = "tool_calls" return model_response + @staticmethod + def _propagate_usage_cost_to_hidden_params( + response: "ModelResponse", + ) -> None: + """ + If the assembled response carries a provider-reported cost on + usage.cost, copy it into _hidden_params so litellm's cost + calculator uses it instead of a token-based estimate. + """ + _usage = getattr(response, "usage", None) + if _usage is not None and hasattr(_usage, "cost") and _usage.cost is not None: + if "additional_headers" not in response._hidden_params: + response._hidden_params["additional_headers"] = {} + response._hidden_params["additional_headers"]["llm_provider-x-litellm-response-cost"] = float(_usage.cost) + def __next__(self) -> "ModelResponseStream": cache_hit = False if self.custom_llm_provider is not None and self.custom_llm_provider == "cached_response": @@ -1753,6 +1773,10 @@ class CustomStreamWrapper: # hasattr(response, "usage") is always True — must check # `is not None` to avoid running this path on every chunk. if getattr(response, "usage", None) is not None: + usage_to_preserve = response.usage + if usage_to_preserve: + response._hidden_params["usage"] = usage_to_preserve + obj_dict = response.model_dump() if "usage" in obj_dict: @@ -1801,6 +1825,8 @@ class CustomStreamWrapper: response = self.model_response_creator() if complete_streaming_response is not None: + self._propagate_usage_cost_to_hidden_params(complete_streaming_response) + setattr( response, "usage", @@ -1986,97 +2012,7 @@ class CustomStreamWrapper: self.chunks.append(processed_chunk) return processed_chunk except (StopAsyncIteration, StopIteration): - if self.sent_last_chunk is True: - # log the final chunk with accurate streaming values - try: - complete_streaming_response = litellm.stream_chunk_builder( - chunks=self.chunks, - messages=self.messages, - logging_obj=self.logging_obj, - ) - except Exception as e: - # see sync __next__: a raise from stream_chunk_builder inside this - # except handler escapes __anext__ and drops the request from SpendLogs. - # Recover best-effort usage from the raw chunks so cost is still tracked - verbose_logger.warning( - "stream_chunk_builder raised at end-of-stream (%s); logging best-effort usage from chunks.", - str(e), - ) - try: - complete_streaming_response = self.model_response_creator( - chunk={"usage": calculate_total_usage(chunks=self.chunks)} - ) - except Exception: - complete_streaming_response = None - - response = self.model_response_creator() - if complete_streaming_response is not None: - setattr( - response, - "usage", - getattr(complete_streaming_response, "usage"), - ) - try: - _copy = complete_streaming_response.model_copy(deep=True) - except RuntimeError: - _copy = complete_streaming_response.model_copy() - asyncio.create_task( - self.async_cache_streaming_response( - processed_chunk=_copy, - cache_hit=cache_hit, - ) - ) - # Update hidden_params with final usage from - # stream_chunk_builder (see sync __next__ for full comment). - if ( - self.stream_options is None - and complete_streaming_response is not None - and self._last_returned_hidden_params is not None - ): - final_usage = getattr(complete_streaming_response, "usage", None) - if final_usage is not None: - self._last_returned_hidden_params["usage"] = final_usage - - if self.sent_stream_usage is False and self.send_stream_usage is True: - self.sent_stream_usage = True - return response - - _deferred_cb = getattr( - self.logging_obj, - "_on_deferred_stream_complete", - None, - ) - if _deferred_cb is not None: - # Proxy has post-call guardrails. Store the assembled - # response so the outer streaming consumer - # (ProxyLogging.async_post_call_streaming_iterator_hook) - # can fire the deferred callback AFTER all guardrail - # end-of-stream blocks complete. Scheduling here via - # create_task would race with unified_guardrail's - # end-of-stream block for short-stream providers. - self.logging_obj._deferred_stream_complete_args = ( # type: ignore[attr-defined] - complete_streaming_response, - cache_hit, - ) - else: - # prefer_async_handlers routes CustomLogger to async_success_handler - # when consumers use ``async for`` on sync-SDK streams. Legacy string - # callbacks still run via executor.submit inside dispatch_success_handlers. - asyncio.create_task( - self.logging_obj.dispatch_success_handlers( - complete_streaming_response, - cache_hit=cache_hit, - start_time=None, - end_time=None, - prefer_async_handlers=True, - ) - ) - - raise StopAsyncIteration # Re-raise StopIteration - else: - self.sent_last_chunk = True - processed_chunk = self.finish_reason_handler() - return processed_chunk + return await self._finalize_completed_stream(cache_hit=cache_hit) except httpx.TimeoutException as e: # if httpx read timeout error occues traceback_exception = traceback.format_exc() ## ADD DEBUG INFORMATION - E.G. LITELLM REQUEST TIMEOUT @@ -2091,20 +2027,122 @@ class CustomStreamWrapper: # Handle any exceptions that might occur during streaming asyncio.create_task(self.logging_obj.async_failure_handler(e, traceback_exception)) self._handle_stream_fallback_error(e) + except (httpx.ReadError, httpx.RemoteProtocolError) as e: + if self.received_finish_reason is None: + self._log_stream_failure_and_raise(e) + return await self._finalize_completed_stream(cache_hit=cache_hit) except Exception as e: - traceback_exception = traceback.format_exc() - if self.logging_obj is not None: - self._record_partial_usage_for_failure() - ## LOGGING - threading.Thread( - target=self.logging_obj.failure_handler, - args=(e, traceback_exception), - ).start() # log response - # Handle any exceptions that might occur during streaming - asyncio.create_task( - self.logging_obj.async_failure_handler(e, traceback_exception) # type: ignore + self._log_stream_failure_and_raise(e) + + async def _finalize_completed_stream(self, cache_hit: bool) -> "ModelResponseStream": + if self.sent_last_chunk is True: + # log the final chunk with accurate streaming values + try: + complete_streaming_response = litellm.stream_chunk_builder( + chunks=self.chunks, + messages=self.messages, + logging_obj=self.logging_obj, ) - self._handle_stream_fallback_error(e) + except Exception as e: + # see sync __next__: a raise from stream_chunk_builder inside this + # except handler escapes __anext__ and drops the request from SpendLogs. + # Recover best-effort usage from the raw chunks so cost is still tracked + verbose_logger.warning( + "stream_chunk_builder raised at end-of-stream (%s); logging best-effort usage from chunks.", + str(e), + ) + try: + complete_streaming_response = self.model_response_creator( + chunk={"usage": calculate_total_usage(chunks=self.chunks)} + ) + except Exception: + complete_streaming_response = None + + response = self.model_response_creator() + if complete_streaming_response is not None: + self._propagate_usage_cost_to_hidden_params(complete_streaming_response) + + setattr( + response, + "usage", + getattr(complete_streaming_response, "usage"), + ) + try: + _copy = complete_streaming_response.model_copy(deep=True) + except RuntimeError: + _copy = complete_streaming_response.model_copy() + asyncio.create_task( + self.async_cache_streaming_response( + processed_chunk=_copy, + cache_hit=cache_hit, + ) + ) + # Update hidden_params with final usage from + # stream_chunk_builder (see sync __next__ for full comment). + if ( + self.stream_options is None + and complete_streaming_response is not None + and self._last_returned_hidden_params is not None + ): + final_usage = getattr(complete_streaming_response, "usage", None) + if final_usage is not None: + self._last_returned_hidden_params["usage"] = final_usage + + if self.sent_stream_usage is False and self.send_stream_usage is True: + self.sent_stream_usage = True + return response + + _deferred_cb = getattr( + self.logging_obj, + "_on_deferred_stream_complete", + None, + ) + if _deferred_cb is not None: + # Proxy has post-call guardrails. Store the assembled + # response so the outer streaming consumer + # (ProxyLogging.async_post_call_streaming_iterator_hook) + # can fire the deferred callback AFTER all guardrail + # end-of-stream blocks complete. Scheduling here via + # create_task would race with unified_guardrail's + # end-of-stream block for short-stream providers. + self.logging_obj._deferred_stream_complete_args = ( # type: ignore[attr-defined] + complete_streaming_response, + cache_hit, + ) + else: + # prefer_async_handlers routes CustomLogger to async_success_handler + # when consumers use ``async for`` on sync-SDK streams. Legacy string + # callbacks still run via executor.submit inside dispatch_success_handlers. + asyncio.create_task( + self.logging_obj.dispatch_success_handlers( + complete_streaming_response, + cache_hit=cache_hit, + start_time=None, + end_time=None, + prefer_async_handlers=True, + ) + ) + + raise StopAsyncIteration # Re-raise StopIteration + else: + self.sent_last_chunk = True + processed_chunk = self.finish_reason_handler() + return processed_chunk + + def _log_stream_failure_and_raise(self, e: Exception) -> NoReturn: + traceback_exception = traceback.format_exc() + if self.logging_obj is not None: + self._record_partial_usage_for_failure() + ## LOGGING + threading.Thread( + target=self.logging_obj.failure_handler, + args=(e, traceback_exception), + ).start() # log response + # Handle any exceptions that might occur during streaming + asyncio.create_task( + self.logging_obj.async_failure_handler(e, traceback_exception) # type: ignore + ) + self._handle_stream_fallback_error(e) def _record_partial_usage_for_failure(self) -> None: """ @@ -2240,12 +2278,16 @@ def calculate_total_usage(chunks: List[ModelResponse]) -> Usage: """Assume most recent usage chunk has total usage uptil then.""" prompt_tokens: int = 0 completion_tokens: int = 0 + latest_usage_chunk = None + for chunk in chunks: if "usage" in chunk and chunk["usage"] is not None: - if "prompt_tokens" in chunk["usage"]: - prompt_tokens = chunk["usage"].get("prompt_tokens", 0) or 0 - if "completion_tokens" in chunk["usage"]: - completion_tokens = chunk["usage"].get("completion_tokens", 0) or 0 + usage = chunk["usage"] + latest_usage_chunk = usage + if "prompt_tokens" in usage: + prompt_tokens = usage.get("prompt_tokens", 0) or 0 + if "completion_tokens" in usage: + completion_tokens = usage.get("completion_tokens", 0) or 0 returned_usage_chunk = Usage( prompt_tokens=prompt_tokens, @@ -2253,6 +2295,15 @@ def calculate_total_usage(chunks: List[ModelResponse]) -> Usage: total_tokens=prompt_tokens + completion_tokens, ) + if latest_usage_chunk is not None: + latest_cost = ( + latest_usage_chunk.get("cost") + if isinstance(latest_usage_chunk, dict) + else getattr(latest_usage_chunk, "cost", None) + ) + if latest_cost is not None: + returned_usage_chunk.cost = latest_cost + return returned_usage_chunk diff --git a/litellm/llms/custom_httpx/aiohttp_transport.py b/litellm/llms/custom_httpx/aiohttp_transport.py index 3172d3667e1..adac6a1b276 100644 --- a/litellm/llms/custom_httpx/aiohttp_transport.py +++ b/litellm/llms/custom_httpx/aiohttp_transport.py @@ -85,29 +85,12 @@ class AiohttpResponseStream(httpx.AsyncByteStream): try: async for chunk in self._aiohttp_response.content.iter_chunked(self.CHUNK_SIZE): yield chunk - except ( - aiohttp.ClientPayloadError, - aiohttp.client_exceptions.ClientPayloadError, - ) as e: - # Handle incomplete transfers more gracefully - # Log the error but don't re-raise if we've already yielded some data - verbose_logger.debug(f"Transfer incomplete, but continuing: {e}") - # If the error is due to incomplete transfer encoding, we can still - # return what we've received so far, similar to how httpx handles it - return except RuntimeError as e: - # Some providers (e.g., SSE streams) may close the connection - # causing aiohttp StreamReader to raise a generic RuntimeError - # with message "Connection closed.". Treat this as a graceful - # end-of-stream so downstream consumers don't error. - if "Connection closed" in str(e): - verbose_logger.debug("Upstream closed streaming connection; ending iterator gracefully") - return - raise + if "Connection closed" not in str(e): + raise + raise httpx.ReadError(str(e)) from e except aiohttp.http_exceptions.TransferEncodingError as e: - # Handle transfer encoding errors gracefully - verbose_logger.debug(f"Transfer encoding error, but continuing: {e}") - return + raise httpx.ReadError(str(e)) from e except Exception: # For other exceptions, use the normal mapping with map_aiohttp_exceptions(): diff --git a/litellm/models/mcp_server.py b/litellm/models/mcp_server.py index af2efa822b0..23b26bd8e89 100644 --- a/litellm/models/mcp_server.py +++ b/litellm/models/mcp_server.py @@ -79,6 +79,7 @@ class LiteLLM_MCPServerTable(LiteLLMPydanticObjectBase): command: Optional[str] = None args: List[str] = Field(default_factory=list) env: Dict[str, str] = Field(default_factory=dict) + issuer: Optional[str] = None authorization_url: Optional[str] = None token_url: Optional[str] = None registration_url: Optional[str] = None diff --git a/litellm/proxy/_experimental/mcp_server/db.py b/litellm/proxy/_experimental/mcp_server/db.py index 97cefb3f2cb..d55eb3ac014 100644 --- a/litellm/proxy/_experimental/mcp_server/db.py +++ b/litellm/proxy/_experimental/mcp_server/db.py @@ -48,6 +48,7 @@ if TYPE_CHECKING: _AUTH_FLOW_SCOPED_FIELDS: frozenset = frozenset( { + "issuer", "authorization_url", "token_url", "registration_url", @@ -60,6 +61,13 @@ _AUTH_FLOW_SCOPED_FIELDS: frozenset = frozenset( } ) + +def _blank_to_none(value: Optional[str]) -> Optional[str]: + if not isinstance(value, str): + return None + return value.strip() or None + + # Token-exchange settings with dedicated columns that also exist on # ``MCPCredentials`` as a legacy shape (rows and REST callers that predate the # columns). Every write lifts blob values into the columns and strips them from @@ -697,13 +705,15 @@ async def update_mcp_server( # of being reset to a schema default (transport=sse, allow_all_keys=False...). data_dict = _prepare_mcp_server_data(data, exclude_unset=True, fields_set=fields_set) - # Pre-fetch existing record once if we need it for auth_type or credential logic + # Pre-fetch existing record once if we need it for auth_type, url, or credential logic existing = None has_credentials = "credentials" in data_dict and data_dict["credentials"] is not None # An explicit token-exchange column write (set or clear) also migrates the # legacy blob copies below, so the existing row is needed for those updates. explicit_te_write = bool(_TOKEN_EXCHANGE_COLUMN_FIELDS & data_dict.keys()) - if data.auth_type or has_credentials or explicit_te_write: + url_provided = "url" in data_dict and data_dict["url"] is not None + issuer_provided = "issuer" in data_dict + if data.auth_type or has_credentials or explicit_te_write or url_provided or issuer_provided: existing = await MCPServerRepository(prisma_client).table.find_unique(where={"server_id": data.server_id}) auth_type_changed = bool( @@ -711,13 +721,30 @@ async def update_mcp_server( and existing and _credential_auth_class(existing.auth_type) != _credential_auth_class(data.auth_type) ) + # A url change re-points the server at a potentially different upstream, so any discovered or + # trust-on-first-use OAuth endpoints/issuer belong to the old upstream and must re-discover. + url_changed = bool(url_provided and existing and existing.url != data_dict["url"]) + old_issuer = _blank_to_none(getattr(existing, "issuer", None)) if existing else None + issuer_changed = bool( + issuer_provided and old_issuer is not None and _blank_to_none(data_dict.get("issuer")) != old_issuer + ) # Clear stale credentials when auth_type changes but no new credentials provided if auth_type_changed and "credentials" not in data_dict: data_dict["credentials"] = None - if auth_type_changed: - data_dict.update({field: None for field in _AUTH_FLOW_SCOPED_FIELDS if field not in data_dict}) + if auth_type_changed or url_changed or issuer_changed: + # Clear each auth-flow-scoped field that the caller either omitted (partial update) or + # resubmitted unchanged. The edit form re-sends every field, so a stale issuer/endpoint + # belonging to the old upstream would otherwise survive a url/auth_type change and win in the + # resolution merge; only a genuinely new submitted value is kept. + data_dict.update( + { + field: None + for field in _AUTH_FLOW_SCOPED_FIELDS + if field not in data_dict or data_dict[field] == getattr(existing, field, None) + } + ) # An explicit column write that does not touch credentials must still migrate # the row's legacy blob copies: lift values for columns the caller left @@ -1181,6 +1208,7 @@ def mcp_oauth_token_identity(server: object) -> tuple[object, ...]: getattr(server, "spec_path", None), getattr(server, "auth_type", None), getattr(server, "oauth2_flow", None), + getattr(server, "issuer", None), getattr(server, "authorization_url", None), getattr(server, "token_url", None), getattr(server, "registration_url", None), diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 8b1d00c2855..115ff2e492c 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -201,6 +201,38 @@ def _blank_to_none(value: str | None) -> str | None: return value.strip() or None +def _uses_issuer_anchor(manual_issuer: str | None, is_discovery_auth_type: bool) -> bool: + """Whether the endpoints are authoritatively anchored to an admin-pinned issuer (RFC 8414 §3.3). + + This is the trust/provenance property, distinct from whether the ``issuer`` field is merely + populated: a trust-on-first-use discovered issuer sets ``issuer`` for token identity but is NOT + anchored, so its endpoints stay resource-rooted. Anchoring holds only when the issuer was pinned + (present on the row/config) on a discovery auth type. Every consumer of "is this anchored" reads + this one definition, so the answer cannot diverge across build paths. + """ + return _blank_to_none(manual_issuer) is not None and is_discovery_auth_type + + +def _endpoints_yield_to_issuer( + issuer: str | None, + is_discovery_auth_type: bool, + authorization_url: str | None, + token_url: str | None, + registration_url: str | None, +) -> tuple[str | None, str | None, str | None]: + """The single rule that makes an admin-configured ``issuer`` the sole authoritative endpoint + source (RFC 8414 §3.3): when it is set for a discovery auth type, the stored/manual + ``authorization_url``/``token_url``/``registration_url`` do not apply. They neither anchor nor + short-circuit discovery, never override the issuer document in the merge, and never substitute for + it when the issuer fetch fails (fail-closed). Returns the endpoint values that remain in force, + i.e. all ``None`` when issuer-anchored, else the inputs unchanged. Called at every resolution site + so the invariant holds in one place instead of being re-derived per merge. + """ + if issuer is not None and is_discovery_auth_type: + return None, None, None + return authorization_url, token_url, registration_url + + def _normalized_authorize_endpoint(url: str) -> str: """Compare authorize endpoints on scheme, host, and path only. The default port is elided and the host is lowercased so ``https://IDP.example.com:443/authorize/`` and @@ -217,6 +249,17 @@ def _normalized_authorize_endpoint(url: str) -> str: return f"{scheme}://{authority}{parsed.path.rstrip('/')}" +def _issuer_matches(claimed_issuer: object, configured_issuer: str) -> bool: + """RFC 8414 §3.3 issuer equality between the metadata document's self-attested ``issuer`` and the + admin-configured issuer, tolerant only of URL-insignificant differences (scheme/host case, the + default port, a trailing slash). A non-string or empty claimed issuer never matches, so a + document that omits ``issuer`` fails closed under issuer-anchored discovery. + """ + if not isinstance(claimed_issuer, str) or not claimed_issuer: + return False + return _normalized_authorize_endpoint(claimed_issuer) == _normalized_authorize_endpoint(configured_issuer) + + def _endpoints_corroborate_authorization_url( source_authorization_url: str | None, trusted_authorization_url: str | None, @@ -260,11 +303,27 @@ def _carry_forward_resolved_oauth_endpoints(new_server: MCPServer, previous_serv incoming build has no pinned authorize endpoint (``None`` -> we adopt the previous one too, a consistent group) or pins the same one. An admin re-pointing ``authorization_url`` to a different server must not keep serving the old server's token endpoint or granted scopes. + + When the server is issuer-anchored (``issuer_is_anchored`` -- a pinned issuer on a discovery auth + type), the endpoints come solely from the §3.3-validated issuer document, so carry-forward is + skipped entirely for its endpoints: a failed issuer fetch leaves them ``None`` and must stay + ``None`` (fail-closed), never resurrected from the previous registry entry. A merely discovered + (trust-on-first-use) issuer is NOT anchored -- ``issuer`` is set for token identity but the + endpoints are resource-rooted, so they still carry forward as last-known-good, gated by the + corroboration check below like any other resource-rooted server. Scopes stay resource-driven and + can carry either way. """ if previous_server is None: return if previous_server.url != new_server.url or previous_server.auth_type != new_server.auth_type: return + if new_server.issuer_is_anchored: + # Endpoints come solely from the §3.3-validated issuer document; a failed fetch stays + # fail-closed and must not be resurrected from the previous entry. Only the resource-driven + # scopes carry as last-known-good. + if not new_server.scopes and previous_server.scopes: + new_server.scopes = previous_server.scopes + return may_carry = _endpoints_corroborate_authorization_url( previous_server.authorization_url, new_server.authorization_url ) @@ -1137,34 +1196,48 @@ class MCPServerManager: ) auth_type = server_config.get("auth_type", None) + manual_issuer = _blank_to_none(server_config.get("issuer")) manual_authorization_url = _blank_to_none(server_config.get("authorization_url")) manual_token_url = _blank_to_none(server_config.get("token_url")) manual_registration_url = _blank_to_none(server_config.get("registration_url")) - if server_url and ( - auth_type in _UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES + is_discovery_auth_type = auth_type in _UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES + use_issuer_anchor = _uses_issuer_anchor(manual_issuer, is_discovery_auth_type) + manual_authorization_url, manual_token_url, manual_registration_url = _endpoints_yield_to_issuer( + manual_issuer, + is_discovery_auth_type, + manual_authorization_url, + manual_token_url, + manual_registration_url, + ) + should_discover = bool(server_url) and ( + is_discovery_auth_type or self._obo_needs_endpoint_discovery( auth_type, server_config.get("token_exchange_endpoint"), manual_token_url, ) - ): + ) + if not should_discover: + mcp_oauth_metadata = None + elif manual_issuer is not None and is_discovery_auth_type: + mcp_oauth_metadata = await self._fetch_issuer_anchored_oauth_metadata(manual_issuer, server_url) + else: mcp_oauth_metadata = await self._descovery_metadata( server_url=server_url, - allow_origin_fallback=auth_type in _UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES, + allow_origin_fallback=is_discovery_auth_type, ) - else: - mcp_oauth_metadata = None - gated_oauth_metadata = ( - _restrict_discovery_to_corroborated_authorization_server( + if use_issuer_anchor: + gated_oauth_metadata = mcp_oauth_metadata + elif is_discovery_auth_type: + gated_oauth_metadata = _restrict_discovery_to_corroborated_authorization_server( mcp_oauth_metadata, manual_authorization_url, server_name or server_id, bool(server_config.get("dcr_bridge")), ) - if auth_type in _UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES - else mcp_oauth_metadata - ) + else: + gated_oauth_metadata = mcp_oauth_metadata # Filter blank scopes (e.g. YAML ``scopes: [""]``) the same way the DB-build path does, so # an all-blank list normalizes to None rather than a ``("",)`` tuple that skips the @@ -1179,6 +1252,12 @@ class MCPServerManager: resolved_registration_url = manual_registration_url or ( gated_oauth_metadata.registration_url if gated_oauth_metadata else None ) + discovered_issuer = ( + gated_oauth_metadata.discovered_issuer + if gated_oauth_metadata and not gated_oauth_metadata.from_origin_fallback + else None + ) + effective_issuer = manual_issuer or discovered_issuer config_oauth2_flow = server_config.get("oauth2_flow", None) if auth_type == MCPAuth.oauth2 and config_oauth2_flow not in ( @@ -1227,6 +1306,8 @@ class MCPServerManager: client_secret=server_config.get("client_secret", None), oauth2_flow=self._explicit_oauth2_flow(config_oauth2_flow), scopes=resolved_scopes, + issuer=effective_issuer, + issuer_is_anchored=use_issuer_anchor, authorization_url=resolved_authorization_url, token_url=resolved_token_url, registration_url=resolved_registration_url, @@ -1487,6 +1568,52 @@ class MCPServerManager: decrypt_global_env_var_values(env_vars_list) return env_vars_list + async def _resolve_table_oauth_metadata( + self, + *, + mcp_server: LiteLLM_MCPServerTable, + auth_type: MCPAuthType, + server_url: Optional[str], + manual_issuer: Optional[str], + manual_authorization_url: Optional[str], + manual_token_url: Optional[str], + is_discovery_auth_type: bool, + use_issuer_anchor: bool, + scopes: Optional[list[str]], + token_exchange_endpoint: Optional[str], + ) -> Optional[MCPOAuthMetadata]: + has_all_upstream_oauth_fields = bool(manual_authorization_url and manual_token_url and scopes) + needs_discovery = bool(server_url) and ( + (is_discovery_auth_type and not has_all_upstream_oauth_fields) + or self._obo_needs_endpoint_discovery(auth_type, token_exchange_endpoint, manual_token_url) + ) + if not needs_discovery: + mcp_oauth_metadata: Optional[MCPOAuthMetadata] = None + elif use_issuer_anchor and manual_issuer is not None: + mcp_oauth_metadata = await self._fetch_issuer_anchored_oauth_metadata(manual_issuer, server_url) + else: + mcp_oauth_metadata = await self._descovery_metadata( + server_url=server_url, # type: ignore[arg-type] + allow_origin_fallback=is_discovery_auth_type, + ) + if needs_discovery and not use_issuer_anchor and mcp_oauth_metadata is None: + verbose_logger.warning( + "MCP OAuth discovery yielded no metadata for server %s (%s); " + "OAuth endpoints/scopes stay unresolved until a rebuild succeeds", + mcp_server.server_id, + server_url, + ) + if use_issuer_anchor: + return mcp_oauth_metadata + if is_discovery_auth_type: + return _restrict_discovery_to_corroborated_authorization_server( + mcp_oauth_metadata, + manual_authorization_url, + mcp_server.server_id, + bool(getattr(mcp_server, "dcr_bridge", None)), + ) + return mcp_oauth_metadata + async def build_mcp_server_from_table( self, mcp_server: LiteLLM_MCPServerTable, @@ -1570,46 +1697,38 @@ class MCPServerManager: auth_type = cast(MCPAuthType, mcp_server.auth_type) server_url = mcp_server.url + manual_issuer = _blank_to_none(mcp_server.issuer) manual_authorization_url = _blank_to_none(mcp_server.authorization_url) manual_token_url = _blank_to_none(mcp_server.token_url) manual_registration_url = _blank_to_none(mcp_server.registration_url) - has_all_upstream_oauth_fields = bool(manual_authorization_url and manual_token_url and scopes) - needs_discovery = bool(server_url) and ( - (auth_type in _UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES and not has_all_upstream_oauth_fields) - or self._obo_needs_endpoint_discovery( - auth_type, - mcp_server.token_exchange_endpoint - or (credentials_dict.get("token_exchange_endpoint") if credentials_dict else None), - manual_token_url, - ) + is_discovery_auth_type = auth_type in _UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES + use_issuer_anchor = _uses_issuer_anchor(manual_issuer, is_discovery_auth_type) + manual_authorization_url, manual_token_url, manual_registration_url = _endpoints_yield_to_issuer( + manual_issuer, is_discovery_auth_type, manual_authorization_url, manual_token_url, manual_registration_url ) - mcp_oauth_metadata = ( - await self._descovery_metadata( - server_url=server_url, # type: ignore[arg-type] - allow_origin_fallback=auth_type in _UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES, - ) - if needs_discovery - else None + token_exchange_endpoint = mcp_server.token_exchange_endpoint or ( + credentials_dict.get("token_exchange_endpoint") if credentials_dict else None ) - if needs_discovery and mcp_oauth_metadata is None: - verbose_logger.warning( - "MCP OAuth discovery yielded no metadata for server %s (%s); " - "OAuth endpoints/scopes stay unresolved until a rebuild succeeds", - mcp_server.server_id, - server_url, - ) - gated_oauth_metadata = ( - _restrict_discovery_to_corroborated_authorization_server( - mcp_oauth_metadata, - manual_authorization_url, - mcp_server.server_id, - bool(getattr(mcp_server, "dcr_bridge", None)), - ) - if auth_type in _UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES - else mcp_oauth_metadata + gated_oauth_metadata = await self._resolve_table_oauth_metadata( + mcp_server=mcp_server, + auth_type=auth_type, + server_url=server_url, + manual_issuer=manual_issuer, + manual_authorization_url=manual_authorization_url, + manual_token_url=manual_token_url, + is_discovery_auth_type=is_discovery_auth_type, + use_issuer_anchor=use_issuer_anchor, + scopes=scopes, + token_exchange_endpoint=token_exchange_endpoint, ) resolved_scopes = scopes or (gated_oauth_metadata.scopes if gated_oauth_metadata else None) + discovered_issuer = ( + gated_oauth_metadata.discovered_issuer + if gated_oauth_metadata and not gated_oauth_metadata.from_origin_fallback + else None + ) + effective_issuer = manual_issuer or discovered_issuer new_server = MCPServer( server_id=mcp_server.server_id, @@ -1629,6 +1748,8 @@ class MCPServerManager: client_secret=client_secret_value or getattr(mcp_server, "client_secret", None), oauth2_flow=self._explicit_oauth2_flow(getattr(mcp_server, "oauth2_flow", None)), scopes=resolved_scopes, + issuer=effective_issuer, + issuer_is_anchored=use_issuer_anchor, authorization_url=manual_authorization_url or getattr(gated_oauth_metadata, "authorization_url", None), token_url=manual_token_url or getattr(gated_oauth_metadata, "token_url", None), registration_url=manual_registration_url or getattr(gated_oauth_metadata, "registration_url", None), @@ -1688,10 +1809,12 @@ class MCPServerManager: await self._persist_discovered_oauth_endpoints( server_id=mcp_server.server_id, auth_type=auth_type, + existing_issuer=manual_issuer, existing_authorization_url=manual_authorization_url, existing_token_url=manual_token_url, existing_scopes=scopes, metadata=gated_oauth_metadata, + is_issuer_anchored=use_issuer_anchor, ) return new_server @@ -1735,10 +1858,12 @@ class MCPServerManager: *, server_id: str, auth_type: MCPAuthType | None, + existing_issuer: str | None, existing_authorization_url: str | None, existing_token_url: str | None, existing_scopes: list[str] | None, metadata: MCPOAuthMetadata | None, + is_issuer_anchored: bool = False, ) -> None: """Write freshly discovered OAuth endpoints back onto the DB row. @@ -1752,19 +1877,37 @@ class MCPServerManager: because ``_dcr_bridge_relays_client_registration`` keys off that column. Best-effort: a failed write re-discovers on the next build. Scopes go through ``update_mcp_server`` so they merge into the credentials blob without touching the stored client credentials. + + For an issuer-anchored server (``is_issuer_anchored``) the endpoints are re-derived from the + §3.3-validated issuer document on every build, so they are NOT persisted into the endpoint + columns: persisting them would make the next build see populated endpoints and treat them as + authoritative stored values, defeating the "endpoints come solely from the issuer" invariant. + Only the resource-driven scopes are persisted for such servers. """ if auth_type not in _UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES: return if metadata is None or metadata.from_origin_fallback: return + issuer_update = ( + {"issuer": metadata.discovered_issuer} if metadata.discovered_issuer and not existing_issuer else {} + ) authorization_url_update = ( {"authorization_url": metadata.authorization_url} - if metadata.authorization_url and not existing_authorization_url + if metadata.authorization_url and not existing_authorization_url and not is_issuer_anchored + else {} + ) + token_url_update = ( + {"token_url": metadata.token_url} + if metadata.token_url and not existing_token_url and not is_issuer_anchored else {} ) - token_url_update = {"token_url": metadata.token_url} if metadata.token_url and not existing_token_url else {} scopes_update = {"credentials": {"scopes": metadata.scopes}} if metadata.scopes and not existing_scopes else {} - updates: dict[str, object] = {**authorization_url_update, **token_url_update, **scopes_update} + updates: dict[str, object] = { + **issuer_update, + **authorization_url_update, + **token_url_update, + **scopes_update, + } if not updates: return from litellm.proxy._experimental.mcp_server.db import ( # noqa: PLC0415 # db.py imports this module at load @@ -3337,8 +3480,41 @@ class MCPServerManager: return metadata return None + async def _fetch_issuer_anchored_oauth_metadata( + self, issuer: str, server_url: Optional[str] + ) -> Optional[MCPOAuthMetadata]: + """RFC 8414 issuer-anchored discovery for the OAuth endpoints, with resource-driven scopes. + + Fetch authorization-server metadata from the admin-configured issuer's own origin and adopt + its ``token_endpoint``/``registration_endpoint`` only when the document self-attests that same + issuer (RFC 8414 §3.3). Because the trust anchor is the pinned issuer rather than anything the + MCP resource advertises, the endpoints are authoritative for that issuer and cannot be + substituted by a compromised resource. Fails closed (returns None) on a §3.3 mismatch or a + fetch failure. The issuer is passed as its own ``server_url`` so the endpoint fetch is treated + as same-authority and is not subject to the resource-scoped SSRF shortcut. + + Scopes are NOT taken from the issuer document. Per the MCP authorization spec Scope Selection + Strategy and RFC 9728, the scopes a client requests are resource-driven (the WWW-Authenticate + challenge or the protected-resource ``scopes_supported``), so the resource's advertised scopes + are fetched separately and used; the resource can influence only the requested scope, which + the authorization server and user consent bound (RFC 6749 §3.3), never the token endpoint. + """ + metadata = await self._fetch_single_authorization_server_metadata(issuer, issuer, require_issuer=issuer) + if metadata is None: + verbose_logger.warning( + "MCP OAuth issuer-anchored discovery for issuer %s yielded no metadata whose issuer " + "matched (RFC 8414 §3.3); OAuth endpoints stay unresolved until a rebuild succeeds", + issuer, + ) + return None + resource_metadata = ( + await self._descovery_metadata(server_url, allow_origin_fallback=False) if server_url else None + ) + resource_scopes = resource_metadata.scopes if resource_metadata else None + return metadata.model_copy(update={"scopes": resource_scopes}) + async def _fetch_single_authorization_server_metadata( - self, issuer_url: str, server_url: str + self, issuer_url: str, server_url: str, require_issuer: Optional[str] = None ) -> Optional[MCPOAuthMetadata]: try: parsed = urlparse(issuer_url) @@ -3382,20 +3558,33 @@ class MCPServerManager: ) continue - scopes = self._extract_scopes(data.get("scopes_supported")) + claimed_issuer = data.get("issuer") verbose_logger.debug( "Authorization server metadata from %s: issuer=%s grant_types_supported=%s " "token_endpoint_auth_methods_supported=%s", url, - data.get("issuer"), + claimed_issuer, data.get("grant_types_supported"), data.get("token_endpoint_auth_methods_supported"), ) + if require_issuer is not None and not _issuer_matches(claimed_issuer, require_issuer): + verbose_logger.warning( + "MCP OAuth issuer-anchored discovery: metadata at %s self-attests issuer %r, which " + "does not match the configured issuer %r (RFC 8414 §3.3); rejecting so a compromised " + "resource cannot substitute an attacker authorization server", + url, + claimed_issuer, + require_issuer, + ) + continue + + scopes = self._extract_scopes(data.get("scopes_supported")) metadata = MCPOAuthMetadata( scopes=scopes, authorization_url=data.get("authorization_endpoint"), token_url=data.get("token_endpoint"), registration_url=data.get("registration_endpoint"), + discovered_issuer=claimed_issuer if isinstance(claimed_issuer, str) and claimed_issuer else None, ) if any( @@ -5116,6 +5305,7 @@ class MCPServerManager: command=getattr(server, "command", None), args=getattr(server, "args", None) or [], env=getattr(server, "env", None) or {}, + issuer=server.issuer, authorization_url=server.authorization_url, token_url=server.token_url, registration_url=server.registration_url, @@ -5225,6 +5415,7 @@ class MCPServerManager: command=getattr(server, "command", None), args=getattr(server, "args", None) or [], env=getattr(server, "env", None) or {}, + issuer=server.issuer, authorization_url=server.authorization_url, token_url=server.token_url, registration_url=server.registration_url, diff --git a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py index 111fde86ea0..7ca4923337c 100644 --- a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py @@ -1138,6 +1138,7 @@ if MCP_AVAILABLE: static_headers=request.static_headers, client_id=client_id, client_secret=client_secret, + issuer=request.issuer, token_url=request.token_url, scopes=scopes, authorization_url=request.authorization_url, diff --git a/litellm/proxy/_experimental/mcp_server/semantic_tool_filter.py b/litellm/proxy/_experimental/mcp_server/semantic_tool_filter.py index e12c6cdbd56..37d3e0aedea 100644 --- a/litellm/proxy/_experimental/mcp_server/semantic_tool_filter.py +++ b/litellm/proxy/_experimental/mcp_server/semantic_tool_filter.py @@ -4,6 +4,7 @@ Semantic MCP Tool Filtering using semantic-router Filters MCP tools semantically for /chat/completions and /responses endpoints. """ +import asyncio from typing import TYPE_CHECKING, Any, Dict, List, Optional from litellm._logging import verbose_logger @@ -76,6 +77,7 @@ class SemanticMCPToolFilter: self.tool_router: Optional["SemanticRouter"] = None self.context_window_error: Optional[str] = None self._tool_map: Dict[str, Any] = {} # MCPTool objects or OpenAI function dicts + self._index_sync_lock = asyncio.Lock() async def build_router_from_mcp_registry(self) -> None: """Build semantic router from all MCP tools in the registry (no auth checks).""" @@ -182,6 +184,81 @@ class SemanticMCPToolFilter: return raise + def _has_tools_missing_from_index(self, tools: list[Any]) -> bool: + """Allocation-free check for any named tool not yet in the semantic index.""" + return any(name and name not in self._tool_map for name in (self._extract_tool_info(t)[0] for t in tools)) + + def _tools_missing_from_index(self, tools: list[Any]) -> dict[str, Any]: + """Map name -> tool for every named tool not yet in the semantic index.""" + return { + name: tool + for name, tool in ((self._extract_tool_info(t)[0], t) for t in tools) + if name and name not in self._tool_map + } + + async def _ensure_tools_indexed(self, available_tools: list[Any]) -> None: + """ + Index request-time tools the startup build never saw. + + The startup index lists every registered MCP server WITHOUT per-user + credentials, so servers requiring per-user auth (interactive OAuth + tokens, user-scoped env vars) contribute zero routes. Tools reaching + the filter came through an authenticated expansion; without indexing + them here they can never be selected, so requests either bypass + filtering entirely (N->N) or lose every tool to unrelated matches. + + Runs async-only (no synchronous embedding on the request path) and + never writes shared error state: an embedding failure here raises and + is scoped to the requesting call, so one request's oversized tool + description cannot poison the filter for other users on the worker. + """ + from semantic_router.routers import SemanticRouter + from semantic_router.routers.base import Route + + from litellm.router_strategy.auto_router.litellm_encoder import ( + LiteLLMRouterEncoder, + ) + + if not self._has_tools_missing_from_index(available_tools): + return + + async with self._index_sync_lock: + missing = self._tools_missing_from_index(available_tools) + if not missing: + return + + descriptions = {name: self._extract_tool_info(tool)[1] for name, tool in missing.items()} + routes = [ + Route( + name=name, + description=description, + utterances=[description], + score_threshold=self.similarity_threshold, + ) + for name, description in descriptions.items() + ] + + if self.tool_router is None: + router = SemanticRouter( + routes=[], + encoder=LiteLLMRouterEncoder( + litellm_router_instance=self.router_instance, + model_name=self.embedding_model, + score_threshold=self.similarity_threshold, + ), + auto_sync="local", + top_k=self.top_k, + ) + await router.aadd(routes) + self.tool_router = router + else: + await self.tool_router.aadd(routes) + + self._tool_map.update(missing) + verbose_logger.info( + f"Semantic tool filter indexed {len(routes)} request-time tools missing from the startup index" + ) + async def filter_tools( self, query: str, @@ -216,22 +293,34 @@ class SemanticMCPToolFilter: if not query or not query.strip(): return available_tools - # Router should be built on startup - if not, something went wrong - if self.tool_router is None: - verbose_logger.warning("Router not initialized - was build_router_from_mcp_registry() called on startup?") - return available_tools - # Run semantic filtering try: + await self._ensure_tools_indexed(available_tools) + + if self.tool_router is None: + verbose_logger.warning("Semantic router could not be built from the request's tools") + return available_tools + + available_names = [name for name in (self._extract_tool_info(t)[0] for t in available_tools) if name] + if not available_names: + return available_tools + limit = top_k or self.top_k - matches = self.tool_router(text=query, limit=limit) + if self.tool_router.top_k < limit: + self.tool_router.top_k = limit + matches = self.tool_router(text=query, limit=limit, route_filter=available_names) matched_tool_names = self._extract_tool_names_from_matches(matches) if not matched_tool_names: return available_tools - return self._get_tools_by_names(matched_tool_names, available_tools) + filtered_tools = self._get_tools_by_names(matched_tool_names, available_tools) + if not filtered_tools: + return available_tools + return filtered_tools + except SemanticToolFilterContextWindowError: + raise except Exception as e: if _is_context_window_error(e): verbose_logger.error( @@ -240,7 +329,7 @@ class SemanticMCPToolFilter: ) raise SemanticToolFilterContextWindowError( embedding_model=self.embedding_model, - stage="the user query", + stage="the user query or the MCP tool descriptions being indexed", original_error=str(e), ) from e verbose_logger.error(f"Semantic tool filter failed: {e}", exc_info=True) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 5e3ea4b7dcb..053f78a3698 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1263,6 +1263,7 @@ class NewMCPServerRequest(LiteLLMPydanticObjectBase): command: Optional[str] = None args: List[str] = Field(default_factory=list) env: Dict[str, str] = Field(default_factory=dict) + issuer: Optional[str] = None authorization_url: Optional[str] = None token_url: Optional[str] = None registration_url: Optional[str] = None @@ -1368,6 +1369,7 @@ class UpdateMCPServerRequest(LiteLLMPydanticObjectBase): command: Optional[str] = None args: List[str] = Field(default_factory=list) env: Dict[str, str] = Field(default_factory=dict) + issuer: Optional[str] = None authorization_url: Optional[str] = None token_url: Optional[str] = None registration_url: Optional[str] = None diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 00f6d44e25a..fa354b8cccb 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -361,7 +361,11 @@ def _global_proxy_budget_check(global_proxy_spend: Optional[float], skip_budget_ and route != "/models" ): if math.isfinite(litellm.max_budget) and global_proxy_spend > litellm.max_budget: - raise litellm.BudgetExceededError(current_cost=global_proxy_spend, max_budget=litellm.max_budget) + raise litellm.BudgetExceededError( + current_cost=global_proxy_spend, + max_budget=litellm.max_budget, + entity_type=Litellm_EntityType.PROXY.value, + ) _GUARDRAIL_MODIFICATION_KEYS: tuple = ( @@ -648,6 +652,8 @@ async def common_checks( current_cost=user_spend, max_budget=user_budget, message=f"ExceededBudget: User={user_object.user_id} over budget. Spend={user_spend}, Budget={user_budget}", + entity_type=Litellm_EntityType.USER.value, + entity_id=user_object.user_id, ) # Each scope reads a distinct counter key with no cross-scope ordering @@ -1093,6 +1099,8 @@ async def _check_end_user_budget( current_cost=end_user_spend, max_budget=end_user_budget, message=f"ExceededBudget: End User={end_user_obj.user_id} over budget. Spend={end_user_spend}, Budget={end_user_budget}", + entity_type=Litellm_EntityType.END_USER.value, + entity_id=end_user_obj.user_id, ) @@ -3552,6 +3560,8 @@ async def _virtual_key_max_budget_check( current_cost=spend, max_budget=valid_token.max_budget, message=f"Budget has been exceeded! Key={key_descriptor} Current cost: {spend}, Max budget: {valid_token.max_budget}", + entity_type=Litellm_EntityType.KEY.value, + entity_id=valid_token.token, ) @@ -3593,6 +3603,8 @@ async def _virtual_key_multi_budget_check( f"ExceededBudget: Key over {w['budget_duration']} budget. " f"Spend=${window_spend:.4f}, Limit=${w['max_budget']:.2f}" ), + entity_type=Litellm_EntityType.KEY.value, + entity_id=valid_token.token, ) @@ -3824,6 +3836,8 @@ async def _check_team_member_budget( current_cost=team_member_spend, max_budget=team_member_budget, message=f"Budget has been exceeded! User={valid_token.user_id} in Team={team_object.team_id} Current cost: {team_member_spend}, Max budget: {team_member_budget}", + entity_type=Litellm_EntityType.TEAM_MEMBER.value, + entity_id=f"{valid_token.user_id}:{team_object.team_id}", ) @@ -3923,6 +3937,8 @@ async def _team_max_budget_check( current_cost=spend, max_budget=team_object.max_budget, message=f"Budget has been exceeded! Team={team_object.team_id} Current cost: {spend}, Max budget: {team_object.max_budget}", + entity_type=Litellm_EntityType.TEAM.value, + entity_id=team_object.team_id, ) @@ -3960,6 +3976,8 @@ async def _team_multi_budget_check( f"ExceededBudget: Team={team_object.team_id} over {w['budget_duration']} budget. " f"Spend=${window_spend:.4f}, Limit=${w['max_budget']:.2f}" ), + entity_type=Litellm_EntityType.TEAM.value, + entity_id=team_object.team_id, ) @@ -4081,6 +4099,8 @@ async def _project_max_budget_check( current_cost=project_object.spend, max_budget=max_budget, message=f"Budget has been exceeded! Project={project_object.project_id} Current cost: {project_object.spend}, Max budget: {max_budget}", + entity_type=Litellm_EntityType.PROJECT.value, + entity_id=project_object.project_id, ) @@ -4269,6 +4289,8 @@ async def _organization_max_budget_check( current_cost=org_spend, max_budget=org_max_budget, message=f"Budget has been exceeded! Organization={org_id} Current cost: {org_spend}, Max budget: {org_max_budget}", + entity_type=Litellm_EntityType.ORGANIZATION.value, + entity_id=org_id, ) @@ -4326,6 +4348,8 @@ async def _tag_max_budget_check( current_cost=tag_spend, max_budget=tag_object.litellm_budget_table.max_budget, message=f"Budget has been exceeded! Tag={tag_name} Current cost: {tag_spend}, Max budget: {tag_object.litellm_budget_table.max_budget}", + entity_type=Litellm_EntityType.TAG.value, + entity_id=tag_name, ) diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index b402212fb2e..0519b5ef0b6 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -1797,6 +1797,8 @@ async def _user_api_key_auth_builder( raise litellm.BudgetExceededError( current_cost=team_member_spend, max_budget=team_member_budget, + entity_type=Litellm_EntityType.TEAM_MEMBER.value, + entity_id=f"{valid_token.user_id}:{valid_token.team_id}", ) # Check 3. If token is expired diff --git a/litellm/proxy/client/cli/commands/auth.py b/litellm/proxy/client/cli/commands/auth.py index 785d3b1e37b..61495403407 100644 --- a/litellm/proxy/client/cli/commands/auth.py +++ b/litellm/proxy/client/cli/commands/auth.py @@ -72,7 +72,7 @@ def display_teams_table(teams: List[Dict[str, Any]]) -> None: console = Console() if not teams: - console.print("❌ No teams found for your user.") + console.print("No teams found for your user.") return table = Table(title="Available Teams") @@ -162,7 +162,7 @@ def display_interactive_team_selection(teams: List[Dict[str, Any]], selected_ind # Clear the screen using Rich's method console.clear() - console.print("🎯 Select a Team (Use ↑↓ arrows, Enter to select, 'q' to skip):\n") + console.print("Select a Team (Use up/down arrows, Enter to select, 'q' to skip):\n") for i, team in enumerate(teams): team_alias = team.get("team_alias") or "N/A" @@ -184,7 +184,7 @@ def display_interactive_team_selection(teams: List[Dict[str, Any]], selected_ind # Highlight the selected item if i == selected_index: - console.print(f"➤ [bold cyan]{team_alias}[/bold cyan] ({team_id})") + console.print(f"> [bold cyan]{team_alias}[/bold cyan] ({team_id})") console.print(f" Models: [yellow]{models_str}[/yellow]") console.print(f" Budget: [blue]{budget_str}[/blue]\n") else: @@ -220,15 +220,13 @@ def prompt_team_selection(teams: List[Dict[str, Any]]) -> Optional[Dict[str, Any # Clear screen and show selection console = Console() console.clear() - click.echo( - f"✅ Selected team: {selected_team.get('team_alias', 'N/A')} ({selected_team.get('team_id')})" - ) + click.echo(f"Selected team: {selected_team.get('team_alias', 'N/A')} ({selected_team.get('team_id')})") return selected_team elif key == "quit" or key == "escape": # Clear screen console = Console() console.clear() - click.echo("ℹ️ Team selection skipped.") + click.echo("Team selection skipped.") return None elif key is None: # If we can't get key input, fall back to simple selection @@ -237,7 +235,7 @@ def prompt_team_selection(teams: List[Dict[str, Any]]) -> Optional[Dict[str, Any except KeyboardInterrupt: console = Console() console.clear() - click.echo("\n❌ Team selection cancelled.") + click.echo("\nTeam selection cancelled.") return None except Exception: # If interactive mode fails, fall back to simple selection @@ -265,15 +263,15 @@ def prompt_team_selection_fallback( if 0 <= index < len(teams): selected_team = teams[index] click.echo( - f"\n✅ Selected team: {selected_team.get('team_alias', 'N/A')} ({selected_team.get('team_id')})" + f"\nSelected team: {selected_team.get('team_alias', 'N/A')} ({selected_team.get('team_id')})" ) return selected_team else: - click.echo(f"❌ Invalid selection. Please enter a number between 1 and {len(teams)}") + click.echo(f"Invalid selection. Please enter a number between 1 and {len(teams)}") except ValueError: - click.echo("❌ Invalid input. Please enter a number or 'skip'") + click.echo("Invalid input. Please enter a number or 'skip'") except KeyboardInterrupt: - click.echo("\n❌ Team selection cancelled.") + click.echo("\nTeam selection cancelled.") return None @@ -437,7 +435,7 @@ def _poll_for_authentication(base_url: str, key_id: str, poll_secret: str) -> Op user_id = data.get("user_id") normalized_teams: List[Dict[str, Any]] = _normalize_teams(teams, team_details) if not normalized_teams: - click.echo("⚠️ No teams available for selection.") + click.echo("Warning: No teams available for selection.") return None # User has multiple teams - let them select @@ -457,7 +455,7 @@ def _poll_for_authentication(base_url: str, key_id: str, poll_secret: str) -> Op "team_id": None, # Set by server in JWT } - click.echo("❌ Team selection cancelled or JWT generation failed.") + click.echo("Team selection cancelled or JWT generation failed.") return None # JWT is ready (single team or team already selected) @@ -468,7 +466,7 @@ def _poll_for_authentication(base_url: str, key_id: str, poll_secret: str) -> Op # Show which team was assigned if team_id and len(teams) == 1: - click.echo(f"\n✅ Automatically assigned to team: {team_id}") + click.echo(f"\nAutomatically assigned to team: {team_id}") if api_key: return { @@ -494,19 +492,19 @@ def _handle_team_selection_during_polling( The JWT token with the selected team, or None if selection was skipped """ if not teams: - click.echo("ℹ️ No teams found. You can create or join teams using the web interface.") + click.echo("No teams found. You can create or join teams using the web interface.") return None click.echo("\n" + "=" * 60) - click.echo("📋 Select a team for your CLI session...") + click.echo("Select a team for your CLI session...") team_id = _render_and_prompt_for_team_selection(teams) if not team_id: - click.echo("ℹ️ No team selected.") + click.echo("No team selected.") return None - click.echo(f"\n🔄 Generating JWT for team: {team_id}") + click.echo(f"\nGenerating JWT for team: {team_id}") poll_url = f"{base_url}/sso/cli/poll/{key_id}?team_id={team_id}" data = _poll_for_ready_data( @@ -520,7 +518,7 @@ def _handle_team_selection_during_polling( return None jwt_token = data.get("key") if jwt_token: - click.echo(f"✅ Successfully generated JWT for team: {team_id}") + click.echo(f"Successfully generated JWT for team: {team_id}") return jwt_token return None @@ -568,14 +566,14 @@ def _render_and_prompt_for_team_selection(teams: List[Dict[str, Any]]) -> Option selected_team = teams[index] team_id = str(selected_team.get("team_id")) team_alias = selected_team.get("team_alias") or team_id - click.echo(f"\n✅ Selected team: {team_alias} ({team_id})") + click.echo(f"\nSelected team: {team_alias} ({team_id})") return team_id - click.echo(f"❌ Invalid selection. Please enter a number between 1 and {len(teams)}") + click.echo(f"Invalid selection. Please enter a number between 1 and {len(teams)}") except ValueError: - click.echo("❌ Invalid input. Please enter a number or 'skip'") + click.echo("Invalid input. Please enter a number or 'skip'") except KeyboardInterrupt: - click.echo("\n❌ Team selection cancelled.") + click.echo("\nTeam selection cancelled.") return None @@ -628,7 +626,7 @@ def login(ctx: click.Context): } ) - click.echo("\n✅ Login successful!") + click.echo("\nLogin successful!") click.echo(f"JWT Token: {api_key[:20]}...") click.echo("You can now use the CLI without specifying --api-key") @@ -637,7 +635,7 @@ def login(ctx: click.Context): show_commands() return else: - click.echo("❌ Authentication timed out. Please try again.") + click.echo("Authentication timed out. Please try again.") click.echo( "The proxy never reported the browser sign-in as finished. If you did complete it, " "check the proxy logs for /sso/callback errors and confirm SSO is configured on the proxy." @@ -645,10 +643,10 @@ def login(ctx: click.Context): return except KeyboardInterrupt: - click.echo("\n❌ Authentication cancelled by user.") + click.echo("\nAuthentication cancelled by user.") return except Exception as e: - click.echo(f"❌ Authentication failed: {e}") + click.echo(f"Authentication failed: {e}") return @@ -656,7 +654,7 @@ def login(ctx: click.Context): def logout(): """Logout and clear stored authentication""" clear_token() - click.echo("✅ Logged out successfully. Authentication token cleared.") + click.echo("Logged out successfully. Authentication token cleared.") @click.command(name="print-token") @@ -703,10 +701,10 @@ def whoami(): token_data = load_token() if not token_data: - click.echo("❌ Not authenticated. Run 'lite login' to authenticate.") + click.echo("Not authenticated. Run 'lite login' to authenticate.") return - click.echo("✅ Authenticated") + click.echo("Authenticated") click.echo(f"User Email: {token_data.get('user_email', 'Unknown')}") click.echo(f"User ID: {token_data.get('user_id', 'Unknown')}") click.echo(f"User Role: {token_data.get('user_role', 'Unknown')}") @@ -717,7 +715,7 @@ def whoami(): click.echo(f"Token age: {age_hours:.1f} hours") if age_hours > CLI_JWT_EXPIRATION_HOURS: - click.echo(f"⚠️ Warning: Token is more than {CLI_JWT_EXPIRATION_HOURS} hours old and may have expired.") + click.echo(f"Warning: Token is more than {CLI_JWT_EXPIRATION_HOURS} hours old and may have expired.") @click.group(name="auth") diff --git a/litellm/proxy/client/cli/commands/chat.py b/litellm/proxy/client/cli/commands/chat.py index 9d1a1aa7a30..d78feb84bd6 100644 --- a/litellm/proxy/client/cli/commands/chat.py +++ b/litellm/proxy/client/cli/commands/chat.py @@ -150,7 +150,7 @@ def chat( f"Max Tokens: [yellow]{max_tokens or 'unlimited'}[/yellow]\n\n" f"Type your messages and press Enter. Type '/quit' or '/exit' to end the session.\n" f"Type '/help' for more commands.", - title="🤖 Chat Session", + title="Chat Session", ) ) diff --git a/litellm/proxy/client/cli/commands/encryption.py b/litellm/proxy/client/cli/commands/encryption.py index f67c9746fa9..4b460bac19c 100644 --- a/litellm/proxy/client/cli/commands/encryption.py +++ b/litellm/proxy/client/cli/commands/encryption.py @@ -32,7 +32,7 @@ def migrate(ctx: click.Context, check_only: bool, dry_run: bool): Requires the proxy to be started with ``general_settings.encryption_algorithm: aes-256-gcm``. Idempotent and - resumable — safe to re-run after an interruption. + resumable; safe to re-run after an interruption. Examples: litellm-proxy encryption migrate --check # attestation scan, no writes diff --git a/litellm/proxy/client/cli/commands/keys.py b/litellm/proxy/client/cli/commands/keys.py index 45e27442708..afbaa3702c1 100644 --- a/litellm/proxy/client/cli/commands/keys.py +++ b/litellm/proxy/client/cli/commands/keys.py @@ -309,12 +309,12 @@ def _import_keys_to_destination( imported_count += 1 key_alias = key.get("key_alias", "N/A") - click.echo(f"✓ Imported key: {key_alias}") + click.echo(f"Imported key: {key_alias}") except Exception as e: failed_count += 1 key_alias = key.get("key_alias", "N/A") - click.echo(f"✗ Failed to import key {key_alias}: {str(e)}", err=True) + click.echo(f"Failed to import key {key_alias}: {str(e)}", err=True) return imported_count, failed_count diff --git a/litellm/proxy/client/cli/commands/teams.py b/litellm/proxy/client/cli/commands/teams.py index c0d45544b11..b0ccdc8f9bf 100644 --- a/litellm/proxy/client/cli/commands/teams.py +++ b/litellm/proxy/client/cli/commands/teams.py @@ -21,7 +21,7 @@ def display_teams_table(teams: List[Dict[str, Any]]) -> None: console = Console() if not teams: - console.print("❌ No teams found for your user.") + console.print("No teams found for your user.") return table = Table(title="Available Teams") @@ -91,10 +91,10 @@ def available(ctx: click.Context): teams = client.teams.get_available() if teams: console = Console() - console.print("\n🎯 Available Teams to Join:") + console.print("\nAvailable Teams to Join:") display_teams_table(teams) else: - click.echo("ℹ️ No available teams to join.") + click.echo("No available teams to join.") except requests.exceptions.HTTPError as e: click.echo(f"Error: HTTP {e.response.status_code}", err=True) error_body = e.response.json() @@ -113,7 +113,7 @@ def assign_key(ctx: click.Context, team_id: Optional[str]): api_key = ctx.obj["api_key"] if not api_key: - click.echo("❌ No API key found. Please login first using 'litellm login'") + click.echo("No API key found. Please login first using 'litellm login'") raise click.Abort() try: @@ -122,7 +122,7 @@ def assign_key(ctx: click.Context, team_id: Optional[str]): teams = client.teams.list() if not teams: - click.echo("❌ No teams found for your user.") + click.echo("No teams found for your user.") return # Use interactive selection from auth module @@ -133,14 +133,14 @@ def assign_key(ctx: click.Context, team_id: Optional[str]): if selected_team: team_id = selected_team.get("team_id") else: - click.echo("❌ Operation cancelled.") + click.echo("Operation cancelled.") return # Update the key with the selected team if team_id: - click.echo(f"\n🔄 Assigning your key to team: {team_id}") + click.echo(f"\nAssigning your key to team: {team_id}") client.keys.update(key=api_key, team_id=team_id) - click.echo(f"✅ Successfully assigned key to team: {team_id}") + click.echo(f"Successfully assigned key to team: {team_id}") # Show team details if available teams = client.teams.list() @@ -148,9 +148,9 @@ def assign_key(ctx: click.Context, team_id: Optional[str]): if team.get("team_id") == team_id: models = team.get("models", []) if models: - click.echo(f"🎯 You can now access models: {', '.join(models)}") + click.echo(f"You can now access models: {', '.join(models)}") else: - click.echo("🎯 You can now access all available models") + click.echo("You can now access all available models") break except requests.exceptions.HTTPError as e: diff --git a/litellm/proxy/client/cli/interface.py b/litellm/proxy/client/cli/interface.py index 33f1f4a4480..e953742f412 100644 --- a/litellm/proxy/client/cli/interface.py +++ b/litellm/proxy/client/cli/interface.py @@ -27,13 +27,13 @@ def styled_prompt(): verbose_logger.debug(f"Error getting terminal size: {e}") click.echo("\n" * 3) - # Unicode box drawing characters - top_left = "┌" - top_right = "┐" - bottom_left = "└" - bottom_right = "┘" - horizontal = "─" - vertical = "│" + # ASCII box drawing characters + top_left = "+" + top_right = "+" + bottom_left = "+" + bottom_right = "+" + horizontal = "-" + vertical = "|" # Create the box with increased width width = 80 diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index ca6875ca800..54a4c2dad91 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -27,7 +27,7 @@ from typing import ( import litellm from litellm._logging import verbose_proxy_logger -from litellm.caching import DualCache, RedisCache +from litellm.caching import RedisCache from litellm.constants import ( DB_SPEND_UPDATE_JOB_NAME, DB_DAILY_TAG_SPEND_UPDATE_JOB_NAME, @@ -44,7 +44,6 @@ from litellm.proxy._types import ( DailyUserSpendTransaction, DBSpendUpdateTransactions, Litellm_EntityType, - LiteLLM_UserTable, SpendLogsMetadata, SpendLogsPayload, SpendUpdateQueueItem, @@ -137,7 +136,6 @@ class DBSpendUpdateWriter: disable_spend_logs, litellm_proxy_budget_name, prisma_client, - user_api_key_cache, ) from litellm.proxy.utils import ProxyUpdateSpend, hash_token @@ -195,7 +193,6 @@ class DBSpendUpdateWriter: org_id=org_id, end_user_id=end_user_id, prisma_client=prisma_client, - user_api_key_cache=user_api_key_cache, litellm_proxy_budget_name=litellm_proxy_budget_name, payload=payload, ) @@ -326,7 +323,6 @@ class DBSpendUpdateWriter: org_id: Optional[str], end_user_id: Optional[str], prisma_client: Optional[PrismaClient], - user_api_key_cache: DualCache, litellm_proxy_budget_name: Optional[str], payload: SpendLogsPayload, ): @@ -345,7 +341,6 @@ class DBSpendUpdateWriter: response_cost=response_cost, user_id=user_id, prisma_client=prisma_client, - user_api_key_cache=user_api_key_cache, litellm_proxy_budget_name=litellm_proxy_budget_name, end_user_id=end_user_id, ) @@ -510,7 +505,6 @@ class DBSpendUpdateWriter: response_cost: Optional[float], user_id: Optional[str], prisma_client: Optional[PrismaClient], - user_api_key_cache: DualCache, litellm_proxy_budget_name: Optional[str], end_user_id: Optional[str] = None, ): @@ -518,10 +512,6 @@ class DBSpendUpdateWriter: - Update that user's row - Update litellm-proxy-budget row (global proxy spend) """ - ## if an end-user is passed in, do an upsert - we can't guarantee they already exist in db - existing_user_obj = await user_api_key_cache.async_get_cache(key=user_id) - if existing_user_obj is not None and isinstance(existing_user_obj, dict): - existing_user_obj = LiteLLM_UserTable(**existing_user_obj) try: if prisma_client is not None: # update user_ids = [user_id] diff --git a/litellm/proxy/hooks/model_max_budget_limiter.py b/litellm/proxy/hooks/model_max_budget_limiter.py index 803fe64c193..e8cf5fbc718 100644 --- a/litellm/proxy/hooks/model_max_budget_limiter.py +++ b/litellm/proxy/hooks/model_max_budget_limiter.py @@ -5,7 +5,7 @@ import litellm from litellm._logging import verbose_proxy_logger from litellm.caching.caching import DualCache from litellm.integrations.custom_logger import Span -from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy._types import Litellm_EntityType, UserAPIKeyAuth from litellm.router_strategy.budget_limiter import RouterBudgetLimiting from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ( @@ -76,6 +76,8 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting): message=f"LiteLLM Virtual Key: {user_api_key_dict.token}, key_alias: {user_api_key_dict.key_alias}, exceeded budget for model={model}", current_cost=_current_spend, max_budget=_current_model_budget_info.max_budget, + entity_type=Litellm_EntityType.KEY.value, + entity_id=user_api_key_dict.token, ) return True @@ -140,6 +142,8 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting): message=f"LiteLLM End User: {end_user_id}, exceeded budget for model={model}", current_cost=_current_spend, max_budget=_current_model_budget_info.max_budget, + entity_type=Litellm_EntityType.END_USER.value, + entity_id=end_user_id, ) return True diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index afd8a437cc1..15d1876e5a2 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -949,6 +949,10 @@ class LiteLLMProxyRequestSetup: user_api_key_alias=user_api_key_dict.key_alias, user_api_key_spend=user_api_key_dict.spend, user_api_key_max_budget=user_api_key_dict.max_budget, + user_api_key_user_spend=user_api_key_dict.user_spend, + user_api_key_user_max_budget=user_api_key_dict.user_max_budget, + user_api_key_team_spend=user_api_key_dict.team_spend, + user_api_key_team_max_budget=user_api_key_dict.team_max_budget, user_api_key_team_id=user_api_key_dict.team_id, user_api_key_project_id=user_api_key_dict.project_id, user_api_key_project_alias=user_api_key_dict.project_alias, diff --git a/litellm/proxy/management_endpoints/mcp_management_endpoints.py b/litellm/proxy/management_endpoints/mcp_management_endpoints.py index 288282dd08b..d920ee474cc 100644 --- a/litellm/proxy/management_endpoints/mcp_management_endpoints.py +++ b/litellm/proxy/management_endpoints/mcp_management_endpoints.py @@ -536,6 +536,7 @@ if MCP_AVAILABLE: sanitized.env = {} sanitized.command = None sanitized.args = [] + sanitized.issuer = None sanitized.authorization_url = None sanitized.token_url = None sanitized.registration_url = None @@ -581,6 +582,7 @@ if MCP_AVAILABLE: sanitized.teams = [] sanitized.env_vars = None + sanitized.issuer = None sanitized.authorization_url = None sanitized.token_url = None sanitized.registration_url = None @@ -686,6 +688,7 @@ if MCP_AVAILABLE: command=payload.command, args=payload.args, env=payload.env, + issuer=payload.issuer, authorization_url=payload.authorization_url, token_url=payload.token_url, registration_url=payload.registration_url, diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index a23cecc3911..f842bf13da9 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -325,6 +325,7 @@ model LiteLLM_MCPServerTable { command String? args String[] @default([]) env Json? @default("{}") + issuer String? authorization_url String? token_url String? registration_url String? diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index e1a093a4a48..80fd8a1594e 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -4,7 +4,7 @@ import asyncio import json from dataclasses import dataclass from datetime import datetime, timedelta, timezone -from typing import Any, Dict, List, Optional, Sequence, cast +from typing import Any, Dict, List, Mapping, Optional, Sequence, cast import litellm from litellm._logging import verbose_proxy_logger @@ -12,6 +12,7 @@ from litellm.caching import DualCache from litellm.litellm_core_utils.duration_parser import duration_in_seconds from litellm.litellm_core_utils.llm_cost_calc.tiered_pricing import select_tier_for_input, tier_rate from litellm.proxy._types import ( + Litellm_EntityType, LiteLLM_TeamMembership, LiteLLM_TeamTable, LiteLLM_UserTable, @@ -36,6 +37,17 @@ class _BudgetCounter: window_start: Optional[datetime] = None +_COUNTER_ENTITY_TYPES: Mapping[str, str] = { + "Key": Litellm_EntityType.KEY.value, + "Team": Litellm_EntityType.TEAM.value, + "TeamMember": Litellm_EntityType.TEAM_MEMBER.value, + "User": Litellm_EntityType.USER.value, + "EndUser": Litellm_EntityType.END_USER.value, + "Tag": Litellm_EntityType.TAG.value, + "Organization": Litellm_EntityType.ORGANIZATION.value, +} + + class _CounterReservationUnavailable(Exception): def __init__( self, @@ -108,6 +120,8 @@ async def _apply_over_budget_reservation_policy( f"Current cost: {current_spend}, " f"Max budget: {counter.max_budget}" ), + entity_type=_COUNTER_ENTITY_TYPES.get(counter.entity_type), + entity_id=counter.spend_log_entity_id or counter.entity_id, ) diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index eb78e6f9c8d..357a7ecefe6 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -718,6 +718,12 @@ class ResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): except StopAsyncIteration: # Normal end of stream - don't log as failure raise + except (httpx.ReadError, httpx.RemoteProtocolError) as e: + self.finished = True + if self.completed_response is None: + self._handle_failure(e) + raise + raise StopAsyncIteration from e except httpx.HTTPError as e: # Handle HTTP errors self.finished = True @@ -794,6 +800,12 @@ class SyncResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): except StopIteration: # Normal end of stream - don't log as failure raise + except (httpx.ReadError, httpx.RemoteProtocolError) as e: + self.finished = True + if self.completed_response is None: + self._handle_failure(e) + raise + raise StopIteration from e except httpx.HTTPError as e: # Handle HTTP errors self.finished = True diff --git a/litellm/router.py b/litellm/router.py index f408d030b8b..809aef695a7 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -251,6 +251,15 @@ else: PreRoutingHookResponse = Any +def _cost_value_as_float(value: Union[str, int, float, None]) -> Optional[float]: + if value is None: + return None + try: + return float(value) + except (TypeError, ValueError): + return None + + class RoutingArgs(enum.Enum): ttl = 60 # 1min (RPM/TPM expire key) @@ -8750,8 +8759,8 @@ class Router: # Get mode from database model_info if available, otherwise default to "chat" db_model_info = model.get("model_info", {}) mode = db_model_info.get("mode", "chat") - input_cost_per_token = db_model_info.get("input_cost_per_token") - output_cost_per_token = db_model_info.get("output_cost_per_token") + input_cost_per_token = _cost_value_as_float(db_model_info.get("input_cost_per_token")) + output_cost_per_token = _cost_value_as_float(db_model_info.get("output_cost_per_token")) model_info = ModelMapInfo( key=model_group, @@ -8802,16 +8811,18 @@ class Router: ) ): model_group_info.max_output_tokens = model_info["max_output_tokens"] - if model_info.get("input_cost_per_token", None) is not None and ( + _input_cost_per_token = _cost_value_as_float(model_info.get("input_cost_per_token")) + if _input_cost_per_token is not None and ( model_group_info.input_cost_per_token is None - or (model_info["input_cost_per_token"] or 0.0) > (model_group_info.input_cost_per_token or 0.0) + or _input_cost_per_token > (model_group_info.input_cost_per_token or 0.0) ): - model_group_info.input_cost_per_token = model_info["input_cost_per_token"] - if model_info.get("output_cost_per_token", None) is not None and ( + model_group_info.input_cost_per_token = _input_cost_per_token + _output_cost_per_token = _cost_value_as_float(model_info.get("output_cost_per_token")) + if _output_cost_per_token is not None and ( model_group_info.output_cost_per_token is None - or (model_info["output_cost_per_token"] or 0.0) > (model_group_info.output_cost_per_token or 0.0) + or _output_cost_per_token > (model_group_info.output_cost_per_token or 0.0) ): - model_group_info.output_cost_per_token = model_info["output_cost_per_token"] + model_group_info.output_cost_per_token = _output_cost_per_token if ( model_info.get("supports_parallel_function_calling", None) is not None and model_info["supports_parallel_function_calling"] is True # type: ignore diff --git a/litellm/types/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/types/litellm_core_utils/streaming_chunk_builder_utils.py index a1f89dac5cf..c9f9d4e6baa 100644 --- a/litellm/types/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/types/litellm_core_utils/streaming_chunk_builder_utils.py @@ -14,3 +14,4 @@ class UsagePerChunk(TypedDict): web_search_requests: Optional[int] completion_tokens_details: Optional[CompletionTokensDetails] prompt_tokens_details: Optional[PromptTokensDetailsWrapper] + cost: Optional[float] diff --git a/litellm/types/mcp_server/mcp_server_manager.py b/litellm/types/mcp_server/mcp_server_manager.py index 801436c774a..d0d8cc4cb28 100644 --- a/litellm/types/mcp_server/mcp_server_manager.py +++ b/litellm/types/mcp_server/mcp_server_manager.py @@ -26,6 +26,11 @@ class MCPOAuthMetadata(BaseModel): authorization_url: Optional[str] = None token_url: Optional[str] = None registration_url: Optional[str] = None + discovered_issuer: Optional[str] = None + """The ``issuer`` the authorization-server metadata document self-attests (RFC 8414). Persisted + trust-on-first-use as the server's ``issuer`` when none is configured, so that later rebuilds + anchor discovery on it (RFC 8414 §3.3) and a subsequently compromised resource cannot re-point + it. Never overwrites an admin-configured issuer.""" from_origin_fallback: bool = False """True when the metadata came from guessing the resource origin as its authorization server rather than from an RFC 9728/8414-advertised document. Guessed endpoints are @@ -60,6 +65,8 @@ class MCPServer(BaseModel): # OAuth-specific fields client_id: Optional[str] = None client_secret: Optional[str] = None + issuer: Optional[str] = None + issuer_is_anchored: bool = False scopes: Optional[List[str]] = None authorization_url: Optional[str] = None token_url: Optional[str] = None diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 6487a8aa33f..db39d58a925 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -1798,14 +1798,17 @@ class ModelResponseStream(ModelResponseBase): else: created = created + usage_to_set = None if "usage" in kwargs and kwargs["usage"] is not None: if isinstance(kwargs["usage"], dict): - kwargs["usage"] = Usage(**kwargs["usage"]) + usage_to_set = Usage(**kwargs["usage"]) + kwargs["usage"] = usage_to_set elif isinstance(kwargs["usage"], BaseModel): dump = ( kwargs["usage"].model_dump() if hasattr(kwargs["usage"], "model_dump") else kwargs["usage"].dict() ) - kwargs["usage"] = Usage(**dump) + usage_to_set = Usage(**dump) + kwargs["usage"] = usage_to_set kwargs["id"] = id kwargs["created"] = created @@ -1814,6 +1817,9 @@ class ModelResponseStream(ModelResponseBase): super().__init__(**kwargs) + if usage_to_set is not None: + self.usage = usage_to_set + def __contains__(self, key): # Define custom behavior for the 'in' operator return hasattr(self, key) @@ -2469,6 +2475,10 @@ class StandardLoggingUserAPIKeyMetadata(TypedDict): user_api_key_spend: Optional[float] user_api_key_max_budget: Optional[float] user_api_key_budget_reset_at: Optional[str] + user_api_key_user_spend: Optional[float] + user_api_key_user_max_budget: Optional[float] + user_api_key_team_spend: Optional[float] + user_api_key_team_max_budget: Optional[float] user_api_key_org_id: Optional[str] user_api_key_org_alias: Optional[str] user_api_key_team_id: Optional[str] @@ -2687,6 +2697,10 @@ class StandardLoggingPayloadErrorInformation(TypedDict, total=False): # hints). Lets dashboards split rate-limit failures by cause without # parsing free-text error messages. error_rate_limit_type: Optional[str] + error_budget_entity_type: Optional[str] + error_budget_entity_id: Optional[str] + error_budget_limit: Optional[float] + error_budget_spend: Optional[float] class GuardrailMode(TypedDict, total=False): diff --git a/schema.prisma b/schema.prisma index a23cecc3911..f842bf13da9 100644 --- a/schema.prisma +++ b/schema.prisma @@ -325,6 +325,7 @@ model LiteLLM_MCPServerTable { command String? args String[] @default([]) env Json? @default("{}") + issuer String? authorization_url String? token_url String? registration_url String? diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index f0d283629b0..0e1eafb5196 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -171,3 +171,16 @@ other... e.g. other.auth.jwt.valid_token_allows other.lifecycle.readiness.reports_db ``` + +## Hard Rules +- no monkeypatching, mock tests or unit tests of any kind. if a contributor asks you to write an end to end test, do NOT stage a unit test with it. if you find a product gap, call it out in the PR description + +- use model management endpoints to create new models for a test. this could be in a conftest / inline for each test. ask the user what they want. + +- do not overengineer a test, i need you to write readable, clean code of what would look like a natural user scenario + +- when it comes to typing an input schema for an api endpoint, have it type X = A | B | C ... where X = exhaustive union of all supported input schemas and A, B, C typically are composed by a base type. types are only pretty for a api request / response body. make sure to compose types instead of repeating the same base attributes over and over again. + +- use the docker-compose to your advantage and spin up a local proxy, make sure all tests pass. if a test fails due to an internally found issue, let users know to create a linear ticket for it. + +- do not use xfail markers, tests should be written in a form that the end user expects it to pass diff --git a/tests/e2e/claude_code/_basic_messaging.py b/tests/e2e/claude_code/_basic_messaging.py index f6b82a38f6a..7c581cc5e38 100644 --- a/tests/e2e/claude_code/_basic_messaging.py +++ b/tests/e2e/claude_code/_basic_messaging.py @@ -27,19 +27,20 @@ collecting this module as a test file. from __future__ import annotations -import os -from typing import Any, Mapping, Sequence +from typing import Any, Callable, Mapping, Sequence import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, + DriverResult, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +ClaudeRunner = Callable[..., Mapping[str, DriverResult | ClaudeCLIError]] # Floor on the number of `stream_event` records (with delta payloads) # we expect to see when the proxy actually streams. With @@ -79,6 +80,8 @@ def run_basic_messaging_cell( models: Sequence[str], prompt: str, verify_streaming: bool = False, + env: Mapping[str, str] | None = None, + runner: ClaudeRunner = run_claude_models_parallel, ) -> None: """Run the shared `basic_messaging_*` × cell body. @@ -99,28 +102,13 @@ def run_basic_messaging_cell( streamed reply to a single ``assistant`` event in ``--print --output-format stream-json`` mode). """ - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", - pytrace=False, - ) + base_url, api_key = require_proxy(compat_result, env=env) extra_args: Sequence[str] = ( ("--include-partial-messages",) if verify_streaming else () ) - outcomes = run_claude_models_parallel( + outcomes = runner( models=models, prompt=prompt, base_url=base_url, diff --git a/tests/e2e/claude_code/_builder_unit_tests/fixtures/expected_matrix.json b/tests/e2e/claude_code/_builder_unit_tests/fixtures/expected_matrix.json index d3aca0142dc..405a3772a90 100644 --- a/tests/e2e/claude_code/_builder_unit_tests/fixtures/expected_matrix.json +++ b/tests/e2e/claude_code/_builder_unit_tests/fixtures/expected_matrix.json @@ -26,7 +26,7 @@ "providers": { "anthropic": { "status": "fail", - "error": "[claude-sonnet-4-6] tool call dropped" + "error": "[claude-sonnet-4-5] tool call dropped" }, "bedrock_invoke": { "status": "not_applicable", diff --git a/tests/e2e/claude_code/_builder_unit_tests/fixtures/results.json b/tests/e2e/claude_code/_builder_unit_tests/fixtures/results.json index a01540c394f..f1b00385f17 100644 --- a/tests/e2e/claude_code/_builder_unit_tests/fixtures/results.json +++ b/tests/e2e/claude_code/_builder_unit_tests/fixtures/results.json @@ -10,7 +10,7 @@ { "feature_id": "basic_messaging_non_streaming", "provider": "anthropic", - "nodeid": "tests/e2e/claude_code/basic_messaging_non_streaming/test_anthropic.py::test_basic_messaging_non_streaming_anthropic[claude-sonnet-4-6]", + "nodeid": "tests/e2e/claude_code/basic_messaging_non_streaming/test_anthropic.py::test_basic_messaging_non_streaming_anthropic[claude-sonnet-4-5]", "result": {"status": "pass"} }, { @@ -28,8 +28,8 @@ { "feature_id": "tool_use", "provider": "anthropic", - "nodeid": "tests/e2e/claude_code/tool_use/test_anthropic.py::test_x[claude-sonnet-4-6]", - "result": {"status": "fail", "error": "[claude-sonnet-4-6] tool call dropped"} + "nodeid": "tests/e2e/claude_code/tool_use/test_anthropic.py::test_x[claude-sonnet-4-5]", + "result": {"status": "fail", "error": "[claude-sonnet-4-5] tool call dropped"} }, { "feature_id": "tool_use", diff --git a/tests/e2e/claude_code/_builder_unit_tests/test_matrix_builder.py b/tests/e2e/claude_code/_builder_unit_tests/test_matrix_builder.py index 9ddbdd29846..95db8acec70 100644 --- a/tests/e2e/claude_code/_builder_unit_tests/test_matrix_builder.py +++ b/tests/e2e/claude_code/_builder_unit_tests/test_matrix_builder.py @@ -393,7 +393,7 @@ def test_build_matrix_6x5_grid_matches_published_sample(): feature_ids = [feature["id"] for feature in manifest["features"]] providers = manifest["providers"] - models = ["claude-haiku-4-5", "claude-sonnet-4-6", "claude-opus-4-7"] + models = ["claude-haiku-4-5", "claude-sonnet-4-5", "claude-opus-4-7"] results = [] for feature_id in feature_ids: diff --git a/tests/e2e/claude_code/_builder_unit_tests/test_v0_layout.py b/tests/e2e/claude_code/_builder_unit_tests/test_v0_layout.py index a3569ebdb49..04e0facff5e 100644 --- a/tests/e2e/claude_code/_builder_unit_tests/test_v0_layout.py +++ b/tests/e2e/claude_code/_builder_unit_tests/test_v0_layout.py @@ -152,7 +152,7 @@ def test_per_provider_test_file_imports_and_parametrizes_three_models( `claude-opus-4-7-bedrock-invoke`), so we check for the tier substrings rather than exact alias names.""" text = (SUITE_ROOT / feature_id / f"test_{provider}.py").read_text() - for tier in ("haiku-4-5", "sonnet-4-6", "opus-4-7"): + for tier in ("haiku-4-5", "sonnet-4-5", "opus-4-7"): assert ( tier in text ), f"{feature_id}/test_{provider}.py does not reference {tier}" diff --git a/tests/e2e/claude_code/_compat_models.py b/tests/e2e/claude_code/_compat_models.py new file mode 100644 index 00000000000..23c9c82e596 --- /dev/null +++ b/tests/e2e/claude_code/_compat_models.py @@ -0,0 +1,86 @@ +"""Load the claude_code compat matrix's deployment list from +``test_config.yaml``. + +``test_config.yaml`` is the ground-truth config the stage deployment +uses; parsing it at fixture time means a change there (new tier, tier +retirement, provider swap, endpoint rename) reaches the fixture with +no extra edit. A drift-check test asserts every ``*_MODELS`` list +referenced by the compat cells is covered by the yaml, so a cell that +adds a probe for a name the yaml doesn't know about fails loudly at +collection instead of at 400-time. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Callable, Mapping + +import yaml + +from models import LiteLLMParamsBody + +CONFIG_PATH = Path(__file__).resolve().parent / "test_config.yaml" + + +@dataclass(frozen=True, slots=True) +class CompatDeployment: + model_name: str + litellm_params: LiteLLMParamsBody + + +# The yaml uses ``vertex_ai_*`` for the vertex project/location fields +# (that is the spelling the proxy config file historically standardized +# on), while ``LiteLLMParamsBody`` names them without the ``_ai`` infix +# (matching the proxy's DB column). Both spellings resolve at call time +# on the proxy side, but pydantic silently drops unknown fields, so a +# raw ``LiteLLMParamsBody(**entry)`` would produce a body with the +# vertex project stripped - the resulting deployment 400s at +# ``/v1/messages`` with "Invalid model name". Normalize the yaml keys +# to the pydantic names in one place. +_YAML_TO_PYDANTIC_ALIASES = { + "vertex_ai_project": "vertex_project", + "vertex_ai_location": "vertex_location", + "vertex_ai_credentials": "vertex_credentials", +} + + +def _normalize_params(raw: Mapping[str, object]) -> dict[str, object]: + return {_YAML_TO_PYDANTIC_ALIASES.get(k, k): v for k, v in raw.items()} + + +ConfigReader = Callable[[Path], str] + + +def _default_reader(path: Path) -> str: + return path.read_text() + + +def load_all_deployments( + config_path: Path = CONFIG_PATH, + reader: ConfigReader = _default_reader, +) -> tuple[CompatDeployment, ...]: + """Every deployment declared in the yaml, in file order.""" + doc = yaml.safe_load(reader(config_path)) + model_list = doc.get("model_list") or [] + return tuple( + CompatDeployment( + model_name=entry["model_name"], + litellm_params=LiteLLMParamsBody( + **_normalize_params(entry["litellm_params"]) + ), + ) + for entry in model_list + ) + + +def all_expected_model_names( + *, + config_path: Path = CONFIG_PATH, + reader: ConfigReader = _default_reader, +) -> frozenset[str]: + """Every virtual name the compat matrix declares - the ground truth + the cells are supposed to probe. Used by the drift-check test.""" + return frozenset( + d.model_name for d in load_all_deployments(config_path, reader) + ) diff --git a/tests/e2e/claude_code/_driver_unit_tests/test_basic_messaging.py b/tests/e2e/claude_code/_driver_unit_tests/test_basic_messaging.py index 018121a8e5c..bb195ac50fe 100644 --- a/tests/e2e/claude_code/_driver_unit_tests/test_basic_messaging.py +++ b/tests/e2e/claude_code/_driver_unit_tests/test_basic_messaging.py @@ -1,20 +1,19 @@ """Unit tests for the shared `run_basic_messaging_cell` helper. -These tests mock `run_claude_models_parallel` so they exercise the -helper's branching (env-missing guard, per-model pass/fail/empty-text, -streaming wire check) without spawning the real CLI. The streaming -check is the regression we care about: a proxy that buffers the -upstream stream must turn the cell red, not green. +These tests inject a fake ``ClaudeRunner`` and a fake env mapping so +they exercise the helper's branching (env-missing guard, per-model +pass/fail/empty-text, streaming wire check) without spawning the real +CLI or touching ``os.environ``. The streaming check is the regression +we care about: a proxy that buffers the upstream stream must turn the +cell red, not green. """ from __future__ import annotations -import os from typing import Any, Dict, List, Mapping, Optional, Sequence import pytest -from claude_code import _basic_messaging from claude_code._basic_messaging import ( MIN_STREAM_DELTA_EVENTS, _count_stream_event_deltas, @@ -23,6 +22,12 @@ from claude_code._basic_messaging import ( from claude_code.cli_driver import DriverResult +_PROXY_ENV: Mapping[str, str] = { + "LITELLM_PROXY_URL": "http://localhost:4000", + "LITELLM_MASTER_KEY": "sk-test", +} + + class _FakeResult: """Stand-in for the test's `compat_result` fixture. @@ -82,16 +87,17 @@ def _buffered_events() -> List[Dict[str, Any]]: ] -def _install_fake_runner(monkeypatch, *, outcomes_by_model): - """Patch `run_claude_models_parallel` to return canned outcomes. +def _make_fake_runner(*, outcomes_by_model): + """Build an injectable runner that returns canned outcomes and + records the kwargs the helper passed in. - Captures the kwargs the cell passed in so tests can assert on - `extra_args` (which is how the streaming variant opts into - `--include-partial-messages`). - """ + Returns a ``(runner, captured)`` pair; ``captured`` is a dict the + test can assert against without any global mutation, which is why + we prefer DI over ``monkeypatch.setattr``: the helper takes a + ``runner=`` kwarg, so tests bind their fake directly.""" captured: Dict[str, Any] = {} - def fake(*, models, prompt, base_url, api_key, extra_args=None, **_kwargs): + def runner(*, models, prompt, base_url, api_key, extra_args=None, **_kwargs): captured["models"] = list(models) captured["prompt"] = prompt captured["base_url"] = base_url @@ -99,14 +105,7 @@ def _install_fake_runner(monkeypatch, *, outcomes_by_model): captured["extra_args"] = list(extra_args) if extra_args else [] return {model: outcomes_by_model[model] for model in models} - monkeypatch.setattr(_basic_messaging, "run_claude_models_parallel", fake) - return captured - - -@pytest.fixture(autouse=True) -def _proxy_env(monkeypatch): - monkeypatch.setenv("LITELLM_PROXY_BASE_URL", "http://localhost:4000") - monkeypatch.setenv("LITELLM_PROXY_API_KEY", "sk-test") + return runner, captured def test_count_stream_event_deltas_only_counts_records_with_event_payload(): @@ -123,28 +122,30 @@ def test_count_stream_event_deltas_only_counts_records_with_event_payload(): assert _count_stream_event_deltas(events) == 2 -def test_verify_streaming_passes_when_proxy_streams(monkeypatch): +def test_verify_streaming_passes_when_proxy_streams(): fake_result = _FakeResult() model = "claude-haiku-4-5" outcome = DriverResult(text="1\n2\n3", events=_streamed_events(n_deltas=5)) - captured = _install_fake_runner(monkeypatch, outcomes_by_model={model: outcome}) + runner, captured = _make_fake_runner(outcomes_by_model={model: outcome}) run_basic_messaging_cell( compat_result=fake_result, models=[model], prompt="Count from 1 to 5, one number per line.", verify_streaming=True, + env=_PROXY_ENV, + runner=runner, ) assert captured["extra_args"] == ["--include-partial-messages"] assert fake_result.rows == [{"status": "pass"}] -def test_verify_streaming_fails_when_proxy_buffers(monkeypatch): +def test_verify_streaming_fails_when_proxy_buffers(): fake_result = _FakeResult() model = "claude-haiku-4-5" outcome = DriverResult(text="1\n2\n3", events=_buffered_events()) - _install_fake_runner(monkeypatch, outcomes_by_model={model: outcome}) + runner, _captured = _make_fake_runner(outcomes_by_model={model: outcome}) with pytest.raises(pytest.fail.Exception): run_basic_messaging_cell( @@ -152,7 +153,9 @@ def test_verify_streaming_fails_when_proxy_buffers(monkeypatch): models=[model], prompt="Count from 1 to 5, one number per line.", verify_streaming=True, - ) + env=_PROXY_ENV, + runner=runner, + ) assert len(fake_result.rows) == 1 row = fake_result.rows[0] @@ -161,33 +164,35 @@ def test_verify_streaming_fails_when_proxy_buffers(monkeypatch): assert f"< {MIN_STREAM_DELTA_EVENTS}" in row["error"] -def test_non_streaming_variant_omits_partial_messages_flag(monkeypatch): +def test_non_streaming_variant_omits_partial_messages_flag(): """Default `verify_streaming=False` keeps the non-streaming wire identical.""" fake_result = _FakeResult() model = "claude-haiku-4-5" outcome = DriverResult(text="pong", events=_buffered_events()) - captured = _install_fake_runner(monkeypatch, outcomes_by_model={model: outcome}) + runner, captured = _make_fake_runner(outcomes_by_model={model: outcome}) run_basic_messaging_cell( compat_result=fake_result, models=[model], prompt="Reply with the single word 'pong' and nothing else.", + env=_PROXY_ENV, + runner=runner, ) assert captured["extra_args"] == [] assert fake_result.rows == [{"status": "pass"}] -def test_verify_streaming_requires_all_models_to_stream(monkeypatch): +def test_verify_streaming_requires_all_models_to_stream(): """If any one tier buffers, the cell fails — same all-must-pass shape as the non-streaming check.""" fake_result = _FakeResult() outcomes = { "claude-haiku-4-5": DriverResult(text="ok", events=_streamed_events(5)), - "claude-sonnet-4-6": DriverResult(text="ok", events=_buffered_events()), + "claude-sonnet-4-5": DriverResult(text="ok", events=_buffered_events()), "claude-opus-4-7": DriverResult(text="ok", events=_streamed_events(5)), } - _install_fake_runner(monkeypatch, outcomes_by_model=outcomes) + runner, _captured = _make_fake_runner(outcomes_by_model=outcomes) with pytest.raises(pytest.fail.Exception): run_basic_messaging_cell( @@ -195,7 +200,30 @@ def test_verify_streaming_requires_all_models_to_stream(monkeypatch): models=list(outcomes.keys()), prompt="Count from 1 to 5, one number per line.", verify_streaming=True, - ) + env=_PROXY_ENV, + runner=runner, + ) statuses = [row["status"] for row in fake_result.rows] assert statuses == ["pass", "fail", "pass"] + + +def test_missing_proxy_env_hard_fails_regardless_of_runner(): + """The env guard fires before the runner is called, and takes the + env from the injected mapping (not os.environ). Passing an empty + env dict must hard-fail even if a happy runner is bound.""" + fake_result = _FakeResult() + runner, captured = _make_fake_runner(outcomes_by_model={}) + + with pytest.raises(pytest.fail.Exception): + run_basic_messaging_cell( + compat_result=fake_result, + models=["claude-haiku-4-5"], + prompt="whatever", + env={}, + runner=runner, + ) + + assert captured == {}, "runner must not be called when env resolution fails" + + diff --git a/tests/e2e/claude_code/_driver_unit_tests/test_cli_driver.py b/tests/e2e/claude_code/_driver_unit_tests/test_cli_driver.py index f1a0534906e..ba6b9502c89 100644 --- a/tests/e2e/claude_code/_driver_unit_tests/test_cli_driver.py +++ b/tests/e2e/claude_code/_driver_unit_tests/test_cli_driver.py @@ -140,7 +140,7 @@ def test_run_claude_inherits_only_allowlisted_os_environ(monkeypatch): monkeypatch.setenv("HOME", "/home/runner") monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "totally-secret") monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-proxy-only") - monkeypatch.setenv("AZURE_FOUNDRY_API_KEY", "azure-secret") + monkeypatch.setenv("AZURE_AI_API_KEY", "azure-secret") monkeypatch.setenv("VERTEXAI_CREDENTIALS", '{"private_key": "leak"}') monkeypatch.setenv("GITHUB_TOKEN", "ghs_xxx") @@ -156,7 +156,7 @@ def test_run_claude_inherits_only_allowlisted_os_environ(monkeypatch): assert env["PATH"] == "/usr/bin:/usr/local/bin" assert "AWS_SECRET_ACCESS_KEY" not in env assert "ANTHROPIC_API_KEY" not in env - assert "AZURE_FOUNDRY_API_KEY" not in env + assert "AZURE_AI_API_KEY" not in env assert "VERTEXAI_CREDENTIALS" not in env assert "GITHUB_TOKEN" not in env diff --git a/tests/e2e/claude_code/_driver_unit_tests/test_passthrough.py b/tests/e2e/claude_code/_driver_unit_tests/test_passthrough.py index 2d17a84d418..7c6c4f189ca 100644 --- a/tests/e2e/claude_code/_driver_unit_tests/test_passthrough.py +++ b/tests/e2e/claude_code/_driver_unit_tests/test_passthrough.py @@ -20,6 +20,10 @@ from typing import Any, Dict, List, Mapping, Optional import pytest +from claude_code._env import ( + PRIMARY_API_KEY_ENV, + PRIMARY_BASE_URL_ENV, +) from claude_code._passthrough import ( ANTHROPIC_PASSTHROUGH_BASE_PATH, CLIENT_SIDE_AWS_REGION, @@ -33,8 +37,8 @@ from claude_code._passthrough import ( from claude_code.cli_driver import ClaudeCLIError, DriverResult PROXY_ENV = { - "LITELLM_PROXY_BASE_URL": "http://localhost:4000", - "LITELLM_PROXY_API_KEY": "sk-test", + PRIMARY_BASE_URL_ENV: "http://localhost:4000", + PRIMARY_API_KEY_ENV: "sk-test", } @@ -73,7 +77,29 @@ def test_env_missing_guard_reports_fail_and_aborts(): ) assert fake_result.single is not None assert fake_result.single["status"] == "fail" - assert "LITELLM_PROXY_BASE_URL" in fake_result.single["error"] + assert PRIMARY_BASE_URL_ENV in fake_result.single["error"] + assert PRIMARY_API_KEY_ENV in fake_result.single["error"] + + +def test_suite_wide_env_reaches_the_proxy(): + """Passthrough cells resolve via the same suite-wide env names as + every other e2e cell, so EKS wiring that only exports + LITELLM_PROXY_URL + LITELLM_MASTER_KEY reaches the ALB.""" + fake_result = _FakeResult() + captured: Dict[str, Any] = {} + outcome = DriverResult(text="pong") + + run_passthrough_cell( + compat_result=fake_result, + models=["claude-haiku-4-5"], + prompt="ping", + run_models=_fake_run_models({"claude-haiku-4-5": outcome}, captured), + env=PROXY_ENV, + ) + + assert captured["base_url"] == "http://localhost:4000" + assert captured["api_key"] == "sk-test" + assert fake_result.rows == [{"status": "pass"}] def test_anthropic_base_path_appended_to_normalized_proxy_url(): @@ -87,7 +113,7 @@ def test_anthropic_base_path_appended_to_normalized_proxy_url(): prompt="ping", passthrough_base_path=ANTHROPIC_PASSTHROUGH_BASE_PATH, run_models=_fake_run_models({"claude-haiku-4-5": outcome}, captured), - env={**PROXY_ENV, "LITELLM_PROXY_BASE_URL": "http://localhost:4000/"}, + env={**PROXY_ENV, PRIMARY_BASE_URL_ENV: "http://localhost:4000/"}, ) assert captured["base_url"] == "http://localhost:4000/anthropic" @@ -111,7 +137,7 @@ def test_extra_env_builder_receives_normalized_base_and_is_forwarded(): prompt="ping", build_extra_env=build, run_models=_fake_run_models({"claude-haiku-4-5": outcome}, captured), - env={**PROXY_ENV, "LITELLM_PROXY_BASE_URL": "http://localhost:4000/"}, + env={**PROXY_ENV, PRIMARY_BASE_URL_ENV: "http://localhost:4000/"}, ) assert seen_bases == ["http://localhost:4000"] @@ -124,7 +150,7 @@ def test_per_model_failures_reported_individually(): captured: Dict[str, Any] = {} outcomes = { "claude-haiku-4-5": DriverResult(text="pong"), - "claude-sonnet-4-6": ClaudeCLIError("claude CLI timed out after 120s"), + "claude-sonnet-4-5": ClaudeCLIError("claude CLI timed out after 120s"), "claude-opus-4-7": DriverResult(text="", exit_code=1), } diff --git a/tests/e2e/claude_code/_driver_unit_tests/test_rate_limiter.py b/tests/e2e/claude_code/_driver_unit_tests/test_rate_limiter.py index 92907eda3c4..32a1c30af98 100644 --- a/tests/e2e/claude_code/_driver_unit_tests/test_rate_limiter.py +++ b/tests/e2e/claude_code/_driver_unit_tests/test_rate_limiter.py @@ -60,10 +60,10 @@ from claude_code.rate_limiter import ( "model, expected", [ ("claude-haiku-4-5", PROVIDER_ANTHROPIC), - ("claude-sonnet-4-6", PROVIDER_ANTHROPIC), + ("claude-sonnet-4-5", PROVIDER_ANTHROPIC), ("claude-opus-4-7", PROVIDER_ANTHROPIC), ("claude-haiku-4-5-azure", PROVIDER_AZURE), - ("claude-sonnet-4-6-azure", PROVIDER_AZURE), + ("claude-sonnet-4-5-azure", PROVIDER_AZURE), ("claude-opus-4-7-vertex", PROVIDER_VERTEX_AI), ("claude-haiku-4-5-bedrock-converse", PROVIDER_BEDROCK_CONVERSE), ("claude-haiku-4-5-bedrock-invoke", PROVIDER_BEDROCK_INVOKE), diff --git a/tests/e2e/claude_code/_env.py b/tests/e2e/claude_code/_env.py new file mode 100644 index 00000000000..4f93cb57fdc --- /dev/null +++ b/tests/e2e/claude_code/_env.py @@ -0,0 +1,74 @@ +"""Proxy-env resolution for the claude_code compat cells. + +Uses the same ``LITELLM_PROXY_URL`` / ``LITELLM_MASTER_KEY`` names as +``e2e_config.py`` and the rest of ``tests/e2e/``. Everything under +``claude_code/`` goes through ``resolve_proxy`` / ``require_proxy`` here +so the naming lives in one place. +""" + +from __future__ import annotations + +import os +from typing import Mapping, NamedTuple + +import pytest + + +class ProxyConfig(NamedTuple): + base_url: str + api_key: str + + +PRIMARY_BASE_URL_ENV = "LITELLM_PROXY_URL" +PRIMARY_API_KEY_ENV = "LITELLM_MASTER_KEY" + + +def resolve_proxy_from(mapping: Mapping[str, str]) -> ProxyConfig | None: + """Pure resolver: takes an env mapping, returns a ProxyConfig if + both a base URL and an API key are present, else None. Extracted so + tests can exercise it without mutating ``os.environ``.""" + base_url = mapping.get(PRIMARY_BASE_URL_ENV) or None + api_key = mapping.get(PRIMARY_API_KEY_ENV) or None + if not base_url or not api_key: + return None + return ProxyConfig(base_url=base_url, api_key=api_key) + + +def resolve_proxy(env: Mapping[str, str] | None = None) -> ProxyConfig | None: + """Convenience wrapper that defaults to ``os.environ``. Prefer + calling ``resolve_proxy_from(env)`` from tests so nothing has to + reach into the process environment.""" + return resolve_proxy_from(os.environ if env is None else env) + + +def _fail_missing_proxy_env(compat_result) -> None: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PRIMARY_BASE_URL_ENV} and " + f"{PRIMARY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PRIMARY_BASE_URL_ENV} / {PRIMARY_API_KEY_ENV} not configured", + pytrace=False, + ) + + +def require_proxy( + compat_result, + *, + env: Mapping[str, str] | None = None, +) -> ProxyConfig: + """Return the proxy config (base URL + master key), or hard-fail + the test. + + ``env`` is injected for tests; production callers pass nothing and + the process env is used. This keeps tests off ``monkeypatch.setenv`` + for a check that is a pure function of its inputs.""" + cfg = resolve_proxy(env) + if cfg is None: + _fail_missing_proxy_env(compat_result) + return cfg diff --git a/tests/e2e/claude_code/_passthrough.py b/tests/e2e/claude_code/_passthrough.py index 24b6d694d3f..be7a475dff7 100644 --- a/tests/e2e/claude_code/_passthrough.py +++ b/tests/e2e/claude_code/_passthrough.py @@ -54,20 +54,17 @@ them unset. from __future__ import annotations -import os from typing import Any, Callable, Dict, Mapping, Optional, Sequence import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" - ANTHROPIC_PASSTHROUGH_BASE_PATH = "/anthropic" CLIENT_SIDE_AWS_REGION = "us-east-1" @@ -140,32 +137,15 @@ def run_passthrough_cell( the trailing-slash-normalized proxy base URL and returns the provider-mode env for the CLI subprocess. """ - environ = env if env is not None else os.environ - base_url = environ.get(PROXY_BASE_URL_ENV) - api_key = environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", - pytrace=False, - ) - - proxy_base = base_url.rstrip("/") + proxy = require_proxy(compat_result, env=env) + proxy_base = proxy.base_url.rstrip("/") extra_env = dict(build_extra_env(proxy_base)) if build_extra_env else None outcomes = run_models( models=models, prompt=prompt, base_url=proxy_base + passthrough_base_path, - api_key=api_key, + api_key=proxy.api_key, extra_env=extra_env, ) diff --git a/tests/e2e/claude_code/_pr_gate_unit_tests/test_bash_tool_restrictions.py b/tests/e2e/claude_code/_pr_gate_unit_tests/test_bash_tool_restrictions.py index d698131670a..ebd6d436d71 100644 --- a/tests/e2e/claude_code/_pr_gate_unit_tests/test_bash_tool_restrictions.py +++ b/tests/e2e/claude_code/_pr_gate_unit_tests/test_bash_tool_restrictions.py @@ -35,8 +35,7 @@ from typing import Iterable import pytest -REPO_ROOT = Path(__file__).resolve().parents[4] -CLAUDE_CODE_DIR = REPO_ROOT / "tests" / "e2e" / "claude_code" +CLAUDE_CODE_DIR = Path(__file__).resolve().parents[1] # Feature directories whose cells drive the `Bash` built-in tool. Add # new entries here when a new Bash-using feature is added; the test @@ -74,14 +73,14 @@ def _has_bare_bash_token(text: str) -> bool: @pytest.mark.parametrize( - "cell", list(_bash_cells()), ids=lambda p: str(p.relative_to(REPO_ROOT)) + "cell", list(_bash_cells()), ids=lambda p: str(p.relative_to(CLAUDE_CODE_DIR)) ) def test_bash_allow_rule_is_pinned_to_exact_echo_pong(cell: Path) -> None: """The cell must pass `Bash(echo pong)` as the allow rule, not the unrestricted `Bash` value that was originally flagged.""" text = cell.read_text() assert '"Bash(echo pong)"' in text, ( - f"{cell.relative_to(REPO_ROOT)} must restrict `--allowed-tools` to " + f"{cell.relative_to(CLAUDE_CODE_DIR)} must restrict `--allowed-tools` to " f'`Bash(echo pong)` (exact-match pattern). Unrestricted `"Bash"` ' f"grants arbitrary host command execution to model-controlled " f"tool_use blocks, which can read `docker inspect compat-proxy` " @@ -95,7 +94,7 @@ def test_bash_allow_rule_is_pinned_to_exact_echo_pong(cell: Path) -> None: # in text` short-circuits to True and lets a stray bare `"Bash"` # slip through silently. assert not _has_bare_bash_token(text), ( - f"{cell.relative_to(REPO_ROOT)} still references the unrestricted " + f"{cell.relative_to(CLAUDE_CODE_DIR)} still references the unrestricted " f'`"Bash"` value outside the `"Bash(echo pong)"` allow rule — ' f"sweep it out before merging." ) @@ -130,7 +129,7 @@ def test_has_bare_bash_token_ignores_unrelated_substrings(): @pytest.mark.parametrize( - "cell", list(_bash_cells()), ids=lambda p: str(p.relative_to(REPO_ROOT)) + "cell", list(_bash_cells()), ids=lambda p: str(p.relative_to(CLAUDE_CODE_DIR)) ) def test_bash_cell_uses_dontask_permission_mode(cell: Path) -> None: """The cell must pair the allow rule with `--permission-mode dontAsk` @@ -139,9 +138,25 @@ def test_bash_cell_uses_dontask_permission_mode(cell: Path) -> None: succeed without ever surfacing the security issue).""" text = cell.read_text() assert '"--permission-mode"' in text and '"dontAsk"' in text, ( - f"{cell.relative_to(REPO_ROOT)} must pass `--permission-mode dontAsk` " + f"{cell.relative_to(CLAUDE_CODE_DIR)} must pass `--permission-mode dontAsk` " f"alongside the `Bash(echo pong)` allow rule. Without dontAsk, " f"commands outside the allow rule fall back to the default ask-" f"mode behavior, which in `--print` (headless) mode is non-" f"interactive — defeating the explicit-allow contract." ) + + +def test_claude_code_dir_anchor_is_layout_independent() -> None: + """CLAUDE_CODE_DIR must resolve to the `claude_code/` directory that + contains this test file, regardless of how deep the repository is + mounted. The previous anchor `Path(__file__).resolve().parents[4]` + baked in the host layout (repo root sits four levels up) and broke + when the suite runs inside the stage container, where tests/e2e/ is + mounted at /app/e2e/ so `parents[4]` resolves to filesystem root and + the BASH_FEATURE_DIRS assertion looks for `/tests/e2e/claude_code/ + tool_use`. Anchoring at `parents[1]` (the sibling of this file's + parent) is the same directory in both layouts. + """ + assert CLAUDE_CODE_DIR.name == "claude_code" + assert CLAUDE_CODE_DIR.is_dir() + assert (CLAUDE_CODE_DIR / "_pr_gate_unit_tests" / Path(__file__).name).is_file() diff --git a/tests/e2e/claude_code/_pr_gate_unit_tests/test_compat_models.py b/tests/e2e/claude_code/_pr_gate_unit_tests/test_compat_models.py new file mode 100644 index 00000000000..6614e75e2f4 --- /dev/null +++ b/tests/e2e/claude_code/_pr_gate_unit_tests/test_compat_models.py @@ -0,0 +1,166 @@ +"""Regression tests for the compat-model registration loader. + +The compat cells hardcode virtual model names like ``claude-sonnet-4-5`` +and expect them to be registered on the proxy before the cell runs. The +session fixture in ``conftest.py`` reads ``test_config.yaml`` and POSTs +those deployments via ``/model/new``. These tests pin the invariants +that make that safe: + +- The yaml declares an entry for every virtual name a cell references - + otherwise a cell probes a name the fixture never registered, and the + cell hits an ``Invalid model name`` 400 that is much harder to trace. + +- The ``vertex_ai_*`` yaml keys get normalized to the ``vertex_*`` + pydantic-body names before ``LiteLLMParamsBody(**)`` sees them, so + the vertex project/location aren't silently dropped by pydantic's + ``extra="ignore"`` default. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +import pytest + +from claude_code._compat_models import ( + all_expected_model_names, + load_all_deployments, +) + + +CLAUDE_CODE_DIR = Path(__file__).resolve().parents[1] + + +def _cell_declared_model_names() -> frozenset[str]: + """Every ``"claude-*"`` model name a compat cell hardcodes in a + ``*_MODELS`` list. Uses a simple regex rather than importing every + cell because the cells depend on the harness which depends on env + the unit-test run does not have.""" + pattern = re.compile(r'"(claude-[a-zA-Z0-9._-]+)"') + found: set[str] = set() + for path in CLAUDE_CODE_DIR.glob("*/test_*.py"): + if path.parent.name.startswith("_"): + continue + for match in pattern.finditer(path.read_text()): + name = match.group(1) + # Skip upstream model references (they carry a version + # suffix or the ``anthropic/`` provider prefix - we only + # want proxy-side virtual names here). + if "/" in name or "@" in name: + continue + found.add(name) + return frozenset(found) + + +def test_yaml_covers_every_cell_declared_model_name() -> None: + """Every ``"claude-..."`` string a cell probes must have a + corresponding ``model_list`` entry in ``test_config.yaml``. A new + cell that adds a probe for a name the yaml doesn't know fails this + test - the alternative is a 400 at runtime that is much harder to + diagnose.""" + yaml_names = all_expected_model_names() + cell_names = _cell_declared_model_names() + missing = cell_names - yaml_names + assert not missing, ( + f"compat cells reference model names not declared in " + f"test_config.yaml: {sorted(missing)}. Add a matching " + f"model_list entry so the session fixture can register them." + ) + + +def test_yaml_has_no_unused_declarations() -> None: + """Every declaration in ``test_config.yaml`` is referenced by at + least one cell. A yaml entry no test exercises is dead + configuration and drift-prone; delete it or add the cell.""" + yaml_names = all_expected_model_names() + cell_names = _cell_declared_model_names() + unused = yaml_names - cell_names + assert not unused, ( + f"test_config.yaml declares model names no cell references: " + f"{sorted(unused)}. Delete them or add the cell." + ) + + +def test_load_returns_fifteen_deployments() -> None: + """The compat matrix is 3 tiers x 5 provider surfaces = 15. Pin the + count so a future edit to the yaml can't silently drop a tier.""" + assert len(load_all_deployments()) == 15 + + +def test_deployments_are_hashable_and_frozen() -> None: + """``CompatDeployment`` is frozen so tests cannot accidentally + mutate the shared list mid-session.""" + d = load_all_deployments()[0] + with pytest.raises((AttributeError, TypeError)): + d.model_name = "mutated" # type: ignore[misc] + + +def test_vertex_yaml_keys_populate_pydantic_body() -> None: + """The yaml spells vertex fields ``vertex_ai_project`` / + ``vertex_ai_location`` but ``LiteLLMParamsBody`` names them + ``vertex_project`` / ``vertex_location``. Without the alias + normalization the pydantic body silently drops the yaml keys, and + the deployment gets registered with no vertex project - a real + incident the drift regressed twice historically.""" + all_deployments = load_all_deployments() + vertex = [ + d for d in all_deployments if d.model_name.endswith("-vertex") + ] + assert vertex, "no vertex deployments found in yaml" + for d in vertex: + assert d.litellm_params.vertex_project, ( + f"{d.model_name} lost its vertex_project after normalization" + ) + assert d.litellm_params.vertex_location, ( + f"{d.model_name} lost its vertex_location after normalization" + ) + + +def test_vertex_deployments_keep_use_in_pass_through() -> None: + """Vertex passthrough cells need the deployment registered with + ``use_in_pass_through: true`` so the proxy wires project/location + credentials into the /vertex_ai passthrough router. ``LiteLLMParamsBody`` + defaults to ``extra="ignore"``, so a missing field on the body silently + strips the yaml flag and every vertex passthrough cell fails at runtime + with "No credentials found on proxy for project_name=...".""" + vertex = [ + d + for d in load_all_deployments() + if d.model_name.endswith("-vertex") + ] + assert vertex, "no vertex deployments found in yaml" + for d in vertex: + assert d.litellm_params.use_in_pass_through is True, ( + f"{d.model_name} lost use_in_pass_through after load; " + f"serialized body would be " + f"{d.litellm_params.model_dump(exclude_none=True)}" + ) + + +def test_yaml_litellm_params_are_all_known_body_fields() -> None: + """Every key under ``litellm_params`` in ``test_config.yaml`` must map + to a ``LiteLLMParamsBody`` field (after the vertex alias rewrite). + Without this pin, a new yaml flag can land in the fixture config and + be silently dropped by pydantic before ``/model/new`` ever sees it.""" + import yaml + from models import LiteLLMParamsBody + + from claude_code._compat_models import ( + CONFIG_PATH, + _YAML_TO_PYDANTIC_ALIASES, + ) + + known = frozenset(LiteLLMParamsBody.model_fields) + doc = yaml.safe_load(CONFIG_PATH.read_text()) + model_list = doc.get("model_list") or [] + unknown = tuple( + (entry["model_name"], key) + for entry in model_list + for key in entry["litellm_params"] + if _YAML_TO_PYDANTIC_ALIASES.get(key, key) not in known + ) + assert not unknown, ( + f"test_config.yaml litellm_params keys not on LiteLLMParamsBody " + f"(will be silently dropped at register time): {unknown}" + ) diff --git a/tests/e2e/claude_code/_pr_gate_unit_tests/test_env_resolution.py b/tests/e2e/claude_code/_pr_gate_unit_tests/test_env_resolution.py new file mode 100644 index 00000000000..f885b4baae2 --- /dev/null +++ b/tests/e2e/claude_code/_pr_gate_unit_tests/test_env_resolution.py @@ -0,0 +1,162 @@ +"""Regression tests for ``claude_code/_env.py``. + +Pin the resolution rules so a future edit cannot silently reintroduce +a private spelling that makes every ``claude_code`` cell fail with +"not configured" even when the surrounding e2e suite has a live proxy +configured under the suite-wide ``LITELLM_PROXY_URL`` / +``LITELLM_MASTER_KEY`` names. +""" + +from __future__ import annotations + +import pytest + +from claude_code._env import ( + PRIMARY_API_KEY_ENV, + PRIMARY_BASE_URL_ENV, + ProxyConfig, + require_proxy, + resolve_proxy_from, +) + + +class _CompatResultStub: + """Minimal stand-in for the compat_result fixture used by cells.""" + + def __init__(self) -> None: + self.calls: list[dict[str, str]] = [] + + def set(self, payload: dict[str, str]) -> None: + self.calls.append(payload) + + +def test_primary_env_names_match_suite_wide_config() -> None: + """The names claude_code reads must exactly match the ones + ``e2e_config.py`` reads for the rest of the suite. Anything else + silently reintroduces the drift this refactor cleaned up.""" + assert PRIMARY_BASE_URL_ENV == "LITELLM_PROXY_URL" + assert PRIMARY_API_KEY_ENV == "LITELLM_MASTER_KEY" + + +def test_returns_none_when_no_env_is_set() -> None: + assert resolve_proxy_from({}) is None + + +def test_returns_none_when_only_url_is_set() -> None: + assert ( + resolve_proxy_from({PRIMARY_BASE_URL_ENV: "http://localhost:4000"}) is None + ) + + +def test_returns_none_when_only_key_is_set() -> None: + assert resolve_proxy_from({PRIMARY_API_KEY_ENV: "sk-1234"}) is None + + +def test_primary_pair_resolves() -> None: + cfg = resolve_proxy_from( + { + PRIMARY_BASE_URL_ENV: "http://localhost:4000", + PRIMARY_API_KEY_ENV: "sk-1234", + } + ) + assert cfg == ProxyConfig("http://localhost:4000", "sk-1234") + + +def test_legacy_pair_is_ignored() -> None: + """``LITELLM_PROXY_BASE_URL`` / ``LITELLM_PROXY_API_KEY`` are not + accepted. A runner that only exports those must fail closed rather + than silently use a private spelling that the rest of the suite + does not know about.""" + assert ( + resolve_proxy_from( + { + "LITELLM_PROXY_BASE_URL": "http://legacy:4000", + "LITELLM_PROXY_API_KEY": "sk-legacy", + } + ) + is None + ) + + +def test_empty_string_env_is_treated_as_unset() -> None: + """``os.environ.get`` on an exported-but-empty var returns "" which + is falsy. The resolver must treat that as unset so a shell that + accidentally exports ``LITELLM_PROXY_URL=`` doesn't turn into a + "" base_url that hits the wrong endpoint.""" + assert ( + resolve_proxy_from( + {PRIMARY_BASE_URL_ENV: "", PRIMARY_API_KEY_ENV: "sk-1234"} + ) + is None + ) + + +def test_require_proxy_fails_with_helpful_message_when_env_empty() -> None: + """The error the user sees must name the suite-wide env vars so + they know exactly what to export.""" + compat = _CompatResultStub() + with pytest.raises(pytest.fail.Exception) as excinfo: + require_proxy(compat, env={}) + assert PRIMARY_BASE_URL_ENV in str(excinfo.value) + assert PRIMARY_API_KEY_ENV in str(excinfo.value) + assert compat.calls and compat.calls[0]["status"] == "fail" + assert PRIMARY_BASE_URL_ENV in compat.calls[0]["error"] + assert PRIMARY_API_KEY_ENV in compat.calls[0]["error"] + assert "LITELLM_PROXY_BASE_URL" not in compat.calls[0]["error"] + + +def test_require_proxy_returns_config_when_primary_env_supplied() -> None: + cfg = require_proxy( + _CompatResultStub(), + env={ + PRIMARY_BASE_URL_ENV: "http://localhost:4000", + PRIMARY_API_KEY_ENV: "sk-1234", + }, + ) + assert cfg == ProxyConfig("http://localhost:4000", "sk-1234") + + +class TestControlGatewayFollowsResolvedProxy: + """The session fixture that registers the compat deployments must talk + to the *same* proxy the cells do. + + Building its Gateway off ``e2e_config``'s own env read instead of the + resolved ``ProxyConfig`` would send ``/model/new`` to whatever + ``e2e_config`` defaults to when the process env is empty, while the + cells drive a different host — so registration silently lands + somewhere else and every cell 400s with "Invalid model name". + """ + + RESOLVED = ProxyConfig("http://eks-alb.internal:4000", "sk-eks") + + def _gateway(self): + from claude_code.conftest import _build_control_gateway + + return _build_control_gateway(self.RESOLVED) + + def test_management_calls_go_to_the_resolved_host_and_key(self) -> None: + control = self._gateway().transport.control + assert control.base_url == self.RESOLVED.base_url + assert control.master_key == self.RESOLVED.api_key + + def test_both_planes_share_the_one_address_the_cells_use(self) -> None: + """The deployment is fronted by a single address that routes + management and LLM paths itself, so a resolved proxy pins both.""" + transport = self._gateway().transport + assert transport.data.base_url == self.RESOLVED.base_url + assert transport.data.master_key == self.RESOLVED.api_key + assert transport.control.base_url == transport.data.base_url + + +def test_require_proxy_leaves_compat_result_untouched_on_success() -> None: + """A successful resolution must NOT append a spurious fail entry. + Would have silently poisoned every compat cell's result rows.""" + compat = _CompatResultStub() + require_proxy( + compat, + env={ + PRIMARY_BASE_URL_ENV: "http://localhost:4000", + PRIMARY_API_KEY_ENV: "sk-1234", + }, + ) + assert compat.calls == [] diff --git a/tests/e2e/claude_code/_publisher_unit_tests/__init__.py b/tests/e2e/claude_code/_publisher_unit_tests/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/e2e/claude_code/_publisher_unit_tests/test_run_daily_pytest_scrubs_env.py b/tests/e2e/claude_code/_publisher_unit_tests/test_run_daily_pytest_scrubs_env.py deleted file mode 100644 index 418d308674b..00000000000 --- a/tests/e2e/claude_code/_publisher_unit_tests/test_run_daily_pytest_scrubs_env.py +++ /dev/null @@ -1,95 +0,0 @@ -"""Pin: the cron `pytest` invocation must run under `env -i`. - -The systemd service `litellm-compat-matrix.service` loads provider -credentials (`ANTHROPIC_API_KEY`, `AWS_BEARER_TOKEN_BEDROCK`, -`AZURE_FOUNDRY_API_KEY`, `VERTEXAI_*`) and the agent-shin GitHub token -(`AGENT_SHIN_GITHUB_TOKEN`) into `run_daily.sh`'s environment from -`/etc/litellm-compat-matrix.env`. Pytest only needs to talk to the -loopback proxy at `127.0.0.1:${PROXY_PORT}` and has no legitimate reason -to see provider creds in its own `os.environ`. Leaving them in would -let a test under `tests/e2e/claude_code/` read them via `os.environ` and -exfiltrate them, and would also let a model-directed `Read` tool call -during a PDF/vision cell reach `/proc//environ`. The -PR-gate's pytest step in `.circleci/config.yml` already runs under -`env -i`; this pin enforces the same scrub on the cron path. -""" - -from __future__ import annotations - -from pathlib import Path - -REPO_ROOT = Path(__file__).resolve().parents[4] -RUN_DAILY = REPO_ROOT / "tests" / "e2e" / "claude_code" / "cron_vm" / "run_daily.sh" - - -def _pytest_invocation_block() -> str: - """Return only the executable lines around the pytest invocation. - - Comment text in run_daily.sh explains *why* certain credential - names must not appear, so a naïve substring scan over the whole - region would false-positive on the rationale itself. Strip lines - whose first non-space character is `#`. - """ - body = RUN_DAILY.read_text() - start = body.index('log "running pytest"') - end = body.index("PYTEST_EXIT=$?", start) - return "\n".join( - line for line in body[start:end].splitlines() - if line.lstrip()[:1] != "#" - ) - - -def test_pytest_invocation_wraps_in_env_i() -> None: - block = _pytest_invocation_block() - assert "env -i" in block, ( - "run_daily.sh: the pytest invocation must run under `env -i` so " - "PR-controlled test code under tests/e2e/claude_code/ cannot read " - "provider/agent-shin credentials out of the systemd service " - "environment, and so a model-directed `Read` tool call cannot " - "reach /proc//environ to pull them out." - ) - assert block.index("env -i") < block.index('"${WORKTREE_UV}" run pytest'), ( - "run_daily.sh: `env -i` must precede the pytest invocation; " - "otherwise pytest inherits the full credential-bearing env." - ) - - -def test_pytest_invocation_env_i_excludes_provider_secrets() -> None: - block = _pytest_invocation_block() - for forbidden in ( - "ANTHROPIC_API_KEY", - "AWS_BEARER_TOKEN_BEDROCK", - "AWS_ACCESS_KEY_ID", - "AWS_SECRET_ACCESS_KEY", - "VERTEXAI_CREDENTIALS", - "VERTEXAI_PROJECT", - "VERTEXAI_LOCATION", - "AZURE_FOUNDRY_API_KEY", - "AZURE_FOUNDRY_API_BASE", - "GITHUB_TOKEN", - "AGENT_SHIN_GITHUB_TOKEN", - ): - assert forbidden not in block, ( - f"run_daily.sh: the pytest-step `env -i` allowlist must not " - f"pass {forbidden} through. Found it inside the pytest " - f"invocation block." - ) - - -def test_pytest_invocation_passes_proxy_url_and_key_explicitly() -> None: - block = _pytest_invocation_block() - assert "LITELLM_PROXY_BASE_URL=" in block, ( - "run_daily.sh: the pytest `env -i` block must still pass " - "LITELLM_PROXY_BASE_URL so the test suite knows where to find " - "the loopback proxy." - ) - assert "LITELLM_PROXY_API_KEY=" in block, ( - "run_daily.sh: the pytest `env -i` block must still pass " - "LITELLM_PROXY_API_KEY so the test suite can authenticate to " - "the loopback proxy." - ) - assert "COMPAT_RESULTS_PATH=" in block, ( - "run_daily.sh: the pytest `env -i` block must still pass " - "COMPAT_RESULTS_PATH so the conftest writes the per-cell " - "tagged-union artifact to the script-managed path." - ) diff --git a/tests/e2e/claude_code/_publisher_unit_tests/test_run_daily_release_pagination.py b/tests/e2e/claude_code/_publisher_unit_tests/test_run_daily_release_pagination.py deleted file mode 100644 index fc733845ba3..00000000000 --- a/tests/e2e/claude_code/_publisher_unit_tests/test_run_daily_release_pagination.py +++ /dev/null @@ -1,289 +0,0 @@ -"""Regression tests for the GitHub release pagination in `run_daily.sh`. - -The cron job resolves "newest LiteLLM v*-stable" via the GitHub Releases -API. A previous version of the loop broke as soon as the current page -contained ANY v*-stable tag. The Releases endpoint orders by -`created_at`, NOT by semver, so a backport on an older series cut today -(e.g. v1.80.1-stable) can land on an earlier page than a higher-version -release cut two weeks ago (e.g. v1.83.0-stable). The early-break would -silently pin the cron to a stale tag because the higher-version release -on a later page never made it into the merged set the final `sort_by` -consumed. - -These tests pin two things: - - 1. The buggy early-break-on-first-stable pattern must not return. - 2. The loop still terminates early on the standard "empty page" guard - so a quiet release feed doesn't burn API quota. - -The shell loop itself is exercised end-to-end with a fake `curl` that -serves canned page JSON, demonstrating that the resolved tag is the -highest-semver stable across all pages even when the highest tag lives -on page 2+. -""" - -from __future__ import annotations - -import os -import shutil -import subprocess -import textwrap -from pathlib import Path - -import pytest - -REPO_ROOT = Path(__file__).resolve().parents[4] -RUN_DAILY = REPO_ROOT / "tests" / "e2e" / "claude_code" / "cron_vm" / "run_daily.sh" - -# The extracted snippet starts AFTER `log`/`die` are defined in run_daily.sh, -# so the test harness has to provide its own stubs. Without them, a failure -# inside the snippet (e.g. jq returning an empty LITELLM_VERSION) would crash -# with `bash: die: command not found` (exit 127) instead of the intended -# diagnostic, making test failures unnecessarily hard to debug. -_PREAMBLE = ( - "set -Eeuo pipefail\n" - "log() { printf '==> %s\\n' \"$*\" >&2; }\n" - "die() { printf 'ERROR: %s\\n' \"$*\" >&2; exit 1; }\n" -) - - -def test_run_daily_does_not_early_break_on_first_stable_page() -> None: - """The regex pattern `select(test("...stable$"))] | length > 0` followed - by `break` is exactly the buggy early-stop. If it ever returns the - cron will silently start testing against a stale stable tag. - """ - body = RUN_DAILY.read_text() - assert ( - "length > 0" not in body - or "break" not in body - or ( - # If both substrings exist, make sure they aren't both inside the - # same release-pagination loop. The current loop only contains - # a `break` for the empty-page guard, not for any "length > 0" - # condition. - not _shares_loop_body(body, "length > 0", "break") - ) - ), ( - "run_daily.sh contains the old early-break-on-stable pattern. The " - "Releases endpoint orders by created_at, not semver, so breaking " - "on first-stable-seen can miss higher-versioned releases sitting " - "on later pages." - ) - - -def _shares_loop_body(body: str, needle_a: str, needle_b: str) -> bool: - """Heuristic: do both needles live inside a `for page in ...; do ... done` - block? Used as a defensive guard for the static check above.""" - in_loop = False - saw_a = False - saw_b = False - for line in body.splitlines(): - stripped = line.strip() - if stripped.startswith("for page in"): - in_loop = True - saw_a = False - saw_b = False - continue - if in_loop and stripped == "done": - if saw_a and saw_b: - return True - in_loop = False - continue - if in_loop: - if needle_a in line: - saw_a = True - if needle_b in line: - saw_b = True - return False - - -def test_run_daily_keeps_empty_page_break_guard() -> None: - """The empty-page break is the only break that should remain in the - pagination loop — without it a quiet release feed wastes API quota - walking past the last real page.""" - body = RUN_DAILY.read_text() - assert "jq 'length' \"${PAGE_JSON}\"" in body, ( - "run_daily.sh must still detect empty pages via `jq 'length' " - "${PAGE_JSON}`; without this the loop walks the full 5-page cap " - "even when there are no more releases." - ) - assert ( - '== "0"' in body - ), 'The empty-page guard must compare jq\'s length output to "0".' - - -def _make_fake_curl(scratch: Path, pages: dict[int, str]) -> Path: - """Build a fake `curl` shim that serves the canned page JSON for - each `page=N` request and an empty array for any page past the - last canned one. - - The shim mimics just enough of curl's CLI surface for the cron - script: it accepts the headers + URL we pass, ignores everything - we don't need, and writes the canned body to either stdout or the - --output target if one is given. - """ - pages_dir = scratch / "pages" - pages_dir.mkdir() - for page_num, body in pages.items(): - (pages_dir / f"page{page_num}.json").write_text(body) - - curl_path = scratch / "curl" - curl_path.write_text( - textwrap.dedent( - f"""\ - #!/usr/bin/env bash - # Fake curl for run_daily.sh release pagination tests. Serves - # page JSON from {pages_dir} keyed by the `page=` query value, - # and returns "[]" for pages past the last canned one (which - # is exactly how the real GitHub API behaves past the end). - url="" - output="" - while [[ $# -gt 0 ]]; do - case "$1" in - -fsS|-fsSL|-H|-o|--output) - if [[ "$1" == "-o" || "$1" == "--output" ]]; then - output="$2"; shift 2 - elif [[ "$1" == "-H" ]]; then - shift 2 - else - shift - fi - ;; - http*) - url="$1"; shift - ;; - *) - shift - ;; - esac - done - page="$(printf '%s' "$url" | sed -n 's/.*[?&]page=\\([0-9]*\\).*/\\1/p')" - [[ -z "$page" ]] && page=1 - file="{pages_dir}/page${{page}}.json" - if [[ -f "$file" ]]; then - if [[ -n "$output" ]]; then cp "$file" "$output"; else cat "$file"; fi - else - if [[ -n "$output" ]]; then printf '[]' > "$output"; else printf '[]'; fi - fi - """ - ) - ) - curl_path.chmod(0o755) - return curl_path - - -def _extract_resolution_snippet() -> str: - """Pull the pagination + sort_by + assignment block out of run_daily.sh - so the test exercises the actual production code path (not a copy). - - The block is everything from the GH_AUTH_HEADER setup down through - the LITELLM_VERSION emission. - """ - body = RUN_DAILY.read_text() - start = body.index("GH_AUTH_HEADER=()") - end = body.index('log "resolved litellm:') - return body[start:end] - - -@pytest.mark.skipif(shutil.which("jq") is None, reason="jq not available") -def test_run_daily_resolves_highest_semver_across_pages(tmp_path: Path) -> None: - """End-to-end: drive the actual run_daily.sh pagination loop with a - fake curl whose page 1 contains a freshly-cut LOW-version backport - (v1.80.1-stable) and page 2 contains a two-weeks-old HIGH-version - release (v1.83.0-stable). The correct behavior is to resolve - v1.83.0-stable. The pre-fix behavior would resolve v1.80.1-stable - because the early-break consumed only page 1. - """ - pages = { - # Page 1: most-recently-created releases. The order here matches - # what /releases?page=1 returns: created-at descending. The - # freshly-cut v1.80.1-stable backport sits at the top, plus a - # bunch of non-stable releases. - 1: """[ - {"tag_name": "v1.84.0-nightly.1"}, - {"tag_name": "v1.80.1-stable"}, - {"tag_name": "v1.84.0-nightly.0"} - ]""", - # Page 2: older releases. The HIGHER-version stable lives here - # because it was cut two weeks ago, before the v1.80.1 backport. - 2: """[ - {"tag_name": "v1.83.0-rc.5"}, - {"tag_name": "v1.83.0-stable"}, - {"tag_name": "v1.82.4-stable"} - ]""", - # Page 3+: empty -> the loop's empty-page guard fires here. - } - fake_curl_dir = tmp_path / "shim" - fake_curl_dir.mkdir() - _make_fake_curl(fake_curl_dir, pages) - - workdir = tmp_path / "work" - workdir.mkdir() - - snippet = _extract_resolution_snippet() - script = ( - _PREAMBLE - + f"WORKDIR={workdir!s}\n" - + snippet - + 'printf "%s" "${LITELLM_VERSION}"\n' - ) - - env = { - **os.environ, - "PATH": f"{fake_curl_dir}:{os.environ.get('PATH', '')}", - } - # Make sure the loop hits the fake curl, not the system one. - env.pop("GITHUB_TOKEN", None) - result = subprocess.run( - ["bash", "-c", script], - capture_output=True, - text=True, - env=env, - check=True, - ) - assert result.stdout == "v1.83.0-stable", ( - f"Expected the highest-semver stable across pages 1-2, got " - f"{result.stdout!r}. stderr={result.stderr!r}" - ) - - -@pytest.mark.skipif(shutil.which("jq") is None, reason="jq not available") -def test_run_daily_terminates_on_empty_page(tmp_path: Path) -> None: - """The empty-page guard must fire so we don't always walk all 5 - pages. With a single populated page and an empty page 2 we should - stop after fetching page 2 (the first empty response).""" - pages = {1: '[{"tag_name": "v1.50.0-stable"}]'} - fake_curl_dir = tmp_path / "shim" - fake_curl_dir.mkdir() - _make_fake_curl(fake_curl_dir, pages) - - workdir = tmp_path / "work" - workdir.mkdir() - - snippet = _extract_resolution_snippet() - script = ( - _PREAMBLE - + f"WORKDIR={workdir!s}\n" - + snippet - + 'printf "%s" "${LITELLM_VERSION}"\n' - ) - - env = { - **os.environ, - "PATH": f"{tmp_path}/shim:{os.environ.get('PATH', '')}", - } - env.pop("GITHUB_TOKEN", None) - result = subprocess.run( - ["bash", "-c", script], - capture_output=True, - text=True, - env=env, - check=True, - ) - assert result.stdout == "v1.50.0-stable" - # Only pages 1 and 2 should have been fetched (2 is empty -> break). - assert (workdir / "releases.page2.json").exists() - assert not (workdir / "releases.page3.json").exists(), ( - "Empty-page guard didn't fire — the loop kept walking past the " - "first empty response." - ) diff --git a/tests/e2e/claude_code/_publisher_unit_tests/test_run_daily_version_probe_scrubs_env.py b/tests/e2e/claude_code/_publisher_unit_tests/test_run_daily_version_probe_scrubs_env.py deleted file mode 100644 index 1c3959764f3..00000000000 --- a/tests/e2e/claude_code/_publisher_unit_tests/test_run_daily_version_probe_scrubs_env.py +++ /dev/null @@ -1,92 +0,0 @@ -"""Pin: the cron `claude --version` probe must run under `env -i`. - -The systemd service `litellm-compat-matrix.service` loads provider -credentials (`ANTHROPIC_API_KEY`, `AWS_BEARER_TOKEN_BEDROCK`, -`AZURE_FOUNDRY_API_KEY`) and the agent-shin GitHub token -(`AGENT_SHIN_GITHUB_TOKEN`) into `run_daily.sh`'s environment from -`/etc/litellm-compat-matrix.env`. Running the npm-installed `claude` -binary directly there would hand that full env to package code, so a -compromised `@anthropic-ai/claude-code` release could read those -secrets out of `os.environ` before the proxy or test harness ever -starts. The version probe must be wrapped in `env -i` with a minimal -PATH/HOME/USER/TERM/LANG/LC_ALL/TMPDIR allowlist — matching the -PR-gate's resolver/npm-install/pytest scrubs. -""" - -from __future__ import annotations - -from pathlib import Path - -REPO_ROOT = Path(__file__).resolve().parents[4] -RUN_DAILY = REPO_ROOT / "tests" / "e2e" / "claude_code" / "cron_vm" / "run_daily.sh" - - -def _version_probe_block() -> str: - body = RUN_DAILY.read_text() - start = body.index("CLAUDE_CODE_VERSION=") - end = body.index('[[ -n "${CLAUDE_CODE_VERSION}" ]]', start) - return body[start:end] - - -def test_version_probe_wraps_claude_in_env_i() -> None: - block = _version_probe_block() - assert "env -i" in block, ( - "run_daily.sh: the `claude --version` probe must run under " - "`env -i` so a compromised @anthropic-ai/claude-code package " - "cannot read provider/GitHub credentials out of the systemd " - "service environment." - ) - assert block.index("env -i") < block.index("claude --version"), ( - "run_daily.sh: `env -i` must precede `claude --version`; " - "otherwise the binary inherits the full credential-bearing env." - ) - - -def test_version_probe_env_i_excludes_provider_secrets() -> None: - block = _version_probe_block() - for forbidden in ( - "ANTHROPIC_API_KEY", - "AWS_BEARER_TOKEN_BEDROCK", - "AWS_ACCESS_KEY_ID", - "AWS_SECRET_ACCESS_KEY", - "VERTEXAI_CREDENTIALS", - "AZURE_FOUNDRY_API_KEY", - "GITHUB_TOKEN", - "AGENT_SHIN_GITHUB_TOKEN", - ): - assert forbidden not in block, ( - f"run_daily.sh: the version-probe `env -i` allowlist must " - f"not pass {forbidden} through. Found it inside the probe " - f"block." - ) - - -def test_version_probe_uses_isolated_home_not_runtime_user_home() -> None: - """Pin: the `claude --version` probe runs under a fresh empty HOME. - - `ProtectHome=read-only` in the systemd unit allows reads of the - runtime user's real home directory. If the probe's `env -i` - block forwards `HOME=${HOME}`, a compromised `claude` package - can `os.path.expanduser("~/.config/gh/hosts.yml")` or - `os.path.expanduser("~/.ssh/...")` and exfiltrate the contents - before the proxy or test harness ever starts. The probe must - point HOME at a per-run tmpdir under `${WORKDIR}` so the CLI - sees an empty HOME instead. - """ - block = _version_probe_block() - body = RUN_DAILY.read_text() - - assert "CLAUDE_PROBE_HOME=" in body, ( - "run_daily.sh: must define a `CLAUDE_PROBE_HOME` per-run tmpdir " - "for the `claude --version` probe so the CLI never sees the " - "runtime user's real $HOME." - ) - assert 'HOME="${CLAUDE_PROBE_HOME}"' in block, ( - "run_daily.sh: the probe's `env -i` block must set HOME to " - "the per-run isolated tmpdir, not to the runtime user's $HOME." - ) - assert 'HOME="${HOME}"' not in block, ( - "run_daily.sh: the probe's `env -i` block must not forward the " - "runtime user's $HOME to `claude --version`. Use the isolated " - "$CLAUDE_PROBE_HOME tmpdir instead." - ) diff --git a/tests/e2e/claude_code/_publisher_unit_tests/test_systemd_unit_credential_isolation.py b/tests/e2e/claude_code/_publisher_unit_tests/test_systemd_unit_credential_isolation.py deleted file mode 100644 index 12edce3cb50..00000000000 --- a/tests/e2e/claude_code/_publisher_unit_tests/test_systemd_unit_credential_isolation.py +++ /dev/null @@ -1,104 +0,0 @@ -"""Pin: the cron systemd unit hides credential-bearing dotdirs. - -`ProtectHome=read-only` blocks writes to /home/mateo but still allows -reads. A model-directed `Read` tool call (the PDF cells pass -`--allowed-tools Read` to the `claude` CLI) or a compromised -`@anthropic-ai/claude-code` package can read absolute paths under -the runtime user's home and exfiltrate the contents — even with the -per-`claude`-invocation HOME isolation in place, because absolute -paths bypass `~`-expansion. - -This file pins the second line of defense: the systemd unit lists -the credential-bearing dotdirs (`~/.config/gh`, `~/.ssh`, `~/.aws`, -`~/.docker`, `~/.kube`, `~/.gnupg`) under `InaccessiblePaths=` so -the kernel hides them from every process in the unit's mount -namespace, including any child of `claude --version` or the pytest -run. It also pins that `~/.config/gh` is *not* in `ReadWritePaths=` -— we pass `GH_TOKEN` inline to every `gh` invocation in -`run_daily.sh`, so the host gh-cli config is unused. -""" - -from __future__ import annotations - -import re -from pathlib import Path - -REPO_ROOT = Path(__file__).resolve().parents[4] -SERVICE = ( - REPO_ROOT / "tests" / "e2e" / "claude_code" / "cron_vm" / "litellm-compat-matrix.service" -) - - -def _service_text() -> str: - return SERVICE.read_text() - - -def _directive(name: str) -> str: - """Return the value of a single-line systemd directive (or empty).""" - text = _service_text() - match = re.search(rf"^\s*{re.escape(name)}\s*=\s*(.*)$", text, re.MULTILINE) - return match.group(1).strip() if match else "" - - -def test_inaccessible_paths_hides_credential_dotdirs() -> None: - """Every credential-bearing dotdir must be under `InaccessiblePaths=`.""" - inaccessible = _directive("InaccessiblePaths") - assert inaccessible, ( - "litellm-compat-matrix.service: must declare `InaccessiblePaths=` " - "to hide credential dotdirs from the `claude` subprocess and the " - "model-directed Read tool. Without this, an absolute-path read " - "like `Read('/home/mateo/.config/gh/hosts.yml')` exfiltrates " - "the gh-cli token despite the per-invocation HOME isolation." - ) - for path in ( - "/home/mateo/.config/gh", - "/home/mateo/.ssh", - "/home/mateo/.aws", - "/home/mateo/.docker", - "/home/mateo/.kube", - "/home/mateo/.gnupg", - ): - # Tolerated `-` prefix means "ignore if missing on host". - assert path in inaccessible, ( - f"litellm-compat-matrix.service: `{path}` must appear in " - f"`InaccessiblePaths=` so the cron `claude` subprocess can " - f"never read it (even via an absolute path that bypasses " - f"the per-invocation HOME override)." - ) - - -def test_gh_config_is_not_writeable() -> None: - """`~/.config/gh` is not whitelisted under `ReadWritePaths=`. - - We pass `GH_TOKEN` inline to every `gh` invocation in - `run_daily.sh` (`gh repo clone`, `gh pr create`, `gh pr edit`). - The host `~/.config/gh/hosts.yml` is therefore never consulted - or written to. Keeping it out of `ReadWritePaths=` is the second - line of defense: a future regression that drops the inline-token - convention will fail loudly (gh writes a new login config and - hits a read-only filesystem) rather than silently re-introduce - the credential exfiltration surface that - `InaccessiblePaths=/home/mateo/.config/gh` is closing. - """ - rw = _directive("ReadWritePaths") - assert ".config/gh" not in rw, ( - "litellm-compat-matrix.service: `/home/mateo/.config/gh` must " - "*not* appear in `ReadWritePaths=`. We pass `GH_TOKEN` inline " - "to every `gh` invocation in run_daily.sh, so the host gh-cli " - "config is never consulted or written to. Keeping the path out " - "of ReadWritePaths means a future regression that drops the " - "inline-token convention will fail loudly instead of silently " - "re-opening the credential exfiltration surface that " - "`InaccessiblePaths=` is closing." - ) - - -def test_protect_home_is_read_only_or_stricter() -> None: - """`ProtectHome=` must be at least `read-only`.""" - value = _directive("ProtectHome") - assert value in ("read-only", "tmpfs", "yes", "true"), ( - f"litellm-compat-matrix.service: `ProtectHome=` must be `read-only`, " - f"`tmpfs`, or `yes`. Got: {value!r}. Without this, the unit can " - f"write anywhere under /home/mateo, including overwriting " - f"~/.config/gh/hosts.yml." - ) diff --git a/tests/e2e/claude_code/basic_messaging_non_streaming/test_anthropic.py b/tests/e2e/claude_code/basic_messaging_non_streaming/test_anthropic.py index c06fff28d2d..21383b85da5 100644 --- a/tests/e2e/claude_code/basic_messaging_non_streaming/test_anthropic.py +++ b/tests/e2e/claude_code/basic_messaging_non_streaming/test_anthropic.py @@ -20,6 +20,7 @@ the matrix builder still sees three rows for this (feature, provider). from __future__ import annotations +import pytest from claude_code._basic_messaging import run_basic_messaging_cell # Per the PRD: each cell is exercised against three Claude tiers via the @@ -27,11 +28,12 @@ from claude_code._basic_messaging import run_basic_messaging_cell # routing config; the driver only sends the alias. ANTHROPIC_MODELS = [ "claude-haiku-4-5", - "claude-sonnet-4-6", + "claude-sonnet-4-5", "claude-opus-4-7", ] +@pytest.mark.covers("llm.messages.anthropic.basic.nonstream.works") def test_basic_messaging_non_streaming_anthropic(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a reply. diff --git a/tests/e2e/claude_code/basic_messaging_non_streaming/test_azure.py b/tests/e2e/claude_code/basic_messaging_non_streaming/test_azure.py index 2a962b244a8..19e88dbe3cb 100644 --- a/tests/e2e/claude_code/basic_messaging_non_streaming/test_azure.py +++ b/tests/e2e/claude_code/basic_messaging_non_streaming/test_azure.py @@ -25,6 +25,7 @@ the matrix builder still sees three rows for this (feature, provider). from __future__ import annotations +import pytest from claude_code._basic_messaging import run_basic_messaging_cell # Per-model aliases registered in the LiteLLM proxy's routing config to @@ -33,11 +34,12 @@ from claude_code._basic_messaging import run_basic_messaging_cell # resource URL and API key. AZURE_MODELS = [ "claude-haiku-4-5-azure", - "claude-sonnet-4-6-azure", + "claude-sonnet-4-5-azure", "claude-opus-4-7-azure", ] +@pytest.mark.covers("llm.messages.azure_foundry.basic.nonstream.works") def test_basic_messaging_non_streaming_azure(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a reply. diff --git a/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_converse.py b/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_converse.py index 2245ed7417a..2b0f49bc205 100644 --- a/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_converse.py +++ b/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_converse.py @@ -20,6 +20,7 @@ the matrix builder still sees three rows for this (feature, provider). from __future__ import annotations +import pytest from claude_code._basic_messaging import run_basic_messaging_cell # Per-model aliases registered in the LiteLLM proxy's routing config to @@ -28,11 +29,12 @@ from claude_code._basic_messaging import run_basic_messaging_cell # strategy. BEDROCK_CONVERSE_MODELS = [ "claude-haiku-4-5-bedrock-converse", - "claude-sonnet-4-6-bedrock-converse", + "claude-sonnet-4-5-bedrock-converse", "claude-opus-4-7-bedrock-converse", ] +@pytest.mark.covers("llm.messages.bedrock_converse.basic.nonstream.works") def test_basic_messaging_non_streaming_bedrock_converse(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a reply.""" run_basic_messaging_cell( diff --git a/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_invoke.py b/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_invoke.py index e0a6e77f3c1..937ea5ee27e 100644 --- a/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_invoke.py @@ -20,6 +20,7 @@ the matrix builder still sees three rows for this (feature, provider). from __future__ import annotations +import pytest from claude_code._basic_messaging import run_basic_messaging_cell # Per-model aliases registered in the LiteLLM proxy's routing config to @@ -28,11 +29,12 @@ from claude_code._basic_messaging import run_basic_messaging_cell # routing strategy. BEDROCK_INVOKE_MODELS = [ "claude-haiku-4-5-bedrock-invoke", - "claude-sonnet-4-6-bedrock-invoke", + "claude-sonnet-4-5-bedrock-invoke", "claude-opus-4-7-bedrock-invoke", ] +@pytest.mark.covers("llm.messages.bedrock_invoke.basic.nonstream.works") def test_basic_messaging_non_streaming_bedrock_invoke(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a reply.""" run_basic_messaging_cell( diff --git a/tests/e2e/claude_code/basic_messaging_non_streaming/test_vertex_ai.py b/tests/e2e/claude_code/basic_messaging_non_streaming/test_vertex_ai.py index e4e2a39e6cd..c46e5a8f762 100644 --- a/tests/e2e/claude_code/basic_messaging_non_streaming/test_vertex_ai.py +++ b/tests/e2e/claude_code/basic_messaging_non_streaming/test_vertex_ai.py @@ -20,6 +20,7 @@ the matrix builder still sees three rows for this (feature, provider). from __future__ import annotations +import pytest from claude_code._basic_messaging import run_basic_messaging_cell # Per-model aliases registered in the LiteLLM proxy's routing config to @@ -28,11 +29,12 @@ from claude_code._basic_messaging import run_basic_messaging_cell # model id and the GCP region. VERTEX_AI_MODELS = [ "claude-haiku-4-5-vertex", - "claude-sonnet-4-6-vertex", + "claude-sonnet-4-5-vertex", "claude-opus-4-7-vertex", ] +@pytest.mark.covers("llm.messages.vertex.basic.nonstream.works") def test_basic_messaging_non_streaming_vertex_ai(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a reply.""" run_basic_messaging_cell( diff --git a/tests/e2e/claude_code/basic_messaging_streaming/test_anthropic.py b/tests/e2e/claude_code/basic_messaging_streaming/test_anthropic.py index 56e3fb6c181..ce453f3e523 100644 --- a/tests/e2e/claude_code/basic_messaging_streaming/test_anthropic.py +++ b/tests/e2e/claude_code/basic_messaging_streaming/test_anthropic.py @@ -25,15 +25,17 @@ sees three rows for this (feature, provider). from __future__ import annotations +import pytest from claude_code._basic_messaging import run_basic_messaging_cell ANTHROPIC_MODELS = [ "claude-haiku-4-5", - "claude-sonnet-4-6", + "claude-sonnet-4-5", "claude-opus-4-7", ] +@pytest.mark.covers("llm.messages.anthropic.basic.stream.works") def test_basic_messaging_streaming_anthropic(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a non-empty streamed reply (one row per Claude tier). diff --git a/tests/e2e/claude_code/basic_messaging_streaming/test_azure.py b/tests/e2e/claude_code/basic_messaging_streaming/test_azure.py index b6c002d0b27..3307194e862 100644 --- a/tests/e2e/claude_code/basic_messaging_streaming/test_azure.py +++ b/tests/e2e/claude_code/basic_messaging_streaming/test_azure.py @@ -19,15 +19,17 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations +import pytest from claude_code._basic_messaging import run_basic_messaging_cell AZURE_MODELS = [ "claude-haiku-4-5-azure", - "claude-sonnet-4-6-azure", + "claude-sonnet-4-5-azure", "claude-opus-4-7-azure", ] +@pytest.mark.covers("llm.messages.azure_foundry.basic.stream.works") def test_basic_messaging_streaming_azure(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a non-empty streamed reply (one row per Claude tier). diff --git a/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_converse.py b/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_converse.py index 44ac54515f0..a8bc0b77a5d 100644 --- a/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_converse.py +++ b/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_converse.py @@ -15,15 +15,17 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations +import pytest from claude_code._basic_messaging import run_basic_messaging_cell BEDROCK_CONVERSE_MODELS = [ "claude-haiku-4-5-bedrock-converse", - "claude-sonnet-4-6-bedrock-converse", + "claude-sonnet-4-5-bedrock-converse", "claude-opus-4-7-bedrock-converse", ] +@pytest.mark.covers("llm.messages.bedrock_converse.basic.stream.works") def test_basic_messaging_streaming_bedrock_converse(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a non-empty streamed reply (one row per Claude tier). diff --git a/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_invoke.py b/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_invoke.py index 1d59d16cdc1..c0ece0e0721 100644 --- a/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_invoke.py @@ -15,15 +15,17 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations +import pytest from claude_code._basic_messaging import run_basic_messaging_cell BEDROCK_INVOKE_MODELS = [ "claude-haiku-4-5-bedrock-invoke", - "claude-sonnet-4-6-bedrock-invoke", + "claude-sonnet-4-5-bedrock-invoke", "claude-opus-4-7-bedrock-invoke", ] +@pytest.mark.covers("llm.messages.bedrock_invoke.basic.stream.works") def test_basic_messaging_streaming_bedrock_invoke(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a non-empty streamed reply (one row per Claude tier). diff --git a/tests/e2e/claude_code/basic_messaging_streaming/test_vertex_ai.py b/tests/e2e/claude_code/basic_messaging_streaming/test_vertex_ai.py index 014a31160a8..13f1a0abf40 100644 --- a/tests/e2e/claude_code/basic_messaging_streaming/test_vertex_ai.py +++ b/tests/e2e/claude_code/basic_messaging_streaming/test_vertex_ai.py @@ -15,15 +15,17 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations +import pytest from claude_code._basic_messaging import run_basic_messaging_cell VERTEX_AI_MODELS = [ "claude-haiku-4-5-vertex", - "claude-sonnet-4-6-vertex", + "claude-sonnet-4-5-vertex", "claude-opus-4-7-vertex", ] +@pytest.mark.covers("llm.messages.vertex.basic.stream.works") def test_basic_messaging_streaming_vertex_ai(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a non-empty streamed reply (one row per Claude tier). diff --git a/tests/e2e/claude_code/conftest.py b/tests/e2e/claude_code/conftest.py index 6ee8b940648..8f5c09fa4e2 100644 --- a/tests/e2e/claude_code/conftest.py +++ b/tests/e2e/claude_code/conftest.py @@ -169,8 +169,9 @@ def _manifest_feature_ids() -> FrozenSet[str]: Used as a positive filter so only directories that correspond to a real matrix row contribute results — utility/support directories - (e.g. `cron_vm`, `_driver_unit_tests`) are dropped regardless of - naming convention, and the rate-limit summary stays clean. + (e.g. `_driver_unit_tests`, `_builder_unit_tests`) are dropped + regardless of naming convention, and the rate-limit summary stays + clean. Returns an empty set if the manifest is missing or malformed; the caller treats that as "no path is a feature path", which is the @@ -199,11 +200,10 @@ def _infer_feature_and_provider(node_path: Path) -> Optional[tuple]: Path shape: tests/e2e/claude_code//test_.py Returns None if the file is not a per-feature test (e.g. unit tests - under `_driver_unit_tests/` or support code under `cron_vm/`), so - those don't pollute the matrix artifact. We positively filter the - parent directory against `manifest.yaml` rather than relying on - naming conventions, because non-feature siblings don't all share - an underscore prefix. + under `_driver_unit_tests/`), so those don't pollute the matrix + artifact. We positively filter the parent directory against + `manifest.yaml` rather than relying on naming conventions, because + non-feature siblings don't all share an underscore prefix. """ name = node_path.name if not name.startswith("test_") or not name.endswith(".py"): @@ -548,3 +548,117 @@ def pytest_sessionfinish(session, exitstatus): ) summary_path.write_text(json.dumps(summary, indent=2, sort_keys=True)) _print_rate_limit_summary(summary) + + +# --------------------------------------------------------------------------- +# Session-scoped compat model registration. +# +# The compat cells probe hardcoded virtual names like ``claude-sonnet-4-5`` +# and ``claude-sonnet-4-5-bedrock-invoke``. On stage those live in the +# gateway's model_list at deploy time; locally the docker-config.yaml +# under tests/e2e/ only declares one of them, so every non-haiku cell +# 400s with ``Invalid model name``. The fixture here reconciles the two: +# it reads ``test_config.yaml`` (the ground-truth compat matrix config) +# and POSTs ``/model/new`` for the subset whose provider credentials are +# actually set in the current environment, then tears them all down at +# session end. +# +# Kept below the rest of the conftest so the compat-artifact hooks stay +# grouped up top. The fixture is opt-in via autouse=True on the session +# scope, so a cell that hits the proxy sees the deployment ready without +# any per-cell wiring, and pure unit tests that never reach the proxy +# pay only one skipped-liveness check. +# --------------------------------------------------------------------------- + +from claude_code._env import ProxyConfig, resolve_proxy # noqa: E402 +from claude_code._compat_models import ( # noqa: E402 + CompatDeployment, + load_all_deployments, +) + + +def _build_control_gateway(proxy: ProxyConfig): + """Local import of the shared harness so the pure-unit-test tree + under ``_driver_unit_tests/`` etc. never has to pull it in. The + control plane transport is what /model/new lives on; SplitTransport + routes it correctly for both monolithic and split deployments. + + The endpoints come from the *resolved* proxy, not from a second + independent env read, so registration and the cells always hit the + same host and key. Both planes get the one URL the cells use; the + deployment is fronted by a single address that routes management + and LLM paths itself.""" + from e2e_gateway import build_gateway + + return build_gateway( + base_url=proxy.base_url, + master_key=proxy.api_key, + control_plane_base_url=proxy.base_url, + ) + + +def _register_deployment(gateway, deployment: CompatDeployment) -> str: + """Register one deployment and return its proxy-assigned model_id + once it is servable on the data plane.""" + return gateway.create_model( + deployment.model_name, + deployment.litellm_params, + ) + + +@pytest.fixture(scope="session", autouse=True) +def _compat_models_registered() -> Any: + """Register every compat deployment against the running proxy, then + tear them all down on session exit. + + Skips silently if the proxy env is not configured (no + ``LITELLM_PROXY_URL``/``LITELLM_MASTER_KEY``) so unit-test runs + stay hermetic. + + Design note: we always attempt to register all 15 deployments, + regardless of what credentials are exported in the test-runner's + shell. The credentials live in the proxy container's environment + (via docker-compose ``env_file``), not the shell running pytest - + so gating on shell env would filter out deployments the proxy can + actually serve. Per-deployment ``/model/new`` failures are printed + but do not abort the session: the cells that need that specific + deployment will 400 with "Invalid model name" and fail loudly, + which is the right signal (missing cred on the proxy side).""" + proxy = resolve_proxy() + if proxy is None: + yield + return + + from requests import RequestException + + gateway = _build_control_gateway(proxy) + registered_ids: list[str] = [] + failures: list[tuple[str, str]] = [] + try: + for deployment in load_all_deployments(): + try: + model_id = _register_deployment(gateway, deployment) + registered_ids.append(model_id) + except (AssertionError, RequestException) as exc: + failures.append((deployment.model_name, str(exc))) + if failures: + summary = "\n".join( + f" - {name}: {reason}" for name, reason in failures + ) + print( + f"[compat fixture] {len(failures)} of " + f"{len(failures) + len(registered_ids)} deployments " + f"failed to register (proxy likely missing that provider's " + f"credentials); cells that target them will fail loudly:\n" + f"{summary}" + ) + yield + finally: + for model_id in registered_ids: + try: + gateway.delete_model(model_id) + except (AssertionError, RequestException): + # Best-effort — teardown surfaces via warnings inside + # ``delete_model`` already; swallowing here so one flaky + # delete does not mask real test failures. + pass diff --git a/tests/e2e/claude_code/count_tokens/test_anthropic.py b/tests/e2e/claude_code/count_tokens/test_anthropic.py index 3508063459c..2fbdd4212c4 100644 --- a/tests/e2e/claude_code/count_tokens/test_anthropic.py +++ b/tests/e2e/claude_code/count_tokens/test_anthropic.py @@ -37,44 +37,27 @@ CLI rows isn't useful here. from __future__ import annotations -import os - import pytest +from claude_code._env import require_proxy from claude_code.http_probe import ( assert_count_tokens_shape, probe_count_tokens, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" ANTHROPIC_MODELS = [ "claude-haiku-4-5", - "claude-sonnet-4-6", + "claude-sonnet-4-5", "claude-opus-4-7", ] +@pytest.mark.covers("llm.messages.anthropic.count_tokens.nonstream.works") def test_count_tokens_anthropic(compat_result): """Probe `/v1/messages/count_tokens` for each Anthropic tier and assert the response shape.""" - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", - pytrace=False, - ) + base_url, api_key = require_proxy(compat_result) failures = [] for model in ANTHROPIC_MODELS: diff --git a/tests/e2e/claude_code/count_tokens/test_azure.py b/tests/e2e/claude_code/count_tokens/test_azure.py index 2b8707b50b0..a9aa168ccea 100644 --- a/tests/e2e/claude_code/count_tokens/test_azure.py +++ b/tests/e2e/claude_code/count_tokens/test_azure.py @@ -37,44 +37,27 @@ CLI rows isn't useful here. from __future__ import annotations -import os - import pytest +from claude_code._env import require_proxy from claude_code.http_probe import ( assert_count_tokens_shape, probe_count_tokens, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" AZURE_MODELS = [ "claude-haiku-4-5-azure", - "claude-sonnet-4-6-azure", + "claude-sonnet-4-5-azure", "claude-opus-4-7-azure", ] +@pytest.mark.covers("llm.messages.azure_foundry.count_tokens.nonstream.works") def test_count_tokens_azure(compat_result): """Probe `/v1/messages/count_tokens` for each Azure (Microsoft Foundry) tier and assert the response shape.""" - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", - pytrace=False, - ) + base_url, api_key = require_proxy(compat_result) failures = [] for model in AZURE_MODELS: diff --git a/tests/e2e/claude_code/count_tokens/test_bedrock_converse.py b/tests/e2e/claude_code/count_tokens/test_bedrock_converse.py index 4221773ead2..6dcff3ecae3 100644 --- a/tests/e2e/claude_code/count_tokens/test_bedrock_converse.py +++ b/tests/e2e/claude_code/count_tokens/test_bedrock_converse.py @@ -37,44 +37,27 @@ CLI rows isn't useful here. from __future__ import annotations -import os - import pytest +from claude_code._env import require_proxy from claude_code.http_probe import ( assert_count_tokens_shape, probe_count_tokens, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" BEDROCK_CONVERSE_MODELS = [ "claude-haiku-4-5-bedrock-converse", - "claude-sonnet-4-6-bedrock-converse", + "claude-sonnet-4-5-bedrock-converse", "claude-opus-4-7-bedrock-converse", ] +@pytest.mark.covers("llm.messages.bedrock_converse.count_tokens.nonstream.works") def test_count_tokens_bedrock_converse(compat_result): """Probe `/v1/messages/count_tokens` for each Bedrock (Converse) tier and assert the response shape.""" - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", - pytrace=False, - ) + base_url, api_key = require_proxy(compat_result) failures = [] for model in BEDROCK_CONVERSE_MODELS: diff --git a/tests/e2e/claude_code/count_tokens/test_bedrock_invoke.py b/tests/e2e/claude_code/count_tokens/test_bedrock_invoke.py index cc70bf12392..ae89067dc00 100644 --- a/tests/e2e/claude_code/count_tokens/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/count_tokens/test_bedrock_invoke.py @@ -37,44 +37,27 @@ CLI rows isn't useful here. from __future__ import annotations -import os - import pytest +from claude_code._env import require_proxy from claude_code.http_probe import ( assert_count_tokens_shape, probe_count_tokens, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" BEDROCK_INVOKE_MODELS = [ "claude-haiku-4-5-bedrock-invoke", - "claude-sonnet-4-6-bedrock-invoke", + "claude-sonnet-4-5-bedrock-invoke", "claude-opus-4-7-bedrock-invoke", ] +@pytest.mark.covers("llm.messages.bedrock_invoke.count_tokens.nonstream.works") def test_count_tokens_bedrock_invoke(compat_result): """Probe `/v1/messages/count_tokens` for each Bedrock (Invoke) tier and assert the response shape.""" - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", - pytrace=False, - ) + base_url, api_key = require_proxy(compat_result) failures = [] for model in BEDROCK_INVOKE_MODELS: diff --git a/tests/e2e/claude_code/count_tokens/test_vertex_ai.py b/tests/e2e/claude_code/count_tokens/test_vertex_ai.py index 8c2678f7010..0f952496566 100644 --- a/tests/e2e/claude_code/count_tokens/test_vertex_ai.py +++ b/tests/e2e/claude_code/count_tokens/test_vertex_ai.py @@ -37,44 +37,27 @@ CLI rows isn't useful here. from __future__ import annotations -import os - import pytest +from claude_code._env import require_proxy from claude_code.http_probe import ( assert_count_tokens_shape, probe_count_tokens, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" VERTEX_AI_MODELS = [ "claude-haiku-4-5-vertex", - "claude-sonnet-4-6-vertex", + "claude-sonnet-4-5-vertex", "claude-opus-4-7-vertex", ] +@pytest.mark.covers("llm.messages.vertex.count_tokens.nonstream.works") def test_count_tokens_vertex_ai(compat_result): """Probe `/v1/messages/count_tokens` for each Vertex AI tier and assert the response shape.""" - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", - pytrace=False, - ) + base_url, api_key = require_proxy(compat_result) failures = [] for model in VERTEX_AI_MODELS: diff --git a/tests/e2e/claude_code/cron_vm/build_matrix.py b/tests/e2e/claude_code/cron_vm/build_matrix.py deleted file mode 100644 index 128f041cced..00000000000 --- a/tests/e2e/claude_code/cron_vm/build_matrix.py +++ /dev/null @@ -1,50 +0,0 @@ -"""Tiny CLI wrapper around `claude_code.matrix_builder.build_from_paths`. - -Exists only so `run_daily.sh` can hand the version metadata + paths into -the matrix builder without re-implementing it in bash. All real logic -lives in `matrix_builder.py`, which has its own unit tests under -`_builder_unit_tests/`. - -Invoked from the cron worktree (where `uv sync` has installed pyyaml), -not the dev checkout — the bash script `cd`s into the worktree before -`uv run python`-ing this file. -""" - -from __future__ import annotations - -import argparse -import datetime -import sys -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parents[2])) - -from claude_code.matrix_builder import build_from_paths # noqa: E402 # import needs the sys.path bootstrap above - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--manifest", type=Path, required=True) - parser.add_argument("--results", type=Path, required=True) - parser.add_argument("--output", type=Path, required=True) - parser.add_argument("--litellm-version", required=True) - parser.add_argument("--claude-code-version", required=True) - args = parser.parse_args() - - generated_at = datetime.datetime.now(datetime.timezone.utc).strftime( - "%Y-%m-%dT%H:%M:%SZ" - ) - build_from_paths( - manifest_path=args.manifest, - results_path=args.results, - litellm_version=args.litellm_version, - claude_code_version=args.claude_code_version, - generated_at=generated_at, - output_path=args.output, - ) - print(f"wrote {args.output}") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.env.example b/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.env.example deleted file mode 100644 index 5ca7937a426..00000000000 --- a/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.env.example +++ /dev/null @@ -1,59 +0,0 @@ -# Environment file consumed by `litellm-compat-matrix.service`. -# -# Install at `/etc/litellm-compat-matrix.env` and chmod 0600. -# `EnvironmentFile=-` in the unit means the service is allowed to start -# even if this file is missing, but the populator will fail at the -# first provider request without these credentials. - -# Anthropic -ANTHROPIC_API_KEY= - -# Bedrock (invoke + converse columns). -# Use Anthropic's Bedrock API-key passthrough (long-lived bearer token). -# No AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY required for the matrix -- -# both the LiteLLM invoke and converse routes pick up -# AWS_BEARER_TOKEN_BEDROCK when present. -AWS_BEARER_TOKEN_BEDROCK= -AWS_REGION_NAME=us-east-1 - -# Vertex AI. -# On the GCP VM, the default service-account ADC from the metadata server -# is used -- no JSON key file is needed. If you ever need to run outside -# GCP, also export GOOGLE_APPLICATION_CREDENTIALS=/path/to/sa.json. -VERTEXAI_PROJECT= -VERTEXAI_LOCATION=global - -# Microsoft Foundry (Azure column) -AZURE_FOUNDRY_API_KEY= -AZURE_FOUNDRY_API_BASE= - -# Azure cell of the `passthrough` row. Foundry-mode Claude Code sends -# the model in the request body, so the proxy's /azure passthrough -# cannot resolve a router alias and falls back to these env vars. -# AZURE_API_BASE is the Foundry resource's Anthropic surface, i.e. -# https://.services.ai.azure.com/anthropic ; AZURE_API_KEY -# is the same key as AZURE_FOUNDRY_API_KEY. -AZURE_API_BASE= -AZURE_API_KEY= - -# REQUIRED for publishing: PAT for the `agent-shin` user, used to push -# the daily compat-matrix branch to its fork (agent-shin/litellm-docs) -# and open the cross-repo PR against BerriAI/litellm-docs. Scopes: -# classic `repo` + `workflow`, or fine-grained on agent-shin/litellm-docs -# with Contents:RW + Pull requests:RW + Workflows:RW. -# Skip by setting SKIP_PUBLISH=1 (publishes nothing; only writes the -# matrix JSON locally). -AGENT_SHIN_GITHUB_TOKEN= - -# Optional: lifts the unauthenticated rate limit on the GitHub Releases -# API used by `resolver.py`. Any token works (read-only). Not required. -# GITHUB_TOKEN= - -# Optional overrides; defaults are sensible for the cron VM. -# PROXY_PORT=4100 -# LITELLM_WORKTREE=/home/mateo/litellm-cron-worktree -# DOCS_REPO=BerriAI/litellm-docs -# DOCS_BRANCH=main -# DOCS_TARGET_PATH=src/data/compatibility-matrix.json -# FORK_OWNER=agent-shin -# FORK_REPO=agent-shin/litellm-docs diff --git a/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.service b/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.service deleted file mode 100644 index c05ece90f50..00000000000 --- a/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.service +++ /dev/null @@ -1,141 +0,0 @@ -# systemd service for the Claude Code compatibility-matrix populator. -# -# Triggered by `litellm-compat-matrix.timer`; not started directly. The -# unit is a `Type=oneshot` so the timer's `OnCalendar=` semantics -# describe "run once per day" cleanly — there's no long-lived daemon to -# supervise; each invocation runs the populator end-to-end and exits. -# -# Install -# ------- -# -# sudo cp tests/e2e/claude_code/cron_vm/litellm-compat-matrix.service /etc/systemd/system/ -# sudo cp tests/e2e/claude_code/cron_vm/litellm-compat-matrix.timer /etc/systemd/system/ -# sudo systemctl daemon-reload -# sudo systemctl enable --now litellm-compat-matrix.timer -# -# Paths are hard-coded to /home/mateo rather than using systemd's %h -# specifier. Why: in *system* units (this one), %h is expanded at -# parse time against the *manager's* home -- which is /root for PID 1 -# -- and *not* against the User= directive. That mismatch makes -# ReadWritePaths point at /root/.cache (which doesn't exist), causing -# the namespace setup to fail with status=226/NAMESPACE before the -# script ever runs. The runtime user (`User=mateo`) must: -# -# * have a checkout of `BerriAI/litellm` at `~/litellm/litellm` so the -# publisher module is importable; -# * have a uv venv at `~/litellm/litellm/.venv` (created by -# `uv sync --frozen` inside that checkout once); -# * have `gh` already authenticated against an account with -# `pull-requests: write` on `BerriAI/litellm-docs`; -# * have provider credentials exported in `/etc/litellm-compat-matrix.env` -# (see `litellm-compat-matrix.env.example` in this directory). - -[Unit] -Description=Claude Code compatibility-matrix populator (oneshot) -Wants=network-online.target -After=network-online.target - -[Service] -Type=oneshot -User=mateo -Group=mateo - -# Provider credentials + any gh/PROXY_PORT overrides live here. Format -# is the standard `KEY=value` one line per env var. -EnvironmentFile=-/etc/litellm-compat-matrix.env - -# systemd starts with a minimal PATH (~/usr/local/bin:/usr/bin:/bin). -# `uv` and `claude` are installed under the runtime user's `~/.local/bin` -# so we have to prepend it explicitly; otherwise run_daily.sh fails at -# the up-front command-presence check. -Environment=PATH=/home/mateo/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin - -# `HOME` is auto-set to /home/mateo when User=mateo is honored, but be -# explicit so anything that reads $HOME (e.g. uv's cache lookup, the -# claude CLI's per-session dir) sees the right value even if a future -# refactor flips DynamicUser= or PrivateUsers= on. -Environment=HOME=/home/mateo - -WorkingDirectory=/home/mateo/litellm/litellm - -ExecStart=/home/mateo/litellm/litellm/tests/e2e/claude_code/cron_vm/run_daily.sh - -# 90 minutes is generous: cold runs do `git clone` + `uv sync` of a new -# tag's lockfile, which can take a couple of minutes on a 2-vCPU VM, -# plus 30 cells of pytest hitting four cloud providers. -TimeoutStartSec=90min - -# A failed run shouldn't restart automatically — the next timer fire is -# the right retry. Reruns of the same day's matrix are idempotent. -Restart=no - -# Security hardening: the populator only reads the litellm checkout and -# the env-file; everything else it writes lives in either the worktree -# (managed) or `/tmp` (cleaned up by tempfile). -# -# `ProtectHome=read-only` blocks writes to /home/mateo but still -# allows reads. That's safe for the trusted run_daily.sh script -# itself, but unsafe for any subprocess we don't control: a -# compromised npm-installed `claude` package, or a model-directed -# `Read` tool call during a PDF/vision cell, could read sensitive -# host files like `~/.config/gh/hosts.yml` (gh-host token), -# `~/.ssh/`, or `~/.bash_history`. We mitigate that at the call -# boundary: every `claude` subprocess (the up-front `claude --version` -# probe in run_daily.sh, plus every CLI invocation routed through -# tests/e2e/claude_code/cli_driver.py) runs with `HOME` pointed at a -# fresh empty per-invocation tmpdir, not at /home/mateo. The CLI -# never sees the runtime user's real dotfiles. `gh` invocations in -# run_daily.sh pass `GH_TOKEN` inline, so they never need to read -# ~/.config/gh either; that path is intentionally NOT in the -# whitelist below — keeping it out is the second line of defense if -# the inline-token convention is ever accidentally regressed. -# -# ReadWritePaths whitelist: -# * litellm-cron-worktree - the long-lived stable-tag checkout + -# its `.venv` (`uv sync` rewrites every -# run) + `.uv-bin` (pinned `uv` binary -# cache). -# * .cache - uv's wheel cache (~/.cache/uv) so we -# don't redownload pinned deps each -# run. Used only by the trusted `uv` -# process; not exposed to `claude`. -# * /tmp - mktemp -d workdir, proxy logs, and -# the per-`claude`-invocation isolated -# HOME tmpdirs. PrivateTmp=true below -# gives the service its own tmpfs view -# so these don't escape to the host. -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=read-only -ReadWritePaths=/home/mateo/litellm-cron-worktree /home/mateo/.cache /tmp -PrivateTmp=true - -# Filesystem-level hiding for credential-bearing dotdirs/files. Even -# though `ProtectHome=read-only` prevents writes, a model-directed -# `Read` tool call (the PDF cells pass `--allowed-tools Read`) or a -# compromised `claude` package can read absolute paths under -# /home/mateo and exfiltrate the contents. `InaccessiblePaths=` makes -# the listed paths look like empty/missing to every process in the -# unit's mount namespace -- including the trusted populator script, -# which is fine because it doesn't need any of these: -# -# * .config/gh - gh CLI host token; we pass GH_TOKEN inline to -# every `gh` invocation (clone/PR/reviewer) so the -# host config is never consulted. -# * .ssh - never used by the populator. -# * .aws - upstream AWS credentials are passed to the proxy -# via the EnvironmentFile (provider env vars), not -# via shared SDK config files. -# * .docker - the populator never talks to a docker socket. -# * .kube - the populator never talks to a k8s API. -# * .gnupg - no GPG signing on the bot's commits. -# -# Leading `-` makes systemd tolerant if a path doesn't exist on the -# host (the unit is portable across VMs that may not have all of -# them set up). Anything else under /home/mateo (the litellm -# checkout, the cron worktree, the uv cache, .local/bin for the -# claude/uv/gh binaries on PATH) stays read-accessible. -InaccessiblePaths=-/home/mateo/.config/gh -/home/mateo/.ssh -/home/mateo/.aws -/home/mateo/.docker -/home/mateo/.kube -/home/mateo/.gnupg - -[Install] -WantedBy=multi-user.target diff --git a/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.timer b/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.timer deleted file mode 100644 index ee22538c6ed..00000000000 --- a/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.timer +++ /dev/null @@ -1,25 +0,0 @@ -# Daily timer for the compatibility-matrix populator. -# -# 06:00 UTC matches the original GitHub Actions cron schedule; chosen so -# operators in US/EU timezones see fresh PRs at the start of their work -# day. -# -# `Persistent=true` causes a missed run (VM was off / suspended) to -# fire the next time the timer is started, which is the property we -# want for a once-a-day job: the matrix should refresh as soon as the -# VM is reachable again, not wait another 24h. -# -# `RandomizedDelaySec=10min` smears load if multiple matrix-style -# pipelines are ever colocated on the same VM in the future. - -[Unit] -Description=Run the Claude Code compatibility-matrix populator daily - -[Timer] -OnCalendar=*-*-* 06:00:00 UTC -Persistent=true -RandomizedDelaySec=10min -Unit=litellm-compat-matrix.service - -[Install] -WantedBy=timers.target diff --git a/tests/e2e/claude_code/cron_vm/run_daily.sh b/tests/e2e/claude_code/cron_vm/run_daily.sh deleted file mode 100755 index ae2d67c070c..00000000000 --- a/tests/e2e/claude_code/cron_vm/run_daily.sh +++ /dev/null @@ -1,590 +0,0 @@ -#!/usr/bin/env bash -# Daily Claude Code compatibility-matrix populator. -# -# Runs from the GCP VM `litellm-compatibility-matrix-populator` via the -# systemd timer in this directory. The flow is: -# -# 1. Resolve the latest LiteLLM v*-stable tag from the GitHub Releases API. -# 2. Update a long-lived worktree at $WORKTREE to that tag and `uv sync` it. -# 3. Boot the proxy as a background subprocess on $PROXY_PORT (default -# 4100; a separate port from the human-tended :4000 proxy). -# 4. Run `pytest tests/e2e/claude_code/` against the proxy. Test failures -# become `fail` cells in the JSON, not script errors. -# 5. Hand the per-test results artifact + manifest to a small Python -# CLI (`build_matrix.py`) that wraps the existing -# `matrix_builder.build_from_paths` to produce the published -# compatibility-matrix.json. -# 6. `gh repo clone` litellm-docs, write the JSON to a deterministic -# branch (`compat-matrix/--`), commit, -# `git push --force`, and `gh pr create`. -# -# Same-day reruns land on the same branch so they update the existing PR -# rather than spawning a new one. If the JSON is byte-identical to the -# docs branch, we skip the push entirely. -# -# Required commands on $PATH: git, uv, gh, jq, curl, claude. -# Required state: ~/litellm/litellm checked out (this file lives in it), -# $WORKTREE is created on first run, gh is already authenticated. -# -# Override any default by setting the matching env var; see the systemd -# unit for the production wiring. - -set -Eeuo pipefail - -LITELLM_REPO="${LITELLM_REPO:-${HOME}/litellm/litellm}" -WORKTREE="${LITELLM_WORKTREE:-${HOME}/litellm-cron-worktree}" -PROXY_PORT="${PROXY_PORT:-4100}" -PROXY_API_KEY="${PROXY_API_KEY:-sk-cron-matrix}" -DOCS_REPO="${DOCS_REPO:-BerriAI/litellm-docs}" -DOCS_BRANCH="${DOCS_BRANCH:-main}" -DOCS_TARGET_PATH="${DOCS_TARGET_PATH:-src/data/compatibility-matrix.json}" -SKIP_PUBLISH="${SKIP_PUBLISH:-0}" -PYTEST_K="${PYTEST_K:-}" -# Comma-separated GitHub usernames to request a review from on every PR. -# Reviewers must have at least read access to ${DOCS_REPO}. PR-author -# (agent-shin) has implicit rights to request reviews from anyone with -# read access, so no extra token scope is needed. Set to empty to skip. -PR_REVIEWERS="${PR_REVIEWERS:-mateo-berri}" - -POPULATOR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -WORKDIR="$(mktemp -d -t litellm-compat-matrix.XXXXXX)" -PROXY_PID_FILE="${WORKDIR}/proxy.pid" - -# Cleanup is intentionally aggressive: it can run on normal exit, on a -# signal received by the script, or after a partial failure where the -# proxy is up but ${PROXY_PID_FILE} is stale. We try four things in -# order and stop as soon as the proxy port is free: -# -# 1. SIGTERM the pid recorded in proxy.pid. -# 2. SIGKILL anything from `pgrep -f "litellm.*--port ${PROXY_PORT}"` -# that survived. This catches the common case where the recorded -# pid was the sh wrapper, not the long-lived python child. -# 3. ss -K on the port (kernel kills sockets but not processes; -# mostly useful for catching lingering CLOSE_WAITs). -# 4. wipe ${WORKDIR}. -cleanup() { - local rc=$? - set +e - local proxy_pid - if [[ -f "${PROXY_PID_FILE}" ]]; then - proxy_pid="$(cat "${PROXY_PID_FILE}")" - if [[ -n "${proxy_pid}" ]]; then - kill -TERM "-${proxy_pid}" 2>/dev/null || kill -TERM "${proxy_pid}" 2>/dev/null || true - for _ in 1 2 3 4 5; do - kill -0 "${proxy_pid}" 2>/dev/null || break - sleep 1 - done - fi - fi - # Belt-and-braces: any python or uv talking to ${PROXY_PORT} that - # survived the SIGTERM gets SIGKILL'd by name. - pgrep -f "litellm.*--port[ =]?${PROXY_PORT}([^0-9]|$)" 2>/dev/null \ - | xargs -r kill -KILL 2>/dev/null || true - pgrep -f "${WORKTREE}/.uv-bin/uv.*run litellm" 2>/dev/null \ - | xargs -r kill -KILL 2>/dev/null || true - rm -rf "${WORKDIR}" - exit "${rc}" -} -trap cleanup EXIT INT TERM - -log() { printf '==> %s\n' "$*" >&2; } -die() { printf 'ERROR: %s\n' "$*" >&2; exit 1; } - -for cmd in git uv gh jq curl claude; do - command -v "${cmd}" >/dev/null 2>&1 || die "missing required command: ${cmd}" -done - -# Publishing is from a fork (agent-shin/litellm-docs) so neither the cron -# host nor the bot identity needs write access to BerriAI/litellm-docs. We -# require the fork token up front -- failing 30 minutes into a run because -# the env file is missing one line is a waste of CI quota. -if [[ "${SKIP_PUBLISH}" != "1" ]]; then - [[ -n "${AGENT_SHIN_GITHUB_TOKEN:-}" ]] \ - || die "AGENT_SHIN_GITHUB_TOKEN required to open PRs from agent-shin/litellm-docs (or set SKIP_PUBLISH=1)" -fi - -# --------------------------------------------------------------------------- -# 1. Resolve versions -# --------------------------------------------------------------------------- - -# Newest v*-stable release on BerriAI/litellm. The `select(...)` filter -# drops drafts/non-stable, the version_key sort handles 1.10 > 1.9. -# -# Paginate through the releases endpoint instead of grabbing only page 1 -# (default page_size=30). LiteLLM ships multiple non-stable releases per -# day, so it's common to need to walk past 30+ entries before hitting -# the most recent v*-stable. We cap at 5 pages (500 releases) which is -# conservatively beyond the worst observed gap. -# -# We deliberately do NOT short-circuit on the first page that contains a -# v*-stable tag. The /releases endpoint orders by `created_at`, not by -# semver, so a backport on an older series (e.g. v1.80.1-stable cut -# today) can show up on an earlier page than a higher-versioned release -# (v1.83.0-stable cut two weeks ago). Breaking early on first-stable-seen -# would silently pin the cron to the stale tag because the -# higher-versioned release still on a later page would never make it -# into the merged set the `sort_by` below consumes. The only break we -# keep is the empty-page guard, which means a quiet period in the -# release feed doesn't waste API quota — we just always walk far enough -# to be confident we've seen the highest stable tag. -GH_AUTH_HEADER=() -if [[ -n "${GITHUB_TOKEN:-}" ]]; then - GH_AUTH_HEADER=(-H "Authorization: Bearer ${GITHUB_TOKEN}") -fi -RELEASES_JSON="${WORKDIR}/releases.json" -echo "[]" >"${RELEASES_JSON}" -for page in 1 2 3 4 5; do - PAGE_JSON="${WORKDIR}/releases.page${page}.json" - curl -fsS \ - -H 'Accept: application/vnd.github+json' \ - -H 'User-Agent: litellm-compat-matrix' \ - "${GH_AUTH_HEADER[@]}" \ - "https://api.github.com/repos/BerriAI/litellm/releases?per_page=100&page=${page}" \ - >"${PAGE_JSON}" - jq -s '.[0] + .[1]' "${RELEASES_JSON}" "${PAGE_JSON}" >"${RELEASES_JSON}.merged" - mv "${RELEASES_JSON}.merged" "${RELEASES_JSON}" - # No more pages? GitHub returns an empty array past the last page. - if [[ "$(jq 'length' "${PAGE_JSON}")" == "0" ]]; then - break - fi -done -LITELLM_VERSION="$( - jq -r ' - [ .[] | .tag_name // empty - | select(test("^v[0-9]+\\.[0-9]+\\.[0-9]+-stable$")) - ] - | sort_by( - capture("^v(?[0-9]+)\\.(?[0-9]+)\\.(?[0-9]+)-stable$") - | [(.a|tonumber), (.b|tonumber), (.c|tonumber)] - ) - | last // empty - ' "${RELEASES_JSON}" -)" -[[ -n "${LITELLM_VERSION}" ]] || die "could not resolve latest v*-stable tag in 5 pages of releases" -log "resolved litellm: ${LITELLM_VERSION}" - -# The systemd unit loads provider credentials and the agent-shin GitHub -# token from /etc/litellm-compat-matrix.env into this script's -# environment. Running the npm-installed `claude` binary directly here -# would hand that full env to package code -- a compromised -# @anthropic-ai/claude-code release could read ANTHROPIC_API_KEY / -# AWS_BEARER_TOKEN_BEDROCK / AZURE_FOUNDRY_API_KEY / -# AGENT_SHIN_GITHUB_TOKEN from os.environ and exfiltrate them before -# the proxy or test harness ever starts. Probe under `env -i` with the -# same minimal allowlist the PR-gate uses (the matrix run itself goes -# through cli_driver.py, which already scrubs the CLI env). -# -# The probe also runs under a fresh empty HOME instead of the runtime -# user's real $HOME. `ProtectHome=read-only` in the systemd unit -# blocks *writes* to /home/mateo but still allows reads, so a -# compromised claude package invoked here with HOME=/home/mateo could -# read ~/.config/gh/hosts.yml (the gh-host token), ~/.bash_history, -# or ~/.ssh/. Pointing HOME at a per-run dir under ${WORKDIR} hides -# those entirely from the subprocess; ${WORKDIR} is rm -rf'd by the -# script-wide cleanup() trap regardless of probe outcome. -CLAUDE_PROBE_HOME="${WORKDIR}/claude-probe-home" -mkdir -p "${CLAUDE_PROBE_HOME}" -CLAUDE_CODE_VERSION="$(env -i \ - PATH="${PATH}" \ - HOME="${CLAUDE_PROBE_HOME}" \ - USER="${USER:-mateo}" \ - TERM="${TERM:-dumb}" \ - LANG="${LANG:-C.UTF-8}" \ - LC_ALL="${LC_ALL:-}" \ - TMPDIR="${TMPDIR:-/tmp}" \ - claude --version 2>/dev/null \ - | grep -oE '[0-9]+\.[0-9]+\.[0-9]+([.-][A-Za-z0-9.-]+)?' \ - | head -n1 || true)" -# `|| true` above keeps `set -Eeuo pipefail` from aborting silently when -# `grep` finds no match (exit 1) — without it the assignment inherits the -# pipeline's non-zero exit, `set -e` kills the script, and the operator -# never sees the helpful diagnostic below. -[[ -n "${CLAUDE_CODE_VERSION}" ]] || die "could not parse semver from 'claude --version'" -log "local claude code: ${CLAUDE_CODE_VERSION}" - -# --------------------------------------------------------------------------- -# 2. Update the worktree to that tag -# --------------------------------------------------------------------------- - -if [[ ! -d "${WORKTREE}/.git" ]]; then - log "first run: cloning litellm into ${WORKTREE}" - mkdir -p "$(dirname "${WORKTREE}")" - git clone https://github.com/BerriAI/litellm.git "${WORKTREE}" -fi - -log "updating worktree to ${LITELLM_VERSION}" -git -C "${WORKTREE}" fetch --tags --force -git -C "${WORKTREE}" reset --hard -# Keep the venv and the .uv-bin cache around — uv sync will reconcile -# the venv on every run, and we don't want to re-download the pinned -# uv binary each time. Drop everything else (including any prior -# tests/e2e/claude_code/ shim) so each run starts clean before the shim -# below rewrites it from the dev checkout. -git -C "${WORKTREE}" clean -fdx -e .venv -e .uv-bin -git -C "${WORKTREE}" checkout --force "${LITELLM_VERSION}" - -# Always overwrite tests/e2e/claude_code/ in the worktree with the copy -# from the dev checkout, regardless of whether the resolved -# ${LITELLM_VERSION} tag already ships a tests/e2e/claude_code/ tree of -# its own. Rationale: the matrix populator's job is to exercise -# today's tests against the latest stable proxy. The dev checkout -# carries the most recent test fixes (e.g. the stream-json vision -# rewrite, the --effort thinking knob, the WebSearch tool_use -# assertion) that haven't yet rolled into a v*-stable, and we want -# every cron run to pick those up the moment they land on -# ${LITELLM_REPO}, not whenever the next stable release happens. -# -# Concretely this means a fresh `rm -rf` + `cp -r` every run so the -# tree is byte-identical to ${LITELLM_REPO}/tests/e2e/claude_code (no -# stale files left over from the tag's own checkout, no drift across -# runs). -if [[ ! -d "${LITELLM_REPO}/tests/e2e/claude_code" ]]; then - die "no shim source at ${LITELLM_REPO}/tests/e2e/claude_code" -fi -log "shimming tests/e2e/claude_code/ from ${LITELLM_REPO} (always-overwrite)" -rm -rf "${WORKTREE}/tests/e2e/claude_code" -mkdir -p "${WORKTREE}/tests/e2e" -cp -r "${LITELLM_REPO}/tests/e2e/claude_code" "${WORKTREE}/tests/e2e/" - -# litellm pins an exact uv version in pyproject.toml's [tool.uv] -# `required-version` field, so a system uv that's newer or older -# refuses to sync. We pin our own local copy at the version the -# checked-out tag asks for, cached under .uv-bin/ inside the worktree -# so subsequent runs skip the download. -PINNED_UV_VERSION="$( - awk -F'"' ' - /^required-version[[:space:]]*=/ { - # Field 2 is the value between the quotes, e.g. ">=0.10.9" or - # "0.10.9". Strip any leading specifier prefix so we end up with - # the bare version string, which is what /releases/download// - # expects. - v = $2 - sub(/^[[:space:]=<>!~]+/, "", v) - if (v != "") { print v; exit } - } - ' "${WORKTREE}/pyproject.toml" -)" -if [[ -z "${PINNED_UV_VERSION}" ]]; then - log "no uv version pin in pyproject.toml; using system uv" - WORKTREE_UV="$(command -v uv)" -else - WORKTREE_UV="${WORKTREE}/.uv-bin/uv-${PINNED_UV_VERSION}" - if [[ ! -x "${WORKTREE_UV}" ]]; then - log "downloading uv ${PINNED_UV_VERSION} for the worktree" - mkdir -p "${WORKTREE}/.uv-bin" - # Detect host arch so the same script works on x86_64 GCP VMs and on - # aarch64 hosts (Astral publishes both `uv-x86_64-unknown-linux-gnu` - # and `uv-aarch64-unknown-linux-gnu` tarballs under the same release - # tag, and `uname -m` already returns the exact token uv uses). - UV_ARCH="$(uname -m)" - UV_TRIPLE="uv-${UV_ARCH}-unknown-linux-gnu" - UV_TARBALL_NAME="${UV_TRIPLE}.tar.gz" - UV_DOWNLOAD_URL="https://github.com/astral-sh/uv/releases/download/${PINNED_UV_VERSION}/${UV_TARBALL_NAME}" - UV_TMPDIR="$(mktemp -d -t uv-download.XXXXXX)" - # Download the tarball and Astral's official .sha256 sidecar to disk - # and verify the digest before extracting/executing anything. This - # closes the supply-chain trust gap of piping a remote binary - # straight into `tar -xzO ... > file ; chmod +x` (see CLAUDE.md - # "CI Supply-Chain Safety"). - curl -fsSL --output "${UV_TMPDIR}/${UV_TARBALL_NAME}" "${UV_DOWNLOAD_URL}" - curl -fsSL --output "${UV_TMPDIR}/${UV_TARBALL_NAME}.sha256" "${UV_DOWNLOAD_URL}.sha256" - (cd "${UV_TMPDIR}" && sha256sum -c "${UV_TARBALL_NAME}.sha256") \ - || { rm -rf "${UV_TMPDIR}"; die "uv ${PINNED_UV_VERSION} sha256 mismatch — refusing to install"; } - tar -xzf "${UV_TMPDIR}/${UV_TARBALL_NAME}" -C "${UV_TMPDIR}" "${UV_TRIPLE}/uv" - mv "${UV_TMPDIR}/${UV_TRIPLE}/uv" "${WORKTREE_UV}.tmp" - chmod +x "${WORKTREE_UV}.tmp" - mv "${WORKTREE_UV}.tmp" "${WORKTREE_UV}" - rm -rf "${UV_TMPDIR}" - fi -fi -# `--extra proxy` pulls fastapi/uvicorn/etc. so `uv run litellm` can -# actually serve. `--group proxy-dev` brings in pytest and the rest of -# what tests/e2e/claude_code/ needs. -log "uv sync --frozen --group proxy-dev --extra proxy (uv ${PINNED_UV_VERSION:-system})" -(cd "${WORKTREE}" && "${WORKTREE_UV}" sync --frozen --group proxy-dev --extra proxy) - -PROXY_CONFIG="${WORKTREE}/tests/e2e/claude_code/test_config.yaml" -[[ -f "${PROXY_CONFIG}" ]] || die "proxy config not found at ${PROXY_CONFIG} (does ${LITELLM_VERSION} predate the compat matrix work?)" - -# --------------------------------------------------------------------------- -# 3. Boot the proxy -# --------------------------------------------------------------------------- - -log "starting proxy on 127.0.0.1:${PROXY_PORT}" -# Bind the proxy to loopback only. The populator proxy is talked to -# exclusively by the pytest run on the same host (the health check and -# the test env set `LITELLM_PROXY_BASE_URL=http://127.0.0.1:...`), -# so there's no reason to expose it on the VM's external interfaces. -# Without `--host`, `litellm` defaults to 0.0.0.0, which combined with -# the predictable default `LITELLM_MASTER_KEY=sk-cron-matrix` would -# allow anything that can reach :${PROXY_PORT} on the VM to authenticate -# and burn upstream provider credentials. -# -# `setsid` puts the proxy in its own session+pgroup so cleanup() can -# SIGTERM the whole tree by passing the pgid as a negative pid. We -# write that pid to a file so cleanup() doesn't need to remember a -# variable that might be stale by the time the trap fires. -# -# Pass the master key as a shell-prefix assignment on `setsid` (inherited -# via the environment) rather than as `env KEY=VAL ...` argv. The argv -# form would land the literal key in /proc//cmdline, where -# any local reader (a model-directed `Read` tool call, another user on -# the VM, a crash dump) could pick it up before the process execs into -# the litellm child. The shell-prefix form keeps the key out of argv at -# every layer (setsid → bash → uv → litellm). -LITELLM_MASTER_KEY="${PROXY_API_KEY}" setsid bash -c ' - echo "$$" > "$0" - cd "$1" - exec "$2" run litellm --config "$3" --host 127.0.0.1 --port "$4" -' "${PROXY_PID_FILE}" "${WORKTREE}" "${WORKTREE_UV}" "${PROXY_CONFIG}" "${PROXY_PORT}" \ - >"${WORKDIR}/proxy.log" 2>&1 & -disown - -HEALTH_URL="http://127.0.0.1:${PROXY_PORT}/health/liveliness" -for _ in $(seq 1 45); do - if curl -fsS "${HEALTH_URL}" >/dev/null 2>&1; then - break - fi - sleep 2 -done -curl -fsS "${HEALTH_URL}" >/dev/null \ - || { tail -50 "${WORKDIR}/proxy.log" >&2; die "proxy did not become healthy"; } - -# --------------------------------------------------------------------------- -# 4. Run pytest -# --------------------------------------------------------------------------- - -RESULTS_JSON="${WORKDIR}/compat-results.json" -PYTEST_ARGS=( - tests/e2e/claude_code/ - --ignore=tests/e2e/claude_code/_driver_unit_tests - --ignore=tests/e2e/claude_code/_builder_unit_tests - --ignore=tests/e2e/claude_code/_publisher_unit_tests - --ignore=tests/e2e/claude_code/_pr_gate_unit_tests -) -if [[ -n "${PYTEST_K}" ]]; then - log "PYTEST_K set; narrowing to: ${PYTEST_K}" - PYTEST_ARGS+=(-k "${PYTEST_K}") -fi - -log "running pytest" -set +e -# Pytest only needs to talk to the loopback proxy at 127.0.0.1:${PROXY_PORT} -# — it has no legitimate reason to see ANTHROPIC_API_KEY / -# AWS_BEARER_TOKEN_BEDROCK / VERTEXAI_* / AZURE_FOUNDRY_* / -# AGENT_SHIN_GITHUB_TOKEN / GITHUB_TOKEN in its own env. The systemd -# unit's EnvironmentFile injects all of those into this script for the -# proxy to consume, and pytest inherits them by default. Wrap the -# invocation in `env -i` so: -# -# 1. test code under tests/e2e/claude_code/ (or anything it imports) -# cannot read provider/agent-shin creds out of `os.environ` and -# exfiltrate them via an outbound call from inside a conftest hook -# or a fixture (a sibling vector to the model-controlled Bash/Read -# concern handled by `cli_driver.py`'s own env scrub); -# 2. a model-directed `Read` tool call during a PDF/vision cell -# cannot reach /proc//environ and pull the creds out -# of the parent process the way it can today; -# 3. this matches the PR-gate pytest step in `.circleci/config.yml`, -# which already runs under `env -i` with the same minimal -# allowlist. -# -# `cli_driver.py` re-allowlists its own subset (PATH/USER/LOGNAME/etc.) -# when spawning the `claude` binary, so the CLI still finds Node + the -# claude shim on PATH and gets a fresh isolated HOME per invocation. -( - cd "${WORKTREE}" \ - && env -i \ - PATH="${PATH}" \ - HOME="${HOME}" \ - USER="${USER:-mateo}" \ - TERM="${TERM:-dumb}" \ - LANG="${LANG:-C.UTF-8}" \ - LC_ALL="${LC_ALL:-}" \ - TMPDIR="${TMPDIR:-/tmp}" \ - LITELLM_PROXY_BASE_URL="http://127.0.0.1:${PROXY_PORT}" \ - LITELLM_PROXY_API_KEY="${PROXY_API_KEY}" \ - COMPAT_RESULTS_PATH="${RESULTS_JSON}" \ - "${WORKTREE_UV}" run pytest "${PYTEST_ARGS[@]}" -) -PYTEST_EXIT=$? -set -e -log "pytest exit code: ${PYTEST_EXIT} (failures become 'fail' cells, not script errors)" -[[ -f "${RESULTS_JSON}" ]] || die "pytest did not produce ${RESULTS_JSON}" - -# --------------------------------------------------------------------------- -# 5. Build the matrix JSON -# --------------------------------------------------------------------------- - -MATRIX_JSON="${WORKDIR}/compatibility-matrix.json" -log "building ${MATRIX_JSON}" -( - cd "${WORKTREE}" \ - && "${WORKTREE_UV}" run python "${POPULATOR_DIR}/build_matrix.py" \ - --manifest "${WORKTREE}/tests/e2e/claude_code/manifest.yaml" \ - --results "${RESULTS_JSON}" \ - --output "${MATRIX_JSON}" \ - --litellm-version "${LITELLM_VERSION}" \ - --claude-code-version "${CLAUDE_CODE_VERSION}" -) - -# --------------------------------------------------------------------------- -# 6. Open a docs-repo PR -# --------------------------------------------------------------------------- - -if [[ "${SKIP_PUBLISH}" == "1" ]]; then - cp "${MATRIX_JSON}" "${LITELLM_REPO}/compatibility-matrix.json" - log "SKIP_PUBLISH=1; matrix written to ${LITELLM_REPO}/compatibility-matrix.json" - exit 0 -fi - -DATE_UTC="$(date -u +%Y-%m-%d)" -BRANCH_NAME="compat-matrix/${LITELLM_VERSION}-${CLAUDE_CODE_VERSION}-${DATE_UTC}" -DOCS_CLONE="${WORKDIR}/litellm-docs" -FORK_OWNER="${FORK_OWNER:-agent-shin}" -FORK_REPO="${FORK_REPO:-${FORK_OWNER}/litellm-docs}" - -log "cloning ${DOCS_REPO}@${DOCS_BRANCH}" -# Use the agent-shin token inline rather than the host gh-cli config. -# `BerriAI/litellm-docs` is a public repo so unauthenticated clone -# would also work, but passing the token explicitly means the systemd -# unit can hide `~/.config/gh` (`InaccessiblePaths=`) without breaking -# this clone — closing the model-directed `Read("/home/mateo/.config/gh/...")` -# exfiltration path on the cron VM. -GH_TOKEN="${AGENT_SHIN_GITHUB_TOKEN}" \ - gh repo clone "${DOCS_REPO}" "${DOCS_CLONE}" -- --depth 1 --branch "${DOCS_BRANCH}" - -cd "${DOCS_CLONE}" -git config user.email "litellm-bot@berri.ai" -git config user.name "litellm-compat-matrix-bot" -git checkout -b "${BRANCH_NAME}" - -mkdir -p "$(dirname "${DOCS_TARGET_PATH}")" -cp "${MATRIX_JSON}" "${DOCS_TARGET_PATH}" -git add "${DOCS_TARGET_PATH}" - -if git diff --cached --quiet; then - log "matrix JSON unchanged from ${DOCS_BRANCH}; skipping PR" - exit 0 -fi - -GENERATED_AT="$(jq -r '.generated_at' "${MATRIX_JSON}")" -COMMIT_MSG="$(cat </dev/null || true -git remote add fork "${FORK_PUSH_URL}" -git push --force --set-upstream fork "${BRANCH_NAME}" -git remote remove fork -unset FORK_PUSH_URL - -# Per-feature status table for the PR body. Reviewers triage from this. -PR_FEATURE_TABLE="$(jq -r ' - .features[] as $f - | "- **\($f.name)**: " + - ([ .providers[] as $p - | "\($p)=\($f.providers[$p].status // "not_tested")" - ] | join(", ")) -' "${MATRIX_JSON}")" - -PR_TITLE="chore(compat-matrix): refresh for ${LITELLM_VERSION} + claude-code ${CLAUDE_CODE_VERSION}" -PR_BODY="$(cat < ${DOCS_REPO}:${DOCS_BRANCH}" -# GH_TOKEN here is scoped to this single subshell so we don't bleed the -# fork token into the rest of the script (release-listing earlier uses -# ${GITHUB_TOKEN}, which may be a different identity). gh's --head accepts -# `OWNER:BRANCH` for cross-repo PRs from a fork. -# -# Reviewer assignment is done in a *separate* call below: as the PR -# author from a fork, agent-shin has no write/triage access on -# ${DOCS_REPO} and the `RequestReviewsByLogin` GraphQL mutation -# (which backs `gh pr create --reviewer` and `gh pr edit --add-reviewer`) -# rejects with "does not have the correct permissions". We use the -# collaborator-scoped ${GITHUB_TOKEN} for that instead. Don't fold -# --reviewer into `gh pr create` here -- it would fail the whole -# create on the very first cron run. -set +e -PR_OUT="$( - GH_TOKEN="${AGENT_SHIN_GITHUB_TOKEN}" gh pr create \ - --repo "${DOCS_REPO}" \ - --base "${DOCS_BRANCH}" \ - --head "${FORK_OWNER}:${BRANCH_NAME}" \ - --title "${PR_TITLE}" \ - --body "${PR_BODY}" 2>&1 -)" -PR_EXIT=$? -set -e -echo "${PR_OUT}" - -if [[ ${PR_EXIT} -ne 0 ]]; then - if grep -q "a pull request for branch.*already exists" <<<"${PR_OUT}"; then - log "PR already exists for ${FORK_OWNER}:${BRANCH_NAME}; updated branch in place" - else - die "gh pr create failed (exit ${PR_EXIT})" - fi -fi - -# Request reviews from PR_REVIEWERS using the collaborator-scoped -# ${GITHUB_TOKEN} (mateo-berri's token, already provisioned for release -# listing). This is idempotent: `gh pr edit --add-reviewer` is a no-op -# on a user who's already in reviewRequests, and silently re-adds -# anyone whose prior review was dismissed -- so same-day reruns stay -# clean. Reviewer-add failures are non-fatal: the matrix JSON has -# already landed on the PR; the worst case is a manual ping. -if [[ -n "${PR_REVIEWERS}" ]]; then - if [[ -z "${GITHUB_TOKEN:-}" ]]; then - log "WARN: PR_REVIEWERS set but GITHUB_TOKEN missing -- cannot request reviews; skipping" - else - log "requesting reviews from: ${PR_REVIEWERS}" - set +e - GH_TOKEN="${GITHUB_TOKEN}" gh pr edit \ - "${FORK_OWNER}:${BRANCH_NAME}" \ - --repo "${DOCS_REPO}" \ - --add-reviewer "${PR_REVIEWERS}" 2>&1 | sed 's/^/ /' - REVIEWER_EXIT=${PIPESTATUS[0]} - set -e - if [[ ${REVIEWER_EXIT} -ne 0 ]]; then - log "WARN: gh pr edit --add-reviewer exited ${REVIEWER_EXIT} (non-fatal)" - fi - fi -fi - -log "done" diff --git a/tests/e2e/claude_code/long_context_1m/test_anthropic.py b/tests/e2e/claude_code/long_context_1m/test_anthropic.py index fb74d5fd40d..b9bbd1c2fe7 100644 --- a/tests/e2e/claude_code/long_context_1m/test_anthropic.py +++ b/tests/e2e/claude_code/long_context_1m/test_anthropic.py @@ -54,25 +54,23 @@ $10/day on this row. from __future__ import annotations -import os from typing import Sequence import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" # Haiku 4.5 is excluded -- only Sonnet 4.6 and Opus 4.7 support the # 1M-context beta. See module docstring for the per-cell-aggregator # rationale. ANTHROPIC_MODELS: Sequence[str] = ( - "claude-sonnet-4-6", + "claude-sonnet-4-5", "claude-opus-4-7", ) @@ -155,26 +153,12 @@ def _build_long_prompt(target_tokens: int = TARGET_INPUT_TOKENS) -> str: return preamble + "".join(pad_lines) + closing +@pytest.mark.covers("llm.messages.anthropic.long_context_1m.nonstream.works") def test_long_context_1m_anthropic(compat_result): """Drive the `claude` CLI with a ~210k-token prompt and the `context-1m-2025-08-07` beta header; assert no 400 / 413 and a non-empty reply for Sonnet + Opus.""" - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", - pytrace=False, - ) + base_url, api_key = require_proxy(compat_result) long_prompt = _build_long_prompt() diff --git a/tests/e2e/claude_code/long_context_1m/test_azure.py b/tests/e2e/claude_code/long_context_1m/test_azure.py index 5800fdadbfc..d62214d2758 100644 --- a/tests/e2e/claude_code/long_context_1m/test_azure.py +++ b/tests/e2e/claude_code/long_context_1m/test_azure.py @@ -54,25 +54,23 @@ $10/day on this row. from __future__ import annotations -import os from typing import Sequence import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" # Haiku 4.5 is excluded -- only Sonnet 4.6 and Opus 4.7 support the # 1M-context beta. See module docstring for the per-cell-aggregator # rationale. AZURE_MODELS: Sequence[str] = ( - "claude-sonnet-4-6-azure", + "claude-sonnet-4-5-azure", "claude-opus-4-7-azure", ) @@ -155,26 +153,12 @@ def _build_long_prompt(target_tokens: int = TARGET_INPUT_TOKENS) -> str: return preamble + "".join(pad_lines) + closing +@pytest.mark.covers("llm.messages.azure_foundry.long_context_1m.nonstream.works") def test_long_context_1m_azure(compat_result): """Drive the `claude` CLI (Azure (Microsoft Foundry)) with a ~210k-token prompt and the `context-1m-2025-08-07` beta header; assert no 400 / 413 and a non-empty reply for Sonnet + Opus.""" - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", - pytrace=False, - ) + base_url, api_key = require_proxy(compat_result) long_prompt = _build_long_prompt() diff --git a/tests/e2e/claude_code/long_context_1m/test_bedrock_converse.py b/tests/e2e/claude_code/long_context_1m/test_bedrock_converse.py index 18587f7c2d6..3c2fd4f02cc 100644 --- a/tests/e2e/claude_code/long_context_1m/test_bedrock_converse.py +++ b/tests/e2e/claude_code/long_context_1m/test_bedrock_converse.py @@ -54,25 +54,23 @@ $10/day on this row. from __future__ import annotations -import os from typing import Sequence import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" # Haiku 4.5 is excluded -- only Sonnet 4.6 and Opus 4.7 support the # 1M-context beta. See module docstring for the per-cell-aggregator # rationale. BEDROCK_CONVERSE_MODELS: Sequence[str] = ( - "claude-sonnet-4-6-bedrock-converse", + "claude-sonnet-4-5-bedrock-converse", "claude-opus-4-7-bedrock-converse", ) @@ -155,26 +153,12 @@ def _build_long_prompt(target_tokens: int = TARGET_INPUT_TOKENS) -> str: return preamble + "".join(pad_lines) + closing +@pytest.mark.covers("llm.messages.bedrock_converse.long_context_1m.nonstream.works") def test_long_context_1m_bedrock_converse(compat_result): """Drive the `claude` CLI (Bedrock (Converse)) with a ~210k-token prompt and the `context-1m-2025-08-07` beta header; assert no 400 / 413 and a non-empty reply for Sonnet + Opus.""" - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", - pytrace=False, - ) + base_url, api_key = require_proxy(compat_result) long_prompt = _build_long_prompt() diff --git a/tests/e2e/claude_code/long_context_1m/test_bedrock_invoke.py b/tests/e2e/claude_code/long_context_1m/test_bedrock_invoke.py index 0270197ce2a..4801d405760 100644 --- a/tests/e2e/claude_code/long_context_1m/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/long_context_1m/test_bedrock_invoke.py @@ -54,25 +54,23 @@ $10/day on this row. from __future__ import annotations -import os from typing import Sequence import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" # Haiku 4.5 is excluded -- only Sonnet 4.6 and Opus 4.7 support the # 1M-context beta. See module docstring for the per-cell-aggregator # rationale. BEDROCK_INVOKE_MODELS: Sequence[str] = ( - "claude-sonnet-4-6-bedrock-invoke", + "claude-sonnet-4-5-bedrock-invoke", "claude-opus-4-7-bedrock-invoke", ) @@ -155,26 +153,12 @@ def _build_long_prompt(target_tokens: int = TARGET_INPUT_TOKENS) -> str: return preamble + "".join(pad_lines) + closing +@pytest.mark.covers("llm.messages.bedrock_invoke.long_context_1m.nonstream.works") def test_long_context_1m_bedrock_invoke(compat_result): """Drive the `claude` CLI (Bedrock (Invoke)) with a ~210k-token prompt and the `context-1m-2025-08-07` beta header; assert no 400 / 413 and a non-empty reply for Sonnet + Opus.""" - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", - pytrace=False, - ) + base_url, api_key = require_proxy(compat_result) long_prompt = _build_long_prompt() diff --git a/tests/e2e/claude_code/long_context_1m/test_vertex_ai.py b/tests/e2e/claude_code/long_context_1m/test_vertex_ai.py index d2db4a1b4ee..efa96bf076d 100644 --- a/tests/e2e/claude_code/long_context_1m/test_vertex_ai.py +++ b/tests/e2e/claude_code/long_context_1m/test_vertex_ai.py @@ -54,25 +54,23 @@ $10/day on this row. from __future__ import annotations -import os from typing import Sequence import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" # Haiku 4.5 is excluded -- only Sonnet 4.6 and Opus 4.7 support the # 1M-context beta. See module docstring for the per-cell-aggregator # rationale. VERTEX_AI_MODELS: Sequence[str] = ( - "claude-sonnet-4-6-vertex", + "claude-sonnet-4-5-vertex", "claude-opus-4-7-vertex", ) @@ -155,26 +153,12 @@ def _build_long_prompt(target_tokens: int = TARGET_INPUT_TOKENS) -> str: return preamble + "".join(pad_lines) + closing +@pytest.mark.covers("llm.messages.vertex.long_context_1m.nonstream.works") def test_long_context_1m_vertex_ai(compat_result): """Drive the `claude` CLI (Vertex AI) with a ~210k-token prompt and the `context-1m-2025-08-07` beta header; assert no 400 / 413 and a non-empty reply for Sonnet + Opus.""" - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", - pytrace=False, - ) + base_url, api_key = require_proxy(compat_result) long_prompt = _build_long_prompt() diff --git a/tests/e2e/claude_code/matrix_builder.py b/tests/e2e/claude_code/matrix_builder.py index 5641e488da2..d9a13d17ea4 100644 --- a/tests/e2e/claude_code/matrix_builder.py +++ b/tests/e2e/claude_code/matrix_builder.py @@ -183,7 +183,7 @@ def build_from_paths( generated_at: str, output_path: Optional[Path] = None, ) -> Dict[str, Any]: - """I/O wrapper around build_matrix used by the publisher script.""" + """I/O wrapper around ``build_matrix``: reads the manifest and per-test results from disk, calls ``build_matrix``, and (optionally) writes the compat-matrix JSON to ``output_path``. Whatever orchestrator publishes the matrix (currently the ECR image) invokes this.""" manifest = load_manifest(manifest_path) results = load_results(results_path) matrix = build_matrix( diff --git a/tests/e2e/claude_code/passthrough/test_anthropic.py b/tests/e2e/claude_code/passthrough/test_anthropic.py index aa0443e0625..8382342ae12 100644 --- a/tests/e2e/claude_code/passthrough/test_anthropic.py +++ b/tests/e2e/claude_code/passthrough/test_anthropic.py @@ -29,7 +29,7 @@ from claude_code._passthrough import ( ANTHROPIC_MODELS = [ "claude-haiku-4-5", - "claude-sonnet-4-6", + "claude-sonnet-4-5", "claude-opus-4-7", ] diff --git a/tests/e2e/claude_code/passthrough/test_azure.py b/tests/e2e/claude_code/passthrough/test_azure.py index 09b0824047a..21100a49c16 100644 --- a/tests/e2e/claude_code/passthrough/test_azure.py +++ b/tests/e2e/claude_code/passthrough/test_azure.py @@ -44,7 +44,7 @@ from claude_code._passthrough import foundry_extra_env, run_passthrough_cell AZURE_MODELS = [ "claude-haiku-4-5", - "claude-sonnet-4-6", + "claude-sonnet-4-5", "claude-opus-4-7", ] diff --git a/tests/e2e/claude_code/passthrough/test_bedrock_invoke.py b/tests/e2e/claude_code/passthrough/test_bedrock_invoke.py index 6e84dea6779..f1f28ab5b4c 100644 --- a/tests/e2e/claude_code/passthrough/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/passthrough/test_bedrock_invoke.py @@ -27,7 +27,7 @@ from claude_code._passthrough import bedrock_extra_env, run_passthrough_cell BEDROCK_INVOKE_MODELS = [ "claude-haiku-4-5-bedrock-invoke", - "claude-sonnet-4-6-bedrock-invoke", + "claude-sonnet-4-5-bedrock-invoke", "claude-opus-4-7-bedrock-invoke", ] diff --git a/tests/e2e/claude_code/passthrough/test_vertex_ai.py b/tests/e2e/claude_code/passthrough/test_vertex_ai.py index 5e3c6bce419..790f8b60c8f 100644 --- a/tests/e2e/claude_code/passthrough/test_vertex_ai.py +++ b/tests/e2e/claude_code/passthrough/test_vertex_ai.py @@ -30,7 +30,7 @@ from claude_code._passthrough import run_passthrough_cell, vertex_extra_env VERTEX_MODELS = [ "claude-haiku-4-5-vertex", - "claude-sonnet-4-6-vertex", + "claude-sonnet-4-5-vertex", "claude-opus-4-7-vertex", ] diff --git a/tests/e2e/claude_code/pdf_input/test_anthropic.py b/tests/e2e/claude_code/pdf_input/test_anthropic.py index 36fb69a1db6..21c8028ef1c 100644 --- a/tests/e2e/claude_code/pdf_input/test_anthropic.py +++ b/tests/e2e/claude_code/pdf_input/test_anthropic.py @@ -22,22 +22,19 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations -import os - import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" ANTHROPIC_MODELS = [ "claude-haiku-4-5", - "claude-sonnet-4-6", + "claude-sonnet-4-5", "claude-opus-4-7", ] @@ -108,24 +105,11 @@ def _build_minimal_pdf(marker: str) -> bytes: return bytes(out) +@pytest.mark.covers("llm.messages.anthropic.pdf_input.nonstream.works") def test_pdf_input_anthropic(compat_result, tmp_path): """Drive the `claude` CLI against the LiteLLM proxy with a PDF attached via the Read tool and assert the reply references it.""" - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False - ) + base_url, api_key = require_proxy(compat_result) pdf_path = tmp_path / "marker.pdf" pdf_path.write_bytes(_build_minimal_pdf(PDF_MARKER)) diff --git a/tests/e2e/claude_code/pdf_input/test_azure.py b/tests/e2e/claude_code/pdf_input/test_azure.py index 810c857e407..34ae3732b99 100644 --- a/tests/e2e/claude_code/pdf_input/test_azure.py +++ b/tests/e2e/claude_code/pdf_input/test_azure.py @@ -15,22 +15,19 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations -import os - import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" AZURE_MODELS = [ "claude-haiku-4-5-azure", - "claude-sonnet-4-6-azure", + "claude-sonnet-4-5-azure", "claude-opus-4-7-azure", ] @@ -85,22 +82,9 @@ def _build_minimal_pdf(marker: str) -> bytes: return bytes(out) +@pytest.mark.covers("llm.messages.azure_foundry.pdf_input.nonstream.works") def test_pdf_input_azure(compat_result, tmp_path): - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False - ) + base_url, api_key = require_proxy(compat_result) pdf_path = tmp_path / "marker.pdf" pdf_path.write_bytes(_build_minimal_pdf(PDF_MARKER)) diff --git a/tests/e2e/claude_code/pdf_input/test_bedrock_converse.py b/tests/e2e/claude_code/pdf_input/test_bedrock_converse.py index 191a27c6d46..76aa84f0f47 100644 --- a/tests/e2e/claude_code/pdf_input/test_bedrock_converse.py +++ b/tests/e2e/claude_code/pdf_input/test_bedrock_converse.py @@ -21,22 +21,19 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations -import os - import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" BEDROCK_CONVERSE_MODELS = [ "claude-haiku-4-5-bedrock-converse", - "claude-sonnet-4-6-bedrock-converse", + "claude-sonnet-4-5-bedrock-converse", "claude-opus-4-7-bedrock-converse", ] @@ -91,22 +88,9 @@ def _build_minimal_pdf(marker: str) -> bytes: return bytes(out) +@pytest.mark.covers("llm.messages.bedrock_converse.pdf_input.nonstream.works") def test_pdf_input_bedrock_converse(compat_result, tmp_path): - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False - ) + base_url, api_key = require_proxy(compat_result) pdf_path = tmp_path / "marker.pdf" pdf_path.write_bytes(_build_minimal_pdf(PDF_MARKER)) diff --git a/tests/e2e/claude_code/pdf_input/test_bedrock_invoke.py b/tests/e2e/claude_code/pdf_input/test_bedrock_invoke.py index 163cabb45a0..4450266bb6b 100644 --- a/tests/e2e/claude_code/pdf_input/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/pdf_input/test_bedrock_invoke.py @@ -20,22 +20,19 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations -import os - import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" BEDROCK_INVOKE_MODELS = [ "claude-haiku-4-5-bedrock-invoke", - "claude-sonnet-4-6-bedrock-invoke", + "claude-sonnet-4-5-bedrock-invoke", "claude-opus-4-7-bedrock-invoke", ] @@ -90,22 +87,9 @@ def _build_minimal_pdf(marker: str) -> bytes: return bytes(out) +@pytest.mark.covers("llm.messages.bedrock_invoke.pdf_input.nonstream.works") def test_pdf_input_bedrock_invoke(compat_result, tmp_path): - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False - ) + base_url, api_key = require_proxy(compat_result) pdf_path = tmp_path / "marker.pdf" pdf_path.write_bytes(_build_minimal_pdf(PDF_MARKER)) diff --git a/tests/e2e/claude_code/pdf_input/test_vertex_ai.py b/tests/e2e/claude_code/pdf_input/test_vertex_ai.py index 0d0573d05b3..b78f58cfda1 100644 --- a/tests/e2e/claude_code/pdf_input/test_vertex_ai.py +++ b/tests/e2e/claude_code/pdf_input/test_vertex_ai.py @@ -15,22 +15,19 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations -import os - import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" VERTEX_AI_MODELS = [ "claude-haiku-4-5-vertex", - "claude-sonnet-4-6-vertex", + "claude-sonnet-4-5-vertex", "claude-opus-4-7-vertex", ] @@ -85,22 +82,9 @@ def _build_minimal_pdf(marker: str) -> bytes: return bytes(out) +@pytest.mark.covers("llm.messages.vertex.pdf_input.nonstream.works") def test_pdf_input_vertex_ai(compat_result, tmp_path): - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False - ) + base_url, api_key = require_proxy(compat_result) pdf_path = tmp_path / "marker.pdf" pdf_path.write_bytes(_build_minimal_pdf(PDF_MARKER)) diff --git a/tests/e2e/claude_code/prompt_caching_1h/test_anthropic.py b/tests/e2e/claude_code/prompt_caching_1h/test_anthropic.py index d81887231d8..637be1c551d 100644 --- a/tests/e2e/claude_code/prompt_caching_1h/test_anthropic.py +++ b/tests/e2e/claude_code/prompt_caching_1h/test_anthropic.py @@ -24,23 +24,21 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations -import os from typing import Any, Mapping, Optional import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" ANTHROPIC_MODELS = [ "claude-haiku-4-5", - "claude-sonnet-4-6", + "claude-sonnet-4-5", "claude-opus-4-7", ] @@ -65,25 +63,12 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: return 0 +@pytest.mark.covers("llm.messages.anthropic.prompt_cache_1h.nonstream.works") def test_prompt_caching_1h_anthropic(compat_result): """Drive the `claude` CLI against the LiteLLM proxy with the 1h TTL opt-in env var set, and assert the upstream usage block surfaces a non-zero cache token count.""" - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False - ) + base_url, api_key = require_proxy(compat_result) outcomes = run_claude_models_parallel( models=ANTHROPIC_MODELS, diff --git a/tests/e2e/claude_code/prompt_caching_1h/test_azure.py b/tests/e2e/claude_code/prompt_caching_1h/test_azure.py index 416757f8691..f34557b3c5f 100644 --- a/tests/e2e/claude_code/prompt_caching_1h/test_azure.py +++ b/tests/e2e/claude_code/prompt_caching_1h/test_azure.py @@ -15,23 +15,21 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations -import os from typing import Any, Mapping, Optional import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" AZURE_MODELS = [ "claude-haiku-4-5-azure", - "claude-sonnet-4-6-azure", + "claude-sonnet-4-5-azure", "claude-opus-4-7-azure", ] @@ -49,22 +47,9 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: return 0 +@pytest.mark.covers("llm.messages.azure_foundry.prompt_cache_1h.nonstream.works") def test_prompt_caching_1h_azure(compat_result): - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False - ) + base_url, api_key = require_proxy(compat_result) outcomes = run_claude_models_parallel( models=AZURE_MODELS, diff --git a/tests/e2e/claude_code/prompt_caching_1h/test_bedrock_converse.py b/tests/e2e/claude_code/prompt_caching_1h/test_bedrock_converse.py index 5bc632c6f1b..bf62a49444c 100644 --- a/tests/e2e/claude_code/prompt_caching_1h/test_bedrock_converse.py +++ b/tests/e2e/claude_code/prompt_caching_1h/test_bedrock_converse.py @@ -20,23 +20,21 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations -import os from typing import Any, Mapping, Optional import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" BEDROCK_CONVERSE_MODELS = [ "claude-haiku-4-5-bedrock-converse", - "claude-sonnet-4-6-bedrock-converse", + "claude-sonnet-4-5-bedrock-converse", "claude-opus-4-7-bedrock-converse", ] @@ -57,22 +55,9 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: return 0 +@pytest.mark.covers("llm.messages.bedrock_converse.prompt_cache_1h.nonstream.works") def test_prompt_caching_1h_bedrock_converse(compat_result): - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False - ) + base_url, api_key = require_proxy(compat_result) outcomes = run_claude_models_parallel( models=BEDROCK_CONVERSE_MODELS, diff --git a/tests/e2e/claude_code/prompt_caching_1h/test_bedrock_invoke.py b/tests/e2e/claude_code/prompt_caching_1h/test_bedrock_invoke.py index 4501834956b..dc3468702d4 100644 --- a/tests/e2e/claude_code/prompt_caching_1h/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/prompt_caching_1h/test_bedrock_invoke.py @@ -22,23 +22,21 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations -import os from typing import Any, Mapping, Optional import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" BEDROCK_INVOKE_MODELS = [ "claude-haiku-4-5-bedrock-invoke", - "claude-sonnet-4-6-bedrock-invoke", + "claude-sonnet-4-5-bedrock-invoke", "claude-opus-4-7-bedrock-invoke", ] @@ -61,22 +59,9 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: return 0 +@pytest.mark.covers("llm.messages.bedrock_invoke.prompt_cache_1h.nonstream.works") def test_prompt_caching_1h_bedrock_invoke(compat_result): - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False - ) + base_url, api_key = require_proxy(compat_result) outcomes = run_claude_models_parallel( models=BEDROCK_INVOKE_MODELS, diff --git a/tests/e2e/claude_code/prompt_caching_1h/test_vertex_ai.py b/tests/e2e/claude_code/prompt_caching_1h/test_vertex_ai.py index 09ded634b45..66cf961fcfc 100644 --- a/tests/e2e/claude_code/prompt_caching_1h/test_vertex_ai.py +++ b/tests/e2e/claude_code/prompt_caching_1h/test_vertex_ai.py @@ -15,23 +15,21 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations -import os from typing import Any, Mapping, Optional import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" VERTEX_AI_MODELS = [ "claude-haiku-4-5-vertex", - "claude-sonnet-4-6-vertex", + "claude-sonnet-4-5-vertex", "claude-opus-4-7-vertex", ] @@ -49,22 +47,9 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: return 0 +@pytest.mark.covers("llm.messages.vertex.prompt_cache_1h.nonstream.works") def test_prompt_caching_1h_vertex_ai(compat_result): - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False - ) + base_url, api_key = require_proxy(compat_result) outcomes = run_claude_models_parallel( models=VERTEX_AI_MODELS, diff --git a/tests/e2e/claude_code/prompt_caching_5m/test_anthropic.py b/tests/e2e/claude_code/prompt_caching_5m/test_anthropic.py index 4b20a65f31b..ef551beb45c 100644 --- a/tests/e2e/claude_code/prompt_caching_5m/test_anthropic.py +++ b/tests/e2e/claude_code/prompt_caching_5m/test_anthropic.py @@ -22,23 +22,21 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations -import os from typing import Any, Mapping, Optional import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" ANTHROPIC_MODELS = [ "claude-haiku-4-5", - "claude-sonnet-4-6", + "claude-sonnet-4-5", "claude-opus-4-7", ] @@ -56,24 +54,11 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: return 0 +@pytest.mark.covers("llm.messages.anthropic.prompt_cache_5m.nonstream.works") def test_prompt_caching_5m_anthropic(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert the upstream usage block surfaces a non-zero cache token count.""" - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False - ) + base_url, api_key = require_proxy(compat_result) outcomes = run_claude_models_parallel( models=ANTHROPIC_MODELS, diff --git a/tests/e2e/claude_code/prompt_caching_5m/test_azure.py b/tests/e2e/claude_code/prompt_caching_5m/test_azure.py index 22bd5aa7048..9d4137e0726 100644 --- a/tests/e2e/claude_code/prompt_caching_5m/test_azure.py +++ b/tests/e2e/claude_code/prompt_caching_5m/test_azure.py @@ -22,23 +22,21 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations -import os from typing import Any, Mapping, Optional import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" AZURE_MODELS = [ "claude-haiku-4-5-azure", - "claude-sonnet-4-6-azure", + "claude-sonnet-4-5-azure", "claude-opus-4-7-azure", ] @@ -54,24 +52,11 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: return 0 +@pytest.mark.covers("llm.messages.azure_foundry.prompt_cache_5m.nonstream.works") def test_prompt_caching_5m_azure(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert the upstream usage block surfaces a non-zero cache token count.""" - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False - ) + base_url, api_key = require_proxy(compat_result) outcomes = run_claude_models_parallel( models=AZURE_MODELS, diff --git a/tests/e2e/claude_code/prompt_caching_5m/test_bedrock_converse.py b/tests/e2e/claude_code/prompt_caching_5m/test_bedrock_converse.py index 681a6ecce10..c9b34c010b0 100644 --- a/tests/e2e/claude_code/prompt_caching_5m/test_bedrock_converse.py +++ b/tests/e2e/claude_code/prompt_caching_5m/test_bedrock_converse.py @@ -15,23 +15,21 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations -import os from typing import Any, Mapping, Optional import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" BEDROCK_CONVERSE_MODELS = [ "claude-haiku-4-5-bedrock-converse", - "claude-sonnet-4-6-bedrock-converse", + "claude-sonnet-4-5-bedrock-converse", "claude-opus-4-7-bedrock-converse", ] @@ -47,24 +45,11 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: return 0 +@pytest.mark.covers("llm.messages.bedrock_converse.prompt_cache_5m.nonstream.works") def test_prompt_caching_5m_bedrock_converse(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert the upstream usage block surfaces a non-zero cache token count.""" - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False - ) + base_url, api_key = require_proxy(compat_result) outcomes = run_claude_models_parallel( models=BEDROCK_CONVERSE_MODELS, diff --git a/tests/e2e/claude_code/prompt_caching_5m/test_bedrock_invoke.py b/tests/e2e/claude_code/prompt_caching_5m/test_bedrock_invoke.py index f1a3109b3a1..b95c509ba3c 100644 --- a/tests/e2e/claude_code/prompt_caching_5m/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/prompt_caching_5m/test_bedrock_invoke.py @@ -15,23 +15,21 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations -import os from typing import Any, Mapping, Optional import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" BEDROCK_INVOKE_MODELS = [ "claude-haiku-4-5-bedrock-invoke", - "claude-sonnet-4-6-bedrock-invoke", + "claude-sonnet-4-5-bedrock-invoke", "claude-opus-4-7-bedrock-invoke", ] @@ -47,24 +45,11 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: return 0 +@pytest.mark.covers("llm.messages.bedrock_invoke.prompt_cache_5m.nonstream.works") def test_prompt_caching_5m_bedrock_invoke(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert the upstream usage block surfaces a non-zero cache token count.""" - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False - ) + base_url, api_key = require_proxy(compat_result) outcomes = run_claude_models_parallel( models=BEDROCK_INVOKE_MODELS, diff --git a/tests/e2e/claude_code/prompt_caching_5m/test_vertex_ai.py b/tests/e2e/claude_code/prompt_caching_5m/test_vertex_ai.py index cc5d337dfbe..f79377b7372 100644 --- a/tests/e2e/claude_code/prompt_caching_5m/test_vertex_ai.py +++ b/tests/e2e/claude_code/prompt_caching_5m/test_vertex_ai.py @@ -15,23 +15,21 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations -import os from typing import Any, Mapping, Optional import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" VERTEX_AI_MODELS = [ "claude-haiku-4-5-vertex", - "claude-sonnet-4-6-vertex", + "claude-sonnet-4-5-vertex", "claude-opus-4-7-vertex", ] @@ -47,24 +45,11 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: return 0 +@pytest.mark.covers("llm.messages.vertex.prompt_cache_5m.nonstream.works") def test_prompt_caching_5m_vertex_ai(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert the upstream usage block surfaces a non-zero cache token count.""" - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False - ) + base_url, api_key = require_proxy(compat_result) outcomes = run_claude_models_parallel( models=VERTEX_AI_MODELS, diff --git a/tests/e2e/claude_code/run_compat.sh b/tests/e2e/claude_code/run_compat.sh index 4d8d0b6d7b2..e383792f45e 100755 --- a/tests/e2e/claude_code/run_compat.sh +++ b/tests/e2e/claude_code/run_compat.sh @@ -11,9 +11,9 @@ # 4. If a provider has `rate_limited > 0`, halve its rate; else, double it. # 5. Repeat until the highest no-429 rate is found. # -# Required env (proxy connection): -# LITELLM_PROXY_BASE_URL e.g. http://localhost:4000 -# LITELLM_PROXY_API_KEY e.g. sk-1234 +# Required env (proxy connection), same names as the rest of tests/e2e: +# LITELLM_PROXY_URL e.g. http://localhost:4000 +# LITELLM_MASTER_KEY e.g. sk-1234 # # Optional env (rate limits, all default to 5 req/s; 0 disables a column): # LITELLM_COMPAT_RATE_ANTHROPIC @@ -32,8 +32,8 @@ set -euo pipefail -if [[ -z "${LITELLM_PROXY_BASE_URL:-}" || -z "${LITELLM_PROXY_API_KEY:-}" ]]; then - echo "error: LITELLM_PROXY_BASE_URL and LITELLM_PROXY_API_KEY must be set" >&2 +if [[ -z "${LITELLM_PROXY_URL:-}" || -z "${LITELLM_MASTER_KEY:-}" ]]; then + echo "error: LITELLM_PROXY_URL and LITELLM_MASTER_KEY must be set" >&2 exit 64 fi diff --git a/tests/e2e/claude_code/structured_outputs/test_anthropic.py b/tests/e2e/claude_code/structured_outputs/test_anthropic.py index 610d8433b72..3dc4c7ab8f2 100644 --- a/tests/e2e/claude_code/structured_outputs/test_anthropic.py +++ b/tests/e2e/claude_code/structured_outputs/test_anthropic.py @@ -48,24 +48,22 @@ tier so the matrix's "all three must pass" rule applies. from __future__ import annotations import json -import os import re from typing import Any, Mapping, Optional, Sequence, Tuple import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" ANTHROPIC_MODELS = [ "claude-haiku-4-5", - "claude-sonnet-4-6", + "claude-sonnet-4-5", "claude-opus-4-7", ] @@ -152,26 +150,12 @@ def _validate_against_schema( return None +@pytest.mark.covers("llm.messages.anthropic.structured_output.nonstream.works") def test_structured_outputs_anthropic(compat_result): """Drive `claude --json-schema ...` against the LiteLLM proxy and assert the trailing `result` event contains a schema-conforming `structured_output`.""" - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", - pytrace=False, - ) + base_url, api_key = require_proxy(compat_result) outcomes = run_claude_models_parallel( models=ANTHROPIC_MODELS, diff --git a/tests/e2e/claude_code/structured_outputs/test_azure.py b/tests/e2e/claude_code/structured_outputs/test_azure.py index 290f9156910..7a776ed55ad 100644 --- a/tests/e2e/claude_code/structured_outputs/test_azure.py +++ b/tests/e2e/claude_code/structured_outputs/test_azure.py @@ -48,24 +48,22 @@ tier so the matrix's "all three must pass" rule applies. from __future__ import annotations import json -import os import re from typing import Any, Mapping, Optional, Sequence, Tuple import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" AZURE_MODELS = [ "claude-haiku-4-5-azure", - "claude-sonnet-4-6-azure", + "claude-sonnet-4-5-azure", "claude-opus-4-7-azure", ] @@ -152,26 +150,12 @@ def _validate_against_schema( return None +@pytest.mark.covers("llm.messages.azure_foundry.structured_output.nonstream.works") def test_structured_outputs_azure(compat_result): """Drive `claude --json-schema ...` against the LiteLLM proxy and assert the trailing `result` event contains a schema-conforming `structured_output`.""" - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", - pytrace=False, - ) + base_url, api_key = require_proxy(compat_result) outcomes = run_claude_models_parallel( models=AZURE_MODELS, diff --git a/tests/e2e/claude_code/structured_outputs/test_bedrock_converse.py b/tests/e2e/claude_code/structured_outputs/test_bedrock_converse.py index 5179014773c..345d7c327cf 100644 --- a/tests/e2e/claude_code/structured_outputs/test_bedrock_converse.py +++ b/tests/e2e/claude_code/structured_outputs/test_bedrock_converse.py @@ -48,24 +48,22 @@ tier so the matrix's "all three must pass" rule applies. from __future__ import annotations import json -import os import re from typing import Any, Mapping, Optional, Sequence, Tuple import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" BEDROCK_CONVERSE_MODELS = [ "claude-haiku-4-5-bedrock-converse", - "claude-sonnet-4-6-bedrock-converse", + "claude-sonnet-4-5-bedrock-converse", "claude-opus-4-7-bedrock-converse", ] @@ -152,26 +150,12 @@ def _validate_against_schema( return None +@pytest.mark.covers("llm.messages.bedrock_converse.structured_output.nonstream.works") def test_structured_outputs_bedrock_converse(compat_result): """Drive `claude --json-schema ...` against the LiteLLM proxy and assert the trailing `result` event contains a schema-conforming `structured_output`.""" - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", - pytrace=False, - ) + base_url, api_key = require_proxy(compat_result) outcomes = run_claude_models_parallel( models=BEDROCK_CONVERSE_MODELS, diff --git a/tests/e2e/claude_code/structured_outputs/test_bedrock_invoke.py b/tests/e2e/claude_code/structured_outputs/test_bedrock_invoke.py index 313a714be34..0cf48c72d4f 100644 --- a/tests/e2e/claude_code/structured_outputs/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/structured_outputs/test_bedrock_invoke.py @@ -48,24 +48,22 @@ tier so the matrix's "all three must pass" rule applies. from __future__ import annotations import json -import os import re from typing import Any, Mapping, Optional, Sequence, Tuple import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" BEDROCK_INVOKE_MODELS = [ "claude-haiku-4-5-bedrock-invoke", - "claude-sonnet-4-6-bedrock-invoke", + "claude-sonnet-4-5-bedrock-invoke", "claude-opus-4-7-bedrock-invoke", ] @@ -152,26 +150,12 @@ def _validate_against_schema( return None +@pytest.mark.covers("llm.messages.bedrock_invoke.structured_output.nonstream.works") def test_structured_outputs_bedrock_invoke(compat_result): """Drive `claude --json-schema ...` against the LiteLLM proxy and assert the trailing `result` event contains a schema-conforming `structured_output`.""" - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", - pytrace=False, - ) + base_url, api_key = require_proxy(compat_result) outcomes = run_claude_models_parallel( models=BEDROCK_INVOKE_MODELS, diff --git a/tests/e2e/claude_code/structured_outputs/test_vertex_ai.py b/tests/e2e/claude_code/structured_outputs/test_vertex_ai.py index ec04c724193..24f5a0c35d4 100644 --- a/tests/e2e/claude_code/structured_outputs/test_vertex_ai.py +++ b/tests/e2e/claude_code/structured_outputs/test_vertex_ai.py @@ -48,24 +48,22 @@ tier so the matrix's "all three must pass" rule applies. from __future__ import annotations import json -import os import re from typing import Any, Mapping, Optional, Sequence, Tuple import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" VERTEX_AI_MODELS = [ "claude-haiku-4-5-vertex", - "claude-sonnet-4-6-vertex", + "claude-sonnet-4-5-vertex", "claude-opus-4-7-vertex", ] @@ -152,26 +150,12 @@ def _validate_against_schema( return None +@pytest.mark.covers("llm.messages.vertex.structured_output.nonstream.works") def test_structured_outputs_vertex_ai(compat_result): """Drive `claude --json-schema ...` against the LiteLLM proxy and assert the trailing `result` event contains a schema-conforming `structured_output`.""" - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", - pytrace=False, - ) + base_url, api_key = require_proxy(compat_result) outcomes = run_claude_models_parallel( models=VERTEX_AI_MODELS, diff --git a/tests/e2e/claude_code/test_config.yaml b/tests/e2e/claude_code/test_config.yaml index e9253da2b3c..ba57ca4ccb7 100644 --- a/tests/e2e/claude_code/test_config.yaml +++ b/tests/e2e/claude_code/test_config.yaml @@ -21,42 +21,54 @@ model_list: litellm_params: model: anthropic/claude-haiku-4-5 api_key: os.environ/ANTHROPIC_API_KEY - - model_name: claude-sonnet-4-6 + - model_name: claude-sonnet-4-5 litellm_params: - model: anthropic/claude-sonnet-4-6 + model: anthropic/claude-sonnet-4-5 api_key: os.environ/ANTHROPIC_API_KEY + extra_headers: + anthropic-beta: "context-1m-2025-08-07" - model_name: claude-opus-4-7 litellm_params: model: anthropic/claude-opus-4-7 api_key: os.environ/ANTHROPIC_API_KEY + extra_headers: + anthropic-beta: "context-1m-2025-08-07" # ---- Bedrock (InvokeModel) ---- - model_name: claude-haiku-4-5-bedrock-invoke litellm_params: model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0 aws_region_name: us-east-1 - - model_name: claude-sonnet-4-6-bedrock-invoke + - model_name: claude-sonnet-4-5-bedrock-invoke litellm_params: - model: bedrock/us.anthropic.claude-sonnet-4-6 + model: bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0 aws_region_name: us-east-1 + extra_headers: + anthropic-beta: "context-1m-2025-08-07" - model_name: claude-opus-4-7-bedrock-invoke litellm_params: model: bedrock/us.anthropic.claude-opus-4-7 aws_region_name: us-east-1 + extra_headers: + anthropic-beta: "context-1m-2025-08-07" # ---- Bedrock (Converse) ---- - model_name: claude-haiku-4-5-bedrock-converse litellm_params: model: bedrock/converse/us.anthropic.claude-haiku-4-5-20251001-v1:0 aws_region_name: us-east-1 - - model_name: claude-sonnet-4-6-bedrock-converse + - model_name: claude-sonnet-4-5-bedrock-converse litellm_params: - model: bedrock/converse/us.anthropic.claude-sonnet-4-6 + model: bedrock/converse/us.anthropic.claude-sonnet-4-5-20250929-v1:0 aws_region_name: us-east-1 + extra_headers: + anthropic-beta: "context-1m-2025-08-07" - model_name: claude-opus-4-7-bedrock-converse litellm_params: model: bedrock/converse/us.anthropic.claude-opus-4-7 aws_region_name: us-east-1 + extra_headers: + anthropic-beta: "context-1m-2025-08-07" # ---- Vertex AI ---- # `use_in_pass_through: true` registers each deployment's @@ -70,37 +82,45 @@ model_list: litellm_params: model: vertex_ai/claude-haiku-4-5 vertex_project: os.environ/VERTEXAI_PROJECT - vertex_location: os.environ/VERTEXAI_LOCATION + vertex_location: global use_in_pass_through: true - - model_name: claude-sonnet-4-6-vertex + - model_name: claude-sonnet-4-5-vertex litellm_params: - model: vertex_ai/claude-sonnet-4-6 + model: vertex_ai/claude-sonnet-4-5 vertex_project: os.environ/VERTEXAI_PROJECT - vertex_location: os.environ/VERTEXAI_LOCATION + vertex_location: global use_in_pass_through: true + extra_headers: + anthropic-beta: "context-1m-2025-08-07" - model_name: claude-opus-4-7-vertex litellm_params: model: vertex_ai/claude-opus-4-7 vertex_project: os.environ/VERTEXAI_PROJECT - vertex_location: os.environ/VERTEXAI_LOCATION + vertex_location: global use_in_pass_through: true + extra_headers: + anthropic-beta: "context-1m-2025-08-07" # ---- Microsoft Foundry (Anthropic deployments on Azure) ---- - model_name: claude-haiku-4-5-azure litellm_params: model: azure_ai/claude-haiku-4-5 - api_base: os.environ/AZURE_FOUNDRY_API_BASE - api_key: os.environ/AZURE_FOUNDRY_API_KEY - - model_name: claude-sonnet-4-6-azure + api_base: os.environ/AZURE_AI_API_BASE + api_key: os.environ/AZURE_AI_API_KEY + - model_name: claude-sonnet-4-5-azure litellm_params: - model: azure_ai/claude-sonnet-4-6 - api_base: os.environ/AZURE_FOUNDRY_API_BASE - api_key: os.environ/AZURE_FOUNDRY_API_KEY + model: azure_ai/claude-sonnet-4-5 + api_base: os.environ/AZURE_AI_API_BASE + api_key: os.environ/AZURE_AI_API_KEY + extra_headers: + anthropic-beta: "context-1m-2025-08-07" - model_name: claude-opus-4-7-azure litellm_params: model: azure_ai/claude-opus-4-7 - api_base: os.environ/AZURE_FOUNDRY_API_BASE - api_key: os.environ/AZURE_FOUNDRY_API_KEY + api_base: os.environ/AZURE_AI_API_BASE + api_key: os.environ/AZURE_AI_API_KEY + extra_headers: + anthropic-beta: "context-1m-2025-08-07" general_settings: # Claude Code sends provider-specific headers (e.g. anthropic-beta) we diff --git a/tests/e2e/claude_code/thinking/test_anthropic.py b/tests/e2e/claude_code/thinking/test_anthropic.py index 1090d1b384e..ebb2445fb6d 100644 --- a/tests/e2e/claude_code/thinking/test_anthropic.py +++ b/tests/e2e/claude_code/thinking/test_anthropic.py @@ -20,23 +20,21 @@ still sees three rows for this (feature, provider). from __future__ import annotations -import os from typing import Any, Mapping, Sequence import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" ANTHROPIC_MODELS = [ "claude-haiku-4-5", - "claude-sonnet-4-6", + "claude-sonnet-4-5", "claude-opus-4-7", ] @@ -76,24 +74,11 @@ def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool: return False +@pytest.mark.covers("llm.messages.anthropic.thinking.nonstream.works") def test_thinking_anthropic(compat_result): """Drive the `claude` CLI against the LiteLLM proxy with thinking enabled and assert a `thinking` content block was emitted.""" - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False - ) + base_url, api_key = require_proxy(compat_result) outcomes = run_claude_models_parallel( models=ANTHROPIC_MODELS, diff --git a/tests/e2e/claude_code/thinking/test_azure.py b/tests/e2e/claude_code/thinking/test_azure.py index 1fd5138d574..ffd5ca92df0 100644 --- a/tests/e2e/claude_code/thinking/test_azure.py +++ b/tests/e2e/claude_code/thinking/test_azure.py @@ -23,23 +23,21 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations -import os from typing import Any, Mapping, Sequence import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" AZURE_MODELS = [ "claude-haiku-4-5-azure", - "claude-sonnet-4-6-azure", + "claude-sonnet-4-5-azure", "claude-opus-4-7-azure", ] @@ -64,24 +62,11 @@ def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool: return False +@pytest.mark.covers("llm.messages.azure_foundry.thinking.nonstream.works") def test_thinking_azure(compat_result): """Drive the `claude` CLI against the LiteLLM proxy with thinking enabled and assert a `thinking` content block was emitted.""" - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False - ) + base_url, api_key = require_proxy(compat_result) outcomes = run_claude_models_parallel( models=AZURE_MODELS, diff --git a/tests/e2e/claude_code/thinking/test_bedrock_converse.py b/tests/e2e/claude_code/thinking/test_bedrock_converse.py index 793ce8542da..0b409f18ea7 100644 --- a/tests/e2e/claude_code/thinking/test_bedrock_converse.py +++ b/tests/e2e/claude_code/thinking/test_bedrock_converse.py @@ -15,23 +15,21 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations -import os from typing import Any, Mapping, Sequence import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" BEDROCK_CONVERSE_MODELS = [ "claude-haiku-4-5-bedrock-converse", - "claude-sonnet-4-6-bedrock-converse", + "claude-sonnet-4-5-bedrock-converse", "claude-opus-4-7-bedrock-converse", ] @@ -56,24 +54,11 @@ def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool: return False +@pytest.mark.covers("llm.messages.bedrock_converse.thinking.nonstream.works") def test_thinking_bedrock_converse(compat_result): """Drive the `claude` CLI against the LiteLLM proxy with thinking enabled and assert a `thinking` content block was emitted.""" - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False - ) + base_url, api_key = require_proxy(compat_result) outcomes = run_claude_models_parallel( models=BEDROCK_CONVERSE_MODELS, diff --git a/tests/e2e/claude_code/thinking/test_bedrock_invoke.py b/tests/e2e/claude_code/thinking/test_bedrock_invoke.py index e31b60eb004..a2c97eae321 100644 --- a/tests/e2e/claude_code/thinking/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/thinking/test_bedrock_invoke.py @@ -15,23 +15,21 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations -import os from typing import Any, Mapping, Sequence import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" BEDROCK_INVOKE_MODELS = [ "claude-haiku-4-5-bedrock-invoke", - "claude-sonnet-4-6-bedrock-invoke", + "claude-sonnet-4-5-bedrock-invoke", "claude-opus-4-7-bedrock-invoke", ] @@ -56,24 +54,11 @@ def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool: return False +@pytest.mark.covers("llm.messages.bedrock_invoke.thinking.nonstream.works") def test_thinking_bedrock_invoke(compat_result): """Drive the `claude` CLI against the LiteLLM proxy with thinking enabled and assert a `thinking` content block was emitted.""" - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False - ) + base_url, api_key = require_proxy(compat_result) outcomes = run_claude_models_parallel( models=BEDROCK_INVOKE_MODELS, diff --git a/tests/e2e/claude_code/thinking/test_vertex_ai.py b/tests/e2e/claude_code/thinking/test_vertex_ai.py index c5c7df1f9b8..f1a1c5b6cee 100644 --- a/tests/e2e/claude_code/thinking/test_vertex_ai.py +++ b/tests/e2e/claude_code/thinking/test_vertex_ai.py @@ -15,23 +15,21 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations -import os from typing import Any, Mapping, Sequence import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" VERTEX_AI_MODELS = [ "claude-haiku-4-5-vertex", - "claude-sonnet-4-6-vertex", + "claude-sonnet-4-5-vertex", "claude-opus-4-7-vertex", ] @@ -56,24 +54,11 @@ def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool: return False +@pytest.mark.covers("llm.messages.vertex.thinking.nonstream.works") def test_thinking_vertex_ai(compat_result): """Drive the `claude` CLI against the LiteLLM proxy with thinking enabled and assert a `thinking` content block was emitted.""" - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False - ) + base_url, api_key = require_proxy(compat_result) outcomes = run_claude_models_parallel( models=VERTEX_AI_MODELS, diff --git a/tests/e2e/claude_code/thinking_with_tool_use/test_anthropic.py b/tests/e2e/claude_code/thinking_with_tool_use/test_anthropic.py index 2c573ea039e..7e39ea26d42 100644 --- a/tests/e2e/claude_code/thinking_with_tool_use/test_anthropic.py +++ b/tests/e2e/claude_code/thinking_with_tool_use/test_anthropic.py @@ -24,23 +24,21 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations -import os from typing import Any, Mapping, Sequence import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" ANTHROPIC_MODELS = [ "claude-haiku-4-5", - "claude-sonnet-4-6", + "claude-sonnet-4-5", "claude-opus-4-7", ] @@ -90,25 +88,12 @@ def _has_block_type( return False +@pytest.mark.covers("llm.messages.anthropic.thinking_with_tool_use.nonstream.works") def test_thinking_with_tool_use_anthropic(compat_result): """Drive the `claude` CLI against the LiteLLM proxy with thinking enabled and tool use, and assert both `thinking` and `tool_use` content blocks landed in the same turn.""" - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False - ) + base_url, api_key = require_proxy(compat_result) outcomes = run_claude_models_parallel( models=ANTHROPIC_MODELS, diff --git a/tests/e2e/claude_code/thinking_with_tool_use/test_azure.py b/tests/e2e/claude_code/thinking_with_tool_use/test_azure.py index 3d65e82cdec..0371a10f8a6 100644 --- a/tests/e2e/claude_code/thinking_with_tool_use/test_azure.py +++ b/tests/e2e/claude_code/thinking_with_tool_use/test_azure.py @@ -18,23 +18,21 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations -import os from typing import Any, Mapping, Sequence import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" AZURE_MODELS = [ "claude-haiku-4-5-azure", - "claude-sonnet-4-6-azure", + "claude-sonnet-4-5-azure", "claude-opus-4-7-azure", ] @@ -71,22 +69,9 @@ def _has_block_type( return False +@pytest.mark.covers("llm.messages.azure_foundry.thinking_with_tool_use.nonstream.works") def test_thinking_with_tool_use_azure(compat_result): - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False - ) + base_url, api_key = require_proxy(compat_result) outcomes = run_claude_models_parallel( models=AZURE_MODELS, diff --git a/tests/e2e/claude_code/thinking_with_tool_use/test_bedrock_converse.py b/tests/e2e/claude_code/thinking_with_tool_use/test_bedrock_converse.py index eb916323546..026d2a3707f 100644 --- a/tests/e2e/claude_code/thinking_with_tool_use/test_bedrock_converse.py +++ b/tests/e2e/claude_code/thinking_with_tool_use/test_bedrock_converse.py @@ -23,23 +23,21 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations -import os from typing import Any, Mapping, Sequence import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" BEDROCK_CONVERSE_MODELS = [ "claude-haiku-4-5-bedrock-converse", - "claude-sonnet-4-6-bedrock-converse", + "claude-sonnet-4-5-bedrock-converse", "claude-opus-4-7-bedrock-converse", ] @@ -76,22 +74,9 @@ def _has_block_type( return False +@pytest.mark.covers("llm.messages.bedrock_converse.thinking_with_tool_use.nonstream.works") def test_thinking_with_tool_use_bedrock_converse(compat_result): - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False - ) + base_url, api_key = require_proxy(compat_result) outcomes = run_claude_models_parallel( models=BEDROCK_CONVERSE_MODELS, diff --git a/tests/e2e/claude_code/thinking_with_tool_use/test_bedrock_invoke.py b/tests/e2e/claude_code/thinking_with_tool_use/test_bedrock_invoke.py index d1a61a59772..1dd4cf0a73c 100644 --- a/tests/e2e/claude_code/thinking_with_tool_use/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/thinking_with_tool_use/test_bedrock_invoke.py @@ -25,23 +25,21 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations -import os from typing import Any, Mapping, Sequence import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" BEDROCK_INVOKE_MODELS = [ "claude-haiku-4-5-bedrock-invoke", - "claude-sonnet-4-6-bedrock-invoke", + "claude-sonnet-4-5-bedrock-invoke", "claude-opus-4-7-bedrock-invoke", ] @@ -78,22 +76,9 @@ def _has_block_type( return False +@pytest.mark.covers("llm.messages.bedrock_invoke.thinking_with_tool_use.nonstream.works") def test_thinking_with_tool_use_bedrock_invoke(compat_result): - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False - ) + base_url, api_key = require_proxy(compat_result) outcomes = run_claude_models_parallel( models=BEDROCK_INVOKE_MODELS, diff --git a/tests/e2e/claude_code/thinking_with_tool_use/test_vertex_ai.py b/tests/e2e/claude_code/thinking_with_tool_use/test_vertex_ai.py index 285419c67f7..b25228edb55 100644 --- a/tests/e2e/claude_code/thinking_with_tool_use/test_vertex_ai.py +++ b/tests/e2e/claude_code/thinking_with_tool_use/test_vertex_ai.py @@ -23,23 +23,21 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations -import os from typing import Any, Mapping, Sequence import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" VERTEX_AI_MODELS = [ "claude-haiku-4-5-vertex", - "claude-sonnet-4-6-vertex", + "claude-sonnet-4-5-vertex", "claude-opus-4-7-vertex", ] @@ -76,22 +74,9 @@ def _has_block_type( return False +@pytest.mark.covers("llm.messages.vertex.thinking_with_tool_use.nonstream.works") def test_thinking_with_tool_use_vertex_ai(compat_result): - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False - ) + base_url, api_key = require_proxy(compat_result) outcomes = run_claude_models_parallel( models=VERTEX_AI_MODELS, diff --git a/tests/e2e/claude_code/tool_search/test_anthropic.py b/tests/e2e/claude_code/tool_search/test_anthropic.py index 3495c882e06..7b8ea07aa07 100644 --- a/tests/e2e/claude_code/tool_search/test_anthropic.py +++ b/tests/e2e/claude_code/tool_search/test_anthropic.py @@ -43,45 +43,28 @@ per-cell aggregator. from __future__ import annotations -import os - import pytest +from claude_code._env import require_proxy from claude_code.http_probe import ( assert_tool_search_shape, probe_tool_search, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" ANTHROPIC_MODELS = [ "claude-haiku-4-5", - "claude-sonnet-4-6", + "claude-sonnet-4-5", "claude-opus-4-7", ] +@pytest.mark.covers("llm.messages.anthropic.tool_search.nonstream.works") def test_tool_search_anthropic(compat_result): """Probe `/v1/messages` with a `tool_search_tool_regex_20251119` tool and assert the proxy + upstream accept it for every Anthropic tier.""" - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", - pytrace=False, - ) + base_url, api_key = require_proxy(compat_result) failures = [] for model in ANTHROPIC_MODELS: diff --git a/tests/e2e/claude_code/tool_search/test_azure.py b/tests/e2e/claude_code/tool_search/test_azure.py index 1d9cb5673c5..4eee13e4ecc 100644 --- a/tests/e2e/claude_code/tool_search/test_azure.py +++ b/tests/e2e/claude_code/tool_search/test_azure.py @@ -43,45 +43,28 @@ per-cell aggregator. from __future__ import annotations -import os - import pytest +from claude_code._env import require_proxy from claude_code.http_probe import ( assert_tool_search_shape, probe_tool_search, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" AZURE_MODELS = [ "claude-haiku-4-5-azure", - "claude-sonnet-4-6-azure", + "claude-sonnet-4-5-azure", "claude-opus-4-7-azure", ] +@pytest.mark.covers("llm.messages.azure_foundry.tool_search.nonstream.works") def test_tool_search_azure(compat_result): """Probe `/v1/messages` with a `tool_search_tool_regex_20251119` tool and assert the proxy + upstream accept it for every Azure (Microsoft Foundry) tier.""" - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", - pytrace=False, - ) + base_url, api_key = require_proxy(compat_result) failures = [] for model in AZURE_MODELS: diff --git a/tests/e2e/claude_code/tool_search/test_bedrock_converse.py b/tests/e2e/claude_code/tool_search/test_bedrock_converse.py index 5ca0792529a..7951f8ecdb4 100644 --- a/tests/e2e/claude_code/tool_search/test_bedrock_converse.py +++ b/tests/e2e/claude_code/tool_search/test_bedrock_converse.py @@ -43,45 +43,28 @@ per-cell aggregator. from __future__ import annotations -import os - import pytest +from claude_code._env import require_proxy from claude_code.http_probe import ( assert_tool_search_shape, probe_tool_search, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" BEDROCK_CONVERSE_MODELS = [ "claude-haiku-4-5-bedrock-converse", - "claude-sonnet-4-6-bedrock-converse", + "claude-sonnet-4-5-bedrock-converse", "claude-opus-4-7-bedrock-converse", ] +@pytest.mark.covers("llm.messages.bedrock_converse.tool_search.nonstream.works") def test_tool_search_bedrock_converse(compat_result): """Probe `/v1/messages` with a `tool_search_tool_regex_20251119` tool and assert the proxy + upstream accept it for every Bedrock (Converse) tier.""" - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", - pytrace=False, - ) + base_url, api_key = require_proxy(compat_result) failures = [] for model in BEDROCK_CONVERSE_MODELS: diff --git a/tests/e2e/claude_code/tool_search/test_bedrock_invoke.py b/tests/e2e/claude_code/tool_search/test_bedrock_invoke.py index 21bb33e34bd..654c2aa18d1 100644 --- a/tests/e2e/claude_code/tool_search/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/tool_search/test_bedrock_invoke.py @@ -43,45 +43,28 @@ per-cell aggregator. from __future__ import annotations -import os - import pytest +from claude_code._env import require_proxy from claude_code.http_probe import ( assert_tool_search_shape, probe_tool_search, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" BEDROCK_INVOKE_MODELS = [ "claude-haiku-4-5-bedrock-invoke", - "claude-sonnet-4-6-bedrock-invoke", + "claude-sonnet-4-5-bedrock-invoke", "claude-opus-4-7-bedrock-invoke", ] +@pytest.mark.covers("llm.messages.bedrock_invoke.tool_search.nonstream.works") def test_tool_search_bedrock_invoke(compat_result): """Probe `/v1/messages` with a `tool_search_tool_regex_20251119` tool and assert the proxy + upstream accept it for every Bedrock (Invoke) tier.""" - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", - pytrace=False, - ) + base_url, api_key = require_proxy(compat_result) failures = [] for model in BEDROCK_INVOKE_MODELS: diff --git a/tests/e2e/claude_code/tool_search/test_vertex_ai.py b/tests/e2e/claude_code/tool_search/test_vertex_ai.py index f91400b1817..f6ff855fa78 100644 --- a/tests/e2e/claude_code/tool_search/test_vertex_ai.py +++ b/tests/e2e/claude_code/tool_search/test_vertex_ai.py @@ -43,45 +43,28 @@ per-cell aggregator. from __future__ import annotations -import os - import pytest +from claude_code._env import require_proxy from claude_code.http_probe import ( assert_tool_search_shape, probe_tool_search, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" VERTEX_AI_MODELS = [ "claude-haiku-4-5-vertex", - "claude-sonnet-4-6-vertex", + "claude-sonnet-4-5-vertex", "claude-opus-4-7-vertex", ] +@pytest.mark.covers("llm.messages.vertex.tool_search.nonstream.works") def test_tool_search_vertex_ai(compat_result): """Probe `/v1/messages` with a `tool_search_tool_regex_20251119` tool and assert the proxy + upstream accept it for every Vertex AI tier.""" - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", - pytrace=False, - ) + base_url, api_key = require_proxy(compat_result) failures = [] for model in VERTEX_AI_MODELS: diff --git a/tests/e2e/claude_code/tool_use/test_anthropic.py b/tests/e2e/claude_code/tool_use/test_anthropic.py index 7d2aa4be683..9ff4c58907f 100644 --- a/tests/e2e/claude_code/tool_use/test_anthropic.py +++ b/tests/e2e/claude_code/tool_use/test_anthropic.py @@ -15,23 +15,21 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations -import os from typing import Any, Mapping, Sequence import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" ANTHROPIC_MODELS = [ "claude-haiku-4-5", - "claude-sonnet-4-6", + "claude-sonnet-4-5", "claude-opus-4-7", ] @@ -73,24 +71,11 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: return False +@pytest.mark.covers("llm.messages.anthropic.tool_use.nonstream.works") def test_tool_use_anthropic(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a tool call was emitted on the wire.""" - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False - ) + base_url, api_key = require_proxy(compat_result) outcomes = run_claude_models_parallel( models=ANTHROPIC_MODELS, diff --git a/tests/e2e/claude_code/tool_use/test_azure.py b/tests/e2e/claude_code/tool_use/test_azure.py index 484f50a5508..9e7398267c4 100644 --- a/tests/e2e/claude_code/tool_use/test_azure.py +++ b/tests/e2e/claude_code/tool_use/test_azure.py @@ -19,23 +19,21 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations -import os from typing import Any, Mapping, Sequence import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" AZURE_MODELS = [ "claude-haiku-4-5-azure", - "claude-sonnet-4-6-azure", + "claude-sonnet-4-5-azure", "claude-opus-4-7-azure", ] @@ -67,24 +65,11 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: return False +@pytest.mark.covers("llm.messages.azure_foundry.tool_use.nonstream.works") def test_tool_use_azure(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a tool call was emitted on the wire.""" - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False - ) + base_url, api_key = require_proxy(compat_result) outcomes = run_claude_models_parallel( models=AZURE_MODELS, diff --git a/tests/e2e/claude_code/tool_use/test_bedrock_converse.py b/tests/e2e/claude_code/tool_use/test_bedrock_converse.py index 7d1b58fce90..33d4d3820d2 100644 --- a/tests/e2e/claude_code/tool_use/test_bedrock_converse.py +++ b/tests/e2e/claude_code/tool_use/test_bedrock_converse.py @@ -15,23 +15,21 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations -import os from typing import Any, Mapping, Sequence import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" BEDROCK_CONVERSE_MODELS = [ "claude-haiku-4-5-bedrock-converse", - "claude-sonnet-4-6-bedrock-converse", + "claude-sonnet-4-5-bedrock-converse", "claude-opus-4-7-bedrock-converse", ] @@ -63,24 +61,11 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: return False +@pytest.mark.covers("llm.messages.bedrock_converse.tool_use.nonstream.works") def test_tool_use_bedrock_converse(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a tool call was emitted on the wire.""" - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False - ) + base_url, api_key = require_proxy(compat_result) outcomes = run_claude_models_parallel( models=BEDROCK_CONVERSE_MODELS, diff --git a/tests/e2e/claude_code/tool_use/test_bedrock_invoke.py b/tests/e2e/claude_code/tool_use/test_bedrock_invoke.py index 7d2b72b951d..47ae3aef1da 100644 --- a/tests/e2e/claude_code/tool_use/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/tool_use/test_bedrock_invoke.py @@ -15,23 +15,21 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations -import os from typing import Any, Mapping, Sequence import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" BEDROCK_INVOKE_MODELS = [ "claude-haiku-4-5-bedrock-invoke", - "claude-sonnet-4-6-bedrock-invoke", + "claude-sonnet-4-5-bedrock-invoke", "claude-opus-4-7-bedrock-invoke", ] @@ -63,24 +61,11 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: return False +@pytest.mark.covers("llm.messages.bedrock_invoke.tool_use.nonstream.works") def test_tool_use_bedrock_invoke(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a tool call was emitted on the wire.""" - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False - ) + base_url, api_key = require_proxy(compat_result) outcomes = run_claude_models_parallel( models=BEDROCK_INVOKE_MODELS, diff --git a/tests/e2e/claude_code/tool_use/test_vertex_ai.py b/tests/e2e/claude_code/tool_use/test_vertex_ai.py index 0a8ecc9f7a7..79a3016345c 100644 --- a/tests/e2e/claude_code/tool_use/test_vertex_ai.py +++ b/tests/e2e/claude_code/tool_use/test_vertex_ai.py @@ -15,23 +15,21 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations -import os from typing import Any, Mapping, Sequence import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" VERTEX_AI_MODELS = [ "claude-haiku-4-5-vertex", - "claude-sonnet-4-6-vertex", + "claude-sonnet-4-5-vertex", "claude-opus-4-7-vertex", ] @@ -63,24 +61,11 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: return False +@pytest.mark.covers("llm.messages.vertex.tool_use.nonstream.works") def test_tool_use_vertex_ai(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a tool call was emitted on the wire.""" - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False - ) + base_url, api_key = require_proxy(compat_result) outcomes = run_claude_models_parallel( models=VERTEX_AI_MODELS, diff --git a/tests/e2e/claude_code/tool_use_streaming/test_anthropic.py b/tests/e2e/claude_code/tool_use_streaming/test_anthropic.py index 9aa94c89241..152652dcf3c 100644 --- a/tests/e2e/claude_code/tool_use_streaming/test_anthropic.py +++ b/tests/e2e/claude_code/tool_use_streaming/test_anthropic.py @@ -25,23 +25,21 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations -import os from typing import Any, Mapping, Sequence import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" ANTHROPIC_MODELS = [ "claude-haiku-4-5", - "claude-sonnet-4-6", + "claude-sonnet-4-5", "claude-opus-4-7", ] @@ -99,24 +97,11 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int: ) +@pytest.mark.covers("llm.messages.anthropic.tool_use.stream.works") def test_tool_use_streaming_anthropic(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert the proxy preserves fine-grained tool streaming end-to-end.""" - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False - ) + base_url, api_key = require_proxy(compat_result) outcomes = run_claude_models_parallel( models=ANTHROPIC_MODELS, diff --git a/tests/e2e/claude_code/tool_use_streaming/test_azure.py b/tests/e2e/claude_code/tool_use_streaming/test_azure.py index c73062b72cd..8a1cc1852dd 100644 --- a/tests/e2e/claude_code/tool_use_streaming/test_azure.py +++ b/tests/e2e/claude_code/tool_use_streaming/test_azure.py @@ -17,23 +17,21 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations -import os from typing import Any, Mapping, Sequence import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" AZURE_MODELS = [ "claude-haiku-4-5-azure", - "claude-sonnet-4-6-azure", + "claude-sonnet-4-5-azure", "claude-opus-4-7-azure", ] @@ -84,22 +82,9 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int: ) +@pytest.mark.covers("llm.messages.azure_foundry.tool_use.stream.works") def test_tool_use_streaming_azure(compat_result): - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False - ) + base_url, api_key = require_proxy(compat_result) outcomes = run_claude_models_parallel( models=AZURE_MODELS, diff --git a/tests/e2e/claude_code/tool_use_streaming/test_bedrock_converse.py b/tests/e2e/claude_code/tool_use_streaming/test_bedrock_converse.py index 3642551c7c3..3b04ed5962f 100644 --- a/tests/e2e/claude_code/tool_use_streaming/test_bedrock_converse.py +++ b/tests/e2e/claude_code/tool_use_streaming/test_bedrock_converse.py @@ -23,23 +23,21 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations -import os from typing import Any, Mapping, Sequence import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" BEDROCK_CONVERSE_MODELS = [ "claude-haiku-4-5-bedrock-converse", - "claude-sonnet-4-6-bedrock-converse", + "claude-sonnet-4-5-bedrock-converse", "claude-opus-4-7-bedrock-converse", ] @@ -90,22 +88,9 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int: ) +@pytest.mark.covers("llm.messages.bedrock_converse.tool_use.stream.works") def test_tool_use_streaming_bedrock_converse(compat_result): - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False - ) + base_url, api_key = require_proxy(compat_result) outcomes = run_claude_models_parallel( models=BEDROCK_CONVERSE_MODELS, diff --git a/tests/e2e/claude_code/tool_use_streaming/test_bedrock_invoke.py b/tests/e2e/claude_code/tool_use_streaming/test_bedrock_invoke.py index af4689b2847..c7b61129782 100644 --- a/tests/e2e/claude_code/tool_use_streaming/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/tool_use_streaming/test_bedrock_invoke.py @@ -21,23 +21,21 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations -import os from typing import Any, Mapping, Sequence import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" BEDROCK_INVOKE_MODELS = [ "claude-haiku-4-5-bedrock-invoke", - "claude-sonnet-4-6-bedrock-invoke", + "claude-sonnet-4-5-bedrock-invoke", "claude-opus-4-7-bedrock-invoke", ] @@ -88,22 +86,9 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int: ) +@pytest.mark.covers("llm.messages.bedrock_invoke.tool_use.stream.works") def test_tool_use_streaming_bedrock_invoke(compat_result): - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False - ) + base_url, api_key = require_proxy(compat_result) outcomes = run_claude_models_parallel( models=BEDROCK_INVOKE_MODELS, diff --git a/tests/e2e/claude_code/tool_use_streaming/test_vertex_ai.py b/tests/e2e/claude_code/tool_use_streaming/test_vertex_ai.py index 19ef9a4e90e..2912e3aae3d 100644 --- a/tests/e2e/claude_code/tool_use_streaming/test_vertex_ai.py +++ b/tests/e2e/claude_code/tool_use_streaming/test_vertex_ai.py @@ -20,23 +20,21 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations -import os from typing import Any, Mapping, Sequence import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" VERTEX_AI_MODELS = [ "claude-haiku-4-5-vertex", - "claude-sonnet-4-6-vertex", + "claude-sonnet-4-5-vertex", "claude-opus-4-7-vertex", ] @@ -87,22 +85,9 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int: ) +@pytest.mark.covers("llm.messages.vertex.tool_use.stream.works") def test_tool_use_streaming_vertex_ai(compat_result): - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False - ) + base_url, api_key = require_proxy(compat_result) outcomes = run_claude_models_parallel( models=VERTEX_AI_MODELS, diff --git a/tests/e2e/claude_code/vision/test_anthropic.py b/tests/e2e/claude_code/vision/test_anthropic.py index 650940248ea..f681b2be5ae 100644 --- a/tests/e2e/claude_code/vision/test_anthropic.py +++ b/tests/e2e/claude_code/vision/test_anthropic.py @@ -25,22 +25,19 @@ the proxy must preserve. from __future__ import annotations import json -import os - import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" ANTHROPIC_MODELS = [ "claude-haiku-4-5", - "claude-sonnet-4-6", + "claude-sonnet-4-5", "claude-opus-4-7", ] @@ -85,24 +82,11 @@ def _build_stdin_input() -> str: return json.dumps(user_event) + "\n" +@pytest.mark.covers("llm.messages.anthropic.vision.nonstream.works") def test_vision_anthropic(compat_result): """Drive the `claude` CLI against the LiteLLM proxy with an image attached via stream-json input and assert a non-empty reply.""" - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False - ) + base_url, api_key = require_proxy(compat_result) outcomes = run_claude_models_parallel( models=ANTHROPIC_MODELS, diff --git a/tests/e2e/claude_code/vision/test_azure.py b/tests/e2e/claude_code/vision/test_azure.py index 3b03c0f2b35..f0eaaad84a2 100644 --- a/tests/e2e/claude_code/vision/test_azure.py +++ b/tests/e2e/claude_code/vision/test_azure.py @@ -25,22 +25,19 @@ the proxy must preserve. from __future__ import annotations import json -import os - import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" AZURE_MODELS = [ "claude-haiku-4-5-azure", - "claude-sonnet-4-6-azure", + "claude-sonnet-4-5-azure", "claude-opus-4-7-azure", ] @@ -85,24 +82,11 @@ def _build_stdin_input() -> str: return json.dumps(user_event) + "\n" +@pytest.mark.covers("llm.messages.azure_foundry.vision.nonstream.works") def test_vision_azure(compat_result): """Drive the `claude` CLI against the LiteLLM proxy with an image attached via stream-json input and assert a non-empty reply.""" - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False - ) + base_url, api_key = require_proxy(compat_result) outcomes = run_claude_models_parallel( models=AZURE_MODELS, diff --git a/tests/e2e/claude_code/vision/test_bedrock_converse.py b/tests/e2e/claude_code/vision/test_bedrock_converse.py index 4201f9e64fc..2a5aba5a393 100644 --- a/tests/e2e/claude_code/vision/test_bedrock_converse.py +++ b/tests/e2e/claude_code/vision/test_bedrock_converse.py @@ -25,22 +25,19 @@ the proxy must preserve. from __future__ import annotations import json -import os - import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" BEDROCK_CONVERSE_MODELS = [ "claude-haiku-4-5-bedrock-converse", - "claude-sonnet-4-6-bedrock-converse", + "claude-sonnet-4-5-bedrock-converse", "claude-opus-4-7-bedrock-converse", ] @@ -85,24 +82,11 @@ def _build_stdin_input() -> str: return json.dumps(user_event) + "\n" +@pytest.mark.covers("llm.messages.bedrock_converse.vision.nonstream.works") def test_vision_bedrock_converse(compat_result): """Drive the `claude` CLI against the LiteLLM proxy with an image attached via stream-json input and assert a non-empty reply.""" - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False - ) + base_url, api_key = require_proxy(compat_result) outcomes = run_claude_models_parallel( models=BEDROCK_CONVERSE_MODELS, diff --git a/tests/e2e/claude_code/vision/test_bedrock_invoke.py b/tests/e2e/claude_code/vision/test_bedrock_invoke.py index d2e641f1462..5c995cd479e 100644 --- a/tests/e2e/claude_code/vision/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/vision/test_bedrock_invoke.py @@ -25,22 +25,19 @@ the proxy must preserve. from __future__ import annotations import json -import os - import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" BEDROCK_INVOKE_MODELS = [ "claude-haiku-4-5-bedrock-invoke", - "claude-sonnet-4-6-bedrock-invoke", + "claude-sonnet-4-5-bedrock-invoke", "claude-opus-4-7-bedrock-invoke", ] @@ -85,24 +82,11 @@ def _build_stdin_input() -> str: return json.dumps(user_event) + "\n" +@pytest.mark.covers("llm.messages.bedrock_invoke.vision.nonstream.works") def test_vision_bedrock_invoke(compat_result): """Drive the `claude` CLI against the LiteLLM proxy with an image attached via stream-json input and assert a non-empty reply.""" - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False - ) + base_url, api_key = require_proxy(compat_result) outcomes = run_claude_models_parallel( models=BEDROCK_INVOKE_MODELS, diff --git a/tests/e2e/claude_code/vision/test_vertex_ai.py b/tests/e2e/claude_code/vision/test_vertex_ai.py index a39ef1a34b7..8d385e295d0 100644 --- a/tests/e2e/claude_code/vision/test_vertex_ai.py +++ b/tests/e2e/claude_code/vision/test_vertex_ai.py @@ -25,22 +25,19 @@ the proxy must preserve. from __future__ import annotations import json -import os - import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" VERTEX_AI_MODELS = [ "claude-haiku-4-5-vertex", - "claude-sonnet-4-6-vertex", + "claude-sonnet-4-5-vertex", "claude-opus-4-7-vertex", ] @@ -85,24 +82,11 @@ def _build_stdin_input() -> str: return json.dumps(user_event) + "\n" +@pytest.mark.covers("llm.messages.vertex.vision.nonstream.works") def test_vision_vertex_ai(compat_result): """Drive the `claude` CLI against the LiteLLM proxy with an image attached via stream-json input and assert a non-empty reply.""" - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False - ) + base_url, api_key = require_proxy(compat_result) outcomes = run_claude_models_parallel( models=VERTEX_AI_MODELS, diff --git a/tests/e2e/claude_code/web_search/test_anthropic.py b/tests/e2e/claude_code/web_search/test_anthropic.py index b8fa806f923..a20a2133dc9 100644 --- a/tests/e2e/claude_code/web_search/test_anthropic.py +++ b/tests/e2e/claude_code/web_search/test_anthropic.py @@ -27,23 +27,21 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations -import os from typing import Any, Mapping, Sequence import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" ANTHROPIC_MODELS = [ "claude-haiku-4-5", - "claude-sonnet-4-6", + "claude-sonnet-4-5", "claude-opus-4-7", ] @@ -86,26 +84,13 @@ def _has_web_search_tool_use(events: Sequence[Mapping[str, Any]]) -> bool: return False +@pytest.mark.covers("llm.messages.anthropic.web_search.nonstream.works") def test_web_search_anthropic(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert the upstream emitted a `tool_use` block calling `WebSearch`, proving the proxy preserved both the request-side tool definition and the response-side tool_use block.""" - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False - ) + base_url, api_key = require_proxy(compat_result) outcomes = run_claude_models_parallel( models=ANTHROPIC_MODELS, diff --git a/tests/e2e/claude_code/web_search/test_azure.py b/tests/e2e/claude_code/web_search/test_azure.py index e70dc848dcf..8f9f638fbee 100644 --- a/tests/e2e/claude_code/web_search/test_azure.py +++ b/tests/e2e/claude_code/web_search/test_azure.py @@ -27,23 +27,21 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations -import os from typing import Any, Mapping, Sequence import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" AZURE_MODELS = [ "claude-haiku-4-5-azure", - "claude-sonnet-4-6-azure", + "claude-sonnet-4-5-azure", "claude-opus-4-7-azure", ] @@ -86,26 +84,13 @@ def _has_web_search_tool_use(events: Sequence[Mapping[str, Any]]) -> bool: return False +@pytest.mark.covers("llm.messages.azure_foundry.web_search.nonstream.works") def test_web_search_azure(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert the upstream emitted a `tool_use` block calling `WebSearch`, proving the proxy preserved both the request-side tool definition and the response-side tool_use block.""" - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False - ) + base_url, api_key = require_proxy(compat_result) outcomes = run_claude_models_parallel( models=AZURE_MODELS, diff --git a/tests/e2e/claude_code/web_search/test_bedrock_converse.py b/tests/e2e/claude_code/web_search/test_bedrock_converse.py index cbeea03df40..32f37b2be79 100644 --- a/tests/e2e/claude_code/web_search/test_bedrock_converse.py +++ b/tests/e2e/claude_code/web_search/test_bedrock_converse.py @@ -27,23 +27,21 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations -import os from typing import Any, Mapping, Sequence import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" BEDROCK_CONVERSE_MODELS = [ "claude-haiku-4-5-bedrock-converse", - "claude-sonnet-4-6-bedrock-converse", + "claude-sonnet-4-5-bedrock-converse", "claude-opus-4-7-bedrock-converse", ] @@ -86,26 +84,13 @@ def _has_web_search_tool_use(events: Sequence[Mapping[str, Any]]) -> bool: return False +@pytest.mark.covers("llm.messages.bedrock_converse.web_search.nonstream.works") def test_web_search_bedrock_converse(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert the upstream emitted a `tool_use` block calling `WebSearch`, proving the proxy preserved both the request-side tool definition and the response-side tool_use block.""" - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False - ) + base_url, api_key = require_proxy(compat_result) outcomes = run_claude_models_parallel( models=BEDROCK_CONVERSE_MODELS, diff --git a/tests/e2e/claude_code/web_search/test_bedrock_invoke.py b/tests/e2e/claude_code/web_search/test_bedrock_invoke.py index 86068e1e22b..68d1b30e83f 100644 --- a/tests/e2e/claude_code/web_search/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/web_search/test_bedrock_invoke.py @@ -27,23 +27,21 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations -import os from typing import Any, Mapping, Sequence import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" BEDROCK_INVOKE_MODELS = [ "claude-haiku-4-5-bedrock-invoke", - "claude-sonnet-4-6-bedrock-invoke", + "claude-sonnet-4-5-bedrock-invoke", "claude-opus-4-7-bedrock-invoke", ] @@ -86,26 +84,13 @@ def _has_web_search_tool_use(events: Sequence[Mapping[str, Any]]) -> bool: return False +@pytest.mark.covers("llm.messages.bedrock_invoke.web_search.nonstream.works") def test_web_search_bedrock_invoke(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert the upstream emitted a `tool_use` block calling `WebSearch`, proving the proxy preserved both the request-side tool definition and the response-side tool_use block.""" - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False - ) + base_url, api_key = require_proxy(compat_result) outcomes = run_claude_models_parallel( models=BEDROCK_INVOKE_MODELS, diff --git a/tests/e2e/claude_code/web_search/test_vertex_ai.py b/tests/e2e/claude_code/web_search/test_vertex_ai.py index a33515771f3..540a8396c98 100644 --- a/tests/e2e/claude_code/web_search/test_vertex_ai.py +++ b/tests/e2e/claude_code/web_search/test_vertex_ai.py @@ -27,23 +27,21 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations -import os from typing import Any, Mapping, Sequence import pytest +from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, failure_diagnostic, run_claude_models_parallel, ) -PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" -PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" VERTEX_AI_MODELS = [ "claude-haiku-4-5-vertex", - "claude-sonnet-4-6-vertex", + "claude-sonnet-4-5-vertex", "claude-opus-4-7-vertex", ] @@ -86,26 +84,13 @@ def _has_web_search_tool_use(events: Sequence[Mapping[str, Any]]) -> bool: return False +@pytest.mark.covers("llm.messages.vertex.web_search.nonstream.works") def test_web_search_vertex_ai(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert the upstream emitted a `tool_use` block calling `WebSearch`, proving the proxy preserved both the request-side tool definition and the response-side tool_use block.""" - base_url = os.environ.get(PROXY_BASE_URL_ENV) - api_key = os.environ.get(PROXY_API_KEY_ENV) - if not base_url or not api_key: - compat_result.set( - { - "status": "fail", - "error": ( - f"missing required env: set {PROXY_BASE_URL_ENV} and " - f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" - ), - } - ) - pytest.fail( - f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False - ) + base_url, api_key = require_proxy(compat_result) outcomes = run_claude_models_parallel( models=VERTEX_AI_MODELS, diff --git a/tests/e2e/coverage_registry/llm_claude_code_compat.yaml b/tests/e2e/coverage_registry/llm_claude_code_compat.yaml new file mode 100644 index 00000000000..6edf890f7ec --- /dev/null +++ b/tests/e2e/coverage_registry/llm_claude_code_compat.yaml @@ -0,0 +1,110 @@ +# Claude Code compatibility matrix: /v1/messages coverage across the five provider surfaces +# claude-code drives (anthropic direct, azure ai foundry, bedrock invoke, bedrock converse, +# vertex ai). Each row is one (feature x provider) cell in the matrix. The seven anthropic-direct +# rows already declared in llm_conversational.yaml are NOT duplicated here; the four other +# provider surfaces plus every feature not already listed for anthropic direct are declared below. +# +# Grammar: llm.messages....works +# route : anthropic | azure_foundry | bedrock_converse | bedrock_invoke | vertex +# capability : basic | tool_use | vision | thinking | prompt_cache_5m | prompt_cache_1h +# | structured_output | pdf_input | long_context_1m +# | thinking_with_tool_use | tool_search | count_tokens | web_search +# streaming : stream | nonstream + +# ---- basic / non-streaming ---- +- {id: llm.messages.azure_foundry.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: azure_foundry, capability: basic, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Basic messaging over Azure AI Foundry Anthropic deployments"} +- {id: llm.messages.bedrock_converse.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Basic messaging over Bedrock Converse Anthropic"} +- {id: llm.messages.bedrock_invoke.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_invoke, capability: basic, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Basic messaging over Bedrock Invoke Anthropic"} +- {id: llm.messages.vertex.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: vertex, capability: basic, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Basic messaging over Vertex AI Anthropic"} + +# ---- basic / streaming ---- +- {id: llm.messages.azure_foundry.basic.stream.works, module: llm, tier: P1, subject_endpoint: messages, route: azure_foundry, capability: basic, streaming: stream, assertions: [works], source: "claude_code compat matrix", rationale: "Basic streaming over Azure AI Foundry"} +- {id: llm.messages.bedrock_converse.basic.stream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_converse, capability: basic, streaming: stream, assertions: [works], source: "claude_code compat matrix", rationale: "Basic streaming over Bedrock Converse"} +- {id: llm.messages.bedrock_invoke.basic.stream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_invoke, capability: basic, streaming: stream, assertions: [works], source: "claude_code compat matrix", rationale: "Basic streaming over Bedrock Invoke"} +- {id: llm.messages.vertex.basic.stream.works, module: llm, tier: P1, subject_endpoint: messages, route: vertex, capability: basic, streaming: stream, assertions: [works], source: "claude_code compat matrix", rationale: "Basic streaming over Vertex AI"} + +# ---- tool_use / non-streaming ---- +- {id: llm.messages.azure_foundry.tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: azure_foundry, capability: tool_use, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Tool use over Azure AI Foundry"} +- {id: llm.messages.bedrock_converse.tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_converse, capability: tool_use, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Tool use over Bedrock Converse"} +- {id: llm.messages.bedrock_invoke.tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_invoke, capability: tool_use, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Tool use over Bedrock Invoke"} +- {id: llm.messages.vertex.tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: vertex, capability: tool_use, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Tool use over Vertex AI"} + +# ---- tool_use / streaming ---- +- {id: llm.messages.azure_foundry.tool_use.stream.works, module: llm, tier: P1, subject_endpoint: messages, route: azure_foundry, capability: tool_use, streaming: stream, assertions: [works], source: "claude_code compat matrix", rationale: "Streaming tool use over Azure AI Foundry"} +- {id: llm.messages.bedrock_converse.tool_use.stream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_converse, capability: tool_use, streaming: stream, assertions: [works], source: "claude_code compat matrix", rationale: "Streaming tool use over Bedrock Converse"} +- {id: llm.messages.bedrock_invoke.tool_use.stream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_invoke, capability: tool_use, streaming: stream, assertions: [works], source: "claude_code compat matrix", rationale: "Streaming tool use over Bedrock Invoke"} +- {id: llm.messages.vertex.tool_use.stream.works, module: llm, tier: P1, subject_endpoint: messages, route: vertex, capability: tool_use, streaming: stream, assertions: [works], source: "claude_code compat matrix", rationale: "Streaming tool use over Vertex AI"} + +# ---- vision ---- +- {id: llm.messages.azure_foundry.vision.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: azure_foundry, capability: vision, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Vision over Azure AI Foundry"} +- {id: llm.messages.bedrock_converse.vision.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_converse, capability: vision, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Vision over Bedrock Converse"} +- {id: llm.messages.bedrock_invoke.vision.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_invoke, capability: vision, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Vision over Bedrock Invoke"} +- {id: llm.messages.vertex.vision.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: vertex, capability: vision, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Vision over Vertex AI"} + +# ---- thinking ---- +- {id: llm.messages.azure_foundry.thinking.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: azure_foundry, capability: thinking, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Extended thinking over Azure AI Foundry"} +- {id: llm.messages.bedrock_converse.thinking.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_converse, capability: thinking, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Extended thinking over Bedrock Converse"} +- {id: llm.messages.bedrock_invoke.thinking.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_invoke, capability: thinking, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Extended thinking over Bedrock Invoke"} +- {id: llm.messages.vertex.thinking.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: vertex, capability: thinking, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Extended thinking over Vertex AI"} + +# ---- prompt_cache_5m ---- +- {id: llm.messages.azure_foundry.prompt_cache_5m.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: azure_foundry, capability: prompt_cache_5m, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "5m prompt cache over Azure AI Foundry"} +- {id: llm.messages.bedrock_converse.prompt_cache_5m.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_converse, capability: prompt_cache_5m, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "5m prompt cache over Bedrock Converse"} +- {id: llm.messages.bedrock_invoke.prompt_cache_5m.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_invoke, capability: prompt_cache_5m, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "5m prompt cache over Bedrock Invoke"} +- {id: llm.messages.vertex.prompt_cache_5m.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: vertex, capability: prompt_cache_5m, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "5m prompt cache over Vertex AI"} + +# ---- prompt_cache_1h ---- +- {id: llm.messages.anthropic.prompt_cache_1h.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: anthropic, capability: prompt_cache_1h, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "1h prompt cache over Anthropic direct"} +- {id: llm.messages.azure_foundry.prompt_cache_1h.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: azure_foundry, capability: prompt_cache_1h, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "1h prompt cache over Azure AI Foundry"} +- {id: llm.messages.bedrock_converse.prompt_cache_1h.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_converse, capability: prompt_cache_1h, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "1h prompt cache over Bedrock Converse"} +- {id: llm.messages.bedrock_invoke.prompt_cache_1h.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_invoke, capability: prompt_cache_1h, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "1h prompt cache over Bedrock Invoke"} +- {id: llm.messages.vertex.prompt_cache_1h.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: vertex, capability: prompt_cache_1h, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "1h prompt cache over Vertex AI"} + +# ---- structured_output ---- +- {id: llm.messages.anthropic.structured_output.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: anthropic, capability: structured_output, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Structured outputs (--json-schema) over Anthropic direct"} +- {id: llm.messages.azure_foundry.structured_output.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: azure_foundry, capability: structured_output, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Structured outputs over Azure AI Foundry"} +- {id: llm.messages.bedrock_converse.structured_output.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_converse, capability: structured_output, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Structured outputs over Bedrock Converse"} +- {id: llm.messages.bedrock_invoke.structured_output.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_invoke, capability: structured_output, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Structured outputs over Bedrock Invoke"} +- {id: llm.messages.vertex.structured_output.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: vertex, capability: structured_output, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Structured outputs over Vertex AI"} + +# ---- pdf_input ---- +- {id: llm.messages.anthropic.pdf_input.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: anthropic, capability: pdf_input, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "PDF document input over Anthropic direct"} +- {id: llm.messages.azure_foundry.pdf_input.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: azure_foundry, capability: pdf_input, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "PDF document input over Azure AI Foundry"} +- {id: llm.messages.bedrock_converse.pdf_input.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_converse, capability: pdf_input, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "PDF document input over Bedrock Converse"} +- {id: llm.messages.bedrock_invoke.pdf_input.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_invoke, capability: pdf_input, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "PDF document input over Bedrock Invoke"} +- {id: llm.messages.vertex.pdf_input.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: vertex, capability: pdf_input, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "PDF document input over Vertex AI"} + +# ---- long_context_1m ---- +- {id: llm.messages.anthropic.long_context_1m.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: anthropic, capability: long_context_1m, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "1M context beta over Anthropic direct"} +- {id: llm.messages.azure_foundry.long_context_1m.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: azure_foundry, capability: long_context_1m, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "1M context beta over Azure AI Foundry"} +- {id: llm.messages.bedrock_converse.long_context_1m.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_converse, capability: long_context_1m, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "1M context beta over Bedrock Converse"} +- {id: llm.messages.bedrock_invoke.long_context_1m.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_invoke, capability: long_context_1m, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "1M context beta over Bedrock Invoke"} +- {id: llm.messages.vertex.long_context_1m.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: vertex, capability: long_context_1m, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "1M context beta over Vertex AI"} + +# ---- thinking_with_tool_use ---- +- {id: llm.messages.anthropic.thinking_with_tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: anthropic, capability: thinking_with_tool_use, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Thinking + tool_use interleaved over Anthropic direct"} +- {id: llm.messages.azure_foundry.thinking_with_tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: azure_foundry, capability: thinking_with_tool_use, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Thinking + tool_use interleaved over Azure AI Foundry"} +- {id: llm.messages.bedrock_converse.thinking_with_tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_converse, capability: thinking_with_tool_use, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Thinking + tool_use interleaved over Bedrock Converse"} +- {id: llm.messages.bedrock_invoke.thinking_with_tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_invoke, capability: thinking_with_tool_use, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Thinking + tool_use interleaved over Bedrock Invoke"} +- {id: llm.messages.vertex.thinking_with_tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: vertex, capability: thinking_with_tool_use, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Thinking + tool_use interleaved over Vertex AI"} + +# ---- tool_search ---- +- {id: llm.messages.anthropic.tool_search.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: anthropic, capability: tool_search, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "tool_search_tool_regex_20251119 discovery tool over Anthropic direct"} +- {id: llm.messages.azure_foundry.tool_search.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: azure_foundry, capability: tool_search, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "tool_search discovery tool over Azure AI Foundry"} +- {id: llm.messages.bedrock_converse.tool_search.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_converse, capability: tool_search, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "tool_search discovery tool over Bedrock Converse"} +- {id: llm.messages.bedrock_invoke.tool_search.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_invoke, capability: tool_search, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "tool_search discovery tool over Bedrock Invoke"} +- {id: llm.messages.vertex.tool_search.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: vertex, capability: tool_search, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "tool_search discovery tool over Vertex AI"} + +# ---- count_tokens ---- +- {id: llm.messages.anthropic.count_tokens.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: anthropic, capability: count_tokens, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "/v1/messages/count_tokens over Anthropic direct"} +- {id: llm.messages.azure_foundry.count_tokens.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: azure_foundry, capability: count_tokens, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "/v1/messages/count_tokens over Azure AI Foundry"} +- {id: llm.messages.bedrock_converse.count_tokens.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_converse, capability: count_tokens, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "/v1/messages/count_tokens over Bedrock Converse"} +- {id: llm.messages.bedrock_invoke.count_tokens.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_invoke, capability: count_tokens, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "/v1/messages/count_tokens over Bedrock Invoke"} +- {id: llm.messages.vertex.count_tokens.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: vertex, capability: count_tokens, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "/v1/messages/count_tokens over Vertex AI"} + +# ---- web_search ---- +- {id: llm.messages.anthropic.web_search.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: anthropic, capability: web_search, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Web search server tool over Anthropic direct"} +- {id: llm.messages.azure_foundry.web_search.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: azure_foundry, capability: web_search, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Web search server tool over Azure AI Foundry"} +- {id: llm.messages.bedrock_converse.web_search.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_converse, capability: web_search, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Web search server tool over Bedrock Converse"} +- {id: llm.messages.bedrock_invoke.web_search.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_invoke, capability: web_search, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Web search server tool over Bedrock Invoke"} +- {id: llm.messages.vertex.web_search.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: vertex, capability: web_search, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Web search server tool over Vertex AI"} diff --git a/tests/e2e/coverage_registry/logging.yaml b/tests/e2e/coverage_registry/logging.yaml index afb6dbc964e..01a44539c88 100644 --- a/tests/e2e/coverage_registry/logging.yaml +++ b/tests/e2e/coverage_registry/logging.yaml @@ -1,11 +1,8 @@ # Logging integration delivery (behavior features). Grounded in litellm/integrations/. -- {id: logging.langfuse.success.logs_spend, module: logging, tier: P0, event: success, assertions: [logs_spend], exercised_on: [chat_completions, messages, embeddings], source: "integrations/langfuse/langfuse.py", rationale: "Primary tracing backend; cost accuracy"} -- {id: logging.langfuse.failure.logs_spend, module: logging, tier: P0, event: failure, assertions: [logs_spend], exercised_on: [chat_completions, messages], source: "integrations/langfuse/langfuse.py", rationale: "Failure path must still track spend"} -- {id: logging.langfuse.stream.logs_spend, module: logging, tier: P0, event: stream, assertions: [logs_spend], exercised_on: [chat_completions, messages], source: "integrations/langfuse/langfuse.py", rationale: "Streaming token counts aggregate"} - {id: logging.s3.success.writes_object, module: logging, tier: P0, event: success, assertions: [writes_object], exercised_on: [chat_completions, messages, embeddings], source: "integrations/s3_v2.py", rationale: "Primary audit trail; batch flush no-drop"} - {id: logging.s3.failure.writes_object, module: logging, tier: P0, event: failure, assertions: [writes_object], exercised_on: [chat_completions, messages], source: "integrations/s3_v2.py", rationale: "Failed calls persisted for compliance"} - {id: logging.gcs_bucket.success.writes_object, module: logging, tier: P0, event: success, assertions: [writes_object], exercised_on: [chat_completions, messages, embeddings], source: "integrations/gcs_bucket/gcs_bucket.py", rationale: "GCS parallel to S3"} -- {id: logging.datadog.success.exports_metric, module: logging, tier: P0, event: success, assertions: [exports_metric], exercised_on: [chat_completions, messages, embeddings], source: "integrations/datadog/datadog.py", rationale: "Powers dashboards/alerts; cardinality regressions common"} +- {id: logging.datadog.success.exports_metric, module: logging, tier: P0, event: success, assertions: [exports_metric], exercised_on: [chat_completions, messages, responses, embeddings], source: "integrations/datadog/datadog.py", rationale: "Powers dashboards/alerts; cardinality regressions common"} - {id: logging.datadog.failure.exports_metric, module: logging, tier: P0, event: failure, assertions: [exports_metric], exercised_on: [chat_completions], source: "integrations/datadog/datadog.py", rationale: "Failure metrics for alerting/SLO"} - {id: logging.prometheus.success.exports_metric, module: logging, tier: P0, event: success, assertions: [exports_metric], exercised_on: [chat_completions, messages, embeddings], source: "integrations/prometheus.py", rationale: "Standard OSS metrics; per-key cardinality (existing e2e)"} - {id: logging.otel.success.exports_metric, module: logging, tier: P0, event: success, assertions: [exports_metric], exercised_on: [chat_completions, messages, responses, embeddings], source: "integrations/otel/logger.py", rationale: "OTEL spans on every call path"} diff --git a/tests/e2e/coverage_registry/schema.py b/tests/e2e/coverage_registry/schema.py index 7482088d93f..a76774c3bde 100644 --- a/tests/e2e/coverage_registry/schema.py +++ b/tests/e2e/coverage_registry/schema.py @@ -54,13 +54,20 @@ LlmRoute = Literal[ LlmCapability = Literal[ "basic", + "count_tokens", + "long_context_1m", "mid_conversation_system", + "pdf_input", + "prompt_cache_1h", "prompt_cache_5m", "service_tier", "structured_output", "thinking", + "thinking_with_tool_use", + "tool_search", "tool_use", "vision", + "web_search", ] diff --git a/tests/e2e/docker-compose.yml b/tests/e2e/docker-compose.yml index b64f3d8dbfd..5f75409f025 100644 --- a/tests/e2e/docker-compose.yml +++ b/tests/e2e/docker-compose.yml @@ -1,5 +1,41 @@ # local setup to run e2e tests configs: + dd_sink_script: + content: | + # Minimal DataDog logs-intake sink for the logging suite: records every + # POST (gunzipping the compressed batches the integration sends) and + # replays them as JSON on GET /requests so tests can assert delivery. + import gzip, json + from http.server import BaseHTTPRequestHandler, HTTPServer + + REQUESTS = [] + + class Handler(BaseHTTPRequestHandler): + def do_POST(self): + body = self.rfile.read(int(self.headers.get("Content-Length", 0))) + if self.headers.get("Content-Encoding") == "gzip": + body = gzip.decompress(body) + REQUESTS.append({"path": self.path, "body": body.decode("utf-8", "replace")}) + self.send_response(202) + self.end_headers() + self.wfile.write(b"{}") + + def do_GET(self): + self.send_response(200) + if self.path == "/health": + self.send_header("Content-Type", "text/plain") + self.end_headers() + self.wfile.write(b"ok") + return + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(json.dumps({"requests": REQUESTS}).encode()) + + def log_message(self, *args): + pass + + HTTPServer(("0.0.0.0", 8080), Handler).serve_forever() + litellm_config: content: | general_settings: @@ -23,7 +59,7 @@ configs: # (PHOENIX_COLLECTOR_HTTP_ENDPOINT below points it at the jaeger service), # so gen-AI spans export through a preset-owned provider - the code path # where trace splits actually happen - with no cloud credentials needed. - callbacks: ["arize_phoenix"] + callbacks: ["arize_phoenix", "datadog"] router_settings: routing_strategy: simple-shuffle @@ -93,10 +129,15 @@ services: condition: service_healthy jaeger: condition: service_healthy + dd-sink: + condition: service_healthy env_file: .env environment: LITELLM_MASTER_KEY: sk-1234 STORE_MODEL_IN_DB: "True" + DD_API_KEY: local-sink-noauth + DD_SITE: datadoghq.com + DD_BASE_URL: http://dd-sink:8080 LITELLM_OTEL_V2: "true" PHOENIX_COLLECTOR_HTTP_ENDPOINT: http://jaeger:4318/v1/traces PHOENIX_API_KEY: local-jaeger-noauth @@ -116,6 +157,8 @@ services: MISTRAL_API_KEY: ${MISTRAL_API_KEY:-} AZURE_API_BASE: ${AZURE_API_BASE:-} AZURE_API_KEY: ${AZURE_API_KEY:-} + AZURE_AI_API_BASE: ${AZURE_AI_API_BASE:-} + AZURE_AI_API_KEY: ${AZURE_AI_API_KEY:-} ports: - "4000:4000" configs: @@ -155,3 +198,19 @@ services: interval: 3s timeout: 3s retries: 20 + +# throwaway DataDog logs-intake sink (records POSTs, replays on GET /requests; +# see E2E_DD_SINK_URL) + dd-sink: + image: python:3.12-alpine + command: ["python", "/sink.py"] + configs: + - source: dd_sink_script + target: /sink.py + ports: + - "9915:8080" + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:8080/health"] + interval: 3s + timeout: 3s + retries: 20 diff --git a/tests/e2e/e2e_config.py b/tests/e2e/e2e_config.py index 6e6c30709de..e84438430fd 100644 --- a/tests/e2e/e2e_config.py +++ b/tests/e2e/e2e_config.py @@ -32,6 +32,10 @@ CHEAP_OPENAI_MODEL = os.environ.get("E2E_CHEAP_OPENAI_MODEL", "gpt-5.5") # read exported spans back through it. OTEL_QUERY_URL = os.environ.get("E2E_OTEL_QUERY_URL", "http://localhost:16686").rstrip("/") +# Query URL of the compose stack's DataDog logs-intake sink (the `dd-sink` +# service records every intake POST and replays them on GET /requests). +DD_SINK_URL = os.environ.get("E2E_DD_SINK_URL", "http://localhost:9915").rstrip("/") + # Writes on the proxy are eventually consistent (e.g. spend rows flush on # proxy_batch_write_at, ~60s). Read-backs poll to this deadline, never sleep-once. POLL_TIMEOUT = float(os.environ.get("E2E_POLL_TIMEOUT", "120")) diff --git a/tests/e2e/e2e_gateway.py b/tests/e2e/e2e_gateway.py index d40b96d60fa..ad8b2e833a8 100644 --- a/tests/e2e/e2e_gateway.py +++ b/tests/e2e/e2e_gateway.py @@ -319,21 +319,31 @@ class Gateway: return self.transport.probe(path, params=params) -def build_gateway() -> Gateway: +def build_gateway( + *, + base_url: str = PROXY_BASE_URL, + master_key: str = MASTER_KEY, + control_plane_base_url: str = CONTROL_PLANE_BASE_URL, +) -> Gateway: """The Gateway every suite's client is built from: a SplitTransport that routes LLM calls to the data plane (PROXY_BASE_URL) and management/admin calls to the control plane (CONTROL_PLANE_BASE_URL), with the shared poll budget. The two - base URLs are the same for a monolithic proxy, so routing is then a no-op.""" + base URLs are the same for a monolithic proxy, so routing is then a no-op. + + The endpoints are injectable for callers that resolve the proxy some other + way than ``e2e_config``'s env names (see ``claude_code/_env.py``); they must + pass all three together, since a caller that overrides only the data plane + would leave management calls pointed at the env default.""" return Gateway( transport=SplitTransport( data=HttpTransport( - base_url=PROXY_BASE_URL, - master_key=MASTER_KEY, + base_url=base_url, + master_key=master_key, request_timeout=REQUEST_TIMEOUT, ), control=HttpTransport( - base_url=CONTROL_PLANE_BASE_URL, - master_key=MASTER_KEY, + base_url=control_plane_base_url, + master_key=master_key, request_timeout=REQUEST_TIMEOUT, ), ), diff --git a/tests/e2e/logging/conftest.py b/tests/e2e/logging/conftest.py index 43d279602ef..5ae791917fd 100644 --- a/tests/e2e/logging/conftest.py +++ b/tests/e2e/logging/conftest.py @@ -11,6 +11,7 @@ import os import pytest from logging_client import LangfuseCreds, LoggingClient, build_logging_client, load_langfuse_creds +from datadog_sink import DdSinkReader, build_dd_sink_reader from otel_client import OtelReader, build_otel_reader @@ -35,6 +36,12 @@ def otel_reader() -> OtelReader: return build_otel_reader() +@pytest.fixture(scope="session") +def dd_sink() -> DdSinkReader: + """Read-back client for the compose stack's DataDog logs-intake sink.""" + return build_dd_sink_reader() + + @pytest.fixture def datadog_creds() -> None: """Require Datadog shipping credentials. Hard-fail when absent; never skip.""" diff --git a/tests/e2e/logging/datadog_sink.py b/tests/e2e/logging/datadog_sink.py new file mode 100644 index 00000000000..5b5059d1428 --- /dev/null +++ b/tests/e2e/logging/datadog_sink.py @@ -0,0 +1,103 @@ +"""Read-back for the DataDog logging tests: typed models over the compose +stack's dd-sink service, which records every logs-intake POST the datadog +callback sends (gunzipped) and replays them as JSON. + +Delivery is judged on what the sink actually received, mirroring how the OTEL +tests read Jaeger; a failed sink query is a hard failure, never an empty +result. External reads go through ``e2e_http``. +""" + +from __future__ import annotations + +import time +from dataclasses import dataclass + +import pytest +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError + +from e2e_config import DD_SINK_URL, POLL_INTERVAL, POLL_TIMEOUT +from e2e_http import URL, NoBody, Success, get + + +class DdSinkRequest(BaseModel): + model_config = ConfigDict(extra="ignore") + + path: str + body: str + + +class DdSinkRequests(BaseModel): + model_config = ConfigDict(extra="ignore") + + requests: list[DdSinkRequest] = [] + + +class DdLogEvent(BaseModel): + model_config = ConfigDict(extra="ignore") + + message: str + ddsource: str | None = None + service: str | None = None + status: str | None = None + + +_EVENT_BATCH: TypeAdapter[list[DdLogEvent]] = TypeAdapter(list[DdLogEvent]) + + +def _parse_batch(request: DdSinkRequest) -> list[DdLogEvent]: + """The intake accepts an array of events or a single event object.""" + try: + return _EVENT_BATCH.validate_json(request.body) + except ValidationError: + try: + return [DdLogEvent.model_validate_json(request.body)] + except ValidationError: + pytest.fail(f"dd-sink recorded a non-log body on {request.path}: {request.body[:200]}") + + +@dataclass(frozen=True, slots=True) +class DdSinkReader: + sink_url: str + + def _recorded_requests(self) -> list[DdSinkRequest]: + result = get( + URL(f"{self.sink_url}/requests"), + headers=NoBody(), + params=NoBody(), + response_type=DdSinkRequests, + timeout=30.0, + ) + match result: + case Success(data=page): + return page.requests + case failure: + pytest.fail(f"dd-sink query at {self.sink_url} failed: {failure}") + + def events_for_marker(self, marker: str) -> list[DdLogEvent]: + """Every log event across every recorded intake batch whose message + carries the marker. More than one hit for one call IS the + duplicate-delivery bug, so this never collapses to a single event.""" + events: list[DdLogEvent] = [] + for request in self._recorded_requests(): + if "/api/v2/logs" not in request.path: + continue + events.extend(event for event in _parse_batch(request) if marker in event.message) + return events + + def poll_events_for_marker(self, marker: str) -> list[DdLogEvent]: + """Poll until at least one matching event lands (the callback flushes + in periodic batches), then re-read after one more interval so a late + duplicate cannot hide from the exactly-one assertion. At the deadline + the last result is returned as-is.""" + deadline = time.monotonic() + POLL_TIMEOUT + while time.monotonic() < deadline: + events = self.events_for_marker(marker) + if events: + time.sleep(POLL_INTERVAL) + return self.events_for_marker(marker) + time.sleep(POLL_INTERVAL) + return self.events_for_marker(marker) + + +def build_dd_sink_reader() -> DdSinkReader: + return DdSinkReader(sink_url=DD_SINK_URL) diff --git a/tests/e2e/logging/logging_client.py b/tests/e2e/logging/logging_client.py index e37d6175705..8be573d72a9 100644 --- a/tests/e2e/logging/logging_client.py +++ b/tests/e2e/logging/logging_client.py @@ -18,7 +18,7 @@ import json import os import time from dataclasses import dataclass -from typing import Literal +from typing import Callable, Literal import pytest from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter, ValidationError @@ -28,6 +28,7 @@ from e2e_gateway import Gateway, build_gateway from e2e_http import ( URL, AuthHeaders, + require_successful_call, NoBody, StreamingResponse, Success, @@ -617,5 +618,20 @@ class LoggingClient: return self.list_langfuse_observations(creds, trace_id=gen.trace_id) or [gen] +def first_ok(client: LoggingClient, send: Callable[[], StreamingResponse]) -> StreamingResponse: + """First successful call on a fresh key. A fresh key may briefly 401 until + the data plane's auth cache picks it up, so retry on 401 to a deadline; a + 401 is rejected before the LLM call, so it cannot contaminate delivery or + trace assertions. Any other failure is behavior under test and fails hard.""" + deadline = time.monotonic() + client.gateway.poll_timeout + while True: + outcome = send() + if outcome.ok: + return outcome + if outcome.status_code != 401 or time.monotonic() >= deadline: + require_successful_call(outcome) + time.sleep(client.gateway.poll_interval) + + def build_logging_client() -> LoggingClient: return LoggingClient(gateway=build_gateway()) diff --git a/tests/e2e/logging/test_datadog_log_e2e.py b/tests/e2e/logging/test_datadog_log_e2e.py new file mode 100644 index 00000000000..4651bbb28ba --- /dev/null +++ b/tests/e2e/logging/test_datadog_log_e2e.py @@ -0,0 +1,162 @@ +"""Live e2e: DataDog log delivery for successful non-streaming calls. + +Covers logging.datadog.success.exports_metric: one successful call on each +route must reach the DataDog logs intake as EXACTLY ONE log event whose +message (the StandardLoggingPayload) carries the model, the token counts, and +the response cost. Delivery is judged on what the intake actually received: +the compose stack's dd-sink service records every batch the datadog callback +ships (DD_BASE_URL override) and the tests read it back, so a dropped event, a +duplicated event, or a payload missing the cost all fail here. + +Both halves of the contract are asserted: the recorded state (the proxy +reports the DataDogLogger callback active via /health/readiness/details) and +the enforced behavior (the event at the intake, with the cost cross-checked +exactly against the x-litellm-response-cost header of the very response the +caller received). +""" + +from __future__ import annotations + +import pytest +from pydantic import BaseModel, ConfigDict + +from datadog_sink import DdLogEvent, DdSinkReader +from e2e_config import CHEAP_ANTHROPIC_MODEL, CHEAP_OPENAI_MODEL, unique_marker +from e2e_http import NoBody, StreamingResponse +from lifecycle import ResourceManager +from logging_client import LoggingClient, first_ok + +pytestmark = pytest.mark.e2e + +#: The active DataDog callback's name in /health/readiness/details success_callbacks. +DD_LOGGER_NAME = "DataDogLogger" + + +class _DdMessagePayload(BaseModel): + """The fields of the StandardLoggingPayload the scenario pins.""" + + model_config = ConfigDict(extra="ignore") + + model_group: str + total_tokens: int + response_cost: float + status: str + call_type: str + + +def _assert_datadog_configured(client: LoggingClient) -> None: + """Recorded state: the proxy reports the DataDog callback among its active + callbacks, so a missing destination config fails here, before any + delivery-based assertion can time out confusingly.""" + result = client.gateway.probe("/health/readiness/details", params=NoBody()) + assert result.status_code == 200, ( + f"/health/readiness/details must answer 200, got {result.status_code}: {result.body[:300]}" + ) + assert DD_LOGGER_NAME in result.body, ( + f"the proxy must report the {DD_LOGGER_NAME} callback active " + f"(callbacks + DD_* env in the compose config); got: {result.body[:400]}" + ) + + +def _assert_exactly_one_event( + events: list[DdLogEvent], *, model_group: str, call_type: str, outcome: StreamingResponse +) -> None: + """The enforced behavior: the intake holds exactly one event for the call, + sourced from litellm, whose payload names the model group and call type, + counts real tokens, and carries the same cost the response header reported.""" + assert events, "no DataDog log event for this call reached the intake within the deadline" + assert len(events) == 1, ( + f"expected exactly ONE DataDog log event for the call, got {len(events)} - " + "more than one event for one call is the duplicate-delivery bug (see LIT-4447 " + "for the currently known /v1/messages instance)" + ) + event = events[0] + assert event.ddsource == "litellm", f"event ddsource must be litellm, got {event.ddsource!r}" + assert event.status == "info", f"success events ship at status info, got {event.status!r}" + + payload = _DdMessagePayload.model_validate_json(event.message) + assert payload.status == "success", f"payload status must be success, got {payload.status!r}" + assert payload.model_group == model_group, ( + f"payload model_group must be {model_group!r}, got {payload.model_group!r}" + ) + assert payload.call_type == call_type, ( + f"payload call_type must be {call_type!r}, got {payload.call_type!r}" + ) + assert payload.total_tokens > 0, f"payload must count real tokens, got {payload.total_tokens}" + assert outcome.response_cost is not None and outcome.response_cost > 0, ( + f"the response must report x-litellm-response-cost, got {outcome.response_cost!r}" + ) + assert abs(payload.response_cost - outcome.response_cost) < 1e-12, ( + f"payload response_cost {payload.response_cost} must equal the response header " + f"cost {outcome.response_cost}" + ) + + +class TestDataDogLogDelivery: + @pytest.mark.covers("logging.datadog.success.exports_metric", exercised_on=["chat_completions"]) + def test_chat_completions_emits_one_log_event( + self, client: LoggingClient, dd_sink: DdSinkReader, resources: ResourceManager + ) -> None: + """One successful non-streaming /chat/completions call must reach the + DataDog logs intake as exactly one log event whose payload carries the + model, the token counts, and the response cost.""" + _assert_datadog_configured(client) + + key = client.key_with_alias(f"dd-chat-{unique_marker()}", models=[CHEAP_ANTHROPIC_MODEL]) + resources.defer(lambda: client.delete_key(key)) + + marker = unique_marker() + outcome = first_ok( + client, + lambda: client.chat_raw(key, CHEAP_ANTHROPIC_MODEL, f"reply with one word {marker}", max_tokens=16), + ) + events = dd_sink.poll_events_for_marker(marker) + _assert_exactly_one_event( + events, model_group=CHEAP_ANTHROPIC_MODEL, call_type="acompletion", outcome=outcome + ) + + @pytest.mark.covers("logging.datadog.success.exports_metric", exercised_on=["messages"]) + def test_messages_emits_one_log_event( + self, client: LoggingClient, dd_sink: DdSinkReader, resources: ResourceManager + ) -> None: + """One successful non-streaming /v1/messages call must reach the + DataDog logs intake as exactly one log event whose payload carries the + model, the token counts, and the response cost. + + This currently fails on the known /v1/messages double-log (LIT-4447); it goes green when the fix lands.""" + _assert_datadog_configured(client) + + key = client.key_with_alias(f"dd-messages-{unique_marker()}", models=[CHEAP_ANTHROPIC_MODEL]) + resources.defer(lambda: client.delete_key(key)) + + marker = unique_marker() + outcome = first_ok( + client, + lambda: client.messages_raw(key, CHEAP_ANTHROPIC_MODEL, f"reply with one word {marker}", max_tokens=16), + ) + events = dd_sink.poll_events_for_marker(marker) + _assert_exactly_one_event( + events, model_group=CHEAP_ANTHROPIC_MODEL, call_type="anthropic_messages", outcome=outcome + ) + + @pytest.mark.covers("logging.datadog.success.exports_metric", exercised_on=["responses"]) + def test_responses_emits_one_log_event( + self, client: LoggingClient, dd_sink: DdSinkReader, resources: ResourceManager + ) -> None: + """One successful non-streaming /v1/responses call must reach the + DataDog logs intake as exactly one log event whose payload carries the + model, the token counts, and the response cost.""" + _assert_datadog_configured(client) + + key = client.key_with_alias(f"dd-responses-{unique_marker()}", models=[CHEAP_OPENAI_MODEL]) + resources.defer(lambda: client.delete_key(key)) + + marker = unique_marker() + outcome = first_ok( + client, + lambda: client.responses_raw(key, CHEAP_OPENAI_MODEL, f"reply with one word {marker}"), + ) + events = dd_sink.poll_events_for_marker(marker) + _assert_exactly_one_event( + events, model_group=CHEAP_OPENAI_MODEL, call_type="aresponses", outcome=outcome + ) diff --git a/tests/e2e/logging/test_langfuse_e2e.py b/tests/e2e/logging/test_langfuse_e2e.py deleted file mode 100644 index d014b5d8291..00000000000 --- a/tests/e2e/logging/test_langfuse_e2e.py +++ /dev/null @@ -1,534 +0,0 @@ -"""Live e2e: Langfuse OTEL logs_spend for registry cells in logging.yaml P0. - -Registry cells: -- logging.langfuse.success.logs_spend (exercised_on chat_completions, messages, embeddings) -- logging.langfuse.failure.logs_spend (exercised_on chat_completions, messages) -- logging.langfuse.stream.logs_spend (exercised_on chat_completions, messages) - -Integration under test is ``langfuse_otel`` (OTLP to Langfuse), not the classic -``langfuse`` SDK callback. StandardLoggingPayload.response_cost is the spend -source of truth. Generations are named ``litellm_request``; correlate by unique -prompt marker and user_api_key_alias in metadata. - -Dynamic credentials by product surface: -- team: POST /team/{id}/callback with callback_name=langfuse_otel -- user/key: key metadata.logging with callback_name=langfuse_otel -- org: organization + team under it + team callback (no org-level callback API) - -Extra success paths assert tool calls and applied guardrails land on the trace. -""" - -from __future__ import annotations - -import json - -import pytest - -from e2e_config import unique_marker -from e2e_http import StreamingResponse, require_successful_call -from lifecycle import ResourceManager -from logging_client import ( - INVALID_UPSTREAM_API_KEY, - WEATHER_TOOL, - LangfuseCreds, - LoggingClient, - completion_response_id, - costs_agree, - observation_has_guardrail, - observation_mentions_tool, - observation_spend, -) -from models import LiteLLMParamsBody - -pytestmark = pytest.mark.e2e - -DRIVER_MODEL = "gemini-2.5-flash" -FAIL_BACKEND = "openai/gpt-4o-mini" - - -def _json_blob(value: object) -> str: - return json.dumps(value, default=str) - - -def _assert_logs_spend( - client: LoggingClient, - *, - key: str, - outcome: StreamingResponse, - obs_cost: float | None, - scope: str, - require_positive: bool = True, -) -> None: - """logs_spend: Langfuse cost matches StandardLogging response_cost and proxy spend. - - Non-stream responses expose response_cost on x-litellm-response-cost. Streaming - sends headers before final cost is known, so stream paths rely on /spend/logs. - """ - if not require_positive: - assert obs_cost is not None, ( - f"{scope}: failure path must still track spend (0 is fine); cost={obs_cost!r}" - ) - return - - assert obs_cost is not None and obs_cost > 0, ( - f"{scope}: Langfuse must log positive spend; calculatedTotalCost={obs_cost!r}" - ) - # Stream responses send headers before final cost is known, so the cost header - # is often absent; non-stream must always expose x-litellm-response-cost. - if not outcome.is_streaming: - assert outcome.response_cost is not None and outcome.response_cost > 0, ( - f"{scope}: proxy must return positive x-litellm-response-cost; " - f"got {outcome.response_cost!r}" - ) - assert costs_agree(outcome.response_cost, obs_cost), ( - f"{scope}: Langfuse cost {obs_cost!r} disagrees with " - f"x-litellm-response-cost {outcome.response_cost!r}" - ) - elif outcome.response_cost is not None and outcome.response_cost > 0: - assert costs_agree(outcome.response_cost, obs_cost), ( - f"{scope}: Langfuse cost {obs_cost!r} disagrees with " - f"x-litellm-response-cost {outcome.response_cost!r}" - ) - spend_row = client.poll_proxy_spend_for_key( - key, - response_id=completion_response_id(outcome.body), - require_positive_spend=True, - ) - assert spend_row is not None and spend_row.spend is not None and spend_row.spend > 0, ( - f"{scope}: proxy /spend/logs never produced a positive spend row for key" - ) - assert costs_agree(spend_row.spend, obs_cost), ( - f"{scope}: Langfuse cost {obs_cost!r} disagrees with proxy spend " - f"{spend_row.spend!r} (request_id={spend_row.request_id!r})" - ) - - -class TestLangfuseTeamLogging: - """Team-scoped callback via POST /team/{id}/callback.""" - - def _team_key( - self, - client: LoggingClient, - resources: ResourceManager, - creds: LangfuseCreds, - *, - models: list[str], - organization_id: str | None = None, - ) -> tuple[str, str, str]: - marker = unique_marker() - key_alias = f"e2e-lf-team-key-{marker}" - team_id = client.create_team( - f"e2e-lf-team-{marker}", - models=models, - organization_id=organization_id, - ) - resources.defer(lambda: client.delete_team(team_id)) - client.add_team_langfuse_callback(team_id, creds) - key = client.key_with_alias(key_alias, models=models, team_id=team_id) - resources.defer(lambda: client.delete_key(key)) - return team_id, key, key_alias - - @pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["chat_completions"]) - def test_success_logs_spend( - self, - client: LoggingClient, - resources: ResourceManager, - langfuse_creds: LangfuseCreds, - ) -> None: - _, key, key_alias = self._team_key( - client, resources, langfuse_creds, models=[DRIVER_MODEL] - ) - prompt_marker = unique_marker() - outcome = client.chat_raw( - key, DRIVER_MODEL, f"reply with one word only {prompt_marker}" - ) - require_successful_call(outcome) - - obs = client.poll_langfuse_observation( - langfuse_creds, - key_alias=key_alias, - prompt_marker=prompt_marker, - require_positive_cost=True, - ) - assert obs is not None, ( - f"team scope: Langfuse never received generation for key_alias={key_alias!r}" - ) - _assert_logs_spend( - client, - key=key, - outcome=outcome, - obs_cost=observation_spend(obs), - scope="team-success", - ) - - @pytest.mark.covers("logging.langfuse.failure.logs_spend", exercised_on=["chat_completions"]) - def test_failure_logs_spend( - self, - client: LoggingClient, - resources: ResourceManager, - langfuse_creds: LangfuseCreds, - ) -> None: - """Provider-auth failure still ships a Langfuse observation with spend tracked. - - Uses a throwaway deployment whose upstream OpenAI key is - INVALID_UPSTREAM_API_KEY (not a LiteLLM virtual key). - """ - prompt_marker = unique_marker() - model_name = f"e2e-lf-fail-{prompt_marker}" - model_id = client.create_model( - model_name, - LiteLLMParamsBody(model=FAIL_BACKEND, api_key=INVALID_UPSTREAM_API_KEY), - ) - resources.defer(lambda: client.delete_model(model_id)) - - _, key, key_alias = self._team_key( - client, resources, langfuse_creds, models=[model_name] - ) - outcome = client.chat_raw(key, model_name, f"this must fail {prompt_marker}") - assert not outcome.ok, ( - f"expected upstream provider failure for {INVALID_UPSTREAM_API_KEY!r}, " - f"got {outcome.status_code}: {outcome.body[:200]}" - ) - - obs = client.poll_langfuse_observation( - langfuse_creds, - key_alias=key_alias, - prompt_marker=prompt_marker, - require_positive_cost=False, - ) - assert obs is not None, ( - f"team failure path: Langfuse never received generation for key_alias={key_alias!r}" - ) - _assert_logs_spend( - client, - key=key, - outcome=outcome, - obs_cost=observation_spend(obs), - scope="team-failure", - require_positive=False, - ) - - @pytest.mark.covers("logging.langfuse.stream.logs_spend", exercised_on=["chat_completions"]) - def test_stream_logs_spend( - self, - client: LoggingClient, - resources: ResourceManager, - langfuse_creds: LangfuseCreds, - ) -> None: - _, key, key_alias = self._team_key( - client, resources, langfuse_creds, models=[DRIVER_MODEL] - ) - prompt_marker = unique_marker() - outcome = client.chat_raw( - key, DRIVER_MODEL, f"reply with one word only {prompt_marker}", stream=True - ) - require_successful_call(outcome) - assert outcome.is_streaming - assert outcome.chunks > 0 - - obs = client.poll_langfuse_observation( - langfuse_creds, - key_alias=key_alias, - prompt_marker=prompt_marker, - require_positive_cost=True, - ) - assert obs is not None - # Streamed body is elided; correlate cost via header + key spend row. - _assert_logs_spend( - client, - key=key, - outcome=outcome, - obs_cost=observation_spend(obs), - scope="team-stream", - ) - - @pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["chat_completions"]) - def test_tool_calls_logged_with_cost( - self, - client: LoggingClient, - resources: ResourceManager, - langfuse_creds: LangfuseCreds, - ) -> None: - _, key, key_alias = self._team_key( - client, resources, langfuse_creds, models=[DRIVER_MODEL] - ) - prompt_marker = unique_marker() - outcome = client.chat_raw( - key, - DRIVER_MODEL, - f"Use get_weather for Paris. marker={prompt_marker}", - tools=[WEATHER_TOOL], - tool_choice="required", - max_tokens=128, - ) - require_successful_call(outcome) - assert "get_weather" in outcome.body or "tool_calls" in outcome.body, ( - f"gateway response must include a tool call; body={outcome.body[:300]}" - ) - - obs = client.poll_langfuse_observation( - langfuse_creds, - key_alias=key_alias, - prompt_marker=prompt_marker, - require_positive_cost=True, - ) - assert obs is not None - assert observation_mentions_tool(obs, "get_weather"), ( - f"Langfuse generation must record the tool; name={obs.name!r} " - f"input={str(obs.input)[:200]} output={str(obs.output)[:200]}" - ) - _assert_logs_spend( - client, - key=key, - outcome=outcome, - obs_cost=observation_spend(obs), - scope="team-tools", - ) - - @pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["chat_completions"]) - def test_tool_permission_guardrail_logged( - self, - client: LoggingClient, - resources: ResourceManager, - langfuse_creds: LangfuseCreds, - ) -> None: - """tool_permission post_call guardrail must appear on the Langfuse trace - (StandardLogging guardrail_information -> Langfuse guardrail span).""" - marker = unique_marker() - guardrail_name = f"e2e-lf-tool-perm-{marker}" - guardrail_id = client.create_tool_permission_guardrail( - guardrail_name, allowed_tool="get_weather" - ) - resources.defer(lambda: client.delete_guardrail(guardrail_id)) - - _, key, key_alias = self._team_key( - client, resources, langfuse_creds, models=[DRIVER_MODEL] - ) - prompt_marker = unique_marker() - outcome = client.chat_raw( - key, - DRIVER_MODEL, - f"Use get_weather for Berlin. marker={prompt_marker}", - tools=[WEATHER_TOOL], - tool_choice="required", - guardrails=[guardrail_name], - max_tokens=128, - ) - require_successful_call(outcome) - - observations = client.poll_langfuse_trace_observations( - langfuse_creds, key_alias=key_alias, prompt_marker=prompt_marker - ) - assert observations, ( - f"team+guardrail: no Langfuse observations for key_alias={key_alias!r}" - ) - gen = next( - ( - o - for o in observations - if prompt_marker in _json_blob(o.input) - or key_alias in _json_blob(o.metadata) - or o.name in (f"litellm:{key_alias}", "litellm_request") - ), - observations[0], - ) - _assert_logs_spend( - client, - key=key, - outcome=outcome, - obs_cost=observation_spend(gen), - scope="team-guardrail", - ) - assert any( - observation_has_guardrail(o, guardrail_name=guardrail_name) - or (o.name is not None and "guardrail" in o.name.lower()) - for o in observations - ), ( - f"Langfuse trace must include applied guardrail {guardrail_name!r}; " - f"observation names={[o.name for o in observations]}" - ) - - -class TestLangfuseUserKeyLogging: - """User-owned key with metadata.logging (key-level dynamic Langfuse credentials). - - Product surface: key metadata.logging on /key/generate, not a separate - /user/.../callback route. The key is bound to a real /user/new user_id. - """ - - def _user_key( - self, - client: LoggingClient, - resources: ResourceManager, - creds: LangfuseCreds, - *, - models: list[str], - ) -> tuple[str, str, str]: - marker = unique_marker() - key_alias = f"e2e-lf-user-key-{marker}" - user_id = client.create_user( - user_email=f"e2e-lf-user-{marker}@example.com", - user_id=f"e2e-lf-user-{marker}", - ) - resources.defer(lambda: client.delete_user(user_id)) - key = client.key_with_alias( - key_alias, - models=models, - user_id=user_id, - metadata=creds.key_logging_metadata(), - ) - resources.defer(lambda: client.delete_key(key)) - return user_id, key, key_alias - - @pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["chat_completions"]) - def test_success_logs_spend( - self, - client: LoggingClient, - resources: ResourceManager, - langfuse_creds: LangfuseCreds, - ) -> None: - user_id, key, key_alias = self._user_key( - client, resources, langfuse_creds, models=[DRIVER_MODEL] - ) - prompt_marker = unique_marker() - outcome = client.chat_raw( - key, DRIVER_MODEL, f"reply with one word only {prompt_marker}" - ) - require_successful_call(outcome) - - obs = client.poll_langfuse_observation( - langfuse_creds, - key_alias=key_alias, - prompt_marker=prompt_marker, - require_positive_cost=True, - ) - assert obs is not None, ( - f"user/key scope: Langfuse never received generation for key_alias={key_alias!r}" - ) - meta_blob = _json_blob(obs.metadata) - assert user_id in meta_blob or key_alias in (obs.name or ""), ( - f"user/key scope should attribute the user or key; metadata={meta_blob[:300]}" - ) - _assert_logs_spend( - client, - key=key, - outcome=outcome, - obs_cost=observation_spend(obs), - scope="user-key", - ) - - @pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["chat_completions"]) - def test_tool_calls_logged_with_cost( - self, - client: LoggingClient, - resources: ResourceManager, - langfuse_creds: LangfuseCreds, - ) -> None: - _, key, key_alias = self._user_key( - client, resources, langfuse_creds, models=[DRIVER_MODEL] - ) - prompt_marker = unique_marker() - outcome = client.chat_raw( - key, - DRIVER_MODEL, - f"Use get_weather for Tokyo. marker={prompt_marker}", - tools=[WEATHER_TOOL], - tool_choice="required", - max_tokens=128, - ) - require_successful_call(outcome) - - obs = client.poll_langfuse_observation( - langfuse_creds, - key_alias=key_alias, - prompt_marker=prompt_marker, - require_positive_cost=True, - ) - assert obs is not None - assert observation_mentions_tool(obs, "get_weather"), ( - f"user/key tool path: tool missing from Langfuse; output={str(obs.output)[:200]}" - ) - _assert_logs_spend( - client, - key=key, - outcome=outcome, - obs_cost=observation_spend(obs), - scope="user-key-tools", - ) - - -class TestLangfuseOrgScopedLogging: - """Org-scoped run: organization + team under it + team Langfuse callback. - - There is no /organization/.../callback today; logging attaches at the team - (or key) under the org. This class proves org-linked team keys still deliver - accurate Langfuse spend and team attribution (StandardLogging metadata - user_api_key_team_id / user_api_key_org_id). - """ - - def _org_team_key( - self, - client: LoggingClient, - resources: ResourceManager, - creds: LangfuseCreds, - *, - models: list[str], - ) -> tuple[str, str, str, str]: - marker = unique_marker() - key_alias = f"e2e-lf-org-key-{marker}" - org_id = client.create_org(f"e2e-lf-org-{marker}", models=models) - resources.defer(lambda: client.delete_org(org_id)) - team_id = client.create_team( - f"e2e-lf-org-team-{marker}", - models=models, - organization_id=org_id, - ) - resources.defer(lambda: client.delete_team(team_id)) - client.add_team_langfuse_callback(team_id, creds) - key = client.key_with_alias( - key_alias, - models=models, - team_id=team_id, - organization_id=org_id, - ) - resources.defer(lambda: client.delete_key(key)) - return org_id, team_id, key, key_alias - - @pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["chat_completions"]) - def test_success_logs_spend_with_team_attribution( - self, - client: LoggingClient, - resources: ResourceManager, - langfuse_creds: LangfuseCreds, - ) -> None: - org_id, team_id, key, key_alias = self._org_team_key( - client, resources, langfuse_creds, models=[DRIVER_MODEL] - ) - prompt_marker = unique_marker() - outcome = client.chat_raw( - key, DRIVER_MODEL, f"reply with one word only {prompt_marker}" - ) - require_successful_call(outcome) - - obs = client.poll_langfuse_observation( - langfuse_creds, - key_alias=key_alias, - prompt_marker=prompt_marker, - require_positive_cost=True, - ) - assert obs is not None, ( - f"org scope: Langfuse never received generation for key_alias={key_alias!r}" - ) - meta_blob = _json_blob(obs.metadata) - assert team_id in meta_blob, ( - f"org-scoped team key must stamp team_id on Langfuse metadata; " - f"team_id={team_id!r} metadata={meta_blob[:400]}" - ) - _ = org_id - _assert_logs_spend( - client, - key=key, - outcome=outcome, - obs_cost=observation_spend(obs), - scope="org-team", - ) diff --git a/tests/e2e/logging/test_otel_trace_e2e.py b/tests/e2e/logging/test_otel_trace_e2e.py index b00fd91be3c..e49dd311b32 100644 --- a/tests/e2e/logging/test_otel_trace_e2e.py +++ b/tests/e2e/logging/test_otel_trace_e2e.py @@ -18,15 +18,14 @@ destination's own query API - never proxy-side "export succeeded" logs). from __future__ import annotations import time -from collections.abc import Callable import pytest from pydantic import BaseModel, ConfigDict, ValidationError from e2e_config import CHEAP_ANTHROPIC_MODEL, CHEAP_OPENAI_MODEL, unique_marker -from e2e_http import NoBody, StreamingResponse, require_successful_call +from e2e_http import NoBody from lifecycle import ResourceManager -from logging_client import INVALID_UPSTREAM_API_KEY, LoggingClient +from logging_client import INVALID_UPSTREAM_API_KEY, LoggingClient, first_ok from models import LiteLLMParamsBody from otel_client import JaegerSpan, JaegerTrace, OtelReader @@ -60,22 +59,6 @@ def _assert_otel_destination_configured(client: LoggingClient) -> None: ) -def _first_ok(client: LoggingClient, send: Callable[[], StreamingResponse]) -> StreamingResponse: - """First successful call on a fresh key. A fresh key may briefly 401 until - the data plane's auth cache picks it up, so retry on 401 to a deadline; a - 401 is rejected before the LLM call so it exports no gen-AI span and cannot - contaminate the trace assertions. Any other failure is behavior under test - and fails hard.""" - deadline = time.monotonic() + client.gateway.poll_timeout - while True: - outcome = send() - if outcome.ok: - return outcome - if outcome.status_code != 401 or time.monotonic() >= deadline: - require_successful_call(outcome) - time.sleep(client.gateway.poll_interval) - - def _parent_ids(span_id: str, trace: JaegerTrace) -> list[str]: span = next(s for s in trace.spans if s.span_id == span_id) return [ref.span_id for ref in span.references if ref.ref_type == "CHILD_OF"] @@ -253,7 +236,7 @@ class TestOtelTraceCompleteness: resources.defer(lambda: client.delete_key(key)) marker = unique_marker() - outcome = _first_ok( + outcome = first_ok( client, lambda: client.chat_raw(key, MODEL, f"reply with one word {marker}", max_tokens=16) ) assert outcome.call_id is not None, "success response must carry x-litellm-call-id" @@ -286,7 +269,7 @@ class TestOtelTraceCompleteness: resources.defer(lambda: client.delete_key(key)) marker = unique_marker() - outcome = _first_ok( + outcome = first_ok( client, lambda: client.messages_raw(key, MODEL, f"reply with one word {marker}", max_tokens=16) ) assert outcome.call_id is not None, "success response must carry x-litellm-call-id" @@ -319,7 +302,7 @@ class TestOtelTraceCompleteness: resources.defer(lambda: client.delete_key(key)) marker = unique_marker() - outcome = _first_ok( + outcome = first_ok( client, lambda: client.responses_raw(key, CHEAP_OPENAI_MODEL, f"reply with one word {marker}"), ) @@ -360,7 +343,7 @@ class TestOtelTraceCompleteness: resources.defer(lambda: client.delete_key(key)) marker = unique_marker() - outcome = _first_ok( + outcome = first_ok( client, lambda: client.chat_raw(key, MODEL, f"reply with one word {marker}", stream=True, max_tokens=16), ) @@ -416,7 +399,7 @@ class TestOtelTraceCompleteness: resources.defer(lambda: client.delete_key(key)) marker = unique_marker() - outcome = _first_ok( + outcome = first_ok( client, lambda: client.messages_raw(key, MODEL, f"reply with one word {marker}", max_tokens=16, stream=True), ) @@ -474,7 +457,7 @@ class TestOtelTraceCompleteness: resources.defer(lambda: client.delete_key(key)) marker = unique_marker() - outcome = _first_ok( + outcome = first_ok( client, lambda: client.responses_raw(key, CHEAP_OPENAI_MODEL, f"reply with one word {marker}", stream=True), ) diff --git a/tests/e2e/management/conftest.py b/tests/e2e/management/conftest.py index 47c1d34baae..264108f6089 100644 --- a/tests/e2e/management/conftest.py +++ b/tests/e2e/management/conftest.py @@ -50,8 +50,10 @@ def ui_page(browser: "Browser") -> "Iterator[Page]": page.goto(f"{PROXY_BASE_URL}/ui/") page.fill("#username", UI_USERNAME) page.fill("#password", UI_PASSWORD) - page.click('input[type="submit"]') - page.wait_for_url("**/ui/**") + page.click('button[type="submit"]') + page.wait_for_function( + "() => document.cookie.includes('token=') || !document.querySelector('#username')" + ) yield page finally: context.close() diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 4140967f3e0..c4ecd0cfa63 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -440,6 +440,8 @@ class LiteLLMParamsBody(BaseModel): aws_batch_role_arn: str | None = None input_cost_per_token: float | None = None output_cost_per_token: float | None = None + extra_headers: dict[str, str] | None = None + use_in_pass_through: bool | None = None ModelMode = Literal["batch", "realtime", "image_generation"] diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index f99cfb953ba..6875894c1bf 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -2247,6 +2247,53 @@ def test_get_error_information_prefers_message_attribute_over_str(): assert result["error_class"] == "ProxyExceptionLike" +def test_get_error_information_budget_exceeded_structured_fields(): + """ + Regression for LIT-4458: a budget-rejected request's failure + StandardLoggingPayload must identify WHICH budget blocked the call + as structured fields, not only inside the free-text error_str + ("ExceededBudget: User=... over budget. Spend=..., Budget=..."). + + Asserts get_error_information copies entity_type / entity_id / + max_budget / current_cost off BudgetExceededError into + error_budget_entity_type / error_budget_entity_id / + error_budget_limit / error_budget_spend, and leaves all four None + for non-budget exceptions. + """ + from litellm.exceptions import BudgetExceededError + from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + + exc = BudgetExceededError( + current_cost=3.4e-05, + max_budget=1e-06, + message="ExceededBudget: User=repro-user over budget. Spend=3.4e-05, Budget=1e-06", + entity_type="user", + entity_id="repro-user", + ) + + result = StandardLoggingPayloadSetup.get_error_information(exc) + assert result["error_budget_entity_type"] == "user" + assert result["error_budget_entity_id"] == "repro-user" + assert result["error_budget_limit"] == 1e-06 + assert result["error_budget_spend"] == 3.4e-05 + assert result["error_code"] == "429" + assert result["error_class"] == "BudgetExceededError" + assert result["error_rate_limit_type"] == "budget" + + legacy_exc = BudgetExceededError(current_cost=2.0, max_budget=1.0) + legacy_result = StandardLoggingPayloadSetup.get_error_information(legacy_exc) + assert legacy_result["error_budget_entity_type"] is None + assert legacy_result["error_budget_entity_id"] is None + assert legacy_result["error_budget_limit"] == 1.0 + assert legacy_result["error_budget_spend"] == 2.0 + + non_budget_result = StandardLoggingPayloadSetup.get_error_information(ValueError("boom")) + assert non_budget_result["error_budget_entity_type"] is None + assert non_budget_result["error_budget_entity_id"] is None + assert non_budget_result["error_budget_limit"] is None + assert non_budget_result["error_budget_spend"] is None + + def test_get_error_information_preserves_explicit_empty_message(): """ An exception that deliberately sets `.message = ""` must surface diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py index ba95c90798e..be8c5a05601 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py @@ -956,3 +956,39 @@ def test_stream_chunk_builder_propagates_vertex_ai_metadata_from_dict_chunks(): assert response.model_dump()["vertex_ai_grounding_metadata"] == [ {"webSearchQueries": ["test query"]} ] + + +def test_cost_field_in_usage_chunks(): + chunk1_usage = Usage(completion_tokens=1, prompt_tokens=10, total_tokens=11) + chunk1 = ModelResponseStream( + id="chatcmpl-1", + created=1745513206, + model="openrouter/claude", + choices=[ + StreamingChoices(finish_reason=None, index=0, delta=Delta(content="Hi")) + ], + usage=chunk1_usage, + ) + + chunk2_usage = Usage( + completion_tokens=5, prompt_tokens=10, total_tokens=15, cost=0.00025 + ) + chunk2 = ModelResponseStream( + id="chatcmpl-1", + created=1745513207, + model="openrouter/claude", + choices=[ + StreamingChoices(finish_reason="stop", index=0, delta=Delta(content="")) + ], + usage=chunk2_usage, + ) + + processor = ChunkProcessor(chunks=[chunk1, chunk2]) + usage = processor.calculate_usage( + chunks=[chunk1, chunk2], model="openrouter/claude", completion_output="Hi" + ) + + assert hasattr(usage, "cost") + assert usage.cost == 0.00025 + assert usage.prompt_tokens == 10 + assert usage.completion_tokens == 5 diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py index a0fd5516e24..50259c1baec 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py @@ -1394,6 +1394,190 @@ def test_has_any_special_delta_attributes( assert result is False +def test_calculate_total_usage_with_cost(): + from litellm.litellm_core_utils.streaming_handler import calculate_total_usage + + chunk1_usage = Usage(completion_tokens=1, prompt_tokens=10, total_tokens=11) + chunk1 = ModelResponseStream( + id="test-1", + created=1745513206, + model="openrouter/test", + choices=[ + StreamingChoices(finish_reason=None, index=0, delta=Delta(content="Hi")) + ], + usage=chunk1_usage, + ) + + chunk2_usage = Usage( + completion_tokens=5, prompt_tokens=10, total_tokens=15, cost=0.00025 + ) + chunk2 = ModelResponseStream( + id="test-1", + created=1745513207, + model="openrouter/test", + choices=[ + StreamingChoices(finish_reason="stop", index=0, delta=Delta(content="")) + ], + usage=chunk2_usage, + ) + + usage = calculate_total_usage([chunk1, chunk2]) + + assert hasattr(usage, "cost") + assert usage.cost == 0.00025 + assert usage.prompt_tokens == 10 + assert usage.completion_tokens == 5 + + +def test_calculate_total_usage_with_dict_usage_cost(): + """Regression: dict-shaped `usage` with a `cost` key must still surface + provider cost even though `hasattr` on a dict does not consult its keys.""" + from litellm.litellm_core_utils.streaming_handler import calculate_total_usage + + chunk = { + "usage": { + "prompt_tokens": 10, + "completion_tokens": 5, + "total_tokens": 15, + "cost": 0.00025, + } + } + + usage = calculate_total_usage([chunk]) + + assert usage.prompt_tokens == 10 + assert usage.completion_tokens == 5 + assert getattr(usage, "cost", None) == 0.00025 + + +@pytest.mark.asyncio +async def test_openrouter_streaming_cost_after_finish_reason(logging_obj: Logging): + from litellm.utils import ModelResponseListIterator + + chunk1 = ModelResponseStream( + id="chatcmpl-or", + created=1742056047, + model="openrouter/claude", + choices=[ + StreamingChoices( + finish_reason=None, index=0, delta=Delta(content="Hi", role="assistant") + ) + ], + usage=None, + ) + chunk2 = ModelResponseStream( + id="chatcmpl-or", + created=1742056048, + model="openrouter/claude", + choices=[ + StreamingChoices(finish_reason="stop", index=0, delta=Delta(content="")) + ], + usage=None, + ) + chunk3_usage = Usage( + completion_tokens=5, prompt_tokens=10, total_tokens=15, cost=0.00025 + ) + chunk3 = ModelResponseStream( + id="chatcmpl-or", + created=1742056049, + model="openrouter/claude", + choices=[ + StreamingChoices(finish_reason=None, index=0, delta=Delta(content="")) + ], + usage=chunk3_usage, + ) + + completion_stream = ModelResponseListIterator( + model_responses=[chunk1, chunk2, chunk3] + ) + response = CustomStreamWrapper( + completion_stream=completion_stream, + model="openrouter/claude", + custom_llm_provider="openrouter", + logging_obj=logging_obj, + stream_options={"include_usage": True}, + ) + + collected_chunks = [] + async for chunk in response: + collected_chunks.append(chunk) + + usage_chunks = [c for c in collected_chunks if hasattr(c, "usage") and c.usage] + assert len(usage_chunks) > 0 + assert hasattr(usage_chunks[-1].usage, "cost") + assert usage_chunks[-1].usage.cost == 0.00025 + + +def test_openrouter_streaming_cost_propagates_to_hidden_params(): + """ + Verify that provider-reported cost from usage.cost flows into + _hidden_params["additional_headers"]["llm_provider-x-litellm-response-cost"] + on the complete streaming response, so litellm's cost calculator uses it. + """ + import litellm + + chunk1 = ModelResponseStream( + id="chatcmpl-or", + created=1742056047, + model="openrouter/claude", + choices=[ + StreamingChoices( + finish_reason=None, index=0, delta=Delta(content="Hi", role="assistant") + ) + ], + usage=None, + ) + chunk2 = ModelResponseStream( + id="chatcmpl-or", + created=1742056048, + model="openrouter/claude", + choices=[ + StreamingChoices(finish_reason="stop", index=0, delta=Delta(content="")) + ], + usage=None, + ) + chunk3 = ModelResponseStream( + id="chatcmpl-or", + created=1742056049, + model="openrouter/claude", + choices=[ + StreamingChoices(finish_reason=None, index=0, delta=Delta(content="")) + ], + usage=Usage( + completion_tokens=5, prompt_tokens=10, total_tokens=15, cost=0.00025 + ), + ) + + # Build the complete response as stream_chunk_builder does + complete_response = litellm.stream_chunk_builder( + chunks=[chunk1, chunk2, chunk3], + messages=[{"role": "user", "content": "test"}], + ) + + assert complete_response is not None + assert hasattr(complete_response.usage, "cost") + assert complete_response.usage.cost == 0.00025 + + # Use the real propagation method from CustomStreamWrapper + CustomStreamWrapper._propagate_usage_cost_to_hidden_params(complete_response) + + assert "additional_headers" in complete_response._hidden_params + assert ( + complete_response._hidden_params["additional_headers"][ + "llm_provider-x-litellm-response-cost" + ] + == 0.00025 + ) + + # Verify the cost calculator would pick this up + from litellm.cost_calculator import get_response_cost_from_hidden_params + + provider_cost = get_response_cost_from_hidden_params( + complete_response._hidden_params + ) + assert provider_cost == 0.00025 + + def test_handle_special_delta_attributes( initialized_custom_stream_wrapper: CustomStreamWrapper, ): @@ -3170,3 +3354,115 @@ async def test_streaming_logprobs_do_not_store_raw_sdk_object(logging_obj: Loggi for lp in logprobs_chunks: assert isinstance(lp, ChoiceLogprobs) assert "openai" not in type(lp).__module__ + + +class TransportErrorAfterChunksIterator: + """Yields the given chunks, then raises the given exception once, then StopAsyncIteration.""" + + def __init__(self, model_responses, exception): + self.model_responses = model_responses + self.exception = exception + self.index = 0 + self.raised = False + + def __aiter__(self): + return self + + async def __anext__(self): + if self.index < len(self.model_responses): + chunk = self.model_responses[self.index] + self.index += 1 + return chunk + if not self.raised: + self.raised = True + raise self.exception + raise StopAsyncIteration + + +def _reset_test_chunk(content: Optional[str] = None, finish_reason: Optional[str] = None) -> ModelResponseStream: + return ModelResponseStream( + id="chatcmpl-reset-test", + created=1783458104, + model="stub-model", + object="chat.completion.chunk", + choices=[ + StreamingChoices( + index=0, + delta=Delta(content=content), + finish_reason=finish_reason, + ) + ], + ) + + +@pytest.mark.asyncio +async def test_transport_read_error_after_finish_reason_ends_stream_gracefully( + logging_obj: Logging, +): + """A trailing connection reset after the provider's finish chunk must not fail the stream.""" + import httpx + + completion_stream = TransportErrorAfterChunksIterator( + model_responses=[ + _reset_test_chunk(content="Hello"), + _reset_test_chunk(finish_reason="stop"), + ], + exception=httpx.ReadError("Response payload is not completed"), + ) + response = CustomStreamWrapper( + completion_stream=completion_stream, + model="hosted_vllm/stub-model", + custom_llm_provider="hosted_vllm", + logging_obj=logging_obj, + ) + + chunks = [chunk async for chunk in response] + + finish_reasons = [ + chunk.choices[0].finish_reason + for chunk in chunks + if chunk.choices and chunk.choices[0].finish_reason + ] + contents = [ + chunk.choices[0].delta.content + for chunk in chunks + if chunk.choices and chunk.choices[0].delta and chunk.choices[0].delta.content + ] + assert finish_reasons == ["stop"] + assert contents == ["Hello"] + + +@pytest.mark.asyncio +async def test_transport_read_error_before_finish_reason_raises(logging_obj: Logging): + """A connection reset before any finish chunk must surface, never end as a clean stop. + + Regression test for silent empty/truncated HTTP 200 streams: the aiohttp + transport used to swallow mid-stream connection resets, so the wrapper saw a + clean end-of-stream and fabricated finish_reason "stop". + """ + import httpx + + from litellm.exceptions import MidStreamFallbackError + + completion_stream = TransportErrorAfterChunksIterator( + model_responses=[_reset_test_chunk(content="Hel")], + exception=httpx.ReadError("Response payload is not completed"), + ) + response = CustomStreamWrapper( + completion_stream=completion_stream, + model="hosted_vllm/stub-model", + custom_llm_provider="hosted_vllm", + logging_obj=logging_obj, + ) + + received = [] + with pytest.raises(MidStreamFallbackError): + async for chunk in response: + received.append(chunk) + + fabricated_finish_reasons = [ + chunk.choices[0].finish_reason + for chunk in received + if chunk.choices and chunk.choices[0].finish_reason + ] + assert fabricated_finish_reasons == [] diff --git a/tests/test_litellm/llms/custom_httpx/test_aiohttp_transport.py b/tests/test_litellm/llms/custom_httpx/test_aiohttp_transport.py index 0c5e386c438..d1bc356662f 100644 --- a/tests/test_litellm/llms/custom_httpx/test_aiohttp_transport.py +++ b/tests/test_litellm/llms/custom_httpx/test_aiohttp_transport.py @@ -81,7 +81,7 @@ class MockContent: def __init__(self, chunks=None, exception_to_raise=None, exception_at_chunk=None): self.chunks = chunks or [b"chunk1", b"chunk2", b"chunk3"] self.exception_to_raise = exception_to_raise - self.exception_at_chunk = exception_at_chunk or (len(self.chunks) - 1) + self.exception_at_chunk = exception_at_chunk if exception_at_chunk is not None else (len(self.chunks) - 1) self.chunk_index = 0 async def iter_chunked(self, chunk_size): @@ -107,15 +107,11 @@ async def test_aiohttp_response_stream_normal_flow(): @pytest.mark.asyncio -async def test_transfer_encoding_error_no_httpx_read_error(): - """Test that TransferEncodingError doesn't get converted to httpx.ReadError""" - - # Create a TransferEncodingError wrapped in ClientPayloadError (like in real scenarios) +async def test_client_payload_error_mid_stream_raises_read_error(): + """A connection reset mid-body must surface as httpx.ReadError, not truncate silently""" transfer_error = aiohttp.http_exceptions.TransferEncodingError( message="400, message: Not enough data for satisfy transfer length header." ) - - # Wrap it in ClientPayloadError as aiohttp does client_payload_error = aiohttp.ClientPayloadError( "Response payload is not completed" ) @@ -124,47 +120,100 @@ async def test_transfer_encoding_error_no_httpx_read_error(): mock_response = MockAiohttpResponse( content_chunks=[b"chunk1", b"chunk2", b"chunk3"], exception_to_raise=client_payload_error, - exception_at_chunk=1, # Error occurs at chunk 1 + exception_at_chunk=1, ) stream = AiohttpResponseStream(mock_response) # type: ignore received_chunks = [] - # This should NOT raise httpx.ReadError or any other exception - # It should handle the error gracefully and just return what was received - async for chunk in stream: - received_chunks.append(chunk) - print(f"received_chunks: {received_chunks}") + with pytest.raises(httpx.ReadError): + async for chunk in stream: + received_chunks.append(chunk) - # Should have received the first chunk before the error assert received_chunks == [b"chunk1"] - assert len(received_chunks) == 1 + assert mock_response.closed is True @pytest.mark.asyncio -async def test_client_payload_error_graceful_handling(): - """Test that ClientPayloadError is handled gracefully without stacktrace""" - # Create a ClientPayloadError directly +async def test_client_payload_error_before_first_chunk_raises_read_error(): + """A connection reset before any body byte must surface, not yield an empty 200 body""" client_error = aiohttp.client_exceptions.ClientPayloadError( "Response payload is not completed" ) mock_response = MockAiohttpResponse( - content_chunks=[b"data1", b"data2", b"data3"], + content_chunks=[b"data1", b"data2"], exception_to_raise=client_error, - exception_at_chunk=2, # Error occurs at chunk 2 + exception_at_chunk=0, ) stream = AiohttpResponseStream(mock_response) # type: ignore received_chunks = [] - # This should handle the error gracefully without raising - async for chunk in stream: - received_chunks.append(chunk) + with pytest.raises(httpx.ReadError): + async for chunk in stream: + received_chunks.append(chunk) - # Should have received chunks before the error - assert received_chunks == [b"data1", b"data2"] - assert len(received_chunks) == 2 + assert received_chunks == [] + assert mock_response.closed is True + + +@pytest.mark.asyncio +async def test_connection_closed_runtime_error_raises_read_error(): + """aiohttp's bare RuntimeError('Connection closed.') must surface as httpx.ReadError""" + mock_response = MockAiohttpResponse( + content_chunks=[b"data1", b"data2"], + exception_to_raise=RuntimeError("Connection closed."), + exception_at_chunk=1, + ) + + stream = AiohttpResponseStream(mock_response) # type: ignore + received_chunks = [] + + with pytest.raises(httpx.ReadError): + async for chunk in stream: + received_chunks.append(chunk) + + assert received_chunks == [b"data1"] + assert mock_response.closed is True + + +@pytest.mark.asyncio +async def test_unrelated_runtime_error_propagates_unmapped(): + """RuntimeErrors other than 'Connection closed' must propagate untouched""" + mock_response = MockAiohttpResponse( + content_chunks=[b"data1"], + exception_to_raise=RuntimeError("something else broke"), + exception_at_chunk=0, + ) + + stream = AiohttpResponseStream(mock_response) # type: ignore + + with pytest.raises(RuntimeError, match="something else broke"): + async for _ in stream: + pass + + +@pytest.mark.asyncio +async def test_transfer_encoding_error_raises_read_error(): + """A raw TransferEncodingError mid-body must surface as httpx.ReadError""" + mock_response = MockAiohttpResponse( + content_chunks=[b"data1", b"data2"], + exception_to_raise=aiohttp.http_exceptions.TransferEncodingError( + message="Not enough data to satisfy transfer length header." + ), + exception_at_chunk=1, + ) + + stream = AiohttpResponseStream(mock_response) # type: ignore + received_chunks = [] + + with pytest.raises(httpx.ReadError): + async for chunk in stream: + received_chunks.append(chunk) + + assert received_chunks == [b"data1"] + assert mock_response.closed is True @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_partial_update.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_partial_update.py index 41d61c8f508..968fafc0e0e 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_partial_update.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_partial_update.py @@ -203,6 +203,7 @@ async def test_auth_type_switch_clears_stale_flow_scoped_fields(): data_dict = await _run_update_with_existing(data, existing_auth_type="oauth2") for stale_field in ( + "issuer", "authorization_url", "token_url", "registration_url", @@ -217,6 +218,165 @@ async def test_auth_type_switch_clears_stale_flow_scoped_fields(): assert _credentials_cleared(data_dict["credentials"]) +@pytest.mark.asyncio +async def test_url_change_clears_stale_discovered_oauth_fields(): + """Re-pointing the server url at a potentially different upstream must clear the discovered or + trust-on-first-use OAuth issuer and endpoints, so the new upstream re-discovers instead of + anchoring on the previous upstream's issuer (RFC 8414 §3.3 against a stale anchor).""" + mock_prisma = _mock_prisma() + existing = MagicMock() + existing.auth_type = "oauth2" + existing.url = "https://old.example.com/mcp" + existing.credentials = None + mock_prisma.db.litellm_mcpservertable.find_unique = AsyncMock(return_value=existing) + + data = UpdateMCPServerRequest(server_id="my-test-server", url="https://new.example.com/mcp") + await update_mcp_server(mock_prisma, data, "test-user") + data_dict = mock_prisma.db.litellm_mcpservertable.update.call_args[1]["data"] + + assert data_dict["url"] == "https://new.example.com/mcp" + for stale_field in ("issuer", "authorization_url", "token_url", "registration_url"): + assert data_dict[stale_field] is None, f"{stale_field} must be cleared on url change" + + +@pytest.mark.asyncio +async def test_url_change_clears_stale_oauth_fields_even_when_resubmitted_unchanged(): + """The edit form re-sends every field, so a URL change arrives WITH the previous upstream's issuer + and endpoints in the payload. Those resubmitted-unchanged values are stale and must still clear + (otherwise they survive the url change and win in the resolution merge). A genuinely new value the + caller changed in the same submit is kept.""" + mock_prisma = _mock_prisma() + existing = MagicMock() + existing.auth_type = "oauth2" + existing.url = "https://old.example.com/mcp" + existing.credentials = None + existing.issuer = "https://old-idp.example.com" + existing.token_url = "https://old-idp.example.com/token" + existing.authorization_url = "https://old-idp.example.com/authorize" + mock_prisma.db.litellm_mcpservertable.find_unique = AsyncMock(return_value=existing) + + data = UpdateMCPServerRequest( + server_id="my-test-server", + url="https://new.example.com/mcp", + issuer="https://old-idp.example.com", # resubmitted unchanged -> stale, must clear + token_url="https://old-idp.example.com/token", # resubmitted unchanged -> stale, must clear + authorization_url="https://new-idp.example.com/authorize", # genuinely changed -> kept + ) + await update_mcp_server(mock_prisma, data, "test-user") + data_dict = mock_prisma.db.litellm_mcpservertable.update.call_args[1]["data"] + + assert data_dict["issuer"] is None + assert data_dict["token_url"] is None + assert data_dict["authorization_url"] == "https://new-idp.example.com/authorize" + + +@pytest.mark.asyncio +async def test_clearing_pinned_issuer_clears_stale_oauth_endpoints(): + """Clearing a previously pinned issuer must not revive the endpoints resolved under it. Under an + issuer anchor the endpoints come solely from the issuer document and are not persisted, but a row + that was resource-rooted before the pin can still hold stale authorization_url/token_url; clearing + the anchor without clearing those would let them win the resolution merge and be posted to without + fresh discovery (RFC 8414 §3.3 provenance).""" + mock_prisma = _mock_prisma() + existing = MagicMock() + existing.auth_type = "oauth2" + existing.url = "https://same.example.com/mcp" + existing.credentials = None + existing.issuer = "https://pinned-idp.example.com" + existing.token_url = "https://pinned-idp.example.com/token" + existing.authorization_url = "https://pinned-idp.example.com/authorize" + mock_prisma.db.litellm_mcpservertable.find_unique = AsyncMock(return_value=existing) + + data = UpdateMCPServerRequest( + server_id="my-test-server", + issuer="", # admin clears the anchor; url and auth_type unchanged + token_url="https://pinned-idp.example.com/token", + authorization_url="https://pinned-idp.example.com/authorize", + ) + await update_mcp_server(mock_prisma, data, "test-user") + data_dict = mock_prisma.db.litellm_mcpservertable.update.call_args[1]["data"] + + assert data_dict["token_url"] is None + assert data_dict["authorization_url"] is None + + +@pytest.mark.asyncio +async def test_repointing_pinned_issuer_clears_stale_endpoints_keeps_new_issuer(): + """Re-pointing the issuer to a different authorization server invalidates the old issuer's + endpoints while keeping the new issuer the admin submitted.""" + mock_prisma = _mock_prisma() + existing = MagicMock() + existing.auth_type = "oauth2" + existing.url = "https://same.example.com/mcp" + existing.credentials = None + existing.issuer = "https://old-idp.example.com" + existing.token_url = "https://old-idp.example.com/token" + existing.authorization_url = "https://old-idp.example.com/authorize" + mock_prisma.db.litellm_mcpservertable.find_unique = AsyncMock(return_value=existing) + + data = UpdateMCPServerRequest( + server_id="my-test-server", + issuer="https://new-idp.example.com", + token_url="https://old-idp.example.com/token", # resubmitted stale -> must clear + authorization_url="https://old-idp.example.com/authorize", # resubmitted stale -> must clear + ) + await update_mcp_server(mock_prisma, data, "test-user") + data_dict = mock_prisma.db.litellm_mcpservertable.update.call_args[1]["data"] + + assert data_dict["issuer"] == "https://new-idp.example.com" + assert data_dict["token_url"] is None + assert data_dict["authorization_url"] is None + + +@pytest.mark.asyncio +async def test_establishing_issuer_first_time_preserves_discovered_fields(): + """Establishing an issuer for the first time (None -> X), which is exactly what the trust-on-first-use + discovery write-back does, must NOT clear the endpoints or oauth2_flow it discovered in the same + write. Only an issuer that was already pinned and is now changed or cleared invalidates its + endpoints, so the discovery persist cannot wipe the fields it just resolved.""" + mock_prisma = _mock_prisma() + existing = MagicMock() + existing.auth_type = "oauth2" + existing.url = "https://same.example.com/mcp" + existing.credentials = None + existing.issuer = None + mock_prisma.db.litellm_mcpservertable.find_unique = AsyncMock(return_value=existing) + + data = UpdateMCPServerRequest( + server_id="my-test-server", + issuer="https://discovered-idp.example.com", + authorization_url="https://discovered-idp.example.com/authorize", + token_url="https://discovered-idp.example.com/token", + oauth2_flow="authorization_code", + ) + await update_mcp_server(mock_prisma, data, "mcp_oauth_discovery") + data_dict = mock_prisma.db.litellm_mcpservertable.update.call_args[1]["data"] + + assert data_dict["issuer"] == "https://discovered-idp.example.com" + assert data_dict["authorization_url"] == "https://discovered-idp.example.com/authorize" + assert data_dict["token_url"] == "https://discovered-idp.example.com/token" + assert data_dict.get("oauth2_flow") == "authorization_code" + + +@pytest.mark.asyncio +async def test_unchanged_url_does_not_clear_discovered_oauth_fields(): + """A partial update that resends the same url (or omits it) must not clear the discovered OAuth + fields, so a routine save does not force needless re-discovery.""" + mock_prisma = _mock_prisma() + existing = MagicMock() + existing.auth_type = "oauth2" + existing.url = "https://same.example.com/mcp" + existing.credentials = None + mock_prisma.db.litellm_mcpservertable.find_unique = AsyncMock(return_value=existing) + + data = UpdateMCPServerRequest(server_id="my-test-server", url="https://same.example.com/mcp") + await update_mcp_server(mock_prisma, data, "test-user") + data_dict = mock_prisma.db.litellm_mcpservertable.update.call_args[1]["data"] + + for preserved_field in ("issuer", "authorization_url", "token_url", "registration_url"): + assert preserved_field not in data_dict, f"{preserved_field} must not be cleared when url is unchanged" + + @pytest.mark.asyncio async def test_auth_type_switch_keeps_explicitly_provided_flow_fields(): """Fields explicitly provided alongside the auth_type switch must survive it.""" diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py index bba00ed1819..80bf08a5eba 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py @@ -1166,6 +1166,67 @@ class TestMCPServerManager: assert built.token_url == "https://idp.example.com/token" assert built.scopes == ["read", "admin"] + @pytest.mark.asyncio + async def test_build_from_table_reflects_discovered_issuer_trust_on_first_use(self): + """An unpinned server resolves endpoints resource-rooted on first discovery and records the + discovered issuer trust-on-first-use. The returned in-memory server must carry that discovered + issuer so the registry matches what gets persisted to the row; otherwise the OAuth token + identity (which includes issuer) differs between this build and the next rebuild, forcing a + spurious re-auth. Endpoints and issuer come from the same authorization-server document, so + they are consistent.""" + manager = MCPServerManager() + row = LiteLLM_MCPServerTable( + server_id="tofu-issuer-1", + alias="tofu_issuer", + description="unpinned, discovers its issuer", + url="https://up.example.com/mcp", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, + created_at=datetime.now(), + updated_at=datetime.now(), + ) + + metadata = MCPOAuthMetadata( + authorization_url="https://idp.example.com/authorize", + token_url="https://idp.example.com/token", + scopes=["read"], + discovered_issuer="https://idp.example.com", + ) + with patch.object(manager, "_descovery_metadata", new=AsyncMock(return_value=metadata)): + built = await manager.build_mcp_server_from_table(row, credentials_are_encrypted=False) + + assert built.issuer == "https://idp.example.com" + assert built.issuer_is_anchored is False + assert built.authorization_url == "https://idp.example.com/authorize" + + @pytest.mark.asyncio + async def test_build_from_table_origin_fallback_issuer_is_not_reflected(self): + """An origin-fallback discovery is a guess that is deliberately never persisted, so the built + server must not claim an issuer the row will not hold; otherwise in-memory and DB would + disagree in the opposite direction.""" + manager = MCPServerManager() + row = LiteLLM_MCPServerTable( + server_id="origin-fallback-1", + alias="origin_fallback", + description="unpinned, origin-fallback discovery", + url="https://up.example.com/mcp", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, + created_at=datetime.now(), + updated_at=datetime.now(), + ) + + metadata = MCPOAuthMetadata( + authorization_url="https://up.example.com/authorize", + token_url="https://up.example.com/token", + discovered_issuer="https://up.example.com", + from_origin_fallback=True, + ) + with patch.object(manager, "_descovery_metadata", new=AsyncMock(return_value=metadata)): + built = await manager.build_mcp_server_from_table(row, credentials_are_encrypted=False) + + assert built.issuer is None + @pytest.mark.asyncio async def test_build_from_table_whitespace_authorization_url_is_not_a_pin(self): """A whitespace-only authorization_url on the row must not be kept for redirects while the @@ -1230,6 +1291,159 @@ class TestMCPServerManager: assert built.registration_url == "https://idp.example.com/register" assert built.scopes == ["read"] + @pytest.mark.asyncio + async def test_build_from_table_uses_issuer_anchored_endpoints_when_issuer_configured(self): + """When an admin configures an issuer, the build takes its endpoints from the issuer-anchored + fetch (RFC 8414 §3.3) rather than the resource-rooted corroboration path. The build path does + not call _descovery_metadata directly; the issuer-anchored helper is responsible for combining + issuer endpoints with resource-driven scopes internally, and is invoked with the server url so + it can fetch those scopes.""" + manager = MCPServerManager() + row = LiteLLM_MCPServerTable( + server_id="issuer-anchored-1", + alias="issuer_anchored", + description="issuer configured, blank endpoints", + url="https://up.example.com/mcp", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, + issuer="https://idp.example.com", + created_at=datetime.now(), + updated_at=datetime.now(), + ) + + resolved = MCPOAuthMetadata( + authorization_url="https://idp.example.com/authorize", + token_url="https://idp.example.com/token", + registration_url="https://idp.example.com/register", + scopes=["read", "write"], + ) + resource_rooted = AsyncMock(return_value=MCPOAuthMetadata(token_url="https://attacker.example.com/steal")) + with ( + patch.object(manager, "_fetch_issuer_anchored_oauth_metadata", new=AsyncMock(return_value=resolved)) as anchored, + patch.object(manager, "_descovery_metadata", new=resource_rooted), + ): + built = await manager.build_mcp_server_from_table(row, credentials_are_encrypted=False) + + anchored.assert_awaited_once_with("https://idp.example.com", "https://up.example.com/mcp") + resource_rooted.assert_not_awaited() + assert built.issuer == "https://idp.example.com" + assert built.issuer_is_anchored is True + assert built.authorization_url == "https://idp.example.com/authorize" + assert built.token_url == "https://idp.example.com/token" + assert built.registration_url == "https://idp.example.com/register" + assert built.scopes == ["read", "write"] + + @pytest.mark.asyncio + async def test_fetch_issuer_anchored_metadata_takes_endpoints_from_issuer_scopes_from_resource(self): + """The issuer-anchored helper adopts token_endpoint/registration_endpoint from the pinned + issuer's own §3.3-validated document, but the scopes are resource-driven: it fetches the + resource's advertised scopes and uses those, not the issuer document's scopes_supported. This + keeps endpoint trust anchored on the issuer while scope selection stays resource-driven per the + MCP Scope Selection Strategy.""" + manager = MCPServerManager() + + issuer_document = MCPOAuthMetadata( + authorization_url="https://idp.example.com/authorize", + token_url="https://idp.example.com/token", + registration_url="https://idp.example.com/register", + scopes=["as.everything"], + ) + resource_document = MCPOAuthMetadata(scopes=["resource.read"]) + with ( + patch.object( + manager, "_fetch_single_authorization_server_metadata", new=AsyncMock(return_value=issuer_document) + ) as issuer_fetch, + patch.object(manager, "_descovery_metadata", new=AsyncMock(return_value=resource_document)) as resource_fetch, + ): + result = await manager._fetch_issuer_anchored_oauth_metadata( + "https://idp.example.com", "https://up.example.com/mcp" + ) + + issuer_fetch.assert_awaited_once_with( + "https://idp.example.com", "https://idp.example.com", require_issuer="https://idp.example.com" + ) + resource_fetch.assert_awaited_once() + assert result is not None + assert result.token_url == "https://idp.example.com/token" + assert result.registration_url == "https://idp.example.com/register" + assert result.scopes == ["resource.read"] + + @pytest.mark.asyncio + async def test_build_from_table_issuer_anchor_fails_closed_without_falling_back_to_resource(self): + """A configured issuer whose metadata does not validate (RFC 8414 §3.3 mismatch or fetch + failure) yields None from the anchored fetch. The build must adopt nothing and must NOT fall + back to resource-rooted discovery, or the fail-closed guarantee would be defeated by the very + resource the issuer anchor exists to distrust.""" + manager = MCPServerManager() + row = LiteLLM_MCPServerTable( + server_id="issuer-anchored-2", + alias="issuer_anchored_failclosed", + description="issuer configured, upstream fails validation", + url="https://up.example.com/mcp", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, + issuer="https://idp.example.com", + created_at=datetime.now(), + updated_at=datetime.now(), + ) + + resource_rooted = AsyncMock(return_value=MCPOAuthMetadata(token_url="https://attacker.example.com/steal")) + with ( + patch.object(manager, "_fetch_issuer_anchored_oauth_metadata", new=AsyncMock(return_value=None)), + patch.object(manager, "_descovery_metadata", new=resource_rooted), + ): + built = await manager.build_mcp_server_from_table(row, credentials_are_encrypted=False) + + resource_rooted.assert_not_awaited() + assert built.issuer == "https://idp.example.com" + assert built.token_url is None + assert built.registration_url is None + assert built.scopes is None + + @pytest.mark.asyncio + async def test_build_from_table_issuer_anchor_overrides_stored_endpoints_even_when_populated(self): + """When an issuer is pinned, the endpoints come SOLELY from the §3.3-validated issuer document + and win over any stored/manual endpoint values, even a fully-populated row. Otherwise an + attacker who controls a stored token endpoint keeps receiving codes/secrets after an admin + pins a trusted issuer: `needs_discovery` must not short-circuit on populated fields, and the + issuer's endpoints must override the stored ones.""" + manager = MCPServerManager() + row = LiteLLM_MCPServerTable( + server_id="issuer-anchored-populated", + alias="issuer_anchored_populated", + description="issuer set, but stale/hostile endpoints already stored", + url="https://up.example.com/mcp", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, + issuer="https://idp.example.com", + authorization_url="https://attacker.example.com/authorize", + token_url="https://attacker.example.com/steal", + credentials={"scopes": ["stale"]}, + created_at=datetime.now(), + updated_at=datetime.now(), + ) + + issuer_resolved = MCPOAuthMetadata( + authorization_url="https://idp.example.com/authorize", + token_url="https://idp.example.com/token", + scopes=["read"], + ) + with ( + patch.object( + manager, "_fetch_issuer_anchored_oauth_metadata", new=AsyncMock(return_value=issuer_resolved) + ) as anchored, + patch.object(manager, "_persist_discovered_oauth_endpoints", new=AsyncMock()) as mock_persist, + ): + built = await manager.build_mcp_server_from_table(row, credentials_are_encrypted=False) + + anchored.assert_awaited_once_with("https://idp.example.com", "https://up.example.com/mcp") + assert built.authorization_url == "https://idp.example.com/authorize" + assert built.token_url == "https://idp.example.com/token" + assert built.token_url != "https://attacker.example.com/steal" + # The issuer-anchored endpoints are never persisted into the endpoint columns, so a later + # build cannot treat them as authoritative stored values. + assert mock_persist.await_args.kwargs["is_issuer_anchored"] is True + @pytest.mark.asyncio @pytest.mark.parametrize( "advertised_authorization_url", @@ -2462,6 +2676,78 @@ class TestMCPServerManager: assert result.token_url == "https://login.microsoftonline.com/test-tenant-id/oauth2/v2.0/token" assert result.scopes == ["api://some-scope/.default"] + @staticmethod + def _issuer_doc_response_builder(well_known_url: str, document: dict): + def build_response(url: str, **kwargs): + mock_response = MagicMock() + if url == well_known_url: + mock_response.json.return_value = document + mock_response.raise_for_status = MagicMock() + else: + request = httpx.Request("GET", url) + response_obj = httpx.Response(status_code=404, request=request) + mock_response.raise_for_status = MagicMock( + side_effect=httpx.HTTPStatusError("not found", request=request, response=response_obj) + ) + return mock_response + + return build_response + + @pytest.mark.asyncio + async def test_fetch_single_authorization_server_metadata_adopts_document_with_matching_issuer(self): + """RFC 8414 §3.3: under require_issuer, a document that self-attests the same issuer it was + fetched from is authoritative and its endpoints and scopes are adopted.""" + manager = MCPServerManager() + issuer = "https://idp.example.com" + build_response = self._issuer_doc_response_builder( + f"{issuer}/.well-known/oauth-authorization-server", + { + "issuer": issuer, + "authorization_endpoint": "https://idp.example.com/authorize", + "token_endpoint": "https://idp.example.com/token", + "scopes_supported": ["read", "write"], + }, + ) + mock_client = MagicMock() + mock_client.get = AsyncMock(side_effect=build_response) + with patch( + "litellm.proxy._experimental.mcp_server.mcp_server_manager.get_async_httpx_client", + return_value=mock_client, + ): + result = await manager._fetch_single_authorization_server_metadata(issuer, issuer, require_issuer=issuer) + + assert result is not None + assert result.authorization_url == "https://idp.example.com/authorize" + assert result.token_url == "https://idp.example.com/token" + assert result.scopes == ["read", "write"] + + @pytest.mark.asyncio + async def test_fetch_single_authorization_server_metadata_rejects_issuer_mismatch(self): + """RFC 8414 §3.3 fail-closed: a document self-attesting a DIFFERENT issuer than the one it was + fetched from is rejected even though it carries valid-looking endpoints, so a compromised + resource cannot point the issuer-anchored fetch at an attacker authorization server that + smuggles its own token_endpoint and inflated scopes.""" + manager = MCPServerManager() + issuer = "https://idp.example.com" + build_response = self._issuer_doc_response_builder( + f"{issuer}/.well-known/oauth-authorization-server", + { + "issuer": "https://attacker.example.com", + "authorization_endpoint": "https://idp.example.com/authorize", + "token_endpoint": "https://attacker.example.com/steal", + "scopes_supported": ["admin"], + }, + ) + mock_client = MagicMock() + mock_client.get = AsyncMock(side_effect=build_response) + with patch( + "litellm.proxy._experimental.mcp_server.mcp_server_manager.get_async_httpx_client", + return_value=mock_client, + ): + result = await manager._fetch_single_authorization_server_metadata(issuer, issuer, require_issuer=issuer) + + assert result is None + @pytest.mark.asyncio async def test_fetch_single_authorization_server_metadata_derives_azure_metadata( self, @@ -2489,6 +2775,37 @@ class TestMCPServerManager: assert result.authorization_url == "https://login.microsoftonline.com/test-tenant-id/oauth2/v2.0/authorize" assert result.token_url == "https://login.microsoftonline.com/test-tenant-id/oauth2/v2.0/token" + @pytest.mark.asyncio + async def test_azure_heuristic_reachable_under_require_issuer(self): + """Under issuer-anchored discovery (require_issuer set), an Entra issuer whose OIDC document + cannot be fetched still gets the deterministic Azure endpoint construction. The heuristic + derives the endpoints from the pinned issuer's own tenant URL, so it is authoritative-by- + construction and safe under require_issuer; only a non-Entra issuer stays fail-closed (None).""" + manager = MCPServerManager() + issuer = "https://login.microsoftonline.com/test-tenant-id/v2.0" + + request = httpx.Request("GET", issuer) + response_obj = httpx.Response(status_code=404, request=request) + mock_response = MagicMock() + mock_response.raise_for_status = MagicMock( + side_effect=httpx.HTTPStatusError("not found", request=request, response=response_obj) + ) + mock_client = MagicMock() + mock_client.get = AsyncMock(return_value=mock_response) + + with patch( + "litellm.proxy._experimental.mcp_server.mcp_server_manager.get_async_httpx_client", + return_value=mock_client, + ): + azure = await manager._fetch_single_authorization_server_metadata(issuer, issuer, require_issuer=issuer) + non_entra = await manager._fetch_single_authorization_server_metadata( + "https://idp.example.com", "https://idp.example.com", require_issuer="https://idp.example.com" + ) + + assert azure is not None + assert azure.token_url == "https://login.microsoftonline.com/test-tenant-id/oauth2/v2.0/token" + assert non_entra is None + @pytest.mark.asyncio async def test_descovery_metadata_falls_back_to_origin_when_no_auth_servers(self): manager = MCPServerManager() @@ -5194,6 +5511,7 @@ class TestMCPServerTimestamps: await manager._persist_discovered_oauth_endpoints( server_id="s", auth_type=MCPAuth.api_key, + existing_issuer=None, existing_authorization_url=None, existing_token_url=None, existing_scopes=None, @@ -5202,6 +5520,7 @@ class TestMCPServerTimestamps: await manager._persist_discovered_oauth_endpoints( server_id="s", auth_type=MCPAuth.oauth2, + existing_issuer=None, existing_authorization_url=None, existing_token_url=None, existing_scopes=None, @@ -5210,6 +5529,7 @@ class TestMCPServerTimestamps: await manager._persist_discovered_oauth_endpoints( server_id="s", auth_type=MCPAuth.oauth2, + existing_issuer=None, existing_authorization_url=None, existing_token_url=None, existing_scopes=None, @@ -5218,6 +5538,7 @@ class TestMCPServerTimestamps: await manager._persist_discovered_oauth_endpoints( server_id="s", auth_type=MCPAuth.oauth2, + existing_issuer=None, existing_authorization_url="https://configured.example.com/authorize", existing_token_url="https://configured.example.com/token", existing_scopes=["configured"], @@ -5243,6 +5564,7 @@ class TestMCPServerTimestamps: await manager._persist_discovered_oauth_endpoints( server_id="s", auth_type=MCPAuth.oauth2, + existing_issuer=None, existing_authorization_url=None, existing_token_url="https://configured.example.com/token", existing_scopes=None, @@ -5259,6 +5581,81 @@ class TestMCPServerTimestamps: assert persisted.credentials == {"scopes": ["s1"]} assert "token_url" not in persisted.fields_set() + @pytest.mark.asyncio + async def test_persist_discovered_oauth_endpoints_writes_discovered_issuer_trust_on_first_use(self): + """A server with no configured issuer records the discovered issuer trust-on-first-use, so the + next rebuild anchors discovery on it (RFC 8414 §3.3) instead of re-trusting the resource. When + an issuer is already set (admin-typed or a prior discovery), it is never overwritten.""" + manager = MCPServerManager() + metadata = MCPOAuthMetadata( + authorization_url="https://idp.example.com/authorize", + token_url="https://idp.example.com/token", + discovered_issuer="https://idp.example.com", + ) + + update_mcp_server_mock = AsyncMock() + with ( + patch("litellm.proxy._experimental.mcp_server.db.update_mcp_server", new=update_mcp_server_mock), + patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), + ): + await manager._persist_discovered_oauth_endpoints( + server_id="s", + auth_type=MCPAuth.oauth2, + existing_issuer=None, + existing_authorization_url=None, + existing_token_url=None, + existing_scopes=None, + metadata=metadata, + ) + await manager._persist_discovered_oauth_endpoints( + server_id="s", + auth_type=MCPAuth.oauth2, + existing_issuer="https://admin-configured.example.com", + existing_authorization_url="https://admin-configured.example.com/authorize", + existing_token_url="https://admin-configured.example.com/token", + existing_scopes=["cfg"], + metadata=metadata, + ) + + assert update_mcp_server_mock.await_count == 1 + persisted = update_mcp_server_mock.call_args.kwargs["data"] + assert persisted.issuer == "https://idp.example.com" + + @pytest.mark.asyncio + async def test_persist_discovered_oauth_endpoints_does_not_persist_endpoints_for_issuer_anchored(self): + """For an issuer-anchored server the endpoints are re-derived from the §3.3-validated issuer + document every build, so they must NOT be written into the endpoint columns: persisting them + would make the next build see populated endpoints and treat them as authoritative stored + values, defeating the issuer-only invariant. Only the resource-driven scopes are persisted.""" + manager = MCPServerManager() + metadata = MCPOAuthMetadata( + authorization_url="https://idp.example.com/authorize", + token_url="https://idp.example.com/token", + scopes=["read"], + ) + + update_mcp_server_mock = AsyncMock() + with ( + patch("litellm.proxy._experimental.mcp_server.db.update_mcp_server", new=update_mcp_server_mock), + patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), + ): + await manager._persist_discovered_oauth_endpoints( + server_id="s", + auth_type=MCPAuth.oauth2, + existing_issuer="https://idp.example.com", + existing_authorization_url=None, + existing_token_url=None, + existing_scopes=None, + metadata=metadata, + is_issuer_anchored=True, + ) + + update_mcp_server_mock.assert_awaited_once() + persisted = update_mcp_server_mock.call_args.kwargs["data"] + assert "authorization_url" not in persisted.fields_set() + assert "token_url" not in persisted.fields_set() + assert persisted.credentials == {"scopes": ["read"]} + @pytest.mark.asyncio async def test_build_mcp_server_from_table_skips_persistence_for_temporary_servers(self): """The session endpoint builds temporary servers whose server_id has no DB row; with @@ -5473,6 +5870,88 @@ class TestMCPServerTimestamps: assert same_authorize.token_url == "https://idp.example.com/token" assert same_authorize.registration_url == "https://idp.example.com/register" + def test_carry_forward_does_not_restore_endpoints_for_issuer_anchored_server(self): + """When the server is issuer-anchored the endpoints come solely from the §3.3-validated issuer + document, so a failed issuer fetch (token_url None) must stay fail-closed. Carry-forward must + NOT resurrect the previous registry entry's token endpoint, or the very attacker-controlled + endpoint the issuer anchor distrusts would keep being served across rebuilds. Resource-driven + scopes still carry as last-known-good. Anchoring is keyed on the explicit issuer_is_anchored + flag, not on issuer truthiness, so a discovered issuer does not trip this fail-closed branch.""" + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + _carry_forward_resolved_oauth_endpoints, + ) + + previous = MCPServer( + server_id="s1", + name="s1", + url="https://up.example.com/mcp", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, + issuer="https://idp.example.com", + issuer_is_anchored=True, + authorization_url="https://idp.example.com/authorize", + token_url="https://idp.example.com/token", + registration_url="https://idp.example.com/register", + scopes=["read"], + ) + failed_rebuild = MCPServer( + server_id="s1", + name="s1", + url="https://up.example.com/mcp", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, + issuer="https://idp.example.com", + issuer_is_anchored=True, + ) + + _carry_forward_resolved_oauth_endpoints(new_server=failed_rebuild, previous_server=previous) + + assert failed_rebuild.authorization_url is None + assert failed_rebuild.token_url is None + assert failed_rebuild.registration_url is None + assert failed_rebuild.scopes == ["read"] + + def test_carry_forward_restores_endpoints_for_discovered_issuer_not_anchored(self): + """A server that merely DISCOVERED its issuer trust-on-first-use is not anchored: issuer is set + for token identity but the endpoints are resource-rooted, so on a transient discovery blip they + must still carry forward as last-known-good, the same as any resource-rooted server. This is the + regression the explicit issuer_is_anchored flag prevents: keying fail-closed on issuer truthiness + alone would drop the working endpoints the moment the server learned its issuer.""" + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + _carry_forward_resolved_oauth_endpoints, + ) + + previous = MCPServer( + server_id="s1", + name="s1", + url="https://up.example.com/mcp", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, + issuer="https://idp.example.com", + issuer_is_anchored=False, + authorization_url="https://idp.example.com/authorize", + token_url="https://idp.example.com/token", + registration_url="https://idp.example.com/register", + scopes=["read"], + ) + blipped_rebuild = MCPServer( + server_id="s1", + name="s1", + url="https://up.example.com/mcp", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, + issuer="https://idp.example.com", + issuer_is_anchored=False, + authorization_url=None, + ) + + _carry_forward_resolved_oauth_endpoints(new_server=blipped_rebuild, previous_server=previous) + + assert blipped_rebuild.authorization_url == "https://idp.example.com/authorize" + assert blipped_rebuild.token_url == "https://idp.example.com/token" + assert blipped_rebuild.registration_url == "https://idp.example.com/register" + assert blipped_rebuild.scopes == ["read"] + def test_normalized_authorize_endpoint_treats_default_port_and_slash_as_identity(self): """The corroboration check must not fail on formatting-only differences an IdP legitimately emits: default port, trailing slash, host case, and query string are not identity, but a @@ -5487,6 +5966,22 @@ class TestMCPServerTimestamps: assert _normalized_authorize_endpoint("https://idp.example.com/authorize?prompt=consent") == canonical assert _normalized_authorize_endpoint("https://idp.example.com:8443/authorize") != canonical + def test_issuer_matches_rfc8414_section_3_3(self): + """Issuer equality tolerates only URL-insignificant differences (scheme/host case, default + port, a trailing slash). A different host, a non-string, an empty string, or a None issuer + never matches, so a document that omits issuer fails closed under issuer-anchored discovery.""" + from litellm.proxy._experimental.mcp_server.mcp_server_manager import _issuer_matches + + assert _issuer_matches("https://mcp.slack.com", "https://mcp.slack.com") + assert _issuer_matches("https://MCP.slack.com/", "https://mcp.slack.com") + assert _issuer_matches("https://mcp.slack.com:443", "https://mcp.slack.com") + assert _issuer_matches("https://login.example.com/tenant/v2.0", "https://login.example.com/tenant/v2.0") + assert not _issuer_matches("https://attacker.example.com", "https://mcp.slack.com") + assert not _issuer_matches("https://login.example.com/other/v2.0", "https://login.example.com/tenant/v2.0") + assert not _issuer_matches(None, "https://mcp.slack.com") + assert not _issuer_matches("", "https://mcp.slack.com") + assert not _issuer_matches(123, "https://mcp.slack.com") + def test_build_mcp_server_table_preserves_timestamps(self): """_build_mcp_server_table must use the MCPServer's stored timestamps, not datetime.now().""" manager = MCPServerManager() diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_semantic_tool_filter.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_semantic_tool_filter.py index 9bc0a525326..0f392a54b4c 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_semantic_tool_filter.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_semantic_tool_filter.py @@ -1664,3 +1664,252 @@ def test_is_context_window_error_detection_variants(): assert _is_context_window_error(ValueError("Invalid 'input[0]': maximum input length is 8192 tokens.")) assert not _is_context_window_error(ValueError("A generic API error occurred.")) assert not _is_context_window_error(None) + + +def _make_keyword_embedding_router(recorded_inputs): + """ + Mock litellm Router whose embeddings are deterministic keyword one-hots: + texts mentioning linear/issue/ticket embed to [1, 0], everything else to + [0, 1]. Lets tests assert real similarity ranking through the actual + semantic-router index. Every embedding input batch is appended to + recorded_inputs. + """ + from litellm.types.utils import Embedding, EmbeddingResponse + + def _vector(text): + lowered = text.lower() + if "kanban" in lowered: + return [0.6, 0.8] + if "linear" in lowered or "issue" in lowered or "ticket" in lowered: + return [1.0, 0.0] + return [0.0, 1.0] + + def mock_embedding_sync(*args, **kwargs): + texts = kwargs["input"] + recorded_inputs.append(list(texts)) + return EmbeddingResponse( + data=[Embedding(embedding=_vector(t), index=i, object="embedding") for i, t in enumerate(texts)], + model="text-embedding-3-small", + object="list", + usage={"prompt_tokens": 10, "total_tokens": 10}, + ) + + async def mock_embedding_async(*args, **kwargs): + return mock_embedding_sync(*args, **kwargs) + + mock_router = Mock() + mock_router.embedding = mock_embedding_sync + mock_router.aembedding = mock_embedding_async + return mock_router + + +def _make_keyword_filter(recorded_inputs, top_k: int = 3): + from litellm.proxy._experimental.mcp_server.semantic_tool_filter import ( + SemanticMCPToolFilter, + ) + + return SemanticMCPToolFilter( + embedding_model="text-embedding-3-small", + litellm_router_instance=_make_keyword_embedding_router(recorded_inputs), + top_k=top_k, + similarity_threshold=0.3, + enabled=True, + ) + + +def _linear_issue_tool(): + return MCPTool( + name="linear_stub-get_issue", + description="Get a Linear issue (ticket) by its identifier such as LIT-1234", + inputSchema={"type": "object"}, + ) + + +def _linear_list_tool(): + return MCPTool( + name="linear_stub-list_issues", + description="List Linear issues (tickets) in the workspace", + inputSchema={"type": "object"}, + ) + + +def _weather_tool(): + return MCPTool( + name="weather_stub-get_weather", + description="Get the current weather conditions for a city", + inputSchema={"type": "object"}, + ) + + +@pytest.mark.asyncio +async def test_filter_indexes_request_tools_when_startup_index_is_empty(): + """ + Regression test: the startup index is built by listing every MCP server + WITHOUT per-user credentials, so a gateway whose servers all require + per-user auth (e.g. interactive OAuth) starts with an empty index + (tool_router is None). filter_tools then failed open and returned all N + tools unfiltered (customer-visible as an N->N header and, past 128 tools, + an OpenAI 400 "tools array too long"). The authed request-time tools must + instead be indexed on first sight so filtering actually runs. + """ + filter_instance = _make_keyword_filter([]) + assert filter_instance.tool_router is None + + tools = [_linear_issue_tool(), _weather_tool()] + filtered = await filter_instance.filter_tools( + query="what is Linear ticket LIT-3794 about", + available_tools=tools, + ) + + assert [t.name for t in filtered] == ["linear_stub-get_issue"] + print("✅ Empty startup index is built from authed request-time tools") + + +@pytest.mark.asyncio +async def test_filter_indexes_tools_missing_from_partial_index(): + """ + Regression test: servers whose tools/list needs per-user auth contribute + zero routes to the startup index while anonymously listable servers are + indexed. Tools reaching the filter through the authed request-time + expansion must be added to the existing router (and only embedded once; + repeat requests embed just the query). + """ + recorded_inputs = [] + filter_instance = _make_keyword_filter(recorded_inputs) + filter_instance._build_router([_weather_tool()]) + assert filter_instance.tool_router is not None + + tools = [_linear_issue_tool(), _weather_tool()] + query = "what is Linear ticket LIT-3794 about" + + filtered = await filter_instance.filter_tools(query=query, available_tools=tools) + assert [t.name for t in filtered] == ["linear_stub-get_issue"] + + calls_after_first = len(recorded_inputs) + filtered_again = await filter_instance.filter_tools(query=query, available_tools=tools) + assert [t.name for t in filtered_again] == ["linear_stub-get_issue"] + assert len(recorded_inputs) == calls_after_first + 1 + + print("✅ Partial startup index is completed from request-time tools, embedding each tool once") + + +@pytest.mark.asyncio +async def test_filter_fails_open_when_matches_are_not_in_available_tools(): + """ + Regression test: when the semantic router's matches are all tools that are + NOT in the request's available_tools (an index/request mismatch), the + filter returned an empty list, stripping every tool from the request and + breaking it outright (observed live as a 3->0 header followed by a + provider 400). It must fail open with the full tool list instead, matching + the zero-match fallback. + """ + filter_instance = _make_keyword_filter([]) + filter_instance._build_router([_weather_tool()]) + + tools = [_linear_issue_tool(), _linear_list_tool()] + filtered = await filter_instance.filter_tools( + query="current weather in San Francisco", + available_tools=tools, + ) + + assert [t.name for t in filtered] == ["linear_stub-get_issue", "linear_stub-list_issues"] + print("✅ Matches outside available_tools fail open instead of dropping every tool") + + +@pytest.mark.asyncio +async def test_request_time_context_window_error_is_request_scoped(): + """ + Regression test: an oversized tool description hitting the embedding + context window while lazily indexing request-time tools must fail only + the requesting call. Previously the lazy path reused the startup build + and recorded the overflow in the shared context_window_error, after + which EVERY user's MCP requests on the worker were blocked with a 400 + until restart (index poisoning via a single request). + """ + from litellm.proxy._experimental.mcp_server.semantic_tool_filter import ( + SemanticToolFilterContextWindowError, + ) + + state = {"raise_context_error": True} + filter_instance = _make_context_window_filter(state) + tools = [ + MCPTool(name="tool_a", description="Tool A", inputSchema={"type": "object"}), + MCPTool(name="tool_b", description="Tool B", inputSchema={"type": "object"}), + ] + + with pytest.raises(SemanticToolFilterContextWindowError): + await filter_instance.filter_tools(query="send an email", available_tools=tools) + + assert filter_instance.context_window_error is None + assert filter_instance.tool_router is None + + state["raise_context_error"] = False + filtered = await filter_instance.filter_tools(query="send an email", available_tools=tools) + + assert len(filtered) > 0 + assert filter_instance.context_window_error is None + assert filter_instance.tool_router is not None + print("✅ Request-time context window overflow is scoped to the request, not the worker") + + +@pytest.mark.asyncio +async def test_foreign_index_routes_cannot_displace_available_tools(): + """ + Regression test: routes indexed from OTHER principals' tool listings must + not occupy the match candidate set for this request. Previously the + router matched over the whole shared index, so foreign routes that + embedded closer to the query displaced the caller's own tools from + top_k, degrading results to the fail-open list (or, before the + empty-result guard, stripping every tool). Matching is now scoped to the + request's own tool names via route_filter. + """ + filter_instance = _make_keyword_filter([], top_k=1) + foreign_tools = [ + MCPTool( + name=f"other_user-linear_tool_{i}", + description=f"Get a Linear issue variant {i}", + inputSchema={"type": "object"}, + ) + for i in range(6) + ] + filter_instance._build_router(foreign_tools) + + my_kanban = MCPTool( + name="mine-kanban_board", + description="Manage kanban board cards", + inputSchema={"type": "object"}, + ) + filtered = await filter_instance.filter_tools( + query="what is Linear ticket LIT-3794 about", + available_tools=[my_kanban, _weather_tool()], + ) + + assert [t.name for t in filtered] == ["mine-kanban_board"] + print("✅ Foreign index routes cannot displace the caller's own tools") + + +@pytest.mark.asyncio +async def test_top_k_above_router_default_is_respected(): + """ + Regression test: semantic-router's SemanticRouter defaults to top_k=5 at + the index-query layer, silently capping any configured filter top_k + above 5 regardless of the limit passed to __call__. The router must be + sized (and resized) to honor the configured top_k. + """ + filter_instance = _make_keyword_filter([], top_k=6) + tools = [ + MCPTool( + name=f"linear_stub-tool_{i}", + description=f"Work with Linear issues part {i}", + inputSchema={"type": "object"}, + ) + for i in range(6) + ] + + filtered = await filter_instance.filter_tools( + query="Linear ticket work", + available_tools=tools, + ) + + assert len(filtered) == 6 + print("✅ Configured top_k above the semantic-router default of 5 is honored") diff --git a/tests/test_litellm/proxy/auth/test_auth_checks.py b/tests/test_litellm/proxy/auth/test_auth_checks.py index 27f43c4948f..8365909f314 100644 --- a/tests/test_litellm/proxy/auth/test_auth_checks.py +++ b/tests/test_litellm/proxy/auth/test_auth_checks.py @@ -2634,6 +2634,8 @@ async def test_virtual_key_budget_check_reads_from_spend_counter(): ) assert exc_info.value.current_cost == 1.5 assert exc_info.value.max_budget == 1.0 + assert exc_info.value.entity_type == "key" + assert exc_info.value.entity_id == "test-hashed-token" @pytest.mark.asyncio @@ -2861,6 +2863,8 @@ async def test_team_budget_check_reads_from_spend_counter(): proxy_logging_obj=proxy_logging_obj, ) assert exc_info.value.current_cost == 1.5 + assert exc_info.value.entity_type == "team" + assert exc_info.value.entity_id == "test-team" @pytest.mark.asyncio @@ -2888,6 +2892,8 @@ async def test_end_user_budget_check_reads_from_spend_counter(): ) assert exc_info.value.current_cost == 1.5 assert exc_info.value.max_budget == 1.0 + assert exc_info.value.entity_type == "end_user" + assert exc_info.value.entity_id == "customer-1" @pytest.mark.asyncio @@ -2926,6 +2932,8 @@ async def test_tag_budget_check_reads_from_spend_counter(): ) assert exc_info.value.current_cost == 1.5 assert exc_info.value.max_budget == 1.0 + assert exc_info.value.entity_type == "tag" + assert exc_info.value.entity_id == "paid-tag" @pytest.mark.asyncio @@ -2976,6 +2984,8 @@ async def test_team_member_budget_check_reads_from_spend_counter(): proxy_logging_obj=proxy_logging_obj, ) assert exc_info.value.current_cost == 1.5 + assert exc_info.value.entity_type == "team_member" + assert exc_info.value.entity_id == "test-user:test-team" class TestGuardrailModificationCheck: diff --git a/tests/test_litellm/proxy/client/cli/test_auth_commands.py b/tests/test_litellm/proxy/client/cli/test_auth_commands.py index e101515d4b1..2fbc9c5c82f 100644 --- a/tests/test_litellm/proxy/client/cli/test_auth_commands.py +++ b/tests/test_litellm/proxy/client/cli/test_auth_commands.py @@ -78,7 +78,7 @@ class TestPollingErrorSurfacing: result = CliRunner().invoke(login, obj=mock_context.obj) assert result.exit_code == 0 - assert "❌ Authentication failed:" in result.output + assert "Authentication failed:" in result.output assert "CLI login session not found or expired." in result.output assert "Authentication timed out" not in result.output @@ -414,7 +414,7 @@ class TestLoginCommand: result = self.runner.invoke(login, obj=mock_context.obj) assert result.exit_code == 0 - assert "✅ Login successful!" in result.output + assert "Login successful!" in result.output assert "Automatically assigned to team: team-1" in result.output # Verify browser was opened with correct URL @@ -456,7 +456,7 @@ class TestLoginCommand: result = self.runner.invoke(login, obj=mock_context.obj) assert result.exit_code == 0 - assert "❌ Authentication timed out" in result.output + assert "Authentication timed out" in result.output def test_login_http_error(self): """Test login with HTTP error""" @@ -476,7 +476,7 @@ class TestLoginCommand: result = self.runner.invoke(login, obj=mock_context.obj) assert result.exit_code == 0 - assert "❌ Authentication timed out" in result.output + assert "Authentication timed out" in result.output def test_login_request_exception(self): """Test login with request exception""" @@ -497,7 +497,7 @@ class TestLoginCommand: result = self.runner.invoke(login, obj=mock_context.obj) assert result.exit_code == 0 - assert "❌ Authentication timed out" in result.output + assert "Authentication timed out" in result.output def test_login_keyboard_interrupt(self): """Test login cancelled by user""" @@ -512,7 +512,7 @@ class TestLoginCommand: result = self.runner.invoke(login, obj=mock_context.obj) assert result.exit_code == 0 - assert "❌ Authentication cancelled by user" in result.output + assert "Authentication cancelled by user" in result.output def test_login_no_api_key_in_response(self): """Test login when response doesn't contain API key""" @@ -536,7 +536,7 @@ class TestLoginCommand: result = self.runner.invoke(login, obj=mock_context.obj) assert result.exit_code == 0 - assert "❌ Authentication timed out" in result.output + assert "Authentication timed out" in result.output def test_login_general_exception(self): """Test login with general exception (not requests exception)""" @@ -551,7 +551,7 @@ class TestLoginCommand: result = self.runner.invoke(login, obj=mock_context.obj) assert result.exit_code == 0 - assert "❌ Authentication failed: Invalid value" in result.output + assert "Authentication failed: Invalid value" in result.output class TestLogoutCommand: @@ -567,7 +567,7 @@ class TestLogoutCommand: result = self.runner.invoke(logout) assert result.exit_code == 0 - assert "✅ Logged out successfully" in result.output + assert "Logged out successfully" in result.output mock_clear.assert_called_once() @@ -591,7 +591,7 @@ class TestWhoamiCommand: result = self.runner.invoke(whoami) assert result.exit_code == 0 - assert "✅ Authenticated" in result.output + assert "Authenticated" in result.output assert "test@example.com" in result.output assert "test-user-123" in result.output assert "admin" in result.output @@ -603,7 +603,7 @@ class TestWhoamiCommand: result = self.runner.invoke(whoami) assert result.exit_code == 0 - assert "❌ Not authenticated" in result.output + assert "Not authenticated" in result.output assert "Run 'lite login'" in result.output def test_whoami_old_token(self): @@ -619,8 +619,8 @@ class TestWhoamiCommand: result = self.runner.invoke(whoami) assert result.exit_code == 0 - assert "✅ Authenticated" in result.output - assert "⚠️ Warning: Token is more than 24 hours old" in result.output + assert "Authenticated" in result.output + assert "Warning: Token is more than 24 hours old" in result.output def test_whoami_missing_fields(self): """Test whoami with token missing some fields""" @@ -633,7 +633,7 @@ class TestWhoamiCommand: result = self.runner.invoke(whoami) assert result.exit_code == 0 - assert "✅ Authenticated" in result.output + assert "Authenticated" in result.output assert "Unknown" in result.output # Should show "Unknown" for missing fields def test_whoami_no_timestamp(self): @@ -655,7 +655,7 @@ class TestWhoamiCommand: result = self.runner.invoke(whoami) assert result.exit_code == 0 - assert "✅ Authenticated" in result.output + assert "Authenticated" in result.output # Should calculate age based on timestamp=0 assert "Token age:" in result.output @@ -714,7 +714,7 @@ class TestCLIKeyRegenerationFlow: result = self.runner.invoke(login, obj=mock_context.obj) assert result.exit_code == 0 - assert "✅ Login successful!" in result.output + assert "Login successful!" in result.output assert "team-beta" in result.output # Ensure we surface the human-readable team alias to the user assert "Beta Team" in result.output @@ -774,7 +774,7 @@ class TestCLIKeyRegenerationFlow: result = self.runner.invoke(login, obj=mock_context.obj) assert result.exit_code == 0 - assert "✅ Login successful!" in result.output + assert "Login successful!" in result.output # Verify browser was opened mock_browser.assert_called_once() diff --git a/tests/test_litellm/proxy/client/cli/test_global_options.py b/tests/test_litellm/proxy/client/cli/test_global_options.py index 53b7e4dbc29..8df763d35c2 100644 --- a/tests/test_litellm/proxy/client/cli/test_global_options.py +++ b/tests/test_litellm/proxy/client/cli/test_global_options.py @@ -1,6 +1,7 @@ # stdlib imports import os import sys +from pathlib import Path from unittest.mock import Mock, patch import pytest @@ -11,6 +12,7 @@ sys.path.insert( ) # Adds the parent directory to the system path +import litellm.proxy.client.cli from litellm._version import version as litellm_version from litellm.proxy.client.cli import cli @@ -36,6 +38,19 @@ def test_cli_version_flag(cli_runner): assert "LiteLLM Proxy Server Version: 1.2.3" in result.output +def test_cli_source_is_ascii_only(): + """Non-ASCII output (emoji, box-drawing chars) raises UnicodeEncodeError on legacy Windows + consoles (cp1252), so the whole CLI package must stay ASCII-only.""" + cli_root = Path(litellm.proxy.client.cli.__file__).parent + offenders = [ + f"{path.relative_to(cli_root)}:{line_number}: {line.strip()}" + for path in sorted(cli_root.rglob("*.py")) + for line_number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1) + if not line.isascii() + ] + assert offenders == [] + + def test_base_url_trailing_slash_normalized(cli_runner): """A trailing slash on --base-url must not produce a double slash (e.g. '//sso/cli/start').""" with ( diff --git a/tests/test_litellm/proxy/client/cli/test_keys_commands.py b/tests/test_litellm/proxy/client/cli/test_keys_commands.py index 2c134f9defb..977aec9f5b7 100644 --- a/tests/test_litellm/proxy/client/cli/test_keys_commands.py +++ b/tests/test_litellm/proxy/client/cli/test_keys_commands.py @@ -262,7 +262,7 @@ def test_keys_import_actual_import_success(mock_keys_client, cli_runner): assert result.exit_code == 0 assert "Found 1 keys in source instance" in result.output - assert "✓ Imported key: import-key-1" in result.output + assert "Imported key: import-key-1" in result.output assert "Successfully imported: 1" in result.output assert "Failed to import: 0" in result.output @@ -481,8 +481,8 @@ def test_keys_import_partial_failure(mock_keys_client, cli_runner): ) assert result.exit_code == 0 # Command completes even with partial failures - assert "✓ Imported key: success-key" in result.output - assert "✗ Failed to import key fail-key" in result.output + assert "Imported key: success-key" in result.output + assert "Failed to import key fail-key" in result.output assert "Successfully imported: 1" in result.output assert "Failed to import: 1" in result.output assert "Total keys processed: 2" in result.output diff --git a/tests/test_litellm/proxy/db/test_db_spend_update_writer.py b/tests/test_litellm/proxy/db/test_db_spend_update_writer.py index c29cdaf4171..4c17c5d3482 100644 --- a/tests/test_litellm/proxy/db/test_db_spend_update_writer.py +++ b/tests/test_litellm/proxy/db/test_db_spend_update_writer.py @@ -13,7 +13,10 @@ from datetime import datetime, timezone from unittest.mock import AsyncMock, MagicMock, call, patch import pytest +from redis.exceptions import DataError +import litellm +from litellm.proxy._types import Litellm_EntityType from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter @@ -1418,7 +1421,6 @@ async def test_batch_database_updates_isolation_on_failure(): org_id="org1", end_user_id="eu1", prisma_client=MagicMock(), - user_api_key_cache=MagicMock(), litellm_proxy_budget_name="budget", payload={"key": "value"}, ) @@ -1818,3 +1820,85 @@ async def test_update_database_does_not_deepcopy_on_request_path(): fake_payload["nested"]["a"] = 999 assert batch_payload["model"] == "gpt-4" assert batch_payload["nested"]["a"] == 1 + + +@pytest.mark.asyncio +async def test_spend_update_path_never_queries_user_cache_with_none_user_id(): + """ + When user_id is None, the spend-update path must not perform a user-cache + lookup at all. With a Redis-backed auth cache (enable_redis_auth_cache), + a lookup with key=None raises redis.exceptions.DataError, which aborted + _update_user_db before any spend updates were enqueued. + + This test fails on the old code twice over: the cache mock records the + forbidden lookup, and the DataError it raises kills the end-user spend + update that must survive. + """ + db_writer = DBSpendUpdateWriter() + + strict_redis_backed_cache = MagicMock() + strict_redis_backed_cache.async_get_cache = AsyncMock( + side_effect=DataError("Invalid input of type: 'NoneType'") + ) + + with ( + patch.object(litellm, "max_budget", 0), + patch("litellm.proxy.proxy_server.disable_spend_logs", True), + patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), + patch("litellm.proxy.proxy_server.user_api_key_cache", strict_redis_backed_cache), + patch("litellm.proxy.proxy_server.litellm_proxy_budget_name", "litellm-proxy-budget"), + patch( + "litellm.proxy.spend_tracking.spend_tracking_utils.get_logging_payload", + return_value={ + "startTime": "2024-01-01T00:00:00", + "endTime": "2024-01-01T00:01:00", + "model": "gpt-4", + "custom_llm_provider": "openai", + "spend": 0.0, + }, + ), + ): + await db_writer.update_database( + token=None, + user_id=None, + end_user_id="end-user-1", + team_id=None, + org_id=None, + kwargs={"model": "gpt-4", "custom_llm_provider": "openai"}, + completion_response=MagicMock(), + start_time=datetime.now(), + end_time=datetime.now(), + response_cost=0.1, + ) + await asyncio.sleep(0) + + strict_redis_backed_cache.async_get_cache.assert_not_called() + + queued = await db_writer.spend_update_queue.flush_all_updates_from_in_memory_queue() + end_user_updates = [u for u in queued if u["entity_type"] == Litellm_EntityType.END_USER] + assert len(end_user_updates) == 1 + assert end_user_updates[0]["entity_id"] == "end-user-1" + assert all(u["entity_type"] != Litellm_EntityType.USER for u in queued) + + +@pytest.mark.asyncio +async def test_update_user_db_enqueues_user_spend_without_cache_dependency(): + """ + _update_user_db needs no cache handle: it enqueues the user spend update + (and the end-user one) purely from the ids it is given. + """ + db_writer = DBSpendUpdateWriter() + + with patch.object(litellm, "max_budget", 0): + await db_writer._update_user_db( + response_cost=0.25, + user_id="user-123", + prisma_client=MagicMock(), + litellm_proxy_budget_name="litellm-proxy-budget", + end_user_id="end-user-9", + ) + + queued = await db_writer.spend_update_queue.flush_all_updates_from_in_memory_queue() + by_type = {u["entity_type"]: u["entity_id"] for u in queued} + assert by_type[Litellm_EntityType.USER] == "user-123" + assert by_type[Litellm_EntityType.END_USER] == "end-user-9" diff --git a/tests/test_litellm/proxy/test_budget_reservation.py b/tests/test_litellm/proxy/test_budget_reservation.py index 540f017ee88..0b304f2fec7 100644 --- a/tests/test_litellm/proxy/test_budget_reservation.py +++ b/tests/test_litellm/proxy/test_budget_reservation.py @@ -150,8 +150,41 @@ async def test_reservation_blocks_over_budget_non_throttled_key( await _reserve(valid_token, 0.6, key_cache, proxy_logging_obj) await _reserve(valid_token, 0.6, key_cache, proxy_logging_obj) # counter -> 1.0 - with pytest.raises(litellm.BudgetExceededError): + with pytest.raises(litellm.BudgetExceededError) as exc_info: await _reserve(valid_token, 0.6, key_cache, proxy_logging_obj) + assert exc_info.value.entity_type == "key" + assert exc_info.value.entity_id == "key-no-optin-over" + + +@pytest.mark.asyncio +async def test_over_budget_window_counter_tags_clean_entity_id(): + from litellm.proxy.spend_tracking.budget_reservation import ( + _apply_over_budget_reservation_policy, + _BudgetCounter, + ) + + counter = _BudgetCounter( + counter_key="spend:key:test-token:window:1d", + max_budget=1.0, + fallback_spend=0.0, + entity_type="Key", + entity_id="test-token:1d", + spend_log_entity_id="test-token", + ) + + with pytest.raises(litellm.BudgetExceededError) as exc_info: + await _apply_over_budget_reservation_policy( + counter=counter, + valid_token=None, + entry={"counter_key": counter.counter_key}, + applied_entries=[], + reservation_cost=0.5, + current_spend=2.0, + ) + assert exc_info.value.entity_type == "key" + assert exc_info.value.entity_id == "test-token" + assert exc_info.value.max_budget == 1.0 + assert exc_info.value.current_cost == 2.0 def test_should_not_serialize_budget_reservation_on_user_api_key_auth(): diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index 913ac116866..d2b8b7ec23d 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -2688,6 +2688,70 @@ def test_get_sanitized_user_information_from_key_includes_guardrails_metadata(): assert result["user_api_key_auth_metadata"]["other_field"] == "value" +def test_user_and_team_spend_and_budget_flow_to_standard_logging_metadata(): + """ + Full flow: UserAPIKeyAuth -> get_sanitized_user_information_from_key -> + get_standard_logging_metadata. User-level and team-level spend + max budget + must reach the StandardLoggingPayload metadata that custom loggers receive, + alongside the key-level values + """ + from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + + user_api_key_dict = UserAPIKeyAuth( + api_key="test-key-hash", + spend=1.5, + max_budget=10.0, + user_id="test-user", + user_spend=25.5, + user_max_budget=100.0, + team_id="test-team", + team_spend=250.75, + team_max_budget=1000.0, + ) + + sanitized = LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key( + user_api_key_dict=user_api_key_dict + ) + + assert sanitized["user_api_key_spend"] == 1.5 + assert sanitized["user_api_key_max_budget"] == 10.0 + assert sanitized["user_api_key_user_spend"] == 25.5 + assert sanitized["user_api_key_user_max_budget"] == 100.0 + assert sanitized["user_api_key_team_spend"] == 250.75 + assert sanitized["user_api_key_team_max_budget"] == 1000.0 + + logging_metadata = StandardLoggingPayloadSetup.get_standard_logging_metadata( + dict(sanitized) + ) + + assert logging_metadata["user_api_key_user_spend"] == 25.5 + assert logging_metadata["user_api_key_user_max_budget"] == 100.0 + assert logging_metadata["user_api_key_team_spend"] == 250.75 + assert logging_metadata["user_api_key_team_max_budget"] == 1000.0 + + +def test_user_and_team_spend_and_budget_default_to_none_in_standard_logging_metadata(): + """ + Keys with no user or team level budgets report None for the new fields in the + StandardLoggingPayload metadata instead of raising + """ + from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + + user_api_key_dict = UserAPIKeyAuth(api_key="test-key-hash") + + sanitized = LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key( + user_api_key_dict=user_api_key_dict + ) + logging_metadata = StandardLoggingPayloadSetup.get_standard_logging_metadata( + dict(sanitized) + ) + + assert logging_metadata["user_api_key_user_spend"] is None + assert logging_metadata["user_api_key_user_max_budget"] is None + assert logging_metadata["user_api_key_team_spend"] is None + assert logging_metadata["user_api_key_team_max_budget"] is None + + @pytest.mark.asyncio async def test_team_guardrails_append_to_key_guardrails(): """ diff --git a/tests/test_litellm/responses/test_streaming_iterator.py b/tests/test_litellm/responses/test_streaming_iterator.py index 3e4b07fce18..5b0f40fdf27 100644 --- a/tests/test_litellm/responses/test_streaming_iterator.py +++ b/tests/test_litellm/responses/test_streaming_iterator.py @@ -5,13 +5,18 @@ completion_start_time = end_time.""" import json from datetime import datetime +from typing import Optional from unittest.mock import Mock +import httpx import pytest from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig -from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator +from litellm.responses.streaming_iterator import ( + ResponsesAPIStreamingIterator, + SyncResponsesAPIStreamingIterator, +) from litellm.types.llms.openai import ( ResponseCompletedEvent, ResponsesAPIResponse, @@ -23,19 +28,7 @@ def _sse_event(payload: dict) -> bytes: return f"data: {json.dumps(payload)}\n\n".encode("utf-8") -def _make_iterator( - *, - sse_events: list[bytes], - logging_obj: LiteLLMLoggingObj, -) -> ResponsesAPIStreamingIterator: - async def aiter_bytes(): - for evt in sse_events: - yield evt - - mock_response = Mock() - mock_response.headers = {} - mock_response.aiter_bytes = aiter_bytes - +def _mock_config() -> Mock: mock_config = Mock(spec=BaseResponsesAPIConfig) mock_responses_api_response = Mock(spec=ResponsesAPIResponse) mock_responses_api_response.id = "resp_ttft" @@ -52,17 +45,68 @@ def _make_iterator( return stub mock_config.transform_streaming_response.side_effect = _transform + return mock_config + + +def _make_iterator( + *, + sse_events: list[bytes], + logging_obj: LiteLLMLoggingObj, + trailing_error: Optional[Exception] = None, +) -> ResponsesAPIStreamingIterator: + async def aiter_bytes(): + for evt in sse_events: + yield evt + if trailing_error is not None: + raise trailing_error + + mock_response = Mock() + mock_response.headers = {} + mock_response.aiter_bytes = aiter_bytes return ResponsesAPIStreamingIterator( response=mock_response, model="gpt-4o-mini", - responses_api_provider_config=mock_config, + responses_api_provider_config=_mock_config(), logging_obj=logging_obj, litellm_metadata={}, custom_llm_provider="openai", ) +def _make_sync_iterator( + *, + sse_events: list[bytes], + logging_obj: LiteLLMLoggingObj, + trailing_error: Optional[Exception] = None, +) -> SyncResponsesAPIStreamingIterator: + def iter_bytes(): + for evt in sse_events: + yield evt + if trailing_error is not None: + raise trailing_error + + mock_response = Mock() + mock_response.headers = {} + mock_response.iter_bytes = iter_bytes + + return SyncResponsesAPIStreamingIterator( + response=mock_response, + model="gpt-4o-mini", + responses_api_provider_config=_mock_config(), + logging_obj=logging_obj, + litellm_metadata={}, + custom_llm_provider="openai", + ) + + +def _logging_obj_stub() -> Mock: + logging_obj = Mock(spec=LiteLLMLoggingObj) + logging_obj.completion_start_time = None + logging_obj.model_call_details = {"litellm_params": {}} + return logging_obj + + @pytest.mark.asyncio async def test_responses_streaming_stamps_completion_start_time_on_first_chunk(): """Without the fix, `logging_obj.completion_start_time` stays None across the @@ -122,3 +166,72 @@ async def test_responses_streaming_does_not_reset_prior_completion_start_time(): logging_obj._update_completion_start_time.assert_not_called() assert logging_obj.completion_start_time == prior + + +_COMPLETE_STREAM_EVENTS = [ + _sse_event({"type": "response.created"}), + _sse_event({"type": "response.output_text.delta", "delta": "hi"}), + _sse_event({"type": "response.completed"}), +] + +_TRAILING_ERRORS = [ + httpx.ReadError("Response payload is not completed"), + httpx.RemoteProtocolError("peer closed connection without sending complete message body"), +] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("trailing_error", _TRAILING_ERRORS, ids=type) +async def test_transport_error_after_completed_event_ends_stream_cleanly(trailing_error): + """A sloppy connection close after `response.completed` must not turn a + complete stream into an error (regression guard for the transport no longer + swallowing ClientPayloadError/TransferEncodingError).""" + iterator = _make_iterator( + sse_events=_COMPLETE_STREAM_EVENTS, + logging_obj=_logging_obj_stub(), + trailing_error=trailing_error, + ) + + seen = [event.type async for event in iterator] + + assert ResponsesAPIStreamEvents.RESPONSE_COMPLETED in seen + + +@pytest.mark.asyncio +async def test_transport_error_before_completed_event_raises(): + """A connection lost before any terminal event is a real failure and must + surface, not end the stream as if it completed.""" + iterator = _make_iterator( + sse_events=_COMPLETE_STREAM_EVENTS[:-1], + logging_obj=_logging_obj_stub(), + trailing_error=httpx.ReadError("Response payload is not completed"), + ) + + with pytest.raises(httpx.ReadError): + async for _ in iterator: + pass + + +@pytest.mark.parametrize("trailing_error", _TRAILING_ERRORS, ids=type) +def test_sync_transport_error_after_completed_event_ends_stream_cleanly(trailing_error): + iterator = _make_sync_iterator( + sse_events=_COMPLETE_STREAM_EVENTS, + logging_obj=_logging_obj_stub(), + trailing_error=trailing_error, + ) + + seen = [event.type for event in iterator] + + assert ResponsesAPIStreamEvents.RESPONSE_COMPLETED in seen + + +def test_sync_transport_error_before_completed_event_raises(): + iterator = _make_sync_iterator( + sse_events=_COMPLETE_STREAM_EVENTS[:-1], + logging_obj=_logging_obj_stub(), + trailing_error=httpx.ReadError("Response payload is not completed"), + ) + + with pytest.raises(httpx.ReadError): + for _ in iterator: + pass diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 9c4d83ff7ea..c2c98c8869c 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -1455,6 +1455,132 @@ def test_model_group_info_cost_none_when_db_model_info_has_no_cost(): assert result.output_cost_per_token is None +@pytest.mark.parametrize( + "value,expected", + [ + ("1e-05", 1e-05), + ("0.00001", 1e-05), + (1e-05, 1e-05), + (5, 5.0), + (None, None), + ("not-a-number", None), + ], +) +def test_cost_value_as_float(value, expected): + from litellm.router import _cost_value_as_float + + assert _cost_value_as_float(value) == expected + + +def test_model_group_info_with_stringified_cost_values(): + """ + YAML 1.2 parsers emit '1e-05' (integer mantissa) as a string, so cost + values in deployment model_info can arrive as str. Aggregating the model + group must not raise TypeError('>' between str and float) and must return + float costs. + """ + router = litellm.Router( + model_list=[ + { + "model_name": "my-custom-model", + "litellm_params": { + "model": "openai/my-custom-backend-1", + "api_key": "fake", + }, + "model_info": { + "input_cost_per_token": "1e-05", + "output_cost_per_token": "1e-05", + }, + }, + { + "model_name": "my-custom-model", + "litellm_params": { + "model": "openai/my-custom-backend-2", + "api_key": "fake", + }, + "model_info": { + "input_cost_per_token": "2e-05", + "output_cost_per_token": "2e-05", + }, + }, + ] + ) + + def _model_info_with_str_costs(model_id: str, model_name: str): + for model in router.model_list: + if model["model_info"]["id"] == model_id: + return { + "key": model_name, + "input_cost_per_token": model["model_info"]["input_cost_per_token"], + "output_cost_per_token": model["model_info"]["output_cost_per_token"], + "litellm_provider": "openai", + "mode": "chat", + } + return None + + with patch.object( + router, "get_deployment_model_info", side_effect=_model_info_with_str_costs + ): + result = router._set_model_group_info( + model_group="my-custom-model", + user_facing_model_group_name="my-custom-model", + ) + + assert result is not None + assert result.input_cost_per_token == 2e-05 + assert result.output_cost_per_token == 2e-05 + assert isinstance(result.input_cost_per_token, float) + assert isinstance(result.output_cost_per_token, float) + + +def test_model_group_info_db_fallback_with_stringified_cost_values(): + """ + Fallback path: when get_deployment_model_info returns nothing, costs are + read straight from the deployment's model_info dict, which can hold + stringified floats parsed from YAML. They must be coerced to float. + """ + router = litellm.Router( + model_list=[ + { + "model_name": "my-custom-model", + "litellm_params": { + "model": "openai/my-custom-backend-1", + "api_key": "fake", + }, + "model_info": { + "input_cost_per_token": "1e-05", + "output_cost_per_token": "3e-05", + }, + }, + { + "model_name": "my-custom-model", + "litellm_params": { + "model": "openai/my-custom-backend-2", + "api_key": "fake", + }, + "model_info": { + "input_cost_per_token": "2e-05", + "output_cost_per_token": "2e-05", + }, + }, + ] + ) + + with patch.object( + router, "get_deployment_model_info", side_effect=Exception("not found") + ): + result = router._set_model_group_info( + model_group="my-custom-model", + user_facing_model_group_name="my-custom-model", + ) + + assert result is not None + assert result.input_cost_per_token == 2e-05 + assert result.output_cost_per_token == 3e-05 + assert isinstance(result.input_cost_per_token, float) + assert isinstance(result.output_cost_per_token, float) + + def test_get_model_access_groups_caching(): """ Test that get_model_access_groups caches the no-args result diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/useAuthorized.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/useAuthorized.test.ts index 94f9d9173f0..7076e69edc2 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/useAuthorized.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/useAuthorized.test.ts @@ -1,7 +1,7 @@ /* @vitest-environment jsdom */ import React from "react"; import { renderHook, waitFor } from "@testing-library/react"; -import { afterEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import useAuthorized from "./useAuthorized"; @@ -26,12 +26,6 @@ const { buildLoginUrlWithReturnMock: vi.fn((baseUrl: string) => baseUrl), })); -vi.mock("next/navigation", () => ({ - useRouter: () => ({ - replace: replaceMock, - }), -})); - vi.mock("@/components/networking", async (importOriginal) => { const actual = await importOriginal(); return { @@ -91,7 +85,28 @@ const clearCookie = () => { }; describe("useAuthorized", () => { + const originalLocation = window.location; + + beforeEach(() => { + Object.defineProperty(window, "location", { + value: { + href: "http://proxy.example/ui/?page=api-keys", + origin: "http://proxy.example", + hostname: "proxy.example", + pathname: "/ui/", + search: "?page=api-keys", + protocol: "http:", + replace: replaceMock, + }, + writable: true, + }); + }); + afterEach(() => { + Object.defineProperty(window, "location", { + value: originalLocation, + writable: true, + }); replaceMock.mockReset(); clearTokenCookiesMock.mockReset(); getProxyBaseUrlMock.mockClear(); @@ -164,7 +179,7 @@ describe("useAuthorized", () => { expect(clearTokenCookiesMock).toHaveBeenCalled(); }); - expect(replaceMock).toHaveBeenCalledWith("http://proxy.example/ui/login"); + expect(replaceMock).toHaveBeenCalledWith("http://proxy.example/ui/login/"); expect(result.current.accessToken).toBeNull(); expect(result.current.userRole).toBe("Undefined Role"); }); @@ -197,7 +212,7 @@ describe("useAuthorized", () => { const { result } = renderHook(() => useAuthorized(), { wrapper }); await waitFor(() => { - expect(replaceMock).toHaveBeenCalledWith("http://proxy.example/ui/login"); + expect(replaceMock).toHaveBeenCalledWith("http://proxy.example/ui/login/"); }); expect(result.current.accessToken).toBe("api-key-123"); @@ -221,7 +236,7 @@ describe("useAuthorized", () => { const { result } = renderHook(() => useAuthorized(), { wrapper }); await waitFor(() => { - expect(replaceMock).toHaveBeenCalledWith("http://proxy.example/ui/login"); + expect(replaceMock).toHaveBeenCalledWith("http://proxy.example/ui/login/"); }); expect(clearTokenCookiesMock).not.toHaveBeenCalled(); @@ -256,7 +271,7 @@ describe("useAuthorized", () => { expect(clearTokenCookiesMock).toHaveBeenCalled(); }); - expect(replaceMock).toHaveBeenCalledWith("http://proxy.example/ui/login"); + expect(replaceMock).toHaveBeenCalledWith("http://proxy.example/ui/login/"); expect(checkTokenValidityMock).toHaveBeenCalledWith(token); }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/useAuthorized.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/useAuthorized.ts index 8f8c403a4e9..bb22ebf5edc 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/useAuthorized.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/useAuthorized.ts @@ -3,14 +3,12 @@ import { getProxyBaseUrl } from "@/components/networking"; import { clearTokenCookies, getCookie } from "@/utils/cookieUtils"; import { checkTokenValidity, decodeToken } from "@/utils/jwtUtils"; -import { buildLoginUrlWithReturn, storeReturnUrl } from "@/utils/returnUrlUtils"; -import { useRouter } from "next/navigation"; +import { buildLoginUrlWithReturn, getLoginUrl, storeReturnUrl } from "@/utils/returnUrlUtils"; import { useCallback, useEffect, useMemo } from "react"; import { formatUserRole } from "@/utils/roles"; import { useUIConfig } from "./uiConfig/useUIConfig"; const useAuthorized = () => { - const router = useRouter(); const { data: uiConfig, isLoading: isUIConfigLoading } = useUIConfig(); const token = typeof document !== "undefined" ? getCookie("token") : null; @@ -23,10 +21,10 @@ const useAuthorized = () => { // Helper function to redirect to login while preserving the current URL const redirectToLogin = useCallback(() => { storeReturnUrl(); - const baseLoginUrl = `${getProxyBaseUrl()}/ui/login`; + const baseLoginUrl = getLoginUrl(getProxyBaseUrl()); const loginUrlWithReturn = buildLoginUrlWithReturn(baseLoginUrl); - router.replace(loginUrlWithReturn); - }, [router]); + window.location.replace(loginUrlWithReturn); + }, []); // Single useEffect for all redirect logic useEffect(() => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OAuthFormFields.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OAuthFormFields.tsx index fc72981edd0..76e4342f52e 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OAuthFormFields.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OAuthFormFields.tsx @@ -167,6 +167,17 @@ const OAuthFormFields: React.FC = ({ > + @@ -1588,6 +1607,7 @@ const MCPServerEdit: React.FC = ({ oauthFlowTypeValue ?? oauth2FlowToFormValue(mcpServer.oauth2_flow) ?? OAUTH_FLOW.INTERACTIVE, static_headers: currentStaticHeaders ?? mcpServer.static_headers, credentials: currentCredentials, + issuer: currentIssuer ?? mcpServer.issuer, authorization_url: currentAuthorizationUrl ?? mcpServer.authorization_url, token_url: currentTokenUrl ?? mcpServer.token_url, registration_url: currentRegistrationUrl ?? mcpServer.registration_url, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/page.tsx index 6c0d780183a..02b2ccf5357 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/page.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/page.tsx @@ -7,6 +7,7 @@ import { useAuth } from "@/contexts/AuthContext"; import { buildLoginUrlWithReturn, consumeReturnUrl, + getLoginUrl, isValidReturnUrl, normalizeUrlForCompare, storeReturnUrl, @@ -33,7 +34,7 @@ function CreateKeyPageContent() { // Store the current URL so we can redirect back after login storeReturnUrl(); // Build login URL with return URL parameter - const baseLoginUrl = (proxyBaseUrl || "") + "/ui/login"; + const baseLoginUrl = getLoginUrl(proxyBaseUrl || ""); const dest = buildLoginUrlWithReturn(baseLoginUrl); // Replace instead of assigning to avoid back-button loops window.location.replace(dest); diff --git a/ui/litellm-dashboard/src/app/login/LoginPage.tsx b/ui/litellm-dashboard/src/app/login/LoginPage.tsx index 1d08e647196..3c3ce42a703 100644 --- a/ui/litellm-dashboard/src/app/login/LoginPage.tsx +++ b/ui/litellm-dashboard/src/app/login/LoginPage.tsx @@ -6,7 +6,7 @@ import LoadingScreen from "@/components/common_components/LoadingScreen"; import { exchangeLoginCode, getProxyBaseUrl, switchToWorkerUrl } from "@/components/networking"; import { clearTokenCookies, getCookieFromDocument } from "@/utils/cookieUtils"; import { isJwtExpired } from "@/utils/jwtUtils"; -import { consumeReturnUrl, getReturnUrl, isValidReturnUrl } from "@/utils/returnUrlUtils"; +import { consumeReturnUrl, getLoginUrl, getReturnUrl, isValidReturnUrl } from "@/utils/returnUrlUtils"; import { InfoCircleOutlined, CloudServerOutlined } from "@ant-design/icons"; import { Alert, Button, Card, Form, Input, Popover, Select, Space, Typography } from "antd"; import { useRouter } from "next/navigation"; @@ -295,7 +295,7 @@ function LoginPageContent() { // SSO on the worker (or this instance if no worker), always // include return_to so the callback redirects back here const ssoBase = selectedWorker?.url ?? getProxyBaseUrl(); - const returnTo = encodeURIComponent(window.location.origin + "/ui/login"); + const returnTo = encodeURIComponent(getLoginUrl(window.location.origin)); router.push(`${ssoBase}/sso/key/generate?return_to=${returnTo}`); }} block diff --git a/ui/litellm-dashboard/src/app/onboarding/OnboardingErrorView.test.tsx b/ui/litellm-dashboard/src/app/onboarding/OnboardingErrorView.test.tsx index 59071f17bf6..bfbfb8fdd5d 100644 --- a/ui/litellm-dashboard/src/app/onboarding/OnboardingErrorView.test.tsx +++ b/ui/litellm-dashboard/src/app/onboarding/OnboardingErrorView.test.tsx @@ -14,10 +14,10 @@ describe("OnboardingErrorView", () => { expect(screen.getByText("The invitation link may be invalid or expired.")).toBeInTheDocument(); }); - it("should render a Back to Login link pointing to /ui/login", () => { + it("should render a Back to Login link pointing to /ui/login/", () => { render(); // antd Button with href renders as an element const link = screen.getByRole("link", { name: "Back to Login" }); - expect(link).toHaveAttribute("href", "/ui/login"); + expect(link).toHaveAttribute("href", "/ui/login/"); }); }); diff --git a/ui/litellm-dashboard/src/app/onboarding/OnboardingErrorView.tsx b/ui/litellm-dashboard/src/app/onboarding/OnboardingErrorView.tsx index ca0f57c56ce..3de9a9ffaae 100644 --- a/ui/litellm-dashboard/src/app/onboarding/OnboardingErrorView.tsx +++ b/ui/litellm-dashboard/src/app/onboarding/OnboardingErrorView.tsx @@ -1,5 +1,6 @@ import React from "react"; import { Alert, Button } from "antd"; +import { getLoginUrl } from "@/utils/returnUrlUtils"; export function OnboardingErrorView() { return ( @@ -11,7 +12,7 @@ export function OnboardingErrorView() { showIcon />
- +
); diff --git a/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.test.tsx b/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.test.tsx index 3a22a55298e..dfe307c7edf 100644 --- a/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.test.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.test.tsx @@ -1,5 +1,5 @@ import * as networking from "@/components/networking"; -import { afterEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { renderWithProviders, screen, waitFor } from "../../../tests/test-utils"; import ModelHubTable from "./ModelHubTable"; @@ -7,6 +7,7 @@ const mockUseUISettings = vi.hoisted(() => vi.fn()); const mockGetCookie = vi.hoisted(() => vi.fn()); const mockCheckTokenValidity = vi.hoisted(() => vi.fn()); const mockRouterReplace = vi.hoisted(() => vi.fn()); +const mockLocationReplace = vi.hoisted(() => vi.fn()); vi.mock("@/components/networking", () => ({ getUiConfig: vi.fn(), @@ -43,7 +44,28 @@ vi.mock("@/utils/jwtUtils", () => ({ })); describe("ModelHubTable", () => { + const originalLocation = window.location; + + beforeEach(() => { + Object.defineProperty(window, "location", { + value: { + href: "http://localhost:4000/ui/model_hub_table", + origin: "http://localhost:4000", + hostname: "localhost", + pathname: "/ui/model_hub_table", + search: "", + protocol: "http:", + replace: mockLocationReplace, + }, + writable: true, + }); + }); + afterEach(() => { + Object.defineProperty(window, "location", { + value: originalLocation, + writable: true, + }); vi.clearAllMocks(); }); @@ -60,6 +82,7 @@ describe("ModelHubTable", () => { mockGetCookie.mockReturnValue(tokenValue); mockCheckTokenValidity.mockReturnValue(isTokenValid); mockRouterReplace.mockClear(); + mockLocationReplace.mockClear(); // Setup other required mocks vi.mocked(networking.getUiConfig).mockResolvedValue({ @@ -92,9 +115,10 @@ describe("ModelHubTable", () => { await waitFor(() => { if (shouldRedirect) { - expect(mockRouterReplace).toHaveBeenCalledWith("http://localhost:4000/ui/login"); - } else { + expect(mockLocationReplace).toHaveBeenCalledWith("http://localhost:4000/ui/login/"); expect(mockRouterReplace).not.toHaveBeenCalled(); + } else { + expect(mockLocationReplace).not.toHaveBeenCalled(); } }); }); diff --git a/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx b/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx index eb47f775d90..1f64d175052 100644 --- a/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx @@ -33,6 +33,7 @@ import { Prism as SyntaxHighlighter } from "react-syntax-highlighter"; import { useUISettings } from "@/app/(dashboard)/hooks/uiSettings/useUISettings"; import { checkTokenValidity } from "@/utils/jwtUtils"; import { getCookie } from "@/utils/cookieUtils"; +import { getLoginUrl } from "@/utils/returnUrlUtils"; interface ModelHubTableProps { accessToken: string | null; @@ -108,12 +109,12 @@ const ModelHubTable: React.FC = ({ accessToken, publicPage, // If token is invalid, redirect to login if (!isTokenValid) { - router.replace(`${getProxyBaseUrl()}/ui/login`); + window.location.replace(getLoginUrl(getProxyBaseUrl())); return; } } // If require_auth_for_public_ai_hub is false, allow public access (no change) - }, [isUISettingsLoading, publicPage, uiSettings, router]); + }, [isUISettingsLoading, publicPage, uiSettings]); useEffect(() => { const fetchData = async (accessToken: string) => { diff --git a/ui/litellm-dashboard/src/components/DashboardHeader.tsx b/ui/litellm-dashboard/src/components/DashboardHeader.tsx index 8b928f8e6fe..babf007d475 100644 --- a/ui/litellm-dashboard/src/components/DashboardHeader.tsx +++ b/ui/litellm-dashboard/src/components/DashboardHeader.tsx @@ -18,7 +18,7 @@ import WorkerDropdown from "@/components/Navbar/WorkerDropdown/WorkerDropdown"; import { useWorker } from "@/hooks/useWorker"; import { useDisableShowPrompts } from "@/app/(dashboard)/hooks/useDisableShowPrompts"; import { clearTokenCookies } from "@/utils/cookieUtils"; -import { clearStoredReturnUrl } from "@/utils/returnUrlUtils"; +import { clearStoredReturnUrl, getLoginUrl } from "@/utils/returnUrlUtils"; interface DashboardHeaderProps { page: string; @@ -37,7 +37,7 @@ export function DashboardHeader({ page }: DashboardHeaderProps) { clearStoredReturnUrl(); localStorage.removeItem("litellm_selected_worker_id"); localStorage.removeItem("litellm_worker_url"); - window.location.href = `/ui/login?worker=${encodeURIComponent(workerId)}`; + window.location.href = `${getLoginUrl()}?worker=${encodeURIComponent(workerId)}`; }; return ( diff --git a/ui/litellm-dashboard/src/components/mcp_tools/types.tsx b/ui/litellm-dashboard/src/components/mcp_tools/types.tsx index 04766aad7b4..dd7ed2bfbe4 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/types.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/types.tsx @@ -81,6 +81,7 @@ export const getOAuthAuthorizationIdentity = (values: Record): client_id: credentials.client_id ?? null, client_secret: credentials.client_secret ?? null, scopes: credentials.scopes ?? null, + issuer: values.issuer ?? null, authorization_url: values.authorization_url ?? null, token_url: values.token_url ?? null, registration_url: values.registration_url ?? null, @@ -341,6 +342,7 @@ export interface MCPServer { transport?: string | null; auth_type?: string | null; oauth2_flow?: string | null; + issuer?: string | null; authorization_url?: string | null; token_url?: string | null; registration_url?: string | null; diff --git a/ui/litellm-dashboard/src/components/navbar.tsx b/ui/litellm-dashboard/src/components/navbar.tsx index 1a0b1c38520..40638e7b8ba 100644 --- a/ui/litellm-dashboard/src/components/navbar.tsx +++ b/ui/litellm-dashboard/src/components/navbar.tsx @@ -5,7 +5,7 @@ import { useWorker } from "@/hooks/useWorker"; import { getProxyBaseUrl } from "@/components/networking"; import { useTheme } from "@/contexts/ThemeContext"; import { clearTokenCookies } from "@/utils/cookieUtils"; -import { clearStoredReturnUrl } from "@/utils/returnUrlUtils"; +import { clearStoredReturnUrl, getLoginUrl } from "@/utils/returnUrlUtils"; import useProxySettings from "@/app/(dashboard)/hooks/proxySettings/useProxySettings"; import { DownOutlined, MenuFoldOutlined, MenuUnfoldOutlined } from "@ant-design/icons"; import { Tag } from "antd"; @@ -56,7 +56,7 @@ const Navbar: React.FC = ({ clearStoredReturnUrl(); localStorage.removeItem("litellm_selected_worker_id"); localStorage.removeItem("litellm_worker_url"); - window.location.href = `/ui/login?worker=${encodeURIComponent(workerId)}`; + window.location.href = `${getLoginUrl()}?worker=${encodeURIComponent(workerId)}`; }; return ( diff --git a/ui/litellm-dashboard/src/utils/returnUrlUtils.test.ts b/ui/litellm-dashboard/src/utils/returnUrlUtils.test.ts index 56de299049c..0f3a8b4bf69 100644 --- a/ui/litellm-dashboard/src/utils/returnUrlUtils.test.ts +++ b/ui/litellm-dashboard/src/utils/returnUrlUtils.test.ts @@ -3,6 +3,7 @@ import { clearStoredReturnUrl, consumeReturnUrl, getCurrentUrl, + getLoginUrl, getReturnUrl, getReturnUrlFromParams, getStoredReturnUrl, @@ -98,6 +99,29 @@ describe("returnUrlUtils", () => { }); }); + describe("getLoginUrl", () => { + it("should build a relative login URL with a trailing slash", () => { + expect(getLoginUrl()).toBe("/ui/login/"); + }); + + it("should prepend the given base URL and keep the trailing slash", () => { + expect(getLoginUrl("http://proxy.example")).toBe("http://proxy.example/ui/login/"); + }); + + it("should keep the trailing slash before the query when composed with buildLoginUrlWithReturn", () => { + Object.defineProperty(window, "location", { + value: { + ...window.location, + href: "http://localhost:3000/ui?page=api-keys", + }, + writable: true, + }); + + const loginUrl = buildLoginUrlWithReturn(getLoginUrl()); + expect(loginUrl).toBe("/ui/login/?redirect_to=http%3A%2F%2Flocalhost%3A3000%2Fui%3Fpage%3Dapi-keys"); + }); + }); + describe("buildLoginUrlWithReturn", () => { it("should build login URL with return URL parameter", () => { Object.defineProperty(window, "location", { diff --git a/ui/litellm-dashboard/src/utils/returnUrlUtils.ts b/ui/litellm-dashboard/src/utils/returnUrlUtils.ts index 76562a0122a..b3cc5345bb1 100644 --- a/ui/litellm-dashboard/src/utils/returnUrlUtils.ts +++ b/ui/litellm-dashboard/src/utils/returnUrlUtils.ts @@ -13,6 +13,10 @@ const RETURN_URL_COOKIE_NAME = "litellm_return_url"; const RETURN_URL_PARAM = "redirect_to"; +export function getLoginUrl(baseUrl: string = ""): string { + return `${baseUrl}/ui/login/`; +} + /** * Gets the current URL with all query parameters. * Returns null if running on server-side. diff --git a/ui/litellm-dashboard/tests/CreateKeyPage.expiredToken.test.tsx b/ui/litellm-dashboard/tests/CreateKeyPage.expiredToken.test.tsx index 62346eff057..211a754dad4 100644 --- a/ui/litellm-dashboard/tests/CreateKeyPage.expiredToken.test.tsx +++ b/ui/litellm-dashboard/tests/CreateKeyPage.expiredToken.test.tsx @@ -232,7 +232,7 @@ describe("CreateKeyPage auth behavior", () => { // Assert: we eventually redirect to SSO login with return URL (single replace, not assign/href) await waitFor(() => { expect(window.location.replace).toHaveBeenCalledWith( - expect.stringContaining("https://example.com/ui/login?redirect_to="), + expect.stringContaining("https://example.com/ui/login/?redirect_to="), ); });