diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 63f4b731871..bf64f537c7f 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -111,6 +111,7 @@ from litellm.repositories.verification_token_repository import ( VerificationTokenRepository, ) from litellm.router import Router +from litellm.secret_managers.base_secret_manager import raise_if_unsafe_secret_name from litellm.secret_managers.main import get_secret from litellm.types.proxy.management_endpoints.key_management_endpoints import ( BulkUpdateKeyRequest, @@ -6291,8 +6292,13 @@ def _validate_key_alias_format(key_alias: Optional[str]) -> None: """ Validate the format of the key_alias. - Gated behind ``litellm.enable_key_alias_format_validation`` (default **False**). - When disabled, no validation is performed so existing workflows are not broken. + A baseline validation always runs, regardless of + ``litellm.enable_key_alias_format_validation``. + + The remaining charset/length rules are gated behind + ``litellm.enable_key_alias_format_validation`` (default **False**). When disabled, + only the baseline validation above is performed, so existing workflows are not + broken. Rules (when enabled): - None is OK (no alias). @@ -6300,10 +6306,20 @@ def _validate_key_alias_format(key_alias: Optional[str]) -> None: - start/end with alphanumeric - only allow a-zA-Z0-9_-/.@ """ - if not litellm.enable_key_alias_format_validation: + if key_alias is None: return - if key_alias is None: + try: + raise_if_unsafe_secret_name(key_alias) + except ValueError: + raise ProxyException( + message="Invalid key_alias", + type=ProxyErrorTypes.bad_request_error, + param="key_alias", + code=400, + ) + + if not litellm.enable_key_alias_format_validation: return if not _KEY_ALIAS_PATTERN.match(key_alias): diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index dbf514d2298..43fdd3ed05a 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -113,7 +113,7 @@ from litellm.proxy.utils import ( from litellm.repositories.table_repositories import SSOConfigRepository from litellm.repositories.team_repository import TeamRepository from litellm.repositories.user_repository import UserRepository -from litellm.secret_managers.main import get_secret_bool, str_to_bool +from litellm.secret_managers.main import get_secret_bool, get_secret_str, str_to_bool from litellm.types.proxy.management_endpoints.ui_sso import * # noqa: F403 from litellm.types.proxy.management_endpoints.ui_sso import ( DefaultTeamSSOParams, @@ -3737,8 +3737,7 @@ class MicrosoftSSOHandler: Handles Microsoft SSO callback response and returns a CustomOpenID object """ - graph_api_base_url = "https://graph.microsoft.com/v1.0" - graph_api_user_groups_endpoint = f"{graph_api_base_url}/me/memberOf" + DEFAULT_GRAPH_API_BASE_URL = "https://graph.microsoft.com/v1.0" """ Constants @@ -3748,6 +3747,19 @@ class MicrosoftSSOHandler: # used for debugging to show the user groups litellm found from Graph API GRAPH_API_RESPONSE_KEY = "graph_api_user_groups" + @staticmethod + def get_graph_api_base_url() -> str: + """ + Returns the Microsoft Graph API base URL, configurable via the + `MICROSOFT_GRAPH_ENDPOINT` env var so non-default clouds such as Azure + Government (GCC High) can point at `https://graph.microsoft.us/v1.0` + """ + return get_secret_str("MICROSOFT_GRAPH_ENDPOINT") or MicrosoftSSOHandler.DEFAULT_GRAPH_API_BASE_URL + + @staticmethod + def get_graph_api_user_groups_endpoint() -> str: + return f"{MicrosoftSSOHandler.get_graph_api_base_url()}/me/memberOf" + @staticmethod async def get_microsoft_callback_response( request: Request, @@ -3924,7 +3936,7 @@ class MicrosoftSSOHandler: # Fetch user membership from Microsoft Graph API all_group_ids = [] - next_link: Optional[str] = MicrosoftSSOHandler.graph_api_user_groups_endpoint + next_link: Optional[str] = MicrosoftSSOHandler.get_graph_api_user_groups_endpoint() auth_headers = {"Authorization": f"Bearer {access_token}"} page_count = 0 @@ -4007,7 +4019,7 @@ class MicrosoftSSOHandler: Users use Enterprise Applications to manage Groups and Users on Microsoft Entra ID """ - base_url = "https://graph.microsoft.com/v1.0" + base_url = MicrosoftSSOHandler.get_graph_api_base_url() # Endpoint to get app role assignments for the given service principal endpoint = f"/servicePrincipals/{service_principal_id}/appRoleAssignedTo" url = base_url + endpoint diff --git a/litellm/secret_managers/base_secret_manager.py b/litellm/secret_managers/base_secret_manager.py index d33d76093c9..2bb8dc73138 100644 --- a/litellm/secret_managers/base_secret_manager.py +++ b/litellm/secret_managers/base_secret_manager.py @@ -1,3 +1,4 @@ +import re from abc import ABC, abstractmethod from typing import Any, Dict, Optional, Union @@ -5,6 +6,20 @@ import httpx from litellm import verbose_logger +_UNSAFE_SECRET_NAME_PATTERN = re.compile(r"(^|/)\.\.(/|$)|[\x00-\x1f\x7f-\x9f…

]") + + +def raise_if_unsafe_secret_name(secret_name: str) -> None: + """ + Validate a secret name before it is used by a secret manager integration. + + Rejects ".." only as a path segment (bounded by "/" or the start/end of the + string, e.g. "../x", "x/..", or exactly ".."), not as a plain substring, so + names like "release-1.0..2" are not rejected. + """ + if _UNSAFE_SECRET_NAME_PATTERN.search(secret_name): + raise ValueError(f"Invalid secret_name {secret_name!r}") + class BaseSecretManager(ABC): """ diff --git a/litellm/secret_managers/cyberark_secret_manager.py b/litellm/secret_managers/cyberark_secret_manager.py index faf6224757f..2b888cb85f6 100644 --- a/litellm/secret_managers/cyberark_secret_manager.py +++ b/litellm/secret_managers/cyberark_secret_manager.py @@ -4,6 +4,7 @@ from typing import Any, Dict, Optional, Union from urllib.parse import quote import httpx +import yaml import litellm from litellm._logging import verbose_logger @@ -15,7 +16,7 @@ from litellm.llms.custom_httpx.http_handler import ( ) from litellm.proxy._types import KeyManagementSystem -from .base_secret_manager import BaseSecretManager +from .base_secret_manager import BaseSecretManager, raise_if_unsafe_secret_name from .main import str_to_bool @@ -125,8 +126,11 @@ class CyberArkSecretManager(BaseSecretManager): """ # In production, we'd check if the variable exists first # For now, we'll attempt to create it and ignore if it already exists + raise_if_unsafe_secret_name(secret_name) policy_url = f"{self.conjur_addr}/policies/{self.conjur_account}/policy/root" - policy_yaml = f"- !variable {secret_name}\n" + # Use a real YAML serializer to build the scalar safely. + quoted_name = yaml.safe_dump(secret_name, default_style='"').strip() + policy_yaml = f"- !variable {quoted_name}\n" try: client = _get_httpx_client(params={"ssl_verify": self.ssl_verify}) diff --git a/litellm/secret_managers/hashicorp_secret_manager.py b/litellm/secret_managers/hashicorp_secret_manager.py index bd1b1097347..039aecb9e58 100644 --- a/litellm/secret_managers/hashicorp_secret_manager.py +++ b/litellm/secret_managers/hashicorp_secret_manager.py @@ -14,7 +14,7 @@ from litellm.llms.custom_httpx.http_handler import ( ) from litellm.proxy._types import KeyManagementSystem -from .base_secret_manager import BaseSecretManager +from .base_secret_manager import BaseSecretManager, raise_if_unsafe_secret_name class HashicorpSecretManager(BaseSecretManager): @@ -220,6 +220,7 @@ class HashicorpSecretManager(BaseSecretManager): - With custom mount: http://127.0.0.1:8200/v1/kv/data/mykey - With path prefix: http://127.0.0.1:8200/v1/secret/data/myapp/mykey """ + raise_if_unsafe_secret_name(secret_name) resolved_namespace = self._sanitize_path_component(namespace if namespace is not None else self.vault_namespace) resolved_mount = self._sanitize_path_component(mount_name if mount_name is not None else self.vault_mount_name) if resolved_mount is None: diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index a4c507ca5ea..502c881c764 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -63,7 +63,7 @@ The harness is fully typed and new code must not add `Any` or widen the basedpyr The set of tests we want is a registry checked into this repo, one row per behavior; that file is the definition of done and the denominator. Each e2e test declares what it covers with `@pytest.mark.covers("...")`, and a small collector diffs the registry against the tests and ships coverage to the existing Grafana. No Allure, no new dependencies -Coverage is organized as module > feature > test. Dashboard modules are Core LLMs, Non-Core LLMs, MCPs, Management/UI, Reliability & Performance, Logging & Guardrails, and Other. A feature is either an endpoint (`/chat/completions`) or a behavior (fallbacks, rate limits; config-driven, with no route of its own). A cell reads like `llm.chat_completions.bedrock_converse.tool_use.stream.works` +Coverage is organized as module > feature > test. Dashboard modules are `Core LLMs`, `Non-Core LLMs`, `MCPs`, `Management/UI`, `Reliability & Performance`, `Logging & Guardrails`, and `Other`. The Loki stdout formatter maps those display modules to log-safe labels (`core_llms`, `non_core_llms`, `mcp`, `management_ui`, `reliability_performance`, `logging_guardrails`, and `other`) without changing JSON or Prometheus labels. A feature is either an endpoint (`/chat/completions`) or a behavior (fallbacks, rate limits; config-driven, with no route of its own). A cell reads like `llm.chat_completions.bedrock_converse.tool_use.stream.works` The metric is coverage: the share of registry rows that have a passing covering test, reported to Grafana per module so a gap surfaces as an uncovered row rather than a silent absence @@ -71,7 +71,7 @@ Tests do not declare a dashboard module directly. They only declare the registry ### Naming grammar per module -LLMs - endpoint features (subject = the route), seeded from the Claude Code compat matrix. `chat_completions`, `messages`, and `responses` are Core LLMs. Other LLM endpoints, including `batches` and `realtime`, roll up as Non-Core LLMs. +LLMs - endpoint features (subject = the route), seeded from the Claude Code compat matrix. `chat_completions`, `messages`, and `responses` roll up to `Core LLMs`. Other LLM endpoints, including `batches` and `realtime`, roll up to `Non-Core LLMs`. ``` llm..... diff --git a/tests/e2e/coverage_registry/README.md b/tests/e2e/coverage_registry/README.md index 4177cba7766..863ce34694d 100644 --- a/tests/e2e/coverage_registry/README.md +++ b/tests/e2e/coverage_registry/README.md @@ -9,18 +9,18 @@ note; the naming grammar lives in `tests/e2e/CLAUDE.md`. A **cell** is one customer-noticeable behavior a single e2e test can assert pass/fail on, for example `llm.chat_completions.bedrock_converse.tool_use.stream.works`. Cells are -grouped `module > feature > test`, with LLM cells split into Core LLMs and Non-Core -LLMs for dashboarding. Each cell carries a tier (P0/P1/P2), a source, and a +grouped `module > feature > test`, with LLM cells split into `Core LLMs` and +`Non-Core LLMs` for dashboarding. Each cell carries a tier (P0/P1/P2), a source, and a `fail_before_fix` flag. The rows live in per-prefix YAML files (`llm_*.yaml`, `mgmt.yaml`, `mcp.yaml`, `reliability.yaml`, `logging.yaml`, `guardrail.yaml`, `other.yaml`) and validate against the discriminated union in `schema.py`, so an LLM row cannot carry a guardrail field and vice versa. `llm` rows with `subject_endpoint` of `chat_completions`, `messages`, or -`responses` roll up to "Core LLMs"; all other LLM endpoints roll up to "Non-Core -LLMs". LLM endpoint, route, and capability values are typed in `schema.py`, so new -taxonomy values require an explicit schema change. `logging` and `guardrail` are two -id-prefixes that roll up into the single "Logging & Guardrails" dashboard module. +`responses` roll up to `Core LLMs`; all other LLM endpoints roll up to `Non-Core LLMs`. +LLM endpoint, route, and capability values are typed in `schema.py`, so new taxonomy +values require an explicit schema change. `logging` and `guardrail` are two id-prefixes +that roll up into the single `Logging & Guardrails` dashboard module. A test declares what it covers with a marker: @@ -40,8 +40,17 @@ proxy. Whether a covered cell currently passes or fails is a separate, live conc cd tests/e2e && PYTHONPATH=. python -m coverage_registry.collector ``` -Use `--format prometheus` or `--format json` for CI jobs that publish coverage to -Grafana. +Use `--format loki` after the e2e pytest run in the same Kubernetes job/pod to print +structured stdout lines for Loki: + +``` +cd tests/e2e && PYTHONPATH=. python -m coverage_registry.collector --format loki --strict +``` + +This emits exactly one `COVERAGE_TOTAL` line and one `COVERAGE_MODULE` line per module +in `MODULE_ORDER`, in that order. Loki uses log-safe `module=` labels from +`LOKI_MODULE_LABELS` (`core_llms`, `management_ui`, etc.) so existing JSON and +Prometheus consumers keep their human-readable module names unchanged. The headline is overall coverage. The collector also lists markers that point at ids not in the registry, so a typo or an unenumerated behavior surfaces instead of being diff --git a/tests/e2e/coverage_registry/collector.py b/tests/e2e/coverage_registry/collector.py index 3b577106605..f6e59ca4a88 100644 --- a/tests/e2e/coverage_registry/collector.py +++ b/tests/e2e/coverage_registry/collector.py @@ -21,7 +21,7 @@ from pathlib import Path import pytest from .registry import load_registry -from .schema import MODULE_ORDER, Cell, Tier, dashboard_module +from .schema import MODULE_ORDER, Cell, Tier, dashboard_module, loki_module_label E2E_DIR = Path(__file__).resolve().parent.parent @@ -239,13 +239,31 @@ def render_prometheus(report: CoverageReport) -> str: return "\n".join(lines) +def render_loki(report: CoverageReport) -> str: + lines = [ + ( + f"COVERAGE_TOTAL percent={report.coverage_percent:.1f} " + f"covered={report.covered} total={report.total}" + ) + ] + lines.extend( + ( + f"COVERAGE_MODULE module={loki_module_label(module.module)} " + f"percent={module.coverage_percent:.1f} " + f"covered={module.covered} total={module.total}" + ) + for module in report.modules + ) + return "\n".join(lines) + + def main() -> int: parser = ArgumentParser() parser.add_argument( "--format", - choices=("text", "json", "prometheus"), + choices=("text", "json", "prometheus", "loki"), default="text", - help="Output format. Use prometheus or json for Grafana ingestion jobs.", + help="Output format. Use loki for structured stdout lines in the e2e job.", ) parser.add_argument( "--strict", @@ -265,9 +283,8 @@ def main() -> int: "text": render, "json": render_json, "prometheus": render_prometheus, - }[ - args.format - ](report) + "loki": render_loki, + }[args.format](report) print(output) # noqa: T201 # CLI entrypoint output if args.strict and report.orphan_markers: return 1 diff --git a/tests/e2e/coverage_registry/schema.py b/tests/e2e/coverage_registry/schema.py index 2e2a00e78ba..bb27fbf0ea0 100644 --- a/tests/e2e/coverage_registry/schema.py +++ b/tests/e2e/coverage_registry/schema.py @@ -157,6 +157,16 @@ MODULE_ORDER: tuple[str, ...] = ( "Other", ) +LOKI_MODULE_LABELS: dict[str, str] = { + "Core LLMs": "core_llms", + "Non-Core LLMs": "non_core_llms", + "MCPs": "mcp", + "Management/UI": "management_ui", + "Reliability & Performance": "reliability_performance", + "Logging & Guardrails": "logging_guardrails", + "Other": "other", +} + def dashboard_module(cell: Cell) -> str: """Return the Grafana/reporting module for a registry cell.""" @@ -165,3 +175,8 @@ def dashboard_module(cell: Cell) -> str: return "Core LLMs" return "Non-Core LLMs" return PREFIX_ROLLUP[cell.module] + + +def loki_module_label(module: str) -> str: + """Return the log-safe Loki label for a dashboard module.""" + return LOKI_MODULE_LABELS[module] diff --git a/tests/e2e/coverage_registry/test_collector.py b/tests/e2e/coverage_registry/test_collector.py index 355bc52730d..079ee215866 100644 --- a/tests/e2e/coverage_registry/test_collector.py +++ b/tests/e2e/coverage_registry/test_collector.py @@ -15,6 +15,7 @@ from coverage_registry.collector import ( compute_coverage, render, render_json, + render_loki, render_prometheus, ) from coverage_registry.registry import load_registry @@ -24,6 +25,7 @@ from coverage_registry.schema import ( LlmEndpoint, LoggingCell, Tier, + loki_module_label, ) @@ -149,6 +151,30 @@ def test_prometheus_render_exposes_module_coverage_timeseries() -> None: assert "litellm_e2e_coverage_orphan_markers 0" in metrics +def test_loki_render_exposes_exact_stdout_lines_for_loki() -> None: + report = compute_coverage( + (_llm("llm.chat", Tier.P0), _llm("llm.batches", Tier.P0, "batches")), + frozenset({"llm.chat"}), + ) + + lines = render_loki(report).splitlines() + + assert len(lines) == 1 + len(report.modules) + assert lines[0] == "COVERAGE_TOTAL percent=50.0 covered=1 total=2" + assert ( + lines[1] == "COVERAGE_MODULE module=core_llms percent=100.0 covered=1 total=1" + ) + assert ( + lines[2] == "COVERAGE_MODULE module=non_core_llms percent=0.0 covered=0 total=1" + ) + assert [line.split("module=", 1)[1].split(" ", 1)[0] for line in lines[1:]] == [ + loki_module_label(module.module) for module in report.modules + ] + assert all( + " " not in line.split("module=", 1)[1].split(" ", 1)[0] for line in lines[1:] + ) + + def test_real_registry_loads_and_ids_are_unique() -> None: cells = load_registry() ids = [c.id for c in cells] diff --git a/tests/litellm_utils_tests/test_cyberark.py b/tests/litellm_utils_tests/test_cyberark.py index 67575d3e781..71daf35a265 100644 --- a/tests/litellm_utils_tests/test_cyberark.py +++ b/tests/litellm_utils_tests/test_cyberark.py @@ -5,6 +5,7 @@ Integration test for CyberArk Conjur Secret Manager. import os import sys import pytest +import yaml from dotenv import load_dotenv load_dotenv() @@ -42,6 +43,82 @@ def create_mock_response(status_code: int, text: str = ""): return mock_response +@pytest.mark.asyncio +async def test_cyberark_write_secret_rejects_yaml_injection(): + """ + Regression test: async_write_secret must reject a secret_name that is not + safe to embed in the Conjur policy body, before any HTTP call is made. + """ + with patch("litellm.proxy.proxy_server.premium_user", True): + malicious_secret_name = "foo\n- !grant\n role: !!admin\n member: attacker" + + mock_sync_client = MagicMock() + mock_async_client = AsyncMock() + + with ( + patch( + "litellm.secret_managers.cyberark_secret_manager._get_httpx_client", + return_value=mock_sync_client, + ), + patch( + "litellm.secret_managers.cyberark_secret_manager.get_async_httpx_client", + return_value=mock_async_client, + ), + ): + cyberark_manager = CyberArkSecretManager() + + response = await cyberark_manager.async_write_secret( + secret_name=malicious_secret_name, + secret_value="sk-1234", + ) + + assert response["status"] == "error" + assert "Invalid secret_name" in response["message"] + # The malicious policy YAML must never reach the wire. + mock_sync_client.client.post.assert_not_called() + mock_async_client.post.assert_not_called() + + +@pytest.mark.parametrize( + "secret_name", + [ + "foo: bar", + "foo # bar", + "plain-alias", + "team/user@example.com", + ], +) +def test_cyberark_ensure_variable_exists_escapes_yaml_metacharacters(secret_name): + """ + Regression test: _ensure_variable_exists must escape secret_name (not just + denylist-check it) so the policy body always parses back to exactly one + '!variable' scalar node holding the untouched secret_name. + """ + with patch("litellm.proxy.proxy_server.premium_user", True): + captured = {} + + def _capture_post(url, headers=None, content=None): + captured["content"] = content + return create_mock_response(status_code=201, text="") + + mock_sync_client = MagicMock() + mock_sync_client.client.post.side_effect = _capture_post + + with patch( + "litellm.secret_managers.cyberark_secret_manager._get_httpx_client", + return_value=mock_sync_client, + ): + cyberark_manager = CyberArkSecretManager() + cyberark_manager._ensure_variable_exists(secret_name) + + policy_yaml = captured["content"] + parsed = yaml.compose(policy_yaml) + assert len(parsed.value) == 1 + node = parsed.value[0] + assert node.tag == "!variable" + assert node.value == secret_name + + @pytest.mark.asyncio async def test_cyberark_write_and_read_secret(): """ diff --git a/tests/litellm_utils_tests/test_hashicorp.py b/tests/litellm_utils_tests/test_hashicorp.py index 3bdf11ea565..9aff7ddc10e 100644 --- a/tests/litellm_utils_tests/test_hashicorp.py +++ b/tests/litellm_utils_tests/test_hashicorp.py @@ -409,6 +409,33 @@ def test_hashicorp_custom_mount_and_prefix(hashicorp_secret_manager): hashicorp_secret_manager.vault_namespace = original_namespace +@pytest.mark.parametrize( + "malicious_secret_name", + [ + "../../../other-app/creds", + "litellm/../../secret", + "foo\nbar", + "foo
bar", + "foo
bar", + "foo\x85bar", + ], +) +def test_hashicorp_get_url_rejects_path_traversal(monkeypatch, malicious_secret_name): + """ + Regression test: get_url must reject an invalid secret_name instead of + building a URL from it. + + Uses monkeypatch + a directly-constructed manager (not the shared + hashicorp_secret_manager fixture) so this runs in CI without real Vault + credentials configured; get_url performs no I/O. + """ + monkeypatch.setenv("HCP_VAULT_TOKEN", "test-token-for-get-url-only") + manager = HashicorpSecretManager() + + with pytest.raises(ValueError): + manager.get_url(malicious_secret_name) + + mock_old_vault_response = { "request_id": "80fafb6a-e96a-4c5b-29fa-ff505ac72201", "lease_id": "", diff --git a/tests/llm_responses_api_testing/base_responses_api.py b/tests/llm_responses_api_testing/base_responses_api.py index 7d2e30f8372..407091a65b3 100644 --- a/tests/llm_responses_api_testing/base_responses_api.py +++ b/tests/llm_responses_api_testing/base_responses_api.py @@ -746,7 +746,8 @@ class BaseResponsesAPITest(ABC): E2E test for Shell tool on OpenAI Responses API. Passes tools=[{"type": "shell", "environment": {"type": "container_auto"}}]; validates that the request is accepted and returns a valid response. - Only runs for OpenAI/Azure (Responses API with shell support). + Only runs for OpenAI; offline coverage for the Azure route lives in + tests/test_litellm/responses/test_responses_api_request_body.py. """ base_completion_call_args = self.get_base_completion_call_args() model = ( @@ -754,8 +755,10 @@ class BaseResponsesAPITest(ABC): or base_completion_call_args.get("model") or "" ) - if "openai/" not in str(model) and "azure/" not in str(model): - pytest.skip("Shell tool e2e is only run for OpenAI/Azure Responses API") + if "openai/" not in str(model): + pytest.skip( + "Shell tool e2e is OpenAI-only; no Azure deployment supports the shell tool yet, re-enable once one exists" + ) tools = [{"type": "shell", "environment": {"type": "container_auto"}}] input_msg = "List files in /mnt/data and show python --version." try: diff --git a/tests/llm_responses_api_testing/test_azure_responses_api.py b/tests/llm_responses_api_testing/test_azure_responses_api.py index fed9e9e11f0..ccef8cbf1e7 100644 --- a/tests/llm_responses_api_testing/test_azure_responses_api.py +++ b/tests/llm_responses_api_testing/test_azure_responses_api.py @@ -2,7 +2,6 @@ import os import sys import pytest import asyncio -from typing import Optional from unittest.mock import patch, AsyncMock sys.path.insert(0, os.path.abspath("../..")) @@ -30,10 +29,6 @@ class TestAzureResponsesAPITest(BaseResponsesAPITest): "api_version": "2025-03-01-preview", } - def get_advanced_model_for_shell_tool(self) -> Optional[str]: - """If specified, overrides the model used by test_responses_api_shell_tool_streaming_sees_shell_output (e.g. openai/gpt-5.2 for shell support).""" - return "azure/gpt-5-mini" - @pytest.mark.asyncio async def test_azure_responses_api_preview_api_version(): diff --git a/tests/test_litellm/expected_responses_api_request/azure_shell_tool.json b/tests/test_litellm/expected_responses_api_request/azure_shell_tool.json new file mode 100644 index 00000000000..b716c518106 --- /dev/null +++ b/tests/test_litellm/expected_responses_api_request/azure_shell_tool.json @@ -0,0 +1,14 @@ +{ + "model": "gpt-5-mini", + "input": "List files in /mnt/data and run python --version.", + "tools": [ + { + "type": "shell", + "environment": { + "type": "container_auto" + } + } + ], + "tool_choice": "auto", + "max_output_tokens": 256 +} diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index d707421aeb6..2fe7725fd12 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -9023,7 +9023,7 @@ class TestValidateKeyAliasFormat: litellm.enable_key_alias_format_validation = False def test_validation_skipped_when_flag_disabled(self): - """When enable_key_alias_format_validation is False (default), no validation occurs.""" + """When enable_key_alias_format_validation is False (default), no charset/length validation occurs.""" from litellm.proxy.management_endpoints.key_management_endpoints import ( _validate_key_alias_format, ) @@ -9034,6 +9034,33 @@ class TestValidateKeyAliasFormat: _validate_key_alias_format("!invalid!") _validate_key_alias_format("a" * 256) + @pytest.mark.parametrize( + "unsafe_alias", + [ + "../../../other-app/creds", + "litellm/../../secret", + "foo\n- !grant\n role: !!admin\n member: attacker", + "foo\rbar", + "foo\x00bar", + ], + ) + def test_validate_key_alias_format_rejects_traversal_and_control_chars_even_when_flag_disabled( + self, unsafe_alias + ): + """ + Regression test: this check must reject an invalid key_alias unconditionally, + even when enable_key_alias_format_validation (the separate, opt-in charset + rule) is disabled. + """ + from litellm.proxy.management_endpoints.key_management_endpoints import ( + _validate_key_alias_format, + ) + + with pytest.raises(ProxyException) as exc: + _validate_key_alias_format(unsafe_alias) + assert str(exc.value.code) == "400" + assert "Invalid key_alias" in str(exc.value.message) + def test_validate_key_alias_format_valid(self): from litellm.proxy.management_endpoints.key_management_endpoints import ( _validate_key_alias_format, diff --git a/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py b/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py index 32229e3e64e..642f20906a0 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py +++ b/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py @@ -389,6 +389,78 @@ async def test_get_user_groups_error_handling(): assert len(result) == 0 +@pytest.mark.asyncio +async def test_get_user_groups_uses_default_graph_endpoint(monkeypatch): + monkeypatch.delenv("MICROSOFT_GRAPH_ENDPOINT", raising=False) + + requested_urls: list[str] = [] + + async def mock_get(url, *args, **kwargs): + requested_urls.append(url) + mock = MagicMock() + mock.json.return_value = {"value": []} + return mock + + with patch( + "litellm.proxy.management_endpoints.ui_sso.get_async_httpx_client" + ) as mock_client: + mock_client.return_value = MagicMock() + mock_client.return_value.get = mock_get + + await MicrosoftSSOHandler.get_user_groups_from_graph_api(access_token="mock_token") + + assert requested_urls == ["https://graph.microsoft.com/v1.0/me/memberOf"] + + +@pytest.mark.asyncio +async def test_get_user_groups_uses_configured_graph_endpoint(monkeypatch): + monkeypatch.setenv("MICROSOFT_GRAPH_ENDPOINT", "https://graph.microsoft.us/v1.0") + + requested_urls: list[str] = [] + + async def mock_get(url, *args, **kwargs): + requested_urls.append(url) + mock = MagicMock() + mock.json.return_value = {"value": []} + return mock + + with patch( + "litellm.proxy.management_endpoints.ui_sso.get_async_httpx_client" + ) as mock_client: + mock_client.return_value = MagicMock() + mock_client.return_value.get = mock_get + + await MicrosoftSSOHandler.get_user_groups_from_graph_api(access_token="mock_token") + + assert requested_urls == ["https://graph.microsoft.us/v1.0/me/memberOf"] + + +@pytest.mark.asyncio +async def test_get_group_ids_from_service_principal_uses_configured_graph_endpoint(monkeypatch): + monkeypatch.setenv("MICROSOFT_GRAPH_ENDPOINT", "https://graph.microsoft.us/v1.0") + + requested_urls: list[str] = [] + + async def mock_get(url, *args, **kwargs): + requested_urls.append(url) + mock = MagicMock() + mock.json.return_value = {"value": []} + return mock + + async_client = MagicMock() + async_client.get = mock_get + + await MicrosoftSSOHandler.get_group_ids_from_service_principal( + service_principal_id="sp-123", + async_client=async_client, + access_token="mock_token", + ) + + assert requested_urls == [ + "https://graph.microsoft.us/v1.0/servicePrincipals/sp-123/appRoleAssignedTo" + ] + + def test_get_group_ids_from_graph_api_response(): # Arrange mock_response = MicrosoftGraphAPIUserGroupResponse( diff --git a/tests/test_litellm/responses/test_responses_api_request_body.py b/tests/test_litellm/responses/test_responses_api_request_body.py index e312a11e893..c39ba75bd97 100644 --- a/tests/test_litellm/responses/test_responses_api_request_body.py +++ b/tests/test_litellm/responses/test_responses_api_request_body.py @@ -1,6 +1,7 @@ """ Test that litellm.responses() / litellm.aresponses() send the expected request body -over the wire. Expected JSON bodies are stored in expected_responses_api_request/. +over the wire and surface provider errors correctly. Expected JSON bodies are stored +in expected_responses_api_request/. """ import json @@ -18,24 +19,20 @@ def _expected_dir() -> Path: return Path(__file__).resolve().parent.parent / "expected_responses_api_request" -@pytest.mark.asyncio -async def test_aresponses_context_management_and_shell_request_body_matches_expected(): - """ - Call litellm.aresponses() with context_management and shell tool; - assert the httpx POST request body matches the expected JSON. - """ - expected_path = _expected_dir() / "context_management_and_shell.json" +def _load_expected_body(filename: str) -> dict: + expected_path = _expected_dir() / filename assert expected_path.exists(), f"Expected file not found: {expected_path}" with open(expected_path) as f: - expected_body = json.load(f) + return json.load(f) - # Minimal Responses API response so parsing succeeds - mock_response = { - "id": "resp_ctx_shell_test", + +def _minimal_responses_api_payload(response_id: str, model: str) -> dict: + return { + "id": response_id, "object": "response", "created_at": 1734366691, "status": "completed", - "model": "gpt-4o", + "model": model, "output": [ { "type": "message", @@ -69,21 +66,41 @@ async def test_aresponses_context_management_and_shell_request_body_matches_expe "user": None, } - class MockResponse: - def __init__(self, json_data, status_code=200): - self._json_data = json_data - self.status_code = status_code - self.text = json.dumps(json_data) - self.headers = httpx.Headers({}) - def json(self): - return self._json_data +class MockResponse: + def __init__(self, json_data, status_code=200): + self._json_data = json_data + self.status_code = status_code + self.text = json.dumps(json_data) + self.headers = httpx.Headers({}) + + def json(self): + return self._json_data + + +def _assert_request_body_matches(request_body: dict, expected_body: dict) -> None: + for key, expected_value in expected_body.items(): + assert key in request_body, f"Missing key in request body: {key}" + assert ( + request_body[key] == expected_value + ), f"Mismatch for key {key}: got {request_body[key]!r}, expected {expected_value!r}" + + +@pytest.mark.asyncio +async def test_aresponses_context_management_and_shell_request_body_matches_expected(): + """ + Call litellm.aresponses() with context_management and shell tool; + assert the httpx POST request body matches the expected JSON. + """ + expected_body = _load_expected_body("context_management_and_shell.json") with patch( "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", new_callable=AsyncMock, ) as mock_post: - mock_post.return_value = MockResponse(mock_response, 200) + mock_post.return_value = MockResponse( + _minimal_responses_api_payload("resp_ctx_shell_test", "gpt-4o"), 200 + ) await litellm.aresponses( model="openai/gpt-4o", @@ -95,10 +112,87 @@ async def test_aresponses_context_management_and_shell_request_body_matches_expe ) mock_post.assert_called_once() - request_body = mock_post.call_args.kwargs["json"] + _assert_request_body_matches(mock_post.call_args.kwargs["json"], expected_body) - for key, expected_value in expected_body.items(): - assert key in request_body, f"Missing key in request body: {key}" - assert ( - request_body[key] == expected_value - ), f"Mismatch for key {key}: got {request_body[key]!r}, expected {expected_value!r}" + +@pytest.mark.asyncio +async def test_aresponses_azure_shell_tool_request_body_matches_expected(): + """ + Call litellm.aresponses() on the Azure route with the shell tool; + assert the httpx POST request body carries the shell tool verbatim. + """ + expected_body = _load_expected_body("azure_shell_tool.json") + + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + new_callable=AsyncMock, + ) as mock_post: + mock_post.return_value = MockResponse( + _minimal_responses_api_payload("resp_azure_shell_test", "gpt-5-mini"), 200 + ) + + await litellm.aresponses( + model="azure/gpt-5-mini", + api_base="https://fake-resource.openai.azure.com", + api_key="fake-api-key", + api_version="2025-03-01-preview", + input=expected_body["input"], + tools=expected_body["tools"], + tool_choice=expected_body["tool_choice"], + max_output_tokens=expected_body["max_output_tokens"], + ) + + mock_post.assert_called_once() + _assert_request_body_matches(mock_post.call_args.kwargs["json"], expected_body) + + +@pytest.mark.asyncio +async def test_aresponses_azure_shell_tool_400_maps_to_bad_request_error(): + """ + Azure rejects the shell tool for unsupported deployments with a 400; + litellm must surface that as litellm.BadRequestError carrying the provider message. + """ + error_body = { + "error": { + "message": "Tool of type 'shell' is not supported with this model.", + "type": "invalid_request_error", + "param": "tools", + "code": None, + } + } + + def _raise_azure_400(*args, **kwargs): + response = httpx.Response( + status_code=400, + json=error_body, + request=httpx.Request( + "POST", + kwargs.get( + "url", + "https://fake-resource.openai.azure.com/openai/responses", + ), + ), + ) + response.raise_for_status() + + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + new_callable=AsyncMock, + ) as mock_post: + mock_post.side_effect = _raise_azure_400 + + with pytest.raises(litellm.BadRequestError) as excinfo: + await litellm.aresponses( + model="azure/gpt-5-mini", + api_base="https://fake-resource.openai.azure.com", + api_key="fake-api-key", + api_version="2025-03-01-preview", + input="List files in /mnt/data and run python --version.", + tools=[{"type": "shell", "environment": {"type": "container_auto"}}], + tool_choice="auto", + max_output_tokens=256, + ) + + assert excinfo.value.status_code == 400 + assert "shell" in str(excinfo.value).lower() + assert "not supported" in str(excinfo.value).lower() diff --git a/tests/test_litellm/secret_managers/test_base_secret_manager.py b/tests/test_litellm/secret_managers/test_base_secret_manager.py new file mode 100644 index 00000000000..cba6a99ab7f --- /dev/null +++ b/tests/test_litellm/secret_managers/test_base_secret_manager.py @@ -0,0 +1,59 @@ +""" +Test raise_if_unsafe_secret_name, the shared guard applied before secret_name +reaches a secret manager backend. +""" + +import os +import sys + +import pytest + +sys.path.insert(0, os.path.abspath("../..")) # Adds the parent directory to the system path + +from litellm.secret_managers.base_secret_manager import raise_if_unsafe_secret_name + + +@pytest.mark.parametrize( + "secret_name", + [ + "..", + "../../../other-app/creds", + "litellm/../../secret", + "foo/../bar", + "foo/..", + "../foo", + "foo\nbar", + "foo\rbar", + "foo\x00bar", + "foo\x7fbar", + "foo\x85bar", + "foo
bar", + "foo
bar", + ], +) +def test_raise_if_unsafe_secret_name_rejects_traversal_and_line_breaks(secret_name): + with pytest.raises(ValueError): + raise_if_unsafe_secret_name(secret_name) + + +@pytest.mark.parametrize( + "secret_name", + [ + "plain-alias", + "my-key-123", + "prod/my-service-key", + "team/user@example.com", + "foo: bar", + "foo # bar", + "foo?evil=1", + "foo#bar", + "a" * 500, + "release-1.0..2", + "my..key", + "..foo", + "foo..", + "v2.0..1-beta", + ], +) +def test_raise_if_unsafe_secret_name_allows_legitimate_aliases(secret_name): + raise_if_unsafe_secret_name(secret_name)