diff --git a/docker/build_from_pip/Dockerfile.build_from_pip b/docker/build_from_pip/Dockerfile.build_from_pip index 05236008ded..37185b307da 100644 --- a/docker/build_from_pip/Dockerfile.build_from_pip +++ b/docker/build_from_pip/Dockerfile.build_from_pip @@ -1,16 +1,14 @@ -FROM python:3.13-alpine +FROM python:3.13-slim WORKDIR /app ENV HOME=/home/litellm ENV PATH="${HOME}/venv/bin:$PATH" -# Install runtime dependencies -# Note: Using Python 3.13 for compatibility with ddtrace and other packages -# rust and cargo are required for building ddtrace from source -# musl-dev and libffi-dev are needed for some Python packages on Alpine -RUN apk update && \ - apk add --no-cache gcc musl-dev libffi-dev openssl openssl-dev rust cargo +# Install build dependencies (slim/Debian provides pre-built wheels for most packages) +RUN apt-get update && \ + apt-get install -y --no-install-recommends gcc libffi-dev && \ + rm -rf /var/lib/apt/lists/* RUN python -m venv ${HOME}/venv RUN ${HOME}/venv/bin/pip install --no-cache-dir --upgrade pip diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 6d20a71f039..80600585362 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -288,6 +288,8 @@ model LiteLLM_MCPServerTable { mcp_info Json? @default("{}") mcp_access_groups String[] allowed_tools String[] @default([]) + tool_name_to_display_name Json? @default("{}") + tool_name_to_description Json? @default("{}") extra_headers String[] @default([]) static_headers Json? @default("{}") // Health check status @@ -303,6 +305,21 @@ model LiteLLM_MCPServerTable { registration_url String? allow_all_keys Boolean @default(false) available_on_public_internet Boolean @default(true) + is_byok Boolean @default(false) + byok_description String[] @default([]) + byok_api_key_help_url String? +} + +// Per-user BYOK credentials for MCP servers +model LiteLLM_MCPUserCredentials { + id String @id @default(uuid()) + user_id String + server_id String + credential_b64 String + created_at DateTime @default(now()) @map("created_at") + updated_at DateTime @default(now()) @updatedAt @map("updated_at") + + @@unique([user_id, server_id]) } // Generate Tokens for Proxy @@ -353,6 +370,7 @@ model LiteLLM_VerificationToken { litellm_organization_table LiteLLM_OrganizationTable? @relation(fields: [organization_id], references: [organization_id]) litellm_project_table LiteLLM_ProjectTable? @relation(fields: [project_id], references: [project_id]) object_permission LiteLLM_ObjectPermissionTable? @relation(fields: [object_permission_id], references: [object_permission_id]) + jwt_key_mappings LiteLLM_JWTKeyMapping[] // SELECT COUNT(*) FROM (SELECT "public"."LiteLLM_VerificationToken"."token" FROM "public"."LiteLLM_VerificationToken" WHERE ("public"."LiteLLM_VerificationToken"."user_id" = $1 AND ("public"."LiteLLM_VerificationToken"."team_id" IS NULL OR "public"."LiteLLM_VerificationToken"."team_id" <> $2)) OFFSET $3 ) AS "sub" // SELECT ... FROM "public"."LiteLLM_VerificationToken" WHERE "public"."LiteLLM_VerificationToken"."user_id" = $1 OFFSET $2 @@ -365,6 +383,24 @@ model LiteLLM_VerificationToken { @@index([budget_reset_at, expires]) } +model LiteLLM_JWTKeyMapping { + id String @id @default(uuid()) + jwt_claim_name String // e.g. "sub", "email" + jwt_claim_value String // The claim value to match + token String // Hashed virtual key (FK) + description String? + is_active Boolean @default(true) + created_at DateTime @default(now()) + created_by String? + updated_at DateTime @default(now()) @updatedAt + updated_by String? + + litellm_verification_token LiteLLM_VerificationToken @relation(fields: [token], references: [token]) + + @@unique([jwt_claim_name, jwt_claim_value]) + @@index([jwt_claim_name, jwt_claim_value, is_active]) +} + // Deprecated keys during grace period - allows old key to work until revoke_at model LiteLLM_DeprecatedVerificationToken { id String @id @default(uuid()) @@ -1077,31 +1113,30 @@ model LiteLLM_PolicyAttachmentTable { updated_by String? } -// Global tool registry - auto-discovered from LLM responses; admins set input_policy/output_policy here +// Global tool registry - auto-discovered from LLM responses; admins set input/output policies here model LiteLLM_ToolTable { - tool_id String @id @default(uuid()) - tool_name String @unique // e.g. "huggingface_remote-mcp__dynamic_space" - origin String? // MCP server name or "user_defined" - input_policy String @default("untrusted") // "trusted" | "untrusted" | "blocked" - output_policy String @default("untrusted") // "trusted" | "untrusted" - call_count Int @default(0) // cumulative number of times this tool was seen - assignments Json? @default("{}") - key_hash String? // hash of the virtual key that first called this tool - team_id String? // team that first called this tool - key_alias String? // human-readable alias of the virtual key - user_agent String? // user-agent of the first request that discovered this tool - last_used_at DateTime? // timestamp of the most recent call - created_at DateTime @default(now()) - created_by String? - updated_at DateTime @default(now()) @updatedAt - updated_by String? + tool_id String @id @default(uuid()) + tool_name String @unique // e.g. "huggingface_remote-mcp__dynamic_space" + origin String? // MCP server name or "user_defined" + input_policy String @default("untrusted") // "trusted" | "untrusted" | "blocked" + output_policy String @default("untrusted") // "trusted" | "untrusted" + call_count Int @default(0) // cumulative number of times this tool was seen + assignments Json? @default("{}") + key_hash String? // hash of the virtual key that first called this tool + team_id String? // team that first called this tool + key_alias String? // human-readable alias of the virtual key + user_agent String? // user-agent of the first request that discovered this tool + last_used_at DateTime? // timestamp of the most recent call + created_at DateTime @default(now()) + created_by String? + updated_at DateTime @default(now()) @updatedAt + updated_by String? @@index([input_policy]) @@index([output_policy]) @@index([team_id]) } -// Per-(tool, team/key) policy overrides. When present, override replaces global tool policy for that scope. //Unified Access Groups table for storing unified access groups model LiteLLM_AccessGroupTable { access_group_id String @id @default(uuid()) diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index b9d07d7c544..518a381dd64 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -209,15 +209,67 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): return params + @staticmethod + def _resolve_refs(schema: Dict[str, Any], defs: Dict[str, Any]) -> Dict[str, Any]: + """ + Recursively resolve $ref references by inlining the referenced definitions. + + Anthropic's output_format API does not support $ref (external schema references). + This method replaces all $ref pointers with the actual definition content. + + Args: + schema: The JSON schema dictionary to process + defs: The $defs dictionary containing all definitions + + Returns: + A new dictionary with all $ref references resolved inline + """ + if not isinstance(schema, dict): + return schema + + # If this node is a $ref, resolve it + if "$ref" in schema: + ref_path = schema["$ref"] + # Handle both "/$defs/Name" and "#/$defs/Name" formats + ref_name = ref_path.split("/")[-1] + if ref_name in defs: + # Recursively resolve refs in the definition itself + return AnthropicConfig._resolve_refs(defs[ref_name], defs) + return schema + + result: Dict[str, Any] = {} + for key, value in schema.items(): + if key == "$defs": + # Skip $defs at the top level - they're inlined now + continue + elif key == "properties" and isinstance(value, dict): + result[key] = { + k: AnthropicConfig._resolve_refs(v, defs) + for k, v in value.items() + } + elif key == "items" and isinstance(value, dict): + result[key] = AnthropicConfig._resolve_refs(value, defs) + elif key in ("anyOf", "allOf", "oneOf") and isinstance(value, list): + result[key] = [ + AnthropicConfig._resolve_refs(item, defs) + for item in value + ] + else: + result[key] = value + + return result + @staticmethod def filter_anthropic_output_schema(schema: Dict[str, Any]) -> Dict[str, Any]: """ - Filter out unsupported fields from JSON schema for Anthropic's output_format API. + Filter and transform JSON schema for Anthropic's output_format API. - Anthropic's output_format doesn't support certain JSON schema properties: + Anthropic's output_format has specific requirements: - maxItems/minItems: Not supported for array types - minimum/maximum: Not supported for numeric types - minLength/maxLength: Not supported for string types + - $ref: External schema references are not supported (must be inlined) + - additionalProperties: Must be explicitly set to false for object types This mirrors the transformation done by the Anthropic Python SDK. See: https://platform.claude.com/docs/en/build-with-claude/structured-outputs#how-sdk-transformation-works @@ -238,6 +290,22 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): if not isinstance(schema, dict): return schema + # First, resolve all $ref references by inlining definitions. + # Anthropic does not support $ref in output_format schemas. + defs = schema.get("$defs", {}) + if defs or "$ref" in schema: + schema = AnthropicConfig._resolve_refs(schema, defs) + + return AnthropicConfig._filter_schema_recursive(schema) + + @staticmethod + def _filter_schema_recursive(schema: Dict[str, Any]) -> Dict[str, Any]: + """ + Recursively filter unsupported fields from a JSON schema for Anthropic. + """ + if not isinstance(schema, dict): + return schema + # All numeric/string/array constraints not supported by Anthropic unsupported_fields = { "maxItems", @@ -288,34 +356,41 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): if key == "properties" and isinstance(value, dict): result[key] = { - k: AnthropicConfig.filter_anthropic_output_schema(v) + k: AnthropicConfig._filter_schema_recursive(v) for k, v in value.items() } elif key == "items" and isinstance(value, dict): - result[key] = AnthropicConfig.filter_anthropic_output_schema(value) + result[key] = AnthropicConfig._filter_schema_recursive(value) elif key == "$defs" and isinstance(value, dict): result[key] = { - k: AnthropicConfig.filter_anthropic_output_schema(v) + k: AnthropicConfig._filter_schema_recursive(v) for k, v in value.items() } elif key == "anyOf" and isinstance(value, list): result[key] = [ - AnthropicConfig.filter_anthropic_output_schema(item) + AnthropicConfig._filter_schema_recursive(item) for item in value ] elif key == "allOf" and isinstance(value, list): result[key] = [ - AnthropicConfig.filter_anthropic_output_schema(item) + AnthropicConfig._filter_schema_recursive(item) for item in value ] elif key == "oneOf" and isinstance(value, list): result[key] = [ - AnthropicConfig.filter_anthropic_output_schema(item) + AnthropicConfig._filter_schema_recursive(item) for item in value ] else: result[key] = value + # Anthropic's output_format requires 'additionalProperties' to be explicitly + # set to false for object types. Pydantic's model_json_schema() (used when + # ref_template is set) does not include this, so we add it here. + # See: https://platform.claude.com/docs/en/build-with-claude/structured-outputs + if result.get("type") == "object" and "additionalProperties" not in result: + result["additionalProperties"] = False + return result def get_json_schema_from_pydantic_object( diff --git a/tests/local_testing/test_stream_chunk_builder.py b/tests/local_testing/test_stream_chunk_builder.py index ddb1546097c..7f711725101 100644 --- a/tests/local_testing/test_stream_chunk_builder.py +++ b/tests/local_testing/test_stream_chunk_builder.py @@ -658,6 +658,7 @@ def test_stream_chunk_builder_openai_audio_output_usage(): except Exception as e: if "openai-internal" in str(e): pytest.skip("Skipping test due to openai-internal error") + raise chunks = [] for chunk in completion: @@ -677,14 +678,28 @@ def test_stream_chunk_builder_openai_audio_output_usage(): print(f"response usage: {response.usage}") check_non_streaming_response(response) print(f"response: {response}") - # Convert both usage objects to dictionaries for easier comparison - usage_dict = usage_obj.model_dump(exclude_none=True) - response_usage_dict = response.usage.model_dump(exclude_none=True) - - # Simple dictionary comparison - assert ( - usage_dict == response_usage_dict - ), f"\nExpected: {usage_dict}\nGot: {response_usage_dict}" + # Compare key usage fields individually to avoid failures from new/extra fields + # that OpenAI may add to the usage object over time + assert usage_obj is not None, "usage_obj should not be None" + assert response.usage is not None, "response.usage should not be None" + assert response.usage.prompt_tokens == usage_obj.prompt_tokens, ( + f"prompt_tokens mismatch: {response.usage.prompt_tokens} != {usage_obj.prompt_tokens}" + ) + assert response.usage.completion_tokens == usage_obj.completion_tokens, ( + f"completion_tokens mismatch: {response.usage.completion_tokens} != {usage_obj.completion_tokens}" + ) + assert response.usage.total_tokens == usage_obj.total_tokens, ( + f"total_tokens mismatch: {response.usage.total_tokens} != {usage_obj.total_tokens}" + ) + # Verify completion_tokens_details are preserved (especially audio_tokens) + if usage_obj.completion_tokens_details is not None: + assert response.usage.completion_tokens_details is not None, ( + "completion_tokens_details should be preserved" + ) + if hasattr(usage_obj.completion_tokens_details, "audio_tokens") and usage_obj.completion_tokens_details.audio_tokens is not None: + assert response.usage.completion_tokens_details.audio_tokens == usage_obj.completion_tokens_details.audio_tokens, ( + f"audio_tokens mismatch: {response.usage.completion_tokens_details.audio_tokens} != {usage_obj.completion_tokens_details.audio_tokens}" + ) def test_stream_chunk_builder_empty_initial_chunk(): diff --git a/tests/local_testing/test_streaming.py b/tests/local_testing/test_streaming.py index f540acec67d..bb9dd4bdd47 100644 --- a/tests/local_testing/test_streaming.py +++ b/tests/local_testing/test_streaming.py @@ -3069,7 +3069,7 @@ def test_unit_test_custom_stream_wrapper_repeating_chunk( loop_amount, chunk_value, expected_chunk_fail ): """ - Test if InternalServerError raised if model enters infinite loop + Test if MidStreamFallbackError raised if model enters infinite loop Test if request passes if model loop is below accepted limit """ diff --git a/tests/mcp_tests/test_mcp_server.py b/tests/mcp_tests/test_mcp_server.py index 4854be02a17..cbec6bfe33c 100644 --- a/tests/mcp_tests/test_mcp_server.py +++ b/tests/mcp_tests/test_mcp_server.py @@ -395,6 +395,7 @@ async def test_mcp_http_transport_tool_not_found(): @pytest.mark.asyncio async def test_streamable_http_mcp_handler_mock(): """Test the streamable HTTP MCP handler functionality""" + from litellm.proxy._types import UserAPIKeyAuth # Mock the session manager and its methods mock_session_manager = AsyncMock() @@ -413,12 +414,29 @@ async def test_streamable_http_mcp_handler_mock(): mock_receive = AsyncMock() mock_send = AsyncMock() + mock_auth_result = ( + UserAPIKeyAuth(), + None, + None, + {}, + {}, + [], + ) + with patch( "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True, ), patch( "litellm.proxy._experimental.mcp_server.server.session_manager", mock_session_manager, + ), patch( + "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", + new=AsyncMock(return_value=mock_auth_result), + ), patch( + "litellm.proxy._experimental.mcp_server.server.set_auth_context", + ), patch( + "litellm.proxy._experimental.mcp_server.server._handle_stale_mcp_session", + new=AsyncMock(return_value=False), ): from litellm.proxy._experimental.mcp_server.server import ( handle_streamable_http_mcp, diff --git a/tests/router_unit_tests/test_router_endpoints.py b/tests/router_unit_tests/test_router_endpoints.py index b6ce6b03c43..c5a1d0b5e5d 100644 --- a/tests/router_unit_tests/test_router_endpoints.py +++ b/tests/router_unit_tests/test_router_endpoints.py @@ -128,12 +128,14 @@ async def test_transcription_on_router(): router_level_clients.append(str(_deployment_openai_client)) ## test 1: user facing function + audio_file.seek(0) response = await router.atranscription( model="whisper", file=audio_file, ) ## test 2: underlying function + audio_file.seek(0) response = await router._atranscription( model="whisper", file=audio_file, diff --git a/tests/test_litellm/llms/openrouter/responses/test_openrouter_responses_transformation.py b/tests/test_litellm/llms/openrouter/responses/test_openrouter_responses_transformation.py index 544ec1ec719..48cda7a65a8 100644 --- a/tests/test_litellm/llms/openrouter/responses/test_openrouter_responses_transformation.py +++ b/tests/test_litellm/llms/openrouter/responses/test_openrouter_responses_transformation.py @@ -60,11 +60,16 @@ class TestOpenRouterResponsesAPIConfig: ) assert headers["Authorization"] == "Bearer sk-or-test-key" - def test_validate_environment_raises_without_key(self): + def test_validate_environment_raises_without_key(self, monkeypatch): """validate_environment should raise when no API key is available.""" config = OpenRouterResponsesAPIConfig() from litellm.types.router import GenericLiteLLMParams + # Clear any API keys that might be set in the environment + monkeypatch.setattr(litellm, "api_key", None) + monkeypatch.delenv("OPENROUTER_API_KEY", raising=False) + monkeypatch.delenv("OR_API_KEY", raising=False) + try: config.validate_environment( headers={}, diff --git a/tests/test_litellm/responses/test_metadata_codex_callback.py b/tests/test_litellm/responses/test_metadata_codex_callback.py index 4c4ea764fe8..94655cfd90e 100644 --- a/tests/test_litellm/responses/test_metadata_codex_callback.py +++ b/tests/test_litellm/responses/test_metadata_codex_callback.py @@ -92,30 +92,33 @@ async def test_metadata_passed_to_custom_callback_codex_models(): original_callbacks = litellm.callbacks.copy() if litellm.callbacks else [] litellm.callbacks = [callback] - with patch( - "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", - new_callable=AsyncMock, - ) as mock_post: - mock_post.return_value = _make_mock_http_response( - mock_response.model_dump() - ) - # gpt-5.1-codex has mode=responses - routes through responses bridge - await litellm.acompletion( - model="gpt-5.1-codex", - messages=[{"role": "user", "content": "Hello"}], - metadata=test_metadata, - ) + try: + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + new_callable=AsyncMock, + ) as mock_post: + mock_post.return_value = _make_mock_http_response( + mock_response.model_dump() + ) + # gpt-5.1-codex has mode=responses - routes through responses bridge + await litellm.acompletion( + model="gpt-5.1-codex", + messages=[{"role": "user", "content": "Hello"}], + metadata=test_metadata, + ) - await asyncio.wait_for(callback.event.wait(), timeout=5.0) + await asyncio.wait_for(callback.event.wait(), timeout=5.0) - assert callback.captured_kwargs is not None, "Callback should have been invoked" + assert callback.captured_kwargs is not None, "Callback should have been invoked" - litellm_params = callback.captured_kwargs.get("litellm_params", {}) - metadata = litellm_params.get("metadata") or {} + litellm_params = callback.captured_kwargs.get("litellm_params", {}) + metadata = litellm_params.get("metadata") or {} - assert "foo" in metadata, "metadata['foo'] should be accessible in callback" - assert metadata["foo"] == "bar" - assert metadata.get("trace_id") == "test-123" + assert "foo" in metadata, "metadata['foo'] should be accessible in callback" + assert metadata["foo"] == "bar" + assert metadata.get("trace_id") == "test-123" + finally: + litellm.callbacks = original_callbacks @pytest.mark.asyncio @@ -152,27 +155,31 @@ async def test_metadata_passed_via_litellm_metadata_responses_api(): test_metadata = {"request_id": "req-456"} callback = MetadataCaptureCallback() + original_callbacks = litellm.callbacks.copy() if litellm.callbacks else [] litellm.callbacks = [callback] - with patch( - "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", - new_callable=AsyncMock, - ) as mock_post: - mock_post.return_value = _make_mock_http_response( - mock_response.model_dump() - ) - await litellm.aresponses( - model="gpt-4o", - input="hi", - litellm_metadata=test_metadata, - ) + try: + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + new_callable=AsyncMock, + ) as mock_post: + mock_post.return_value = _make_mock_http_response( + mock_response.model_dump() + ) + await litellm.aresponses( + model="gpt-4o", + input="hi", + litellm_metadata=test_metadata, + ) - await asyncio.wait_for(callback.event.wait(), timeout=5.0) + await asyncio.wait_for(callback.event.wait(), timeout=5.0) - assert callback.captured_kwargs is not None + assert callback.captured_kwargs is not None - litellm_params = callback.captured_kwargs.get("litellm_params", {}) - metadata = litellm_params.get("metadata") or {} + litellm_params = callback.captured_kwargs.get("litellm_params", {}) + metadata = litellm_params.get("metadata") or {} - assert "request_id" in metadata - assert metadata["request_id"] == "req-456" + assert "request_id" in metadata + assert metadata["request_id"] == "req-456" + finally: + litellm.callbacks = original_callbacks