diff --git a/litellm-rust/crates/ai-gateway/src/messages/common_utils.rs b/litellm-rust/crates/ai-gateway/src/messages/common_utils.rs index fe4ac4cf26f..bc4b13bfe3d 100644 --- a/litellm-rust/crates/ai-gateway/src/messages/common_utils.rs +++ b/litellm-rust/crates/ai-gateway/src/messages/common_utils.rs @@ -1,5 +1,6 @@ use litellm_core::error::{json_type_name, CoreError}; use litellm_core::messages::transformation::AnthropicMessagesProviderConfig; +use litellm_core::providers::anthropic::messages::transformation::ANTHROPIC_MESSAGES_CONFIG; use litellm_core::providers::azure_ai::messages::transformation::AZURE_ANTHROPIC_MESSAGES_CONFIG; use litellm_core::CoreResult; use serde_json::{Map, Value}; @@ -19,6 +20,7 @@ pub(super) fn messages_provider_config( ) -> Option<&'static dyn AnthropicMessagesProviderConfig> { match provider { "azure_ai" => Some(&AZURE_ANTHROPIC_MESSAGES_CONFIG), + "anthropic" => Some(&ANTHROPIC_MESSAGES_CONFIG), _ => None, } } diff --git a/litellm-rust/crates/ai-gateway/src/messages/tests.rs b/litellm-rust/crates/ai-gateway/src/messages/tests.rs index 30f6642400e..1175d800e1c 100644 --- a/litellm-rust/crates/ai-gateway/src/messages/tests.rs +++ b/litellm-rust/crates/ai-gateway/src/messages/tests.rs @@ -52,9 +52,9 @@ fn write_response(body: &str) -> String { } #[test] -fn provider_config_only_resolves_azure_ai() { +fn provider_config_resolves_anthropic_and_azure_ai() { assert!(messages_provider_config("azure_ai").is_some()); - assert!(messages_provider_config("anthropic").is_none()); + assert!(messages_provider_config("anthropic").is_some()); assert!(messages_provider_config("openai").is_none()); } @@ -148,6 +148,52 @@ async fn messages_round_trip_builds_azure_request_and_passes_response_through() ); } +#[tokio::test] +async fn messages_round_trip_builds_native_anthropic_request() { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("binds"); + let addr = listener.local_addr().expect("addr"); + + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.expect("accepts request"); + let request = read_http_request(&mut socket).await; + let response_body = r#"{"id":"msg_1","type":"message","role":"assistant","content":[{"type":"text","text":"hi"}],"model":"claude-sonnet-4-5","stop_reason":"end_turn","usage":{"input_tokens":1,"output_tokens":2}}"#; + socket + .write_all(write_response(response_body).as_bytes()) + .await + .expect("writes response"); + request + }); + + let response = messages(MessagesRequest { + model: "claude-sonnet-4-5", + body: json!({ + "model": "claude-sonnet-4-5", + "max_tokens": 1024, + "messages": [{"role": "user", "content": "hi"}] + }), + api_key: Some("sk-ant"), + api_base: Some(&format!("http://{addr}")), + custom_llm_provider: Some("anthropic"), + extra_headers: None, + timeout: Some(Duration::from_secs(5)), + }) + .await + .expect("messages request succeeds"); + + assert_eq!(response["content"][0]["text"], "hi"); + assert_eq!(response["stop_reason"], "end_turn"); + + let request = server.await.expect("server task completes"); + let (head, _) = request.split_once("\r\n\r\n").expect("has body"); + assert!(head.starts_with("POST /v1/messages "), "{head}"); + let head_lower = head.to_ascii_lowercase(); + assert!(head_lower.contains("x-api-key: sk-ant"), "{head}"); + assert!( + head_lower.contains("anthropic-version: 2023-06-01"), + "{head}" + ); +} + #[tokio::test] async fn messages_does_not_duplicate_auth_when_x_api_key_supplied() { let listener = TcpListener::bind("127.0.0.1:0").await.expect("binds"); @@ -248,12 +294,12 @@ async fn messages_rejects_unsupported_provider() { body: json!({"model": "claude-3-5-sonnet", "max_tokens": 8, "messages": []}), api_key: Some("sk"), api_base: Some("http://127.0.0.1:1"), - custom_llm_provider: Some("anthropic"), + custom_llm_provider: Some("openai"), extra_headers: None, timeout: Some(Duration::from_millis(50)), }) .await .expect_err("unsupported provider errors"); - assert!(matches!(err, CoreError::InvalidProvider(provider) if provider == "anthropic")); + assert!(matches!(err, CoreError::InvalidProvider(provider) if provider == "openai")); } diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 3e6f9ee08ee..116ab905b88 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -1,5 +1,6 @@ import asyncio import json +import os import ssl from functools import lru_cache from typing import ( @@ -2247,6 +2248,10 @@ class BaseLLMHTTPHandler: "anthropic_messages", ) + @staticmethod + def _rust_env_enabled() -> bool: + return os.getenv("LITELLM_RUST", "").strip().lower() in {"1", "true", "yes", "on"} + @staticmethod async def _maybe_rust_anthropic_messages( *, @@ -2261,7 +2266,9 @@ class BaseLLMHTTPHandler: request_body: dict, timeout: float | httpx.Timeout | None, ) -> AnthropicMessagesResponse | None: - if custom_llm_provider != "azure_ai" or litellm_params.get("rust") is not True: + if custom_llm_provider not in ("azure_ai", "anthropic"): + return None + if litellm_params.get("rust") is not True and not BaseLLMHTTPHandler._rust_env_enabled(): return None if stream and not rust_stream_eligible: return None diff --git a/tests/documentation_tests/test_env_keys.py b/tests/documentation_tests/test_env_keys.py index 60fbd505d67..3bf2c88a848 100644 --- a/tests/documentation_tests/test_env_keys.py +++ b/tests/documentation_tests/test_env_keys.py @@ -28,6 +28,7 @@ EXCLUDED_GUARD_ONLY_VARS = { # environment settings docs until the feature is ready for broad use. EXCLUDED_ROLLOUT_FLAGS = { "LITELLM_USE_RUST_OCR", + "LITELLM_RUST", } EXCLUDED_TERMINAL_VARS = { diff --git a/tests/e2e/coverage_registry/mgmt.yaml b/tests/e2e/coverage_registry/mgmt.yaml index 4fbaac0a205..a43e7103523 100644 --- a/tests/e2e/coverage_registry/mgmt.yaml +++ b/tests/e2e/coverage_registry/mgmt.yaml @@ -66,3 +66,4 @@ - {id: mgmt.config_override.hashicorp_vault.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "config_override_endpoints.py", rationale: "Vault integration (smoke)"} - {id: mgmt.workflow.list.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "workflow_management_endpoints.py", rationale: "Workflow tracking (smoke)"} - {id: mgmt.credential_migration.check.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "key_management_endpoints.py:4252", rationale: "Encryption migration (smoke)"} +- {id: mgmt.credential.new.serves_request, module: mgmt, tier: P1, surface: api, assertions: [serves_request], source: "credential_endpoints/endpoints.py:42", rationale: "Stored credential resolves into a deployment and serves a live /messages request"} diff --git a/tests/e2e/llm_translation/test_credential_messages_e2e.py b/tests/e2e/llm_translation/test_credential_messages_e2e.py new file mode 100644 index 00000000000..49ea748430e --- /dev/null +++ b/tests/e2e/llm_translation/test_credential_messages_e2e.py @@ -0,0 +1,49 @@ +"""Live e2e: stored credentials resolve into a deployment serving /v1/messages.""" + +from __future__ import annotations + +import os + +import pytest + +from e2e_config import unique_marker +from e2e_http import require_successful_call +from endpoints_client import EndpointsClient, MessagesResult +from lifecycle import ResourceManager +from models import CredentialCreateBody, LiteLLMParamsBody + +pytestmark = pytest.mark.e2e + + +class TestCredentialBackedMessages: + @pytest.mark.covers("mgmt.credential.new.serves_request") + def test_credential_backed_messages(self, endpoints_client: EndpointsClient, resources: ResourceManager) -> None: + marker = unique_marker() + credential_name = f"e2e-cred-{marker}" + model = f"e2e-cred-messages-{marker}" + anthropic_api_key = os.getenv("ANTHROPIC_API_KEY") + assert anthropic_api_key, "ANTHROPIC_API_KEY must be set for this live e2e test" + + endpoints_client.proxy.create_credential( + CredentialCreateBody( + credential_name=credential_name, + credential_values={"api_key": anthropic_api_key}, + ) + ) + resources.defer(lambda: endpoints_client.proxy.delete_credential(credential_name)) + + model_id = endpoints_client.create_model( + model, + LiteLLMParamsBody( + model="anthropic/claude-haiku-4-5", + litellm_credential_name=credential_name, + ), + ) + resources.defer(lambda: endpoints_client.delete_model(model_id)) + + key = resources.key() + result = endpoints_client.messages(key, model, "reply with one word") + require_successful_call(result) + parsed = MessagesResult.model_validate_json(result.body) + assert parsed.role == "assistant", f"unexpected role: {result.body[:300]}" + assert parsed.text.strip(), f"/v1/messages returned no text: {result.body[:300]}" diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 97f72c487b7..2e7bfe41e30 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -498,6 +498,7 @@ class LiteLLMParamsBody(BaseModel): model: str api_key: str | None = None + litellm_credential_name: str | None = None api_base: str | None = None api_version: str | None = None realtime_protocol: str | None = None @@ -562,6 +563,16 @@ class ModelDeleteBody(BaseModel): id: str +class CredentialCreateBody(BaseModel): + credential_name: str + credential_values: dict[str, str] + credential_info: dict[str, str] = {} + + +class CredentialCreateResponse(BaseModel): + success: bool + + # ---------- key / team / user / organization management ---------- diff --git a/tests/e2e/proxy_client.py b/tests/e2e/proxy_client.py index 7eb86046375..f1039257193 100644 --- a/tests/e2e/proxy_client.py +++ b/tests/e2e/proxy_client.py @@ -31,6 +31,8 @@ from models import ( ChatResponse, CountTokensBody, CountTokensResponse, + CredentialCreateBody, + CredentialCreateResponse, CustomerDeleteBody, EmbedBody, EmbedResponse, @@ -219,6 +221,26 @@ class ProxyClient: if not is_ok(result): warnings.warn(f"delete_model({model_id!r}) failed: {result}", stacklevel=2) + def create_credential(self, body: CredentialCreateBody) -> None: + unwrap( + self.transport.post( + "/credentials", + headers=self.transport.master, + json=body, + response_type=CredentialCreateResponse, + ) + ) + + def delete_credential(self, credential_name: str) -> None: + result = self.transport.delete( + f"/credentials/{credential_name}", + headers=self.transport.master, + json=NoBody(), + response_type=NoBody, + ) + if not is_ok(result): + warnings.warn(f"delete_credential({credential_name!r}) failed: {result}", stacklevel=2) + # ---- LLM calls ------------------------------------------------------ def chat(self, key: str, body: ChatBody) -> Result[ChatResponse]: diff --git a/tests/test_litellm/anthropic_interface/test_rust_bridge_messages.py b/tests/test_litellm/anthropic_interface/test_rust_bridge_messages.py index 26ca7d27210..b745ca8eadf 100644 --- a/tests/test_litellm/anthropic_interface/test_rust_bridge_messages.py +++ b/tests/test_litellm/anthropic_interface/test_rust_bridge_messages.py @@ -285,11 +285,60 @@ async def test_gate_skips_rust_when_flag_false(): @pytest.mark.asyncio -async def test_gate_skips_rust_for_non_azure_provider(): +async def test_gate_invokes_rust_for_native_anthropic_provider(): + bridge = RecordingAsyncMessages() + litellm.use_litellm_rust(True, amessages=bridge) + + response = await _gate( + custom_llm_provider="anthropic", + litellm_params=GenericLiteLLMParams(api_key="sk-ant", rust=True), + api_key="sk-ant", + api_base="https://api.anthropic.com", + headers={"x-api-key": "sk-ant", "anthropic-version": "2023-06-01"}, + ) + + assert response is not None + assert response["_hidden_params"]["additional_headers"] == {"x-litellm-rust": "true"} + assert bridge.calls[0]["custom_llm_provider"] == "anthropic" + assert bridge.calls[0]["api_key"] == "sk-ant" + + +@pytest.mark.asyncio +async def test_gate_invokes_rust_when_env_var_set(monkeypatch): + bridge = RecordingAsyncMessages() + litellm.use_litellm_rust(True, amessages=bridge) + monkeypatch.setenv("LITELLM_RUST", "1") + + response = await _gate( + custom_llm_provider="anthropic", + litellm_params=GenericLiteLLMParams(api_key="sk-ant"), + ) + + assert response is not None + assert bridge.calls[0]["custom_llm_provider"] == "anthropic" + + +@pytest.mark.asyncio +async def test_gate_env_var_falsey_does_not_enable(monkeypatch): + bridge = ExplodingAsyncMessages() + litellm.use_litellm_rust(True, amessages=bridge) + monkeypatch.setenv("LITELLM_RUST", "0") + + response = await _gate( + custom_llm_provider="anthropic", + litellm_params=GenericLiteLLMParams(api_key="sk-ant"), + ) + + assert response is None + assert bridge.calls == 0 + + +@pytest.mark.asyncio +async def test_gate_skips_rust_for_unsupported_provider(): bridge = ExplodingAsyncMessages() litellm.use_litellm_rust(True, amessages=bridge) - response = await _gate(custom_llm_provider="anthropic") + response = await _gate(custom_llm_provider="openai") assert response is None assert bridge.calls == 0