Merge remote-tracking branch 'origin/litellm_internal_staging' into litellm_otel_v2_admin_owned_destinations

litellm-rust/crates/ai-gateway/src/messages/common_utils.rs

@@ -1,5 +1,6 @@
 use litellm_core::error::{json_type_name, CoreError};
 use litellm_core::messages::transformation::AnthropicMessagesProviderConfig;
+use litellm_core::providers::anthropic::messages::transformation::ANTHROPIC_MESSAGES_CONFIG;
 use litellm_core::providers::azure_ai::messages::transformation::AZURE_ANTHROPIC_MESSAGES_CONFIG;
 use litellm_core::CoreResult;
 use serde_json::{Map, Value};
@@ -19,6 +20,7 @@ pub(super) fn messages_provider_config(
 ) -> Option<&'static dyn AnthropicMessagesProviderConfig> {
     match provider {
         "azure_ai" => Some(&AZURE_ANTHROPIC_MESSAGES_CONFIG),
+        "anthropic" => Some(&ANTHROPIC_MESSAGES_CONFIG),
         _ => None,
     }
 }

litellm-rust/crates/ai-gateway/src/messages/tests.rs

@@ -52,9 +52,9 @@ fn write_response(body: &str) -> String {
 }
 
 #[test]
-fn provider_config_only_resolves_azure_ai() {
+fn provider_config_resolves_anthropic_and_azure_ai() {
     assert!(messages_provider_config("azure_ai").is_some());
-    assert!(messages_provider_config("anthropic").is_none());
+    assert!(messages_provider_config("anthropic").is_some());
     assert!(messages_provider_config("openai").is_none());
 }
 
@@ -148,6 +148,52 @@ async fn messages_round_trip_builds_azure_request_and_passes_response_through()
     );
 }
 
+#[tokio::test]
+async fn messages_round_trip_builds_native_anthropic_request() {
+    let listener = TcpListener::bind("127.0.0.1:0").await.expect("binds");
+    let addr = listener.local_addr().expect("addr");
+
+    let server = tokio::spawn(async move {
+        let (mut socket, _) = listener.accept().await.expect("accepts request");
+        let request = read_http_request(&mut socket).await;
+        let response_body = r#"{"id":"msg_1","type":"message","role":"assistant","content":[{"type":"text","text":"hi"}],"model":"claude-sonnet-4-5","stop_reason":"end_turn","usage":{"input_tokens":1,"output_tokens":2}}"#;
+        socket
+            .write_all(write_response(response_body).as_bytes())
+            .await
+            .expect("writes response");
+        request
+    });
+
+    let response = messages(MessagesRequest {
+        model: "claude-sonnet-4-5",
+        body: json!({
+            "model": "claude-sonnet-4-5",
+            "max_tokens": 1024,
+            "messages": [{"role": "user", "content": "hi"}]
+        }),
+        api_key: Some("sk-ant"),
+        api_base: Some(&format!("http://{addr}")),
+        custom_llm_provider: Some("anthropic"),
+        extra_headers: None,
+        timeout: Some(Duration::from_secs(5)),
+    })
+    .await
+    .expect("messages request succeeds");
+
+    assert_eq!(response["content"][0]["text"], "hi");
+    assert_eq!(response["stop_reason"], "end_turn");
+
+    let request = server.await.expect("server task completes");
+    let (head, _) = request.split_once("\r\n\r\n").expect("has body");
+    assert!(head.starts_with("POST /v1/messages "), "{head}");
+    let head_lower = head.to_ascii_lowercase();
+    assert!(head_lower.contains("x-api-key: sk-ant"), "{head}");
+    assert!(
+        head_lower.contains("anthropic-version: 2023-06-01"),
+        "{head}"
+    );
+}
+
 #[tokio::test]
 async fn messages_does_not_duplicate_auth_when_x_api_key_supplied() {
     let listener = TcpListener::bind("127.0.0.1:0").await.expect("binds");
@@ -248,12 +294,12 @@ async fn messages_rejects_unsupported_provider() {
         body: json!({"model": "claude-3-5-sonnet", "max_tokens": 8, "messages": []}),
         api_key: Some("sk"),
         api_base: Some("http://127.0.0.1:1"),
-        custom_llm_provider: Some("anthropic"),
+        custom_llm_provider: Some("openai"),
         extra_headers: None,
         timeout: Some(Duration::from_millis(50)),
     })
     .await
     .expect_err("unsupported provider errors");
 
-    assert!(matches!(err, CoreError::InvalidProvider(provider) if provider == "anthropic"));
+    assert!(matches!(err, CoreError::InvalidProvider(provider) if provider == "openai"));
 }

litellm/llms/custom_httpx/llm_http_handler.py

@@ -1,5 +1,6 @@
 import asyncio
 import json
+import os
 import ssl
 from functools import lru_cache
 from typing import (
@@ -2247,6 +2248,10 @@ class BaseLLMHTTPHandler:
             "anthropic_messages",
         )
 
+    @staticmethod
+    def _rust_env_enabled() -> bool:
+        return os.getenv("LITELLM_RUST", "").strip().lower() in {"1", "true", "yes", "on"}
+
     @staticmethod
     async def _maybe_rust_anthropic_messages(
         *,
@@ -2261,7 +2266,9 @@ class BaseLLMHTTPHandler:
         request_body: dict,
         timeout: float | httpx.Timeout | None,
     ) -> AnthropicMessagesResponse | None:
-        if custom_llm_provider != "azure_ai" or litellm_params.get("rust") is not True:
+        if custom_llm_provider not in ("azure_ai", "anthropic"):
+            return None
+        if litellm_params.get("rust") is not True and not BaseLLMHTTPHandler._rust_env_enabled():
             return None
         if stream and not rust_stream_eligible:
             return None

tests/documentation_tests/test_env_keys.py

@@ -28,6 +28,7 @@ EXCLUDED_GUARD_ONLY_VARS = {
 # environment settings docs until the feature is ready for broad use.
 EXCLUDED_ROLLOUT_FLAGS = {
     "LITELLM_USE_RUST_OCR",
+    "LITELLM_RUST",
 }
 
 EXCLUDED_TERMINAL_VARS = {

tests/e2e/coverage_registry/mgmt.yaml

@@ -66,3 +66,4 @@
 - {id: mgmt.config_override.hashicorp_vault.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "config_override_endpoints.py", rationale: "Vault integration (smoke)"}
 - {id: mgmt.workflow.list.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "workflow_management_endpoints.py", rationale: "Workflow tracking (smoke)"}
 - {id: mgmt.credential_migration.check.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "key_management_endpoints.py:4252", rationale: "Encryption migration (smoke)"}
+- {id: mgmt.credential.new.serves_request, module: mgmt, tier: P1, surface: api, assertions: [serves_request], source: "credential_endpoints/endpoints.py:42", rationale: "Stored credential resolves into a deployment and serves a live /messages request"}

tests/e2e/llm_translation/test_credential_messages_e2e.py

@@ -0,0 +1,49 @@
+"""Live e2e: stored credentials resolve into a deployment serving /v1/messages."""
+
+from __future__ import annotations
+
+import os
+
+import pytest
+
+from e2e_config import unique_marker
+from e2e_http import require_successful_call
+from endpoints_client import EndpointsClient, MessagesResult
+from lifecycle import ResourceManager
+from models import CredentialCreateBody, LiteLLMParamsBody
+
+pytestmark = pytest.mark.e2e
+
+
+class TestCredentialBackedMessages:
+    @pytest.mark.covers("mgmt.credential.new.serves_request")
+    def test_credential_backed_messages(self, endpoints_client: EndpointsClient, resources: ResourceManager) -> None:
+        marker = unique_marker()
+        credential_name = f"e2e-cred-{marker}"
+        model = f"e2e-cred-messages-{marker}"
+        anthropic_api_key = os.getenv("ANTHROPIC_API_KEY")
+        assert anthropic_api_key, "ANTHROPIC_API_KEY must be set for this live e2e test"
+
+        endpoints_client.proxy.create_credential(
+            CredentialCreateBody(
+                credential_name=credential_name,
+                credential_values={"api_key": anthropic_api_key},
+            )
+        )
+        resources.defer(lambda: endpoints_client.proxy.delete_credential(credential_name))
+
+        model_id = endpoints_client.create_model(
+            model,
+            LiteLLMParamsBody(
+                model="anthropic/claude-haiku-4-5",
+                litellm_credential_name=credential_name,
+            ),
+        )
+        resources.defer(lambda: endpoints_client.delete_model(model_id))
+
+        key = resources.key()
+        result = endpoints_client.messages(key, model, "reply with one word")
+        require_successful_call(result)
+        parsed = MessagesResult.model_validate_json(result.body)
+        assert parsed.role == "assistant", f"unexpected role: {result.body[:300]}"
+        assert parsed.text.strip(), f"/v1/messages returned no text: {result.body[:300]}"

tests/e2e/models.py

@@ -498,6 +498,7 @@ class LiteLLMParamsBody(BaseModel):
 
     model: str
     api_key: str | None = None
+    litellm_credential_name: str | None = None
     api_base: str | None = None
     api_version: str | None = None
     realtime_protocol: str | None = None
@@ -562,6 +563,16 @@ class ModelDeleteBody(BaseModel):
     id: str
 
 
+class CredentialCreateBody(BaseModel):
+    credential_name: str
+    credential_values: dict[str, str]
+    credential_info: dict[str, str] = {}
+
+
+class CredentialCreateResponse(BaseModel):
+    success: bool
+
+
 # ---------- key / team / user / organization management ----------
 
 

tests/e2e/proxy_client.py

@@ -31,6 +31,8 @@ from models import (
     ChatResponse,
     CountTokensBody,
     CountTokensResponse,
+    CredentialCreateBody,
+    CredentialCreateResponse,
     CustomerDeleteBody,
     EmbedBody,
     EmbedResponse,
@@ -219,6 +221,26 @@ class ProxyClient:
         if not is_ok(result):
             warnings.warn(f"delete_model({model_id!r}) failed: {result}", stacklevel=2)
 
+    def create_credential(self, body: CredentialCreateBody) -> None:
+        unwrap(
+            self.transport.post(
+                "/credentials",
+                headers=self.transport.master,
+                json=body,
+                response_type=CredentialCreateResponse,
+            )
+        )
+
+    def delete_credential(self, credential_name: str) -> None:
+        result = self.transport.delete(
+            f"/credentials/{credential_name}",
+            headers=self.transport.master,
+            json=NoBody(),
+            response_type=NoBody,
+        )
+        if not is_ok(result):
+            warnings.warn(f"delete_credential({credential_name!r}) failed: {result}", stacklevel=2)
+
     # ---- LLM calls ------------------------------------------------------
 
     def chat(self, key: str, body: ChatBody) -> Result[ChatResponse]:

tests/test_litellm/anthropic_interface/test_rust_bridge_messages.py

@@ -285,11 +285,60 @@ async def test_gate_skips_rust_when_flag_false():
 
 
 @pytest.mark.asyncio
-async def test_gate_skips_rust_for_non_azure_provider():
+async def test_gate_invokes_rust_for_native_anthropic_provider():
+    bridge = RecordingAsyncMessages()
+    litellm.use_litellm_rust(True, amessages=bridge)
+
+    response = await _gate(
+        custom_llm_provider="anthropic",
+        litellm_params=GenericLiteLLMParams(api_key="sk-ant", rust=True),
+        api_key="sk-ant",
+        api_base="https://api.anthropic.com",
+        headers={"x-api-key": "sk-ant", "anthropic-version": "2023-06-01"},
+    )
+
+    assert response is not None
+    assert response["_hidden_params"]["additional_headers"] == {"x-litellm-rust": "true"}
+    assert bridge.calls[0]["custom_llm_provider"] == "anthropic"
+    assert bridge.calls[0]["api_key"] == "sk-ant"
+
+
+@pytest.mark.asyncio
+async def test_gate_invokes_rust_when_env_var_set(monkeypatch):
+    bridge = RecordingAsyncMessages()
+    litellm.use_litellm_rust(True, amessages=bridge)
+    monkeypatch.setenv("LITELLM_RUST", "1")
+
+    response = await _gate(
+        custom_llm_provider="anthropic",
+        litellm_params=GenericLiteLLMParams(api_key="sk-ant"),
+    )
+
+    assert response is not None
+    assert bridge.calls[0]["custom_llm_provider"] == "anthropic"
+
+
+@pytest.mark.asyncio
+async def test_gate_env_var_falsey_does_not_enable(monkeypatch):
+    bridge = ExplodingAsyncMessages()
+    litellm.use_litellm_rust(True, amessages=bridge)
+    monkeypatch.setenv("LITELLM_RUST", "0")
+
+    response = await _gate(
+        custom_llm_provider="anthropic",
+        litellm_params=GenericLiteLLMParams(api_key="sk-ant"),
+    )
+
+    assert response is None
+    assert bridge.calls == 0
+
+
+@pytest.mark.asyncio
+async def test_gate_skips_rust_for_unsupported_provider():
     bridge = ExplodingAsyncMessages()
     litellm.use_litellm_rust(True, amessages=bridge)
 
-    response = await _gate(custom_llm_provider="anthropic")
+    response = await _gate(custom_llm_provider="openai")
 
     assert response is None
     assert bridge.calls == 0