mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-03 07:23:33 +00:00
057c45f23f
The deleted wrapper put cache: {"no-cache": true} on every request body, so the
gateway's response cache never answered a re-sent prompt. The SDKs send nothing
of the sort, and the mid-conversation prompt-cache priming loop re-sends an
identical body until the provider reports a warm cache, which a cached reply
never does. NO_PROXY_CACHE in sdk_clients.py restores the field as extra_body
on every messages, responses, completions, and embeddings call.
The wrapper also returned a 4xx as a value where the SDKs raise. The Bedrock
safety_identifier test judges the captured Converse body, and Claude on Bedrock
rejects the forwarded field with a 400, so the /v1/responses leg now suppresses
openai.BadRequestError the way the chat leg carries the same 400 as a Result.
62 lines
2.2 KiB
Python
62 lines
2.2 KiB
Python
"""Real provider SDK clients pointed at the proxy, connected the way customers
|
|
connect (LIT-4577).
|
|
|
|
The OpenAI SDK drives the OpenAI-compatible surface (/responses, /embeddings,
|
|
/images/generations, /moderations, /audio/*) and the Anthropic SDK drives
|
|
/v1/messages, each authenticated with a litellm virtual key. Errors surface as
|
|
the SDK's own exceptions, exactly what an end user sees. Retries are disabled
|
|
so a proxy fault fails the test instead of being papered over, and the timeout
|
|
matches the shared transport's request budget.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from collections.abc import Mapping
|
|
from dataclasses import dataclass
|
|
from types import MappingProxyType
|
|
from typing import Final
|
|
|
|
from anthropic import Anthropic
|
|
from openai import OpenAI
|
|
|
|
from e2e_config import PROXY_BASE_URL, REQUEST_TIMEOUT
|
|
|
|
NO_PROXY_CACHE: Final = MappingProxyType({"cache": {"no-cache": True}})
|
|
"""``extra_body`` for every cacheable SDK call (messages, responses, completions,
|
|
embeddings): the gateway under test caches those call types, so an identical
|
|
re-send would otherwise be served from Redis instead of reaching the provider,
|
|
which hides provider-side behavior such as prompt-cache warm-up. The SDKs
|
|
themselves cannot bypass it (``Cache-Control`` only sets a TTL on the proxy)."""
|
|
|
|
|
|
def response_header(headers: Mapping[str, str], name: str) -> str | None:
|
|
"""Typed read of an SDK response header: httpx.Headers.get returns Any and
|
|
httpx itself is a banned import in suite code, so tests read headers through
|
|
the Mapping[str, str] interface Headers fulfils."""
|
|
return headers[name] if name in headers else None
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class SdkClients:
|
|
base_url: str
|
|
request_timeout: float
|
|
|
|
def openai(self, key: str) -> OpenAI:
|
|
return OpenAI(
|
|
base_url=self.base_url,
|
|
api_key=key,
|
|
timeout=self.request_timeout,
|
|
max_retries=0,
|
|
)
|
|
|
|
def anthropic(self, key: str) -> Anthropic:
|
|
return Anthropic(
|
|
base_url=self.base_url,
|
|
api_key=key,
|
|
timeout=self.request_timeout,
|
|
max_retries=0,
|
|
)
|
|
|
|
|
|
def build_sdk_clients() -> SdkClients:
|
|
return SdkClients(base_url=PROXY_BASE_URL, request_timeout=REQUEST_TIMEOUT)
|