mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-09 22:31:41 +00:00
08fa25042c
The shared proxy wrapper in tests/e2e/e2e_gateway.py was misnamed: Gateway is not a gateway server, it is the client every suite uses to talk to the proxy (keys, models, chat/embed/ocr, spend read-backs, poll helpers). Rename the module to proxy_client.py and the class to ProxyClient, with build_gateway becoming build_proxy_client and the GatewayProvider protocol becoming ProxyClientProvider. The .gateway attribute suites held is now .proxy. Only identifiers changed; prose and string literals that use the word gateway for the proxy-server concept were left alone. Each suite previously built its own instance through a per-suite build_client() that called build_gateway() inside, duplicating the proxy wiring across suites. There is now one session-scoped proxy fixture in tests/e2e/conftest.py; every suite's client fixture depends on it and injects it, so the wiring lives in one place. claude_code keeps building its own client directly since it has its own harness and does not use the shared fixtures. Behavior is unchanged: shared transport, data-plane/control-plane split routing, poll budget, typed request/response models, and resource cleanup all go through the same object.
63 lines
2.2 KiB
Python
63 lines
2.2 KiB
Python
"""Live e2e for model-specific request features: service_tier.
|
|
|
|
Each case asserts the feature took effect, not just a 200.
|
|
|
|
service_tier is an OpenAI concept. The proxy forwards it and the provider echoes
|
|
the tier back on the response, so sending a non-default tier ("priority") and
|
|
reading it back off ``service_tier`` proves the param was honored end to end;
|
|
litellm's own default injection (and service_tier="auto") both report "default",
|
|
so a "priority" echo can only come from the request being forwarded. "flex" is
|
|
avoided here because it is capacity-constrained and returns a transient 429 when
|
|
flex resources are unavailable. Bedrock and Vertex do not accept service_tier, so
|
|
that cell is OpenAI-only by design.
|
|
|
|
Prompt caching lives in test_cache_control.py.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from e2e_config import unique_marker
|
|
from e2e_http import unwrap
|
|
from lifecycle import ResourceManager
|
|
from models import ChatBody, ChatMessage, LiteLLMParamsBody
|
|
from passthrough_client import PassthroughClient
|
|
|
|
pytestmark = pytest.mark.e2e
|
|
|
|
SERVICE_TIER = "priority"
|
|
|
|
|
|
class TestServiceTier:
|
|
@pytest.mark.covers(
|
|
"llm.chat_completions.openai.service_tier.nonstream.works", exercised_on=[]
|
|
)
|
|
def test_openai_service_tier_is_echoed(
|
|
self, client: PassthroughClient, resources: ResourceManager
|
|
) -> None:
|
|
model = f"e2e-service-tier-{unique_marker()}"
|
|
model_id = client.proxy.create_model(
|
|
model,
|
|
LiteLLMParamsBody(
|
|
model="openai/gpt-5.5", api_key="os.environ/OPENAI_API_KEY"
|
|
),
|
|
)
|
|
resources.defer(lambda: client.proxy.delete_model(model_id))
|
|
key = resources.key()
|
|
|
|
response = unwrap(
|
|
client.proxy.chat(
|
|
key,
|
|
ChatBody(
|
|
model=model,
|
|
messages=[ChatMessage(role="user", content="reply with one word")],
|
|
max_tokens=64,
|
|
service_tier=SERVICE_TIER,
|
|
),
|
|
)
|
|
)
|
|
assert response.service_tier == SERVICE_TIER, (
|
|
f"service_tier not honored: sent {SERVICE_TIER!r}, response reported "
|
|
f"{response.service_tier!r} ({response})"
|
|
)
|