mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-25 05:54:53 +00:00
44d99475a5
* test(integration): stop GET /__observations from draining the upstream log GET /__observations used to empty the queue, so a second read in a test saw nothing and a "never reached upstream" check could pass whether or not the request was forwarded. GET now returns every request since the last reset and DELETE /__observations is the reset. An autouse fixture resets the worker upstream before each test, which also covers tests that read without the gateway fixture. Discarded reads used as resets become DELETEs, helpers that accumulated drained reads now read one snapshot, and tests with several phases reset between them. The three vacuous negative checks (rag allowlist, guardrail opt-out chaos, xAI batch output) now assert the absence on the same read that shows a request which did reach the upstream. * style(integration): space the observations unit test like its neighbours
374 lines
16 KiB
Python
374 lines
16 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import uuid
|
|
from collections.abc import Iterator
|
|
from pathlib import Path
|
|
from typing import Final
|
|
|
|
import httpx
|
|
import pytest
|
|
from integration._support.client import (
|
|
JSON_OBJECT,
|
|
Gateway,
|
|
Scenario,
|
|
eventually,
|
|
gateway_from_environment,
|
|
object_value,
|
|
)
|
|
from integration._support.process import owned_proxy_process
|
|
from integration._support.upstream import ScenarioHandle, delete_scenario, register_scenario
|
|
from pydantic import JsonValue
|
|
|
|
from tests.integration.cost_calculation.cost_tracking_case import BinaryResponse, JsonResponse
|
|
|
|
_ROUTER_DEFAULTS: Final[dict[str, JsonValue]] = {
|
|
"prompt": "router default prompt",
|
|
"input": "router default input",
|
|
"max_tokens": 32,
|
|
}
|
|
_MESSAGE: Final[dict[str, JsonValue]] = {
|
|
"id": "msg-$UNIQUE_ID",
|
|
"type": "message",
|
|
"role": "assistant",
|
|
"model": "claude-haiku-4-5",
|
|
"content": [{"type": "text", "text": "scripted"}],
|
|
"stop_reason": "end_turn",
|
|
"stop_sequence": None,
|
|
"usage": {"input_tokens": 1, "output_tokens": 1},
|
|
}
|
|
_CHAT: Final[dict[str, JsonValue]] = {
|
|
"id": "chatcmpl-$UNIQUE_ID",
|
|
"object": "chat.completion",
|
|
"created": 1,
|
|
"model": "gpt-4o-mini",
|
|
"choices": [
|
|
{"index": 0, "message": {"role": "assistant", "content": "scripted"}, "finish_reason": "stop"},
|
|
],
|
|
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
|
|
}
|
|
_RESPONSE: Final[dict[str, JsonValue]] = {
|
|
"id": "resp_$UNIQUE_ID",
|
|
"object": "response",
|
|
"created_at": 1,
|
|
"status": "completed",
|
|
"model": "gpt-4o-mini",
|
|
"output": [],
|
|
"usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2},
|
|
}
|
|
_VECTOR_STORE_FILE: Final[dict[str, JsonValue]] = {
|
|
"id": "file-audit",
|
|
"object": "vector_store.file",
|
|
"created_at": 1,
|
|
"usage_bytes": 0,
|
|
"vector_store_id": "vs-audit",
|
|
"status": "completed",
|
|
"last_error": None,
|
|
"attributes": {},
|
|
}
|
|
_SEVENTEEN_ATTRIBUTES: Final[dict[str, JsonValue]] = {f"key_{index:02d}": "v" for index in range(17)}
|
|
|
|
|
|
def _openai_error(message: str, param: str) -> JsonResponse:
|
|
return JsonResponse(
|
|
content_type="application/json",
|
|
status=400,
|
|
body={"error": {"message": message, "type": "invalid_request_error", "param": param, "code": None}},
|
|
)
|
|
|
|
|
|
class _Observations:
|
|
def __init__(self, url: str) -> None:
|
|
self.url = url.rstrip("/")
|
|
self.items: tuple[dict[str, JsonValue], ...] = ()
|
|
|
|
def read(self) -> tuple[dict[str, JsonValue], ...]:
|
|
with httpx.Client(timeout=10, trust_env=False) as client:
|
|
payload: Final = JSON_OBJECT.validate_python(
|
|
client.get(f"{self.url}/__observations?include_method=true").json()
|
|
)
|
|
requests: Final = payload.get("requests")
|
|
assert isinstance(requests, list)
|
|
self.items = tuple(object_value(item) for item in requests if isinstance(item, dict))
|
|
return self.items
|
|
|
|
def provider_calls(self, identity: str) -> tuple[dict[str, JsonValue], ...]:
|
|
return tuple(
|
|
item
|
|
for item in self.items
|
|
if f"/{identity}/" in str(item.get("path"))
|
|
and not (item.get("method") == "GET" and str(item.get("path", "")).endswith(("/v1/models", "/models")))
|
|
)
|
|
|
|
|
|
def _scripted(scenario: Scenario, prefix: str, response: JsonResponse | BinaryResponse) -> tuple[str, ScenarioHandle]:
|
|
identity: Final = f"{prefix}-{uuid.uuid4().hex}"
|
|
handle: Final = register_scenario(identity, response)
|
|
scenario.cleanups.callback(delete_scenario, handle)
|
|
return identity, handle
|
|
|
|
|
|
def _outbound(gateway: Gateway, identity: str) -> dict[str, JsonValue]:
|
|
observations: Final = _Observations(gateway.upstream_url)
|
|
eventually(observations.read, lambda _items: len(observations.provider_calls(identity)) == 1, seconds=20)
|
|
return object_value(observations.provider_calls(identity)[0]["body"])
|
|
|
|
|
|
def _assert_no_provider_call(gateway: Gateway, identity: str) -> None:
|
|
observations: Final = _Observations(gateway.upstream_url)
|
|
observations.read()
|
|
assert observations.provider_calls(identity) == ()
|
|
|
|
|
|
def _assert_no_call_to_deployment(gateway: Gateway, model_name: str) -> None:
|
|
observations: Final = _Observations(gateway.upstream_url)
|
|
observations.read()
|
|
calls: Final = tuple(
|
|
f"{item.get('method')} {item.get('path')}"
|
|
for item in observations.items
|
|
if f"/{model_name}-" in str(item.get("path")) and not str(item.get("path")).endswith("/models")
|
|
)
|
|
assert calls == (), calls
|
|
|
|
|
|
def _post(gateway: Gateway, path: str, body: dict[str, JsonValue]) -> httpx.Response:
|
|
return gateway.client.post(path, json=body, headers={"Authorization": f"Bearer {gateway.key}"})
|
|
|
|
|
|
def _invalid_request(response: httpx.Response) -> dict[str, JsonValue]:
|
|
assert response.status_code == 400, response.text
|
|
error: Final = object_value(JSON_OBJECT.validate_python(response.json())["error"])
|
|
assert error.get("type") == "invalid_request_error", response.text
|
|
return error
|
|
|
|
|
|
def _write_config(path: Path, config: dict[str, JsonValue]) -> Path:
|
|
path.write_text(json.dumps(config), encoding="utf-8")
|
|
return path
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def router_default_proxy(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]:
|
|
"""A proxy whose router-wide defaults name positional and route-required params,
|
|
with client-side `user_config` allowed so a null value can be sent."""
|
|
directory: Final = tmp_path_factory.mktemp("router-defaults")
|
|
with gateway_from_environment() as gateway, gateway.scenario() as scenario:
|
|
image_identity, image_handle = _scripted(
|
|
scenario, "followup-image", JsonResponse(content_type="application/json", body={"created": 1, "data": []})
|
|
)
|
|
text_identity, text_handle = _scripted(
|
|
scenario, "followup-text", JsonResponse(content_type="application/json", body={"choices": []})
|
|
)
|
|
speech_identity, speech_handle = _scripted(
|
|
scenario, "followup-speech", BinaryResponse(content_type="audio/mpeg", length=16)
|
|
)
|
|
message_identity, message_handle = _scripted(
|
|
scenario, "followup-message", JsonResponse(content_type="application/json", body=_MESSAGE)
|
|
)
|
|
config: Final = _write_config(
|
|
directory / "router-defaults.yaml",
|
|
{
|
|
"model_list": [
|
|
{
|
|
"model_name": "followup-image",
|
|
"litellm_params": {
|
|
"model": "openai/gpt-image-1",
|
|
"api_base": image_handle.api_base(),
|
|
"api_key": image_identity,
|
|
},
|
|
},
|
|
{
|
|
"model_name": "followup-text",
|
|
"litellm_params": {
|
|
"model": "openai/gpt-3.5-turbo-instruct",
|
|
"api_base": text_handle.api_base(),
|
|
"api_key": text_identity,
|
|
},
|
|
},
|
|
{
|
|
"model_name": "followup-speech",
|
|
"litellm_params": {
|
|
"model": "openai/gpt-4o-mini-tts",
|
|
"api_base": speech_handle.api_base(),
|
|
"api_key": speech_identity,
|
|
},
|
|
},
|
|
{
|
|
"model_name": "followup-message",
|
|
"litellm_params": {
|
|
"model": "anthropic/claude-haiku-4-5",
|
|
"api_base": message_handle.api_base(),
|
|
"api_key": message_identity,
|
|
},
|
|
},
|
|
],
|
|
"router_settings": {"default_litellm_params": _ROUTER_DEFAULTS},
|
|
"general_settings": {"allow_client_side_credentials": True},
|
|
},
|
|
)
|
|
with owned_proxy_process(gateway, directory, {}, config=config) as owned:
|
|
yield owned.gateway
|
|
|
|
|
|
def test_ocr_multipart_without_file_returns_400(gateway: Gateway) -> None:
|
|
with gateway.scenario() as scenario:
|
|
identity, handle = _scripted(
|
|
scenario, "followup-ocr", JsonResponse(content_type="application/json", body={"pages": []})
|
|
)
|
|
model: Final = scenario.model(model="mistral/mistral-ocr-latest", api_base=handle.api_base(), api_key=identity)
|
|
response: Final = gateway.client.post(
|
|
"/v1/ocr", files={"model": (None, model)}, headers={"Authorization": f"Bearer {gateway.key}"}
|
|
)
|
|
error: Final = _invalid_request(response)
|
|
assert error.get("param") == "file", response.text
|
|
_assert_no_provider_call(gateway, identity)
|
|
|
|
|
|
def test_ocr_empty_file_returns_400(gateway: Gateway) -> None:
|
|
with gateway.scenario() as scenario:
|
|
identity, handle = _scripted(
|
|
scenario, "followup-ocr-empty", JsonResponse(content_type="application/json", body={"pages": []})
|
|
)
|
|
model: Final = scenario.model(model="mistral/mistral-ocr-latest", api_base=handle.api_base(), api_key=identity)
|
|
response: Final = gateway.request_multipart(
|
|
"/v1/ocr", {"model": model}, {"file": ("empty.pdf", b"", "application/pdf")}
|
|
)
|
|
error: Final = _invalid_request(response)
|
|
assert error.get("message") == "Uploaded file is empty", response.text
|
|
_assert_no_provider_call(gateway, identity)
|
|
|
|
|
|
@pytest.mark.parametrize("max_output_tokens", (-1, 0), ids=("negative", "zero"))
|
|
def test_responses_non_positive_max_output_tokens_returns_provider_400(
|
|
gateway: Gateway, max_output_tokens: int
|
|
) -> None:
|
|
message: Final = f"Invalid 'max_output_tokens': integer below minimum value. Expected a value >= 16, but got {max_output_tokens} instead."
|
|
with gateway.scenario() as scenario:
|
|
identity, handle = _scripted(scenario, "followup-max-output", _openai_error(message, "max_output_tokens"))
|
|
model: Final = scenario.model(model="openai/gpt-4o-mini", api_base=handle.api_base(), api_key=identity)
|
|
response: Final = _post(
|
|
gateway,
|
|
"/v1/responses",
|
|
{"model": model, "input": "Say pong", "store": False, "max_output_tokens": max_output_tokens},
|
|
)
|
|
assert response.status_code == 400, response.text
|
|
assert message in response.text, response.text
|
|
assert _outbound(gateway, identity).get("max_output_tokens") == max_output_tokens
|
|
|
|
|
|
def test_responses_small_positive_max_output_tokens_is_still_raised_to_minimum(gateway: Gateway) -> None:
|
|
with gateway.scenario() as scenario:
|
|
identity, handle = _scripted(
|
|
scenario, "followup-max-output-min", JsonResponse(content_type="application/json", body=_RESPONSE)
|
|
)
|
|
model: Final = scenario.model(model="openai/gpt-4o-mini", api_base=handle.api_base(), api_key=identity)
|
|
response: Final = _post(
|
|
gateway, "/v1/responses", {"model": model, "input": "Say pong", "store": False, "max_output_tokens": 1}
|
|
)
|
|
assert response.status_code == 200, response.text
|
|
assert _outbound(gateway, identity).get("max_output_tokens") == 16
|
|
|
|
|
|
def test_team_daily_activity_end_before_start_returns_400(gateway: Gateway) -> None:
|
|
response: Final = gateway.request(
|
|
"GET",
|
|
"/team/daily/activity",
|
|
params={"start_date": "2026-10-01", "end_date": "2026-09-24", "page": "1"},
|
|
)
|
|
assert response.status_code == 400, response.text
|
|
assert response.json() == {"detail": {"error": "end_date must be on or after start_date"}}, response.text
|
|
|
|
|
|
def _registered_store(gateway: Gateway, scenario: Scenario, identity: str, handle: ScenarioHandle) -> str:
|
|
store: Final = f"vs-{uuid.uuid4().hex}"
|
|
model: Final = scenario.model(model="openai/text-embedding-3-small", api_base=handle.api_base(), api_key=identity)
|
|
gateway.post(
|
|
"/vector_store/new",
|
|
{
|
|
"vector_store_id": store,
|
|
"custom_llm_provider": "openai",
|
|
"litellm_params": {"model": model, "api_base": handle.api_base(), "api_key": identity},
|
|
},
|
|
)
|
|
scenario.cleanups.callback(gateway.post, "/vector_store/delete", {"vector_store_id": store})
|
|
return store
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("path", "body"),
|
|
(
|
|
("/v1/vector_stores/{store}/files/file-audit", {"attributes": _SEVENTEEN_ATTRIBUTES}),
|
|
("/v1/vector_stores/{store}/files", {"file_id": "file-audit", "attributes": _SEVENTEEN_ATTRIBUTES}),
|
|
),
|
|
ids=("update", "create"),
|
|
)
|
|
def test_vector_store_file_attributes_over_provider_limit_reach_provider(
|
|
gateway: Gateway, path: str, body: dict[str, JsonValue]
|
|
) -> None:
|
|
message: Final = (
|
|
"Invalid 'attributes': too many properties. "
|
|
"Expected an object with at most 16 properties, but got an object with 17 properties instead."
|
|
)
|
|
with gateway.scenario() as scenario:
|
|
identity, handle = _scripted(scenario, "followup-attributes", _openai_error(message, "attributes"))
|
|
store: Final = _registered_store(gateway, scenario, identity, handle)
|
|
response: Final = _post(gateway, path.format(store=store), body)
|
|
assert response.status_code == 400, response.text
|
|
assert message in response.text, response.text
|
|
assert _outbound(gateway, identity).get("attributes") == _SEVENTEEN_ATTRIBUTES
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("path", "body", "missing"),
|
|
(
|
|
("/v1/images/generations", {"model": "followup-image"}, "prompt"),
|
|
("/v1/completions", {"model": "followup-text"}, "prompt"),
|
|
("/v1/audio/speech", {"model": "followup-speech", "voice": "alloy"}, "input"),
|
|
),
|
|
ids=("image-prompt", "text-completion-prompt", "speech-input"),
|
|
)
|
|
def test_router_default_for_positional_param_returns_400(
|
|
router_default_proxy: Gateway, path: str, body: dict[str, JsonValue], missing: str
|
|
) -> None:
|
|
response: Final = _post(router_default_proxy, path, body)
|
|
error: Final = _invalid_request(response)
|
|
assert error.get("param") == missing, response.text
|
|
_assert_no_call_to_deployment(router_default_proxy, str(body["model"]))
|
|
|
|
|
|
def test_router_default_for_model_outside_router_returns_400(router_default_proxy: Gateway) -> None:
|
|
response: Final = _post(router_default_proxy, "/v1/moderations", {"model": "omni-moderation-2024-09-26"})
|
|
error: Final = _invalid_request(response)
|
|
assert error.get("param") == "input", response.text
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("body", "expected_max_tokens"),
|
|
(
|
|
({"messages": [{"role": "user", "content": "router default"}]}, 32),
|
|
({"messages": [{"role": "user", "content": "explicit"}], "max_tokens": 8}, 8),
|
|
),
|
|
ids=("router-default", "explicit"),
|
|
)
|
|
def test_null_user_config_is_treated_as_absent(
|
|
router_default_proxy: Gateway, body: dict[str, JsonValue], expected_max_tokens: int
|
|
) -> None:
|
|
observations: Final = _Observations(router_default_proxy.upstream_url)
|
|
observations.read()
|
|
before: Final = len(observations.items)
|
|
response: Final = _post(
|
|
router_default_proxy, "/v1/messages", {"model": "followup-message", "user_config": None, **body}
|
|
)
|
|
assert response.status_code == 200, response.text
|
|
assert JSON_OBJECT.validate_python(response.json()).get("type") == "message", response.text
|
|
eventually(
|
|
observations.read,
|
|
lambda items: any("followup-message-" in str(item.get("path")) for item in items[before:]),
|
|
seconds=20,
|
|
)
|
|
outbound: Final = next(
|
|
object_value(item["body"])
|
|
for item in observations.items[before:]
|
|
if "followup-message-" in str(item.get("path"))
|
|
)
|
|
assert outbound.get("max_tokens") == expected_max_tokens
|