mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
test(s3_v2): audit matrix across chat, messages and responses surfaces with sink faults
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
15a051ac27
commit
9dd742891f
3 changed files with 853 additions and 1 deletions
338
tests/integration/observability/_s3_v2_support.py
Normal file
338
tests/integration/observability/_s3_v2_support.py
Normal file
|
|
@ -0,0 +1,338 @@
|
|||
import asyncio
|
||||
import json
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Mapping
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import anthropic
|
||||
import openai
|
||||
import yaml
|
||||
from integration._support.client import Gateway, JsonValue, eventually, object_value
|
||||
from integration._support.wire import Reply, Request
|
||||
|
||||
BUCKET: Final = "integration-bucket"
|
||||
PREFIX: Final = "integration-logs"
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class RecordingS3Sink:
|
||||
"""Records every accepted PUT body by target, tracks peak concurrency, and can reject a leading
|
||||
run of PUT attempts with a chosen status before accepting. Serves stored bodies back on GET."""
|
||||
|
||||
fail_attempts: int = 0
|
||||
fail_until: float = 0.0
|
||||
fail_status: int = 503
|
||||
delay_seconds: float = 0.5
|
||||
lock: threading.Lock = field(default_factory=threading.Lock)
|
||||
in_flight: int = 0
|
||||
peak: int = 0
|
||||
attempts: int = 0
|
||||
store: dict[str, bytes] = field(default_factory=dict) # mutable-ok: GET reads must see writes from earlier PUTs
|
||||
|
||||
def respond(self, request: Request) -> Reply:
|
||||
if request.method == "GET":
|
||||
body: Final = self.store.get(request.target)
|
||||
if body is None:
|
||||
return Reply(status=404)
|
||||
return Reply(body=body)
|
||||
assert request.method == "PUT", request.method
|
||||
assert request.target.startswith(f"/{BUCKET}/{PREFIX}/"), request.target
|
||||
with self.lock:
|
||||
self.attempts += 1
|
||||
if self.attempts <= self.fail_attempts or time.time() < self.fail_until:
|
||||
return Reply(
|
||||
status=self.fail_status,
|
||||
body=b"<Error><Code>SinkFailure</Code></Error>",
|
||||
content_type="application/xml",
|
||||
)
|
||||
self.in_flight += 1
|
||||
self.peak = max(self.peak, self.in_flight)
|
||||
self.store[request.target] = request.body
|
||||
time.sleep(self.delay_seconds)
|
||||
with self.lock:
|
||||
self.in_flight -= 1
|
||||
return Reply()
|
||||
|
||||
|
||||
def s3_config(
|
||||
path: Path, sink_url: str, extra: Mapping[str, JsonValue], settings: Mapping[str, JsonValue] | None = None
|
||||
) -> Path:
|
||||
config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
|
||||
config["litellm_settings"].update(
|
||||
{
|
||||
"callbacks": ["s3_v2"],
|
||||
"s3_callback_params": {
|
||||
"s3_bucket_name": BUCKET,
|
||||
"s3_region_name": "us-east-1",
|
||||
"s3_endpoint_url": sink_url,
|
||||
"s3_path": PREFIX,
|
||||
"s3_aws_access_key_id": "AKIAIOSFODNN7EXAMPLE",
|
||||
"s3_aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
|
||||
**extra,
|
||||
},
|
||||
**(settings or {}),
|
||||
}
|
||||
)
|
||||
target: Final = path / "s3_v2.yaml"
|
||||
target.write_text(yaml.safe_dump(config))
|
||||
return target
|
||||
|
||||
|
||||
def _chat_completion(identity: str) -> dict[str, JsonValue]:
|
||||
return {
|
||||
"id": identity,
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": "gpt-4o-mini",
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 11, "completion_tokens": 4, "total_tokens": 15},
|
||||
}
|
||||
|
||||
|
||||
def _chat_stream_frames(identity: str) -> tuple[bytes, ...]:
|
||||
chunks: Final = (
|
||||
{
|
||||
"id": identity,
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1,
|
||||
"model": "gpt-4o-mini",
|
||||
"choices": [{"index": 0, "delta": {"role": "assistant", "content": "ok"}, "finish_reason": None}],
|
||||
},
|
||||
{
|
||||
"id": identity,
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1,
|
||||
"model": "gpt-4o-mini",
|
||||
"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 11, "completion_tokens": 4, "total_tokens": 15},
|
||||
},
|
||||
)
|
||||
return tuple(f"data: {json.dumps(chunk)}\n\n".encode() for chunk in chunks) + (b"data: [DONE]\n\n",)
|
||||
|
||||
|
||||
def _messages_completion(identity: str) -> dict[str, JsonValue]:
|
||||
return {
|
||||
"id": identity,
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-sonnet-4-5-20250929",
|
||||
"content": [{"type": "text", "text": "ok"}],
|
||||
"stop_reason": "end_turn",
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 11, "output_tokens": 4},
|
||||
}
|
||||
|
||||
|
||||
def _messages_stream_frames(identity: str) -> tuple[bytes, ...]:
|
||||
events: Final = (
|
||||
(
|
||||
"message_start",
|
||||
{
|
||||
"type": "message_start",
|
||||
"message": {
|
||||
"id": identity,
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-sonnet-4-5-20250929",
|
||||
"content": [],
|
||||
"stop_reason": None,
|
||||
"usage": {"input_tokens": 11, "output_tokens": 1},
|
||||
},
|
||||
},
|
||||
),
|
||||
(
|
||||
"content_block_start",
|
||||
{"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}},
|
||||
),
|
||||
(
|
||||
"content_block_delta",
|
||||
{"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "ok"}},
|
||||
),
|
||||
("content_block_stop", {"type": "content_block_stop", "index": 0}),
|
||||
(
|
||||
"message_delta",
|
||||
{"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 4}},
|
||||
),
|
||||
("message_stop", {"type": "message_stop"}),
|
||||
)
|
||||
return tuple(f"event: {name}\ndata: {json.dumps(payload)}\n\n".encode() for name, payload in events)
|
||||
|
||||
|
||||
def _responses_completion(identity: str) -> dict[str, JsonValue]:
|
||||
return {
|
||||
"id": identity,
|
||||
"object": "response",
|
||||
"created_at": 1,
|
||||
"status": "completed",
|
||||
"model": "gpt-4o-mini",
|
||||
"output": [
|
||||
{
|
||||
"type": "message",
|
||||
"id": f"msg_{identity}",
|
||||
"status": "completed",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": "ok", "annotations": []}],
|
||||
}
|
||||
],
|
||||
"usage": {"input_tokens": 11, "output_tokens": 4, "total_tokens": 15},
|
||||
}
|
||||
|
||||
|
||||
def _responses_stream_frames(identity: str) -> tuple[bytes, ...]:
|
||||
events: Final = (
|
||||
(
|
||||
"response.created",
|
||||
{
|
||||
"type": "response.created",
|
||||
"response": {**_responses_completion(identity), "status": "in_progress", "output": []},
|
||||
},
|
||||
),
|
||||
(
|
||||
"response.output_text.delta",
|
||||
{
|
||||
"type": "response.output_text.delta",
|
||||
"item_id": f"msg_{identity}",
|
||||
"output_index": 0,
|
||||
"content_index": 0,
|
||||
"delta": "ok",
|
||||
},
|
||||
),
|
||||
("response.completed", {"type": "response.completed", "response": _responses_completion(identity)}),
|
||||
)
|
||||
return tuple(f"event: {name}\ndata: {json.dumps(payload)}\n\n".encode() for name, payload in events)
|
||||
|
||||
|
||||
def surface_reply(request: Request) -> Reply:
|
||||
"""Scripted upstream that echoes the caller's marker string back as the response id."""
|
||||
body: Final = json.loads(request.body)
|
||||
if request.target.endswith("/chat/completions"):
|
||||
identity: Final = body["messages"][0]["content"]
|
||||
if body.get("stream"):
|
||||
return Reply(content_type="text/event-stream", chunks=_chat_stream_frames(identity))
|
||||
return Reply(body=json.dumps(_chat_completion(identity)).encode())
|
||||
if request.target.endswith("/messages"):
|
||||
identity_messages: Final = body["messages"][0]["content"]
|
||||
if body.get("stream"):
|
||||
return Reply(content_type="text/event-stream", chunks=_messages_stream_frames(identity_messages))
|
||||
return Reply(body=json.dumps(_messages_completion(identity_messages)).encode())
|
||||
assert request.target.endswith("/responses"), request.target
|
||||
identity_responses: Final = body["input"]
|
||||
if body.get("stream"):
|
||||
return Reply(content_type="text/event-stream", chunks=_responses_stream_frames(identity_responses))
|
||||
return Reply(body=json.dumps(_responses_completion(identity_responses)).encode())
|
||||
|
||||
|
||||
SURFACES: Final = ("chat", "chat_stream", "messages", "messages_stream", "responses", "responses_stream")
|
||||
|
||||
|
||||
def call_surface(
|
||||
candidate: Gateway, surface: str, openai_model: str, anthropic_model: str, key: str, marker: str
|
||||
) -> tuple[str, str | None]:
|
||||
"""Drive one request through the given surface; return (client-visible response id, x-litellm-call-id)."""
|
||||
base: Final = str(candidate.client.base_url).rstrip("/")
|
||||
headers: Final = {"Authorization": f"Bearer {key}"}
|
||||
if surface == "chat":
|
||||
reply: Final = openai.OpenAI(base_url=f"{base}/v1", api_key=key).chat.completions.create(
|
||||
model=openai_model,
|
||||
messages=[{"role": "user", "content": marker}],
|
||||
extra_body={"cache": {"no-cache": True}},
|
||||
)
|
||||
return reply.id, None
|
||||
|
||||
async def chat_stream() -> str:
|
||||
stream = await openai.AsyncOpenAI(base_url=f"{base}/v1", api_key=key).chat.completions.create(
|
||||
model=openai_model,
|
||||
messages=[{"role": "user", "content": marker}],
|
||||
stream=True,
|
||||
extra_body={"cache": {"no-cache": True}},
|
||||
)
|
||||
seen = ""
|
||||
async for chunk in stream:
|
||||
seen = chunk.id # rebind-ok: the stream yields one chunk at a time
|
||||
return seen
|
||||
|
||||
if surface == "chat_stream":
|
||||
return asyncio.run(chat_stream()), None
|
||||
if surface in ("messages", "messages_stream"):
|
||||
client: Final = anthropic.Anthropic(base_url=base, api_key="anthropic-placeholder", default_headers=headers)
|
||||
if surface == "messages":
|
||||
reply_messages: Final = client.messages.create(
|
||||
model=anthropic_model, max_tokens=16, messages=[{"role": "user", "content": marker}]
|
||||
)
|
||||
return reply_messages.id, None
|
||||
with client.messages.stream(
|
||||
model=anthropic_model, max_tokens=16, messages=[{"role": "user", "content": marker}]
|
||||
) as stream:
|
||||
final: Final = stream.get_final_message()
|
||||
return final.id, None
|
||||
if surface == "responses":
|
||||
response: Final = candidate.request(
|
||||
"POST",
|
||||
"/v1/responses",
|
||||
{"model": openai_model, "input": marker, "cache": {"no-cache": True}},
|
||||
key=key,
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
return str(response.json()["id"]), response.headers.get("x-litellm-call-id")
|
||||
assert surface == "responses_stream", surface
|
||||
with candidate.client.stream(
|
||||
"POST",
|
||||
"/v1/responses",
|
||||
json={"model": openai_model, "input": marker, "stream": True},
|
||||
headers=headers,
|
||||
) as response:
|
||||
text: Final = response.read().decode()
|
||||
assert response.status_code == 200, text
|
||||
call_id: Final = response.headers.get("x-litellm-call-id")
|
||||
assert marker in text, text
|
||||
return marker, call_id
|
||||
|
||||
|
||||
def collect_payloads(sink: RecordingS3Sink, count: int, seconds: float = 60) -> tuple[dict[str, JsonValue], ...]:
|
||||
"""Wait until `count` stored payload lines exist, then return every stored payload object."""
|
||||
|
||||
def delivered() -> int:
|
||||
with sink.lock:
|
||||
bodies: Final = tuple(sink.store.values())
|
||||
return sum(len(body.splitlines()) for body in bodies)
|
||||
|
||||
eventually(delivered, lambda total: total >= count, seconds=seconds)
|
||||
with sink.lock:
|
||||
stored: Final = tuple(sink.store.values())
|
||||
return tuple(object_value(json.loads(line)) for body in stored for line in body.splitlines())
|
||||
|
||||
|
||||
def mixed_burst(
|
||||
candidate: Gateway, openai_model: str, anthropic_model: str, key: str, marker: str, per_surface: int = 8
|
||||
) -> tuple[tuple[str, str | None], ...]:
|
||||
"""Fire `per_surface` requests on every surface; returns (response id, x-litellm-call-id) per request."""
|
||||
jobs: Final = tuple(
|
||||
(surface, f"{marker}-{surface}-{index}") for surface in SURFACES for index in range(per_surface)
|
||||
)
|
||||
|
||||
def call(job: tuple[str, str]) -> tuple[str, str | None]:
|
||||
surface, identity = job
|
||||
return call_surface(candidate, surface, openai_model, anthropic_model, key, identity)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=48) as pool:
|
||||
return tuple(pool.map(call, jobs))
|
||||
|
||||
|
||||
def matched_ids(
|
||||
payloads: tuple[dict[str, JsonValue], ...], answered: tuple[tuple[str, str | None], ...]
|
||||
) -> frozenset[str]:
|
||||
"""Every payload must be accountable to an answered request by response id or litellm_call_id."""
|
||||
response_ids: Final = frozenset(observed for observed, _ in answered)
|
||||
call_ids: Final = frozenset(call_id for _, call_id in answered if call_id is not None)
|
||||
landed: Final = []
|
||||
for payload in payloads:
|
||||
if payload["id"] in response_ids:
|
||||
landed.append(payload["id"])
|
||||
continue
|
||||
assert payload["litellm_call_id"] in call_ids, f"unmatched payload {payload['id']!r}"
|
||||
landed.append(str(payload["id"]))
|
||||
return frozenset(landed)
|
||||
97
tests/integration/observability/test_s3_v2_flush_surfaces.py
Normal file
97
tests/integration/observability/test_s3_v2_flush_surfaces.py
Normal file
|
|
@ -0,0 +1,97 @@
|
|||
import re
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from _s3_v2_support import (
|
||||
BUCKET,
|
||||
PREFIX,
|
||||
RecordingS3Sink,
|
||||
collect_payloads,
|
||||
matched_ids,
|
||||
mixed_burst,
|
||||
s3_config,
|
||||
surface_reply,
|
||||
)
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.process import owned_proxy
|
||||
from integration._support.wire import wire_server
|
||||
|
||||
PER_REQUEST_KEY: Final = re.compile(rf"^/{BUCKET}/{PREFIX}/\d{{4}}-\d{{2}}-\d{{2}}/.+\.json$")
|
||||
BATCH_KEY: Final = re.compile(
|
||||
rf"^/{BUCKET}/{PREFIX}/\d{{4}}-\d{{2}}-\d{{2}}/batch_\d{{2}}-\d{{2}}-\d{{2}}_[0-9a-f]{{32}}\.jsonl$"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.covers("other.observability.s3_v2.mixed_surface_burst_bounds_puts_one_object_per_response_id")
|
||||
def test_s3_v2_mixed_surface_burst_bounds_puts_one_object_per_response_id(gateway: Gateway, tmp_path: Path) -> None:
|
||||
marker: Final = "s3mix" + uuid.uuid4().hex[:8]
|
||||
sink: Final = RecordingS3Sink()
|
||||
with wire_server(surface_reply) as provider, wire_server(sink.respond) as bucket:
|
||||
config: Final = s3_config(tmp_path, bucket.url, {})
|
||||
with (
|
||||
owned_proxy(gateway, tmp_path, {"DEFAULT_S3_FLUSH_INTERVAL_SECONDS": "3"}, config=config) as candidate,
|
||||
candidate.scenario() as scenario,
|
||||
):
|
||||
openai_model: Final = scenario.model(api_base=provider.url + "/v1", api_key="synthetic-provider-key")
|
||||
anthropic_model: Final = scenario.model(
|
||||
model="anthropic/claude-sonnet-4-5-20250929", api_base=provider.url, api_key="synthetic-provider-key"
|
||||
)
|
||||
key: Final = scenario.key(models=[openai_model, anthropic_model])
|
||||
answered: Final = mixed_burst(candidate, openai_model, anthropic_model, key, marker)
|
||||
payloads: Final = collect_payloads(sink, len(answered))
|
||||
targets: Final = tuple(sink.store.keys())
|
||||
assert len(provider.drain()) == 48
|
||||
assert sink.peak <= 16, f"peak concurrent PUTs {sink.peak} exceeded the default bound"
|
||||
assert all(PER_REQUEST_KEY.match(target) for target in targets), list(targets)
|
||||
assert len(targets) == 48
|
||||
assert matched_ids(payloads, answered)
|
||||
|
||||
|
||||
@pytest.mark.covers("other.observability.s3_v2.mixed_surface_batch_writes_ndjson_lines_per_response_id")
|
||||
def test_s3_v2_mixed_surface_batch_writes_ndjson_lines_per_response_id(gateway: Gateway, tmp_path: Path) -> None:
|
||||
marker: Final = "s3mixb" + uuid.uuid4().hex[:8]
|
||||
sink: Final = RecordingS3Sink()
|
||||
with wire_server(surface_reply) as provider, wire_server(sink.respond) as bucket:
|
||||
config: Final = s3_config(tmp_path, bucket.url, {"s3_batch_file_upload": True})
|
||||
with (
|
||||
owned_proxy(gateway, tmp_path, {"DEFAULT_S3_FLUSH_INTERVAL_SECONDS": "3"}, config=config) as candidate,
|
||||
candidate.scenario() as scenario,
|
||||
):
|
||||
openai_model: Final = scenario.model(api_base=provider.url + "/v1", api_key="synthetic-provider-key")
|
||||
anthropic_model: Final = scenario.model(
|
||||
model="anthropic/claude-sonnet-4-5-20250929", api_base=provider.url, api_key="synthetic-provider-key"
|
||||
)
|
||||
key: Final = scenario.key(models=[openai_model, anthropic_model])
|
||||
answered: Final = mixed_burst(candidate, openai_model, anthropic_model, key, marker)
|
||||
payloads: Final = collect_payloads(sink, len(answered))
|
||||
targets: Final = tuple(sink.store.keys())
|
||||
puts: Final = bucket.drain()
|
||||
assert len(provider.drain()) == 48
|
||||
assert all(BATCH_KEY.match(target) for target in targets), list(targets)
|
||||
assert all(put.headers["content-type"] == "application/x-ndjson" for put in puts), [put.headers for put in puts]
|
||||
assert matched_ids(payloads, answered)
|
||||
assert len(payloads) == 48
|
||||
|
||||
|
||||
@pytest.mark.covers("other.observability.s3_v2.sink_outage_mid_mixed_burst_recovers_every_response_id")
|
||||
def test_s3_v2_sink_outage_mid_mixed_burst_recovers_every_response_id(gateway: Gateway, tmp_path: Path) -> None:
|
||||
marker: Final = "s3mixo" + uuid.uuid4().hex[:8]
|
||||
sink: Final = RecordingS3Sink(fail_attempts=30, fail_status=503, delay_seconds=0.2)
|
||||
with wire_server(surface_reply) as provider, wire_server(sink.respond) as bucket:
|
||||
config: Final = s3_config(tmp_path, bucket.url, {})
|
||||
with (
|
||||
owned_proxy(gateway, tmp_path, {"DEFAULT_S3_FLUSH_INTERVAL_SECONDS": "3"}, config=config) as candidate,
|
||||
candidate.scenario() as scenario,
|
||||
):
|
||||
openai_model: Final = scenario.model(api_base=provider.url + "/v1", api_key="synthetic-provider-key")
|
||||
anthropic_model: Final = scenario.model(
|
||||
model="anthropic/claude-sonnet-4-5-20250929", api_base=provider.url, api_key="synthetic-provider-key"
|
||||
)
|
||||
key: Final = scenario.key(models=[openai_model, anthropic_model])
|
||||
answered: Final = mixed_burst(candidate, openai_model, anthropic_model, key, marker)
|
||||
payloads: Final = collect_payloads(sink, len(answered), seconds=90)
|
||||
assert len(provider.drain()) == 48
|
||||
assert matched_ids(payloads, answered)
|
||||
assert len(payloads) == 48, "a stored id was overwritten or duplicated"
|
||||
|
|
@ -9,10 +9,13 @@ from dataclasses import dataclass, field
|
|||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import yaml
|
||||
from _s3_v2_support import RecordingS3Sink, collect_payloads
|
||||
from _s3_v2_support import s3_config as _recording_s3_config
|
||||
from integration._support.client import Gateway, JsonValue, eventually
|
||||
from integration._support.process import owned_proxy
|
||||
from integration._support.process import group_members, owned_proxy, owned_proxy_process
|
||||
from integration._support.wire import Reply, Request, Wire, wire_server
|
||||
|
||||
BUCKET: Final = "integration-bucket"
|
||||
|
|
@ -212,3 +215,417 @@ def test_s3_v2_batch_file_upload_keeps_team_alias_prefix(gateway: Gateway, tmp_p
|
|||
lines: Final = tuple(line for put in puts for line in put.body.decode().splitlines())
|
||||
assert frozenset(json.loads(line)["id"] for line in lines) == ids
|
||||
assert len(lines) == REQUESTS
|
||||
|
||||
|
||||
@pytest.mark.covers("other.observability.s3_v2.upstream_failure_events_land_alongside_successes")
|
||||
def test_s3_v2_upstream_failure_events_land_alongside_successes(gateway: Gateway, tmp_path: Path) -> None:
|
||||
marker: Final = "s3fail" + uuid.uuid4().hex[:8]
|
||||
sink: Final = RecordingS3Sink()
|
||||
|
||||
def provider(request: Request) -> Reply:
|
||||
text: Final = json.loads(request.body)["messages"][0]["content"]
|
||||
if text.endswith("-fail"):
|
||||
return Reply(
|
||||
status=401,
|
||||
body=b'{"error": {"message": "synthetic upstream rejection", "code": "synthetic_401"}}',
|
||||
)
|
||||
return _chat_reply(request)
|
||||
|
||||
with wire_server(provider) as upstream, wire_server(sink.respond) as bucket:
|
||||
config: Final = _s3_config(tmp_path, bucket.url, {})
|
||||
with (
|
||||
owned_proxy(gateway, tmp_path, {"DEFAULT_S3_FLUSH_INTERVAL_SECONDS": "3"}, config=config) as candidate,
|
||||
candidate.scenario() as scenario,
|
||||
):
|
||||
model: Final = scenario.model(api_base=upstream.url + "/v1", api_key="synthetic-provider-key")
|
||||
key: Final = scenario.key(models=[model])
|
||||
success_ids: Final = tuple(f"{marker}-{index}" for index in range(8))
|
||||
failure_ids: Final = tuple(f"{marker}-{index}-fail" for index in range(4))
|
||||
|
||||
def send(identity: str) -> httpx.Response:
|
||||
return candidate.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{"model": model, "messages": [{"role": "user", "content": identity}], "cache": {"no-cache": True}},
|
||||
key=key,
|
||||
)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=12) as pool:
|
||||
responses: Final = tuple(pool.map(send, (*success_ids, *failure_ids)))
|
||||
ok: Final = responses[:8]
|
||||
rejected: Final = responses[8:]
|
||||
assert all(response.status_code == 200 for response in ok), [r.text for r in ok]
|
||||
assert tuple(response.json()["id"] for response in ok) == success_ids
|
||||
for response in rejected:
|
||||
assert response.status_code in (400, 401), response.status_code
|
||||
assert "synthetic upstream rejection" in response.text, response.text
|
||||
failure_call_ids: Final = frozenset(response.headers["x-litellm-call-id"] for response in rejected)
|
||||
payloads: Final = collect_payloads(sink, len(success_ids) + len(failure_ids))
|
||||
assert len(upstream.drain()) == len(success_ids) + len(failure_ids)
|
||||
delivered: Final = frozenset(payload["id"] for payload in payloads if payload["status"] == "success")
|
||||
assert delivered == frozenset(success_ids)
|
||||
failures: Final = tuple(payload for payload in payloads if payload["status"] == "failure")
|
||||
assert len(failures) == len(failure_ids)
|
||||
assert frozenset(payload["litellm_call_id"] for payload in failures) == failure_call_ids
|
||||
assert all("synthetic upstream rejection" in json.dumps(payload["error_information"]) for payload in failures)
|
||||
|
||||
|
||||
@pytest.mark.covers("other.observability.s3_v2.invalid_or_empty_bound_falls_back_to_sixteen")
|
||||
@pytest.mark.parametrize(
|
||||
("bad", "warns"),
|
||||
[
|
||||
pytest.param("abc", True, id="non_integer"),
|
||||
pytest.param(0, True, id="below_one"),
|
||||
pytest.param("", False, id="empty"),
|
||||
],
|
||||
)
|
||||
def test_s3_v2_invalid_or_empty_bound_falls_back_to_sixteen(
|
||||
gateway: Gateway, tmp_path: Path, bad: JsonValue, warns: bool
|
||||
) -> None:
|
||||
marker: Final = "s3bound" + uuid.uuid4().hex[:8]
|
||||
sink: Final = RecordingS3Sink()
|
||||
with wire_server(_chat_reply) as provider, wire_server(sink.respond) as bucket:
|
||||
config: Final = _s3_config(tmp_path, bucket.url, {"s3_max_concurrent_uploads": bad})
|
||||
with (
|
||||
owned_proxy_process(gateway, tmp_path, {"DEFAULT_S3_FLUSH_INTERVAL_SECONDS": "3"}, config=config) as owned,
|
||||
owned.gateway.scenario() as scenario,
|
||||
):
|
||||
model: Final = scenario.model(api_base=provider.url + "/v1", api_key="synthetic-provider-key")
|
||||
key: Final = scenario.key(models=[model])
|
||||
ids: Final = _burst(owned.gateway, model, key, marker)
|
||||
payloads: Final = collect_payloads(sink, REQUESTS)
|
||||
if warns:
|
||||
eventually(
|
||||
lambda: owned.log.read_text(),
|
||||
lambda text: "s3_max_concurrent_uploads" in text,
|
||||
seconds=15,
|
||||
)
|
||||
else:
|
||||
assert "s3_max_concurrent_uploads" not in owned.log.read_text()
|
||||
assert len(provider.drain()) == REQUESTS
|
||||
assert sink.peak <= 16, f"peak concurrent PUTs {sink.peak} exceeded the fallback bound"
|
||||
assert frozenset(payload["id"] for payload in payloads) == ids
|
||||
|
||||
|
||||
@pytest.mark.covers("other.observability.s3_v2.sink_rejection_requeues_and_delivers_every_id_once")
|
||||
def test_s3_v2_sink_rejection_requeues_and_delivers_every_id_once(gateway: Gateway, tmp_path: Path) -> None:
|
||||
marker: Final = "s3deny" + uuid.uuid4().hex[:8]
|
||||
sink: Final = RecordingS3Sink(fail_status=403, delay_seconds=0.2)
|
||||
with wire_server(_chat_reply) as provider, wire_server(sink.respond) as bucket:
|
||||
config: Final = _s3_config(tmp_path, bucket.url, {})
|
||||
with (
|
||||
owned_proxy_process(gateway, tmp_path, {"DEFAULT_S3_FLUSH_INTERVAL_SECONDS": "3"}, config=config) as owned,
|
||||
owned.gateway.scenario() as scenario,
|
||||
):
|
||||
model: Final = scenario.model(api_base=provider.url + "/v1", api_key="synthetic-provider-key")
|
||||
key: Final = scenario.key(models=[model])
|
||||
sink.fail_until = time.time() + 10
|
||||
ids: Final = _burst(owned.gateway, model, key, marker)
|
||||
payloads: Final = collect_payloads(sink, REQUESTS, seconds=90)
|
||||
eventually(
|
||||
lambda: owned.log.read_text(),
|
||||
lambda text: "S3BatchUploadError" in text,
|
||||
seconds=15,
|
||||
)
|
||||
readiness: Final = owned.gateway.client.get("/health/readiness")
|
||||
assert readiness.status_code == 200, readiness.text
|
||||
assert len(provider.drain()) == REQUESTS
|
||||
assert len(sink.store) == REQUESTS
|
||||
assert frozenset(payload["id"] for payload in payloads) == ids
|
||||
|
||||
|
||||
@pytest.mark.covers("other.observability.s3_v2.batch_retry_resends_identical_key_and_body")
|
||||
def test_s3_v2_batch_retry_resends_identical_key_and_body(gateway: Gateway, tmp_path: Path) -> None:
|
||||
marker: Final = "s3retry" + uuid.uuid4().hex[:8]
|
||||
sink: Final = RecordingS3Sink(fail_status=500, delay_seconds=0.2)
|
||||
with wire_server(_chat_reply) as provider, wire_server(sink.respond) as bucket:
|
||||
config: Final = _s3_config(tmp_path, bucket.url, {"s3_batch_file_upload": True})
|
||||
with (
|
||||
owned_proxy(gateway, tmp_path, {"DEFAULT_S3_FLUSH_INTERVAL_SECONDS": "3"}, config=config) as candidate,
|
||||
candidate.scenario() as scenario,
|
||||
):
|
||||
model: Final = scenario.model(api_base=provider.url + "/v1", api_key="synthetic-provider-key")
|
||||
key: Final = scenario.key(models=[model])
|
||||
sink.fail_until = time.time() + 8
|
||||
ids: Final = _burst(candidate, model, key, marker)
|
||||
payloads: Final = collect_payloads(sink, REQUESTS, seconds=90)
|
||||
puts: Final = bucket.drain()
|
||||
assert len(provider.drain()) == REQUESTS
|
||||
by_target: Final = {}
|
||||
for put in puts:
|
||||
by_target.setdefault(put.target, set()).add(put.body) # mutable-ok: grouping attempts seen so far per target
|
||||
assert all(len(bodies) == 1 for bodies in by_target.values()), "a retried batch PUT changed key or body"
|
||||
assert max(sum(1 for put in puts if put.target == target) for target in by_target) >= 2, "no retried PUT observed"
|
||||
assert frozenset(payload["id"] for payload in payloads) == ids
|
||||
assert len(payloads) == REQUESTS
|
||||
|
||||
|
||||
@pytest.mark.covers("other.observability.s3_v2.unknown_model_rejection_keeps_other_requests_logging")
|
||||
def test_s3_v2_unknown_model_rejection_keeps_other_requests_logging(gateway: Gateway, tmp_path: Path) -> None:
|
||||
marker: Final = "s3ghost" + uuid.uuid4().hex[:8]
|
||||
sink: Final = RecordingS3Sink()
|
||||
with wire_server(_chat_reply) as provider, wire_server(sink.respond) as bucket:
|
||||
config: Final = _s3_config(tmp_path, bucket.url, {})
|
||||
with (
|
||||
owned_proxy(gateway, tmp_path, {"DEFAULT_S3_FLUSH_INTERVAL_SECONDS": "3"}, config=config) as candidate,
|
||||
candidate.scenario() as scenario,
|
||||
):
|
||||
model: Final = scenario.model(api_base=provider.url + "/v1", api_key="synthetic-provider-key")
|
||||
key: Final = scenario.key(models=[model])
|
||||
ghost: Final = candidate.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{"model": f"ghost-{uuid.uuid4().hex}", "messages": [{"role": "user", "content": "hi"}]},
|
||||
key=key,
|
||||
)
|
||||
assert ghost.status_code in (400, 403, 404), ghost.text
|
||||
ids: Final = _burst(candidate, model, key, marker)
|
||||
payloads: Final = collect_payloads(sink, REQUESTS)
|
||||
assert len(provider.drain()) == REQUESTS
|
||||
assert ids <= frozenset(payload["id"] for payload in payloads)
|
||||
extras: Final = tuple(payload for payload in payloads if payload["id"] not in ids)
|
||||
assert all(payload["status"] == "failure" for payload in extras), extras
|
||||
|
||||
|
||||
@pytest.mark.covers("other.observability.s3_v2.batch_flag_ignored_when_s3_v2_is_cold_storage_logger")
|
||||
def test_s3_v2_batch_flag_ignored_when_s3_v2_is_cold_storage_logger(gateway: Gateway, tmp_path: Path) -> None:
|
||||
marker: Final = "s3cold" + uuid.uuid4().hex[:8]
|
||||
sink: Final = RecordingS3Sink()
|
||||
with wire_server(_chat_reply) as provider, wire_server(sink.respond) as bucket:
|
||||
config: Final = _recording_s3_config(
|
||||
tmp_path,
|
||||
bucket.url,
|
||||
{"s3_batch_file_upload": True},
|
||||
{"cold_storage_custom_logger": "s3_v2"},
|
||||
)
|
||||
with (
|
||||
owned_proxy_process(gateway, tmp_path, {"DEFAULT_S3_FLUSH_INTERVAL_SECONDS": "3"}, config=config) as owned,
|
||||
owned.gateway.scenario() as scenario,
|
||||
):
|
||||
model: Final = scenario.model(api_base=provider.url + "/v1", api_key="synthetic-provider-key")
|
||||
key: Final = scenario.key(models=[model])
|
||||
response: Final = owned.gateway.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{"model": model, "messages": [{"role": "user", "content": marker}], "cache": {"no-cache": True}},
|
||||
key=key,
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
request_id: Final = str(response.json()["id"])
|
||||
payloads: Final = collect_payloads(sink, 1)
|
||||
assert all(PER_REQUEST_KEY.match(target) for target in sink.store), list(sink.store)
|
||||
eventually(
|
||||
lambda: owned.log.read_text(),
|
||||
lambda text: "s3_batch_file_upload is ignored because s3_v2 is the cold storage logger" in text,
|
||||
seconds=15,
|
||||
)
|
||||
spend: Final = eventually(
|
||||
lambda: owned.gateway.request("GET", f"/spend/logs/ui/{request_id}"),
|
||||
lambda reply: reply.status_code == 200 and bool(reply.json().get("messages")),
|
||||
seconds=60,
|
||||
)
|
||||
assert spend.status_code == 200, spend.text
|
||||
body: Final = spend.json()
|
||||
assert body["messages"], spend.text
|
||||
assert body["response"], spend.text
|
||||
assert payloads[0]["id"] == request_id
|
||||
|
||||
|
||||
@pytest.mark.covers("other.observability.s3_v2.identical_requests_land_distinct_objects")
|
||||
def test_s3_v2_identical_requests_land_distinct_objects(gateway: Gateway, tmp_path: Path) -> None:
|
||||
marker: Final = "s3same" + uuid.uuid4().hex[:8]
|
||||
sink: Final = RecordingS3Sink()
|
||||
with wire_server(_chat_reply) as provider, wire_server(sink.respond) as bucket:
|
||||
config: Final = _s3_config(tmp_path, bucket.url, {})
|
||||
with (
|
||||
owned_proxy(gateway, tmp_path, {"DEFAULT_S3_FLUSH_INTERVAL_SECONDS": "3"}, config=config) as candidate,
|
||||
candidate.scenario() as scenario,
|
||||
):
|
||||
model: Final = scenario.model(api_base=provider.url + "/v1", api_key="synthetic-provider-key")
|
||||
key: Final = scenario.key(models=[model])
|
||||
|
||||
def send(_: int) -> str:
|
||||
response: Final = candidate.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{"model": model, "messages": [{"role": "user", "content": marker}], "cache": {"no-cache": True}},
|
||||
key=key,
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
return str(response.json()["id"])
|
||||
|
||||
with ThreadPoolExecutor(max_workers=16) as pool:
|
||||
returned: Final = frozenset(pool.map(send, range(16)))
|
||||
payloads: Final = collect_payloads(sink, 16)
|
||||
assert len(provider.drain()) == 16
|
||||
assert returned == {marker}, "the upstream echo keeps the same id for identical requests"
|
||||
assert len(sink.store) == 16, "identical requests must still land as distinct objects"
|
||||
assert all(payload["id"] == marker for payload in payloads)
|
||||
|
||||
|
||||
@pytest.mark.covers("other.observability.s3_v2.two_workers_bound_and_deliver_every_id")
|
||||
def test_s3_v2_two_workers_bound_and_deliver_every_id(gateway: Gateway, tmp_path: Path) -> None:
|
||||
marker: Final = "s3work" + uuid.uuid4().hex[:8]
|
||||
sink: Final = RecordingS3Sink()
|
||||
with wire_server(_chat_reply) as provider, wire_server(sink.respond) as bucket:
|
||||
config: Final = _s3_config(tmp_path, bucket.url, {})
|
||||
with (
|
||||
owned_proxy(
|
||||
gateway, tmp_path, {"DEFAULT_S3_FLUSH_INTERVAL_SECONDS": "3"}, config=config, workers=2
|
||||
) as candidate,
|
||||
candidate.scenario() as scenario,
|
||||
):
|
||||
model: Final = scenario.model(api_base=provider.url + "/v1", api_key="synthetic-provider-key")
|
||||
key: Final = scenario.key(models=[model])
|
||||
ids: Final = _burst(candidate, model, key, marker)
|
||||
payloads: Final = collect_payloads(sink, REQUESTS)
|
||||
assert len(provider.drain()) == REQUESTS
|
||||
assert sink.peak <= 32, f"peak concurrent PUTs {sink.peak} exceeded two workers at the default bound"
|
||||
assert len(sink.store) == REQUESTS
|
||||
assert frozenset(payload["id"] for payload in payloads) == ids
|
||||
|
||||
|
||||
@pytest.mark.covers("other.observability.s3_v2.slow_sink_never_duplicates_or_stalls_readiness")
|
||||
def test_s3_v2_slow_sink_never_duplicates_or_stalls_readiness(gateway: Gateway, tmp_path: Path) -> None:
|
||||
marker: Final = "s3slow" + uuid.uuid4().hex[:8]
|
||||
sink: Final = RecordingS3Sink(delay_seconds=1.5)
|
||||
with wire_server(_chat_reply) as provider, wire_server(sink.respond) as bucket:
|
||||
config: Final = _s3_config(tmp_path, bucket.url, {"s3_batch_file_upload": True})
|
||||
with (
|
||||
owned_proxy(gateway, tmp_path, {"DEFAULT_S3_FLUSH_INTERVAL_SECONDS": "1"}, config=config) as candidate,
|
||||
candidate.scenario() as scenario,
|
||||
):
|
||||
model: Final = scenario.model(api_base=provider.url + "/v1", api_key="synthetic-provider-key")
|
||||
key: Final = scenario.key(models=[model])
|
||||
ids: Final = _burst(candidate, model, key, marker)
|
||||
|
||||
def delivered() -> int:
|
||||
readiness: Final = candidate.client.get("/health/readiness")
|
||||
assert readiness.status_code == 200, readiness.text
|
||||
with sink.lock:
|
||||
return sum(len(body.splitlines()) for body in sink.store.values())
|
||||
|
||||
eventually(delivered, lambda total: total >= REQUESTS, seconds=90)
|
||||
payloads: Final = tuple(
|
||||
json.loads(line) for body in tuple(sink.store.values()) for line in body.splitlines()
|
||||
)
|
||||
puts: Final = bucket.drain()
|
||||
targets: Final = tuple(put.target for put in puts)
|
||||
assert len(provider.drain()) == REQUESTS
|
||||
assert len(set(targets)) == len(targets), "the same object was PUT more than once"
|
||||
assert frozenset(payload["id"] for payload in payloads) == ids
|
||||
assert len(payloads) == REQUESTS
|
||||
|
||||
|
||||
@pytest.mark.covers("other.observability.s3_v2.worker_kill_mid_burst_keeps_surviving_deliveries")
|
||||
def test_s3_v2_worker_kill_mid_burst_keeps_surviving_deliveries(gateway: Gateway, tmp_path: Path) -> None:
|
||||
marker: Final = "s3kill" + uuid.uuid4().hex[:8]
|
||||
sink: Final = RecordingS3Sink()
|
||||
with wire_server(_chat_reply) as provider, wire_server(sink.respond) as bucket:
|
||||
config: Final = _s3_config(tmp_path, bucket.url, {})
|
||||
with (
|
||||
owned_proxy_process(
|
||||
gateway, tmp_path, {"DEFAULT_S3_FLUSH_INTERVAL_SECONDS": "3"}, config=config, workers=2
|
||||
) as owned,
|
||||
owned.gateway.scenario() as scenario,
|
||||
):
|
||||
model: Final = scenario.model(api_base=provider.url + "/v1", api_key="synthetic-provider-key")
|
||||
key: Final = scenario.key(models=[model])
|
||||
sent: Final = tuple(f"{marker}-{index}" for index in range(REQUESTS))
|
||||
|
||||
def send(identity: str) -> tuple[str, bool]:
|
||||
try:
|
||||
response: Final = owned.gateway.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": identity}],
|
||||
"cache": {"no-cache": True},
|
||||
},
|
||||
key=key,
|
||||
)
|
||||
except Exception:
|
||||
return identity, False
|
||||
return identity, response.status_code == 200
|
||||
|
||||
with ThreadPoolExecutor(max_workers=32) as pool:
|
||||
futures: Final = tuple(pool.submit(send, identity) for identity in sent)
|
||||
time.sleep(0.5)
|
||||
children: Final = tuple(
|
||||
process for process in group_members(owned.process.pid) if process.pid != owned.process.pid
|
||||
)
|
||||
assert children, "no worker children found to kill"
|
||||
children[0].kill()
|
||||
results: Final = tuple(future.result() for future in futures)
|
||||
survivors: Final = frozenset(identity for identity, ok in results if ok)
|
||||
assert survivors, "no request survived the worker kill"
|
||||
readiness: Final = owned.gateway.client.get("/health/readiness")
|
||||
assert readiness.status_code == 200, readiness.text
|
||||
payloads: Final = collect_payloads(sink, len(survivors), seconds=90)
|
||||
landed: Final = frozenset(payload["id"] for payload in payloads)
|
||||
assert survivors <= landed, "an id whose response succeeded never landed"
|
||||
assert landed <= frozenset(sent), "an id that was never sent landed"
|
||||
|
||||
|
||||
@pytest.mark.covers("other.observability.s3_v2.sigterm_mid_burst_loses_only_inflight_without_duplicates")
|
||||
def test_s3_v2_sigterm_mid_burst_loses_only_inflight_without_duplicates(gateway: Gateway, tmp_path: Path) -> None:
|
||||
marker: Final = "s3term" + uuid.uuid4().hex[:8]
|
||||
sink: Final = RecordingS3Sink()
|
||||
with wire_server(_chat_reply) as provider, wire_server(sink.respond) as bucket:
|
||||
config: Final = _s3_config(tmp_path, bucket.url, {})
|
||||
owned: Final = owned_proxy_process(gateway, tmp_path, {"DEFAULT_S3_FLUSH_INTERVAL_SECONDS": "3"}, config=config)
|
||||
candidate_owned: Final = owned.__enter__()
|
||||
try:
|
||||
created: Final = candidate_owned.gateway.post(
|
||||
"/model/new",
|
||||
{
|
||||
"model_name": f"integration-{marker}",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-4o-mini",
|
||||
"api_key": "synthetic-provider-key",
|
||||
"api_base": provider.url + "/v1",
|
||||
},
|
||||
"model_info": {},
|
||||
},
|
||||
)
|
||||
model: Final = str(created["model_name"])
|
||||
key: Final = str(candidate_owned.gateway.post("/key/generate", {"models": [model]})["key"])
|
||||
sent: Final = tuple(f"{marker}-{index}" for index in range(REQUESTS))
|
||||
|
||||
def send(identity: str) -> tuple[str, bool]:
|
||||
try:
|
||||
response: Final = candidate_owned.gateway.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": identity}],
|
||||
"cache": {"no-cache": True},
|
||||
},
|
||||
key=key,
|
||||
)
|
||||
except Exception:
|
||||
return identity, False
|
||||
return identity, response.status_code == 200
|
||||
|
||||
with ThreadPoolExecutor(max_workers=32) as pool:
|
||||
futures: Final = tuple(pool.submit(send, identity) for identity in sent)
|
||||
time.sleep(0.5)
|
||||
candidate_owned.process.terminate()
|
||||
results: Final = tuple(future.result() for future in futures)
|
||||
candidate_owned.process.wait(timeout=30)
|
||||
finally:
|
||||
owned.__exit__(None, None, None)
|
||||
answered: Final = frozenset(identity for identity, ok in results if ok)
|
||||
landed: Final = frozenset(
|
||||
payload["id"]
|
||||
for body in tuple(sink.store.values())
|
||||
for line in body.splitlines()
|
||||
for payload in [json.loads(line)]
|
||||
)
|
||||
assert landed <= answered, (
|
||||
"a delivered object has no matching answered request; lost in-flight ids are expected, extras are not"
|
||||
)
|
||||
targets: Final = tuple(sink.store.keys())
|
||||
assert len(set(targets)) == len(targets), "the same object was PUT more than once"
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue