mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-22 10:40:46 +00:00
test(otel): flush every worker before the restart cell kills it and cover a failed call after a dropped one
This commit is contained in:
parent
abaafd5fc4
commit
debff6eed5
1 changed files with 72 additions and 31 deletions
|
|
@ -13,8 +13,9 @@ from collections.abc import Callable, Generator, Iterator, Mapping, Sequence
|
|||
from concurrent.futures import ThreadPoolExecutor
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass, replace
|
||||
from itertools import chain, product
|
||||
from pathlib import Path
|
||||
from typing import Final, Literal
|
||||
from typing import Final, Literal, TypeAlias
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import httpx
|
||||
|
|
@ -43,12 +44,12 @@ CREDENTIALS: Final = {"arize_space_key": SPACE, "arize_api_key": "arize-sampled"
|
|||
FAILING: Final = re.compile(rb"sampling-fail")
|
||||
JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
|
||||
TRACES: Final = 16
|
||||
Endpoint = Literal["chat", "responses", "messages"]
|
||||
Verdicts = dict[str, tuple[bool, ...]]
|
||||
Endpoint: TypeAlias = Literal["chat", "responses", "messages"]
|
||||
Verdicts: TypeAlias = Mapping[str, tuple[bool, ...]]
|
||||
|
||||
|
||||
def _chat_reply(stream: bool) -> Reply:
|
||||
usage: Final[dict[str, JsonValue]] = {"prompt_tokens": 7, "completion_tokens": 2, "total_tokens": 9}
|
||||
usage: Final[JsonValue] = {"prompt_tokens": 7, "completion_tokens": 2, "total_tokens": 9}
|
||||
if not stream:
|
||||
return Reply(
|
||||
body=json.dumps(
|
||||
|
|
@ -65,7 +66,7 @@ def _chat_reply(stream: bool) -> Reply:
|
|||
).encode()
|
||||
)
|
||||
chunk: Final = {"id": "chatcmpl-sampling", "object": "chat.completion.chunk", "created": 1, "model": "gpt-4o-mini"}
|
||||
deltas: Final[tuple[dict[str, JsonValue], ...]] = (
|
||||
deltas: Final[tuple[JsonValue, ...]] = (
|
||||
{**chunk, "choices": [{"index": 0, "delta": {"role": "assistant", "content": "ok"}}]},
|
||||
{**chunk, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]},
|
||||
{**chunk, "choices": [], "usage": usage},
|
||||
|
|
@ -77,7 +78,7 @@ def _chat_reply(stream: bool) -> Reply:
|
|||
|
||||
|
||||
def _responses_reply(stream: bool) -> Reply:
|
||||
response: Final[dict[str, JsonValue]] = {
|
||||
response: Final[Mapping[str, JsonValue]] = {
|
||||
"id": "resp_sampling",
|
||||
"object": "response",
|
||||
"created_at": 1,
|
||||
|
|
@ -96,7 +97,7 @@ def _responses_reply(stream: bool) -> Reply:
|
|||
}
|
||||
if not stream:
|
||||
return Reply(body=json.dumps(response).encode())
|
||||
events: Final[tuple[dict[str, JsonValue], ...]] = (
|
||||
events: Final[tuple[Mapping[str, JsonValue], ...]] = (
|
||||
{
|
||||
"type": "response.created",
|
||||
"sequence_number": 0,
|
||||
|
|
@ -110,7 +111,7 @@ def _responses_reply(stream: bool) -> Reply:
|
|||
"content_index": 0,
|
||||
"delta": "ok",
|
||||
},
|
||||
{"type": "response.completed", "sequence_number": 2, "response": response},
|
||||
{"type": "response.completed", "sequence_number": 2, "response": dict(response)},
|
||||
)
|
||||
return Reply(
|
||||
content_type="text/event-stream",
|
||||
|
|
@ -137,8 +138,8 @@ class Sent:
|
|||
@dataclass(frozen=True, slots=True)
|
||||
class Call:
|
||||
path: str
|
||||
body: dict[str, JsonValue]
|
||||
headers: dict[str, str]
|
||||
body: Mapping[str, JsonValue]
|
||||
headers: Mapping[str, str]
|
||||
sent: Sent
|
||||
expected: int
|
||||
|
||||
|
|
@ -148,7 +149,7 @@ def _call(
|
|||
) -> Call:
|
||||
parent: Final = uuid.uuid4().hex[:16]
|
||||
call_id: Final = str(uuid.uuid4())
|
||||
messages: Final[list[JsonValue]] = [{"role": "user", "content": content}]
|
||||
messages: Final[JsonValue] = [{"role": "user", "content": content}]
|
||||
path, body = {
|
||||
"chat": ("/v1/chat/completions", {"model": MODEL, "messages": messages, "stream": stream}),
|
||||
"responses": ("/v1/responses", {"model": MODEL, "input": content, "stream": stream}),
|
||||
|
|
@ -161,7 +162,7 @@ def _call(
|
|||
**({} if trace_id is None else {"traceparent": f"00-{trace_id}-{parent}-01"}),
|
||||
}
|
||||
expected: Final = 500 if FAILING.search(content.encode()) else 200
|
||||
return Call(path, dict(body), headers, Sent(trace_id or "", parent if trace_id else "", call_id), expected)
|
||||
return Call(path, body, headers, Sent(trace_id or "", parent if trace_id else "", call_id), expected)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -224,18 +225,18 @@ def _suspended_all_but(pod: OwnedProxy, workers: Sequence[int], serving: int) ->
|
|||
yield
|
||||
|
||||
|
||||
def _space_id(request: Mapping[str, JsonValue]) -> JsonValue:
|
||||
return {name.lower(): value for name, value in object_value(request.get("headers") or {}).items()}.get(
|
||||
"arize-space-id"
|
||||
)
|
||||
|
||||
|
||||
def _space_span_ids(url: str) -> frozenset[str]:
|
||||
"""Span ids the Arize sink received in export requests sent with the team's space id."""
|
||||
return frozenset(
|
||||
span_id
|
||||
for request in recorded_requests(url)
|
||||
if {name.lower(): value for name, value in object_value(request.get("headers") or {}).items()}.get(
|
||||
"arize-space-id"
|
||||
)
|
||||
== SPACE
|
||||
for span_id in list_value(request.get("span_ids") or [])
|
||||
if isinstance(span_id, str)
|
||||
ours: Final = (
|
||||
list_value(request.get("span_ids") or []) for request in recorded_requests(url) if _space_id(request) == SPACE
|
||||
)
|
||||
return frozenset(span_id for span_id in chain.from_iterable(ours) if isinstance(span_id, str))
|
||||
|
||||
|
||||
def _team_spans(pods: Pods, since: int) -> tuple[Span, ...]:
|
||||
|
|
@ -280,6 +281,17 @@ def _cursors(pods: Pods) -> Cursors:
|
|||
return Cursors(recorded_spans(pods.sinks.operator)[0], recorded_spans(pods.sinks.arize)[0])
|
||||
|
||||
|
||||
def _await_flushed(pods: Pods, cursors: Cursors, fence_key: str) -> None:
|
||||
"""Wait until every worker of both pods has sent the team's exporter everything it decided so far."""
|
||||
fences: Final = pods.fence(fence_key)
|
||||
|
||||
def arrived() -> None:
|
||||
delivered: Final = _delivered(_team_spans(pods, cursors.arize), fences)
|
||||
assert {fence.call_id for fence in fences} <= delivered, "the unsampled team's calls have not arrived yet"
|
||||
|
||||
until_asserted(arrived, seconds=30)
|
||||
|
||||
|
||||
def _settled(
|
||||
pods: Pods,
|
||||
cursors: Cursors,
|
||||
|
|
@ -289,13 +301,12 @@ def _settled(
|
|||
) -> Verdicts:
|
||||
"""Per trace, which of its calls reached the team's space, once every worker has sent what it decided
|
||||
and ``holds`` is true of every trace."""
|
||||
calls: Final = [call for sent in by_trace.values() for call in sent]
|
||||
calls: Final = tuple(chain.from_iterable(by_trace.values()))
|
||||
_await_logged(pods, cursors.operator, calls)
|
||||
fences: Final = pods.fence(fence_key)
|
||||
_await_flushed(pods, cursors, fence_key)
|
||||
|
||||
def check() -> Verdicts:
|
||||
delivered: Final = _delivered(_team_spans(pods, cursors.arize), [*calls, *fences])
|
||||
assert {fence.call_id for fence in fences} <= delivered, "the unsampled team's calls have not arrived yet"
|
||||
delivered: Final = _delivered(_team_spans(pods, cursors.arize), calls)
|
||||
verdicts: Final = {
|
||||
trace_id: tuple(call.call_id in delivered for call in sent) for trace_id, sent in by_trace.items()
|
||||
}
|
||||
|
|
@ -399,11 +410,13 @@ def test_a_replaced_worker_reaches_the_verdict_its_predecessor_reached(pods: Pod
|
|||
traces still answer the way the first ones did."""
|
||||
sampled: Final = pods.team_key({**CREDENTIALS, "arize_success_sampling_rate": "0.5"})
|
||||
cursors: Final = _cursors(pods)
|
||||
fence_key: Final = pods.team_key(CREDENTIALS)
|
||||
traces: Final = _trace_ids("replaced")
|
||||
before: Final = {
|
||||
trace_id: pods.on_each_worker(pods.second, lambda t=trace_id: _call(sampled, t)) for trace_id in traces
|
||||
}
|
||||
_await_logged(pods, cursors.operator, [call for sent in before.values() for call in sent])
|
||||
_await_logged(pods, cursors.operator, tuple(chain.from_iterable(before.values())))
|
||||
_await_flushed(pods, cursors, fence_key)
|
||||
for victim in sorted(pods.second.worker_pids()):
|
||||
pods.second.kill_worker(victim)
|
||||
by_trace: Final = {
|
||||
|
|
@ -411,7 +424,7 @@ def test_a_replaced_worker_reaches_the_verdict_its_predecessor_reached(pods: Pod
|
|||
for trace_id in traces
|
||||
}
|
||||
|
||||
verdicts: Final = _settled(pods, cursors, by_trace, pods.team_key(CREDENTIALS))
|
||||
verdicts: Final = _settled(pods, cursors, by_trace, fence_key)
|
||||
|
||||
assert 0 < _kept(verdicts) < TRACES
|
||||
|
||||
|
|
@ -440,6 +453,30 @@ def test_a_failed_call_is_kept_wherever_a_successful_call_of_its_trace_is(pods:
|
|||
assert any(kept == (True, True) for kept in verdicts.values()), "no trace sat under both rates"
|
||||
|
||||
|
||||
def test_a_failed_call_answers_to_the_error_rate_on_a_pod_that_dropped_its_trace(pods: Pods) -> None:
|
||||
"""A pod that already dropped a successful call of the trace still keeps a later failed call of it at an
|
||||
unset error rate, as the pod that only served the failed call does."""
|
||||
sampled: Final = pods.team_key({**CREDENTIALS, "arize_success_sampling_rate": "0.3"})
|
||||
cursors: Final = _cursors(pods)
|
||||
traces: Final = _trace_ids("failed-after-dropped")
|
||||
succeeded: Final = {trace_id: pods.send(pods.first, _call(sampled, trace_id)) for trace_id in traces}
|
||||
_await_logged(pods, cursors.operator, tuple(succeeded.values()))
|
||||
by_trace: Final = {
|
||||
trace_id: (
|
||||
succeeded[trace_id],
|
||||
pods.send(pods.first, _call(sampled, trace_id, content="sampling-fail")),
|
||||
pods.send(pods.second, _call(sampled, trace_id, content="sampling-fail")),
|
||||
)
|
||||
for trace_id in traces
|
||||
}
|
||||
|
||||
verdicts: Final = _settled(
|
||||
pods, cursors, by_trace, pods.team_key(CREDENTIALS), holds=lambda kept: kept[1] and kept[2]
|
||||
)
|
||||
|
||||
assert any(not kept[0] for kept in verdicts.values()), "no successful call was dropped before its trace failed"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("variables", "expected"),
|
||||
[({"arize_success_sampling_rate": "1.0"}, TRACES), ({"arize_success_sampling_rate": "0.0"}, 0), ({}, TRACES)],
|
||||
|
|
@ -465,7 +502,9 @@ def test_a_request_without_a_traceparent_is_sampled_whole_at_the_rate(pods: Pods
|
|||
together and the rate still keeps some requests and drops others."""
|
||||
sampled: Final = pods.team_key({**CREDENTIALS, "arize_success_sampling_rate": "0.5"})
|
||||
cursors: Final = _cursors(pods)
|
||||
sent: Final = tuple(pods.send(pod, _call(sampled, None)) for _ in range(12) for pod in (pods.first, pods.second))
|
||||
sent: Final = tuple(
|
||||
pods.send(pod, _call(sampled, None)) for _, pod in product(range(12), (pods.first, pods.second))
|
||||
)
|
||||
_await_logged(pods, cursors.operator, sent)
|
||||
trace_of: Final = {
|
||||
str(span["attributes"][CALL_ID]): span["trace_id"]
|
||||
|
|
@ -474,9 +513,11 @@ def test_a_request_without_a_traceparent_is_sampled_whole_at_the_rate(pods: Pods
|
|||
}
|
||||
fences: Final = pods.fence(pods.team_key(CREDENTIALS))
|
||||
|
||||
def check() -> dict[str, tuple[bool, bool]]:
|
||||
def check() -> Mapping[str, tuple[bool, bool]]:
|
||||
spans: Final = _team_spans(pods, cursors.arize)
|
||||
assert {fence.call_id for fence in fences} <= _delivered(spans, fences)
|
||||
assert {fence.call_id for fence in fences} <= _delivered(spans, fences), (
|
||||
"the unsampled team's calls have not arrived yet"
|
||||
)
|
||||
kinds: Final = {
|
||||
call.call_id: (
|
||||
any(span["kind"] == SERVER for span in spans if span["trace_id"] == trace_of[call.call_id]),
|
||||
|
|
@ -522,7 +563,7 @@ def test_a_concurrent_burst_keeps_every_trace_whole(pods: Pods) -> None:
|
|||
sampled: Final = pods.team_key({**CREDENTIALS, "arize_success_sampling_rate": "0.5"})
|
||||
cursors: Final = _cursors(pods)
|
||||
traces: Final = _trace_ids("burst", 24)
|
||||
plan: Final = [(trace_id, pod) for trace_id in traces for pod in (pods.first, pods.second, pods.first, pods.second)]
|
||||
plan: Final = tuple(product(traces, (pods.first, pods.second, pods.first, pods.second)))
|
||||
|
||||
def send(job: tuple[str, OwnedProxy]) -> Sent:
|
||||
return pods.send(job[1], _call(sampled, job[0]))
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue