test(otel): cover recursive fallback and concurrent retry traces

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
shivam 2026-11-14 02:01:47 +00:00
parent b193d2a8f4
commit b4161834a6
3 changed files with 150 additions and 25 deletions

View file

@ -2987,7 +2987,9 @@ class Logging(LiteLLMLoggingBaseClass):
exception, traceback_exception, start_time, end_time, estimate_prompt_tokens=False
)
if attempt._should_estimate_failed_prompt():
attempt.model_call_details[FAILED_PROMPT_TOKEN_ESTIMATE_KEY] = await attempt._offloaded_prompt_token_estimate()
attempt.model_call_details[
FAILED_PROMPT_TOKEN_ESTIMATE_KEY
] = await attempt._offloaded_prompt_token_estimate()
for callback in callbacks:
try:
if not attempt.should_run_callback(callback, params, "async_failure_handler"):

View file

@ -31,11 +31,10 @@ from urllib.parse import urlparse
import httpx
import psutil
import yaml
from integration._support.ports import allocate_port
from pydantic import JsonValue, TypeAdapter
from typing_extensions import NotRequired, ReadOnly, TypedDict
from integration._support.ports import allocate_port
INTERNAL_MARKERS: Final = ("gen_ai.operation.name", "mcp.method.name", "litellm.guardrail_name")

View file

@ -1,6 +1,9 @@
import json
from collections import Counter
from collections.abc import Iterator
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass
from pathlib import Path
from functools import partial
from typing import Final
from urllib.parse import urlsplit
@ -13,12 +16,15 @@ from integration._support.process import owned_proxy
from integration._support.wire import Reply, Request, Wire, wire_server
def _unavailable(request: Request) -> Reply:
assert "/model/" in request.target, request.target
def _unavailable(request: Request, *, message: str = "primary region unavailable") -> Reply:
if request.method == "GET":
assert request.target == "/v1/models", request.target
return Reply(body=b'{"object":"list","data":[]}')
assert "/model/" in request.target or request.target == "/chat/completions", request.target
return Reply(
status=503,
headers={"x-amzn-errortype": "ServiceUnavailableException"},
body=b'{"message":"primary region unavailable"}',
body=json.dumps({"message": message}).encode(),
)
@ -32,7 +38,7 @@ def _completed(request: Request) -> Reply:
)
return Reply(
body=b'{"id":"msg-fallback","type":"message","role":"assistant",'
b'"model":"claude-sonnet-5","content":[{"type":"text","text":"fallback ok"}],'
b'"model":"claude-sonnet-5-5","content":[{"type":"text","text":"fallback ok"}],'
b'"stop_reason":"end_turn","usage":{"input_tokens":7,"output_tokens":2}}'
)
@ -41,25 +47,48 @@ def _completed(request: Request) -> Reply:
class RetryRig:
proxy: Gateway
primary: Wire
secondary: Wire
fallback: Wire
broken_stream: Wire
sink: str
def _failed_stream(request: Request) -> Reply:
if request.method == "GET":
assert request.target == "/v1/models", request.target
return Reply(body=b'{"object":"list","data":[]}')
assert request.target == "/chat/completions", request.target
assert json.loads(request.body)["stream"] is True
return Reply(
content_type="text/event-stream",
body=b'data: {"id":"chatcmpl-partial","object":"chat.completion.chunk","created":0,'
b'"model":"attempt-test","choices":[{"index":0,"delta":{"role":"assistant",'
b'"content":"partial"},"finish_reason":null}]}\n\n'
b'data: {"error":{"message":"stream attempt failed","type":"server_error",'
b'"code":"internal_server_error"}}\n\n',
)
@pytest.fixture(scope="module", params=("false", "true"), ids=("legacy", "v2"))
def retry_rig(
request: pytest.FixtureRequest, tmp_path_factory: pytest.TempPathFactory, audit_sinks: SpanSinks
) -> Iterator[RetryRig]:
directory: Final = tmp_path_factory.mktemp("otel-retry")
with wire_server(_unavailable) as primary, wire_server(_completed) as fallback:
with (
wire_server(_unavailable) as primary,
wire_server(partial(_unavailable, message="secondary region unavailable")) as secondary,
wire_server(_completed) as fallback,
wire_server(_failed_stream) as broken_stream,
):
config: Final = directory / "proxy.yaml"
config.write_text(
yaml.safe_dump(
{
"model_list": [
{
"model_name": "regional-retry",
"model_name": name,
"litellm_params": {
"model": "bedrock/anthropic.claude-sonnet-5",
"model": "bedrock/anthropic.claude-sonnet-5-5",
"api_base": wire.url,
"aws_region_name": region,
"aws_access_key_id": "synthetic-access-key",
@ -67,12 +96,42 @@ def retry_rig(
"order": order,
},
}
for wire, region, order in ((primary, "eu-west-1", 1), (fallback, "eu-central-1", 2))
for name, wire, region, order in (
("regional-retry", primary, "eu-west-1", 1),
("regional-retry", fallback, "eu-central-1", 2),
("recursive-primary", primary, "eu-west-1", 1),
("recursive-secondary", secondary, "eu-north-1", 1),
("recursive-fallback", fallback, "eu-central-1", 1),
)
]
+ [
{
"model_name": "stream-retry",
"litellm_params": {
"model": "openai/attempt-test",
"api_base": wire.url,
"api_key": "synthetic-openai-key",
"order": order,
},
}
for wire, order in ((primary, 1), (broken_stream, 2))
],
"router_settings": {"num_retries": 3, "retry_after": 0, "disable_cooldowns": True},
"router_settings": {
"num_retries": 3,
"retry_after": 0,
"disable_cooldowns": True,
"fallbacks": [
{"recursive-primary": ["recursive-secondary"]},
{"recursive-secondary": ["recursive-fallback"]},
],
},
"litellm_settings": {"callbacks": ["otel"]},
"callback_settings": {
"otel": {"exporter": "http/json", "endpoint": audit_sinks.operator, "use_simple_processor": True}
"otel": {
"exporter": "http/json",
"endpoint": audit_sinks.operator,
"use_simple_processor": True,
}
},
"general_settings": {"master_key": "sk-integration-otel-retry", "store_model_in_db": True},
}
@ -87,7 +146,8 @@ def retry_rig(
with httpx.Client(base_url="http://127.0.0.1:1", trust_env=False) as client:
base: Final = Gateway(client, "sk-integration-otel-retry", primary.url)
with owned_proxy(base, directory, overrides, config=config, workers=2) as proxy:
yield RetryRig(proxy, primary, fallback, audit_sinks.operator)
proxy.client.timeout = 120
yield RetryRig(proxy, primary, secondary, fallback, broken_stream, audit_sinks.operator)
def _llm_spans(sink: str, start: int) -> tuple[Span, ...]:
@ -99,29 +159,93 @@ def _llm_spans(sink: str, start: int) -> tuple[Span, ...]:
@pytest.mark.parametrize("endpoint", ("/v1/messages", "/v1/chat/completions"))
def test_every_failed_bedrock_attempt_has_its_own_destination(retry_rig: RetryRig, endpoint: str) -> None:
@pytest.mark.parametrize("recursive", (False, True), ids=("ordered", "recursive"))
def test_every_failed_bedrock_attempt_has_its_own_destination(
retry_rig: RetryRig, endpoint: str, recursive: bool
) -> None:
start, _ = recorded_spans(retry_rig.sink)
response: Final = retry_rig.proxy.request(
"POST",
endpoint,
{"model": "regional-retry", "messages": [{"role": "user", "content": "retry regions"}], "max_tokens": 16},
{
"model": "recursive-primary" if recursive else "regional-retry",
"messages": [{"role": "user", "content": "retry regions"}],
"max_tokens": 16,
},
)
assert response.status_code == 200, response.text
assert "fallback ok" in response.text, response.text
attempts: Final = retry_rig.primary.drain()
attempts: Final = tuple(request for request in retry_rig.primary.drain() if request.method == "POST")
assert len(attempts) == 4, attempts
assert len(retry_rig.secondary.drain()) == (4 if recursive else 0)
assert len(retry_rig.fallback.drain()) == 1
expected_failures: Final = 8 if recursive else 4
spans: Final = eventually(
lambda: _llm_spans(retry_rig.sink, start),
lambda values: len(values) >= expected_failures + 1,
seconds=15,
return_last_on_timeout=True,
)
assert len(spans) == expected_failures + 1, spans
ports: Final = sorted(int(str(span["attributes"].get("server.port"))) for span in spans)
expected_ports: Final = (
[urlsplit(retry_rig.primary.url).port] * 4
+ ([urlsplit(retry_rig.secondary.url).port] * 4 if recursive else [])
+ [urlsplit(retry_rig.fallback.url).port]
)
assert ports == sorted(expected_ports), spans
assert all(span["attributes"].get("server.address") == "127.0.0.1" for span in spans), spans
failures: Final = tuple(span for span in spans if span.get("status_code") == 2)
assert len(failures) == expected_failures, spans
for span in failures:
region: Final = (
"primary"
if int(str(span["attributes"]["server.port"])) == urlsplit(retry_rig.primary.url).port
else "secondary"
)
assert f"{region} region unavailable" in str(span["attributes"].get("error.message")), span
def test_failed_retry_stream_has_one_span_per_upstream_attempt(retry_rig: RetryRig) -> None:
start, _ = recorded_spans(retry_rig.sink)
response: Final = retry_rig.proxy.request(
"POST",
"/v1/chat/completions",
{"model": "stream-retry", "messages": [{"role": "user", "content": "stream retries"}], "stream": True},
)
assert response.status_code == 200, response.text
assert "partial" in response.text and "stream attempt failed" in response.text, response.text
assert len(tuple(request for request in retry_rig.primary.drain() if request.method == "POST")) == 4
assert len(tuple(request for request in retry_rig.broken_stream.drain() if request.method == "POST")) == 1
spans: Final = eventually(
lambda: _llm_spans(retry_rig.sink, start),
lambda values: len(values) >= 5,
seconds=15,
return_last_on_timeout=True,
)
assert len(spans) == 5, spans
assert all(span.get("status_code") == 2 for span in spans), spans
ports: Final = sorted(int(str(span["attributes"].get("server.port"))) for span in spans)
assert ports == sorted([urlsplit(retry_rig.primary.url).port] * 4 + [urlsplit(retry_rig.fallback.url).port]), spans
assert all(span["attributes"].get("server.address") == "127.0.0.1" for span in spans), spans
failures: Final = tuple(span for span in spans if span.get("status_code") == 2)
assert len(failures) == 4, spans
assert all("primary region unavailable" in str(span["attributes"].get("error.message")) for span in failures), failures
assert all(int(str(span["attributes"]["server.port"])) == urlsplit(retry_rig.primary.url).port for span in failures)
assert ports == sorted([urlsplit(retry_rig.primary.url).port] * 4 + [urlsplit(retry_rig.broken_stream.url).port]), (
spans
)
def test_concurrent_retries_keep_every_attempt_in_its_request_trace(retry_rig: RetryRig) -> None:
start, _ = recorded_spans(retry_rig.sink)
with ThreadPoolExecutor(max_workers=3) as pool:
futures: Final = tuple(
pool.submit(
retry_rig.proxy.request,
"POST",
"/v1/chat/completions",
{"model": "regional-retry", "messages": [{"role": "user", "content": f"concurrent retries {index}"}]},
)
for index in range(3)
)
responses: Final = tuple(future.result() for future in futures)
assert all(response.status_code == 200 for response in responses), tuple(response.text for response in responses)
assert len(tuple(request for request in retry_rig.primary.drain() if request.method == "POST")) == 12
assert len(retry_rig.fallback.drain()) == 3
spans: Final = eventually(lambda: _llm_spans(retry_rig.sink, start), lambda values: len(values) >= 15, seconds=15)
assert sorted(Counter(span["trace_id"] for span in spans).values()) == [5, 5, 5], spans
assert sum(span.get("status_code") == 2 for span in spans) == 12, spans