scriptoriumadmin/strix
1
0
Fork 0
mirror of https://github.com/usestrix/strix.git synced 2026-10-12 03:47:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

refactor(llm): keep every details key; no cf-ray fallback for the request id

Browse source
This commit is contained in:
Ahmed Allam 2026-09-23 02:25:34 +00:00
parent a7f0a49dcf
commit 26c580a6c2
2 changed files with 37 additions and 91 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

91
strix/llm/request_log.py
View file

@ -11,10 +11,9 @@ The typed fields are the common ground every provider shares (status, ids,
timing, sizes, tokens, cost). Everything else a provider or SDK reports about
the attempt travels free-form: the response headers, and a ``details`` object
with the request parameters, the full usage object, the SDK's hidden
parameters and the error body. Content (prompts, completions, tool arguments)
and the caller's own credentials are left out of both by key name, and every
string, list and object is bounded. Raw request and response bodies are never
captured.
parameters and the error body. Every string, list and object is bounded. Raw
request and response bodies are never captured: prompts and completions are
only ever measured for their size.
Sinks are plain callables. The built-in sink writes one log line per event;
deployments register their own (a database, a queue) with
@ -113,7 +112,7 @@ class LlmRequestEvent:
# Every response header the provider sent, name-normalized, values capped.
response_headers: dict[str, str] | None = None
# Free-form: request parameters, full usage object, SDK hidden params,
# error body ... with content removed and sizes bounded.
# error body ... sizes bounded.
details: dict[str, Any] | None = None
def to_dict(self) -> dict[str, Any]:
@ -236,9 +235,6 @@ _REQUEST_ID_HEADERS: tuple[str, ...] = (
"x-goog-request-id",
"x-generation-id",
)
# The Cloudflare edge id: the only handle a reply carries when the provider
# sends none of its own, in the header or the body.
_EDGE_ID_HEADERS: tuple[str, ...] = ("cf-ray",)
_LITELLM_HEADER_PREFIX = "llm_provider-"
# Anthropic error bodies carry the id even when a gateway strips the header.
@ -265,11 +261,6 @@ def request_id_from_headers(headers: Mapping[str, Any] | None) -> str | None:
return _header_value(headers, _REQUEST_ID_HEADERS)
def edge_request_id(headers: Mapping[str, Any] | None) -> str | None:
"""The edge network's id, for replies where nothing better exists."""
return _header_value(headers, _EDGE_ID_HEADERS)
def request_id_from_text(text: str | None) -> str | None:
if not text:
return None
@ -278,10 +269,8 @@ def request_id_from_text(text: str | None) -> str | None:
def request_id_from_reply(headers: Mapping[str, Any] | None, text: str | None = None) -> str | None:
"""Provider header id, then the id in the error body, then the edge id."""
return (
request_id_from_headers(headers) or request_id_from_text(text) or edge_request_id(headers)
)
"""Provider header id, then the id in the error body."""
return request_id_from_headers(headers) or request_id_from_text(text)
# --------------------------------------------------------------------------- #
@ -293,45 +282,6 @@ FINISH_REASON_MAX_CHARS = 128
HEADERS_MAX_COUNT = 64
HEADER_VALUE_MAX_CHARS = 512
# Keys of a request/response structure whose value is content: the prompt, the
# completion, tool schemas and arguments. ``details`` is metadata about the
# exchange, never the exchange itself; bodies are measured, never kept.
_CONTENT_KEYS: frozenset[str] = frozenset(
{
"messages",
"message",
"content",
"text",
"input",
"output",
"instructions",
"system",
"prompt",
"completion",
"choices",
"delta",
"tool_calls",
"function_call",
"arguments",
"tools",
"functions",
"thinking_blocks",
"reasoning_content",
"citations",
"image_url",
"data",
"b64_json",
"embedding",
"audio",
"complete_input_dict",
"original_response",
}
)
# Request parameters that hold the caller's own credential: the key itself and
# the header block (``extra_headers``) the authorization header travels in.
_CREDENTIAL_KEYS: frozenset[str] = frozenset(
{"api_key", "authorization", "extra_headers", "headers"}
)
DETAILS_MAX_BYTES = 16 * 1024
DETAILS_MAX_DEPTH = 6
DETAILS_MAX_ITEMS = 32
@ -362,7 +312,7 @@ def headers_from_response(headers: Mapping[str, Any] | None) -> dict[str, str] |
return picked or None
def _sanitize_value(value: object, depth: int) -> object: # noqa: PLR0911
def _bounded_value(value: object, depth: int) -> object: # noqa: PLR0911
if value is None or isinstance(value, bool | int | float):
return value
if isinstance(value, str):
@ -372,17 +322,14 @@ def _sanitize_value(value: object, depth: int) -> object: # noqa: PLR0911
if depth >= DETAILS_MAX_DEPTH:
return "…"
if isinstance(value, BaseModel):
return _sanitize_value(value.model_dump(exclude_none=True), depth)
return _bounded_value(value.model_dump(exclude_none=True), depth)
if dataclasses.is_dataclass(value) and not isinstance(value, type):
return _sanitize_value(dataclasses.asdict(value), depth)
return _bounded_value(dataclasses.asdict(value), depth)
if isinstance(value, Mapping):
out: dict[str, object] = {}
for raw_key, item in cast("Mapping[object, object]", value).items():
key = str(raw_key)
lowered = key.lower()
if lowered in _CONTENT_KEYS or lowered in _CREDENTIAL_KEYS:
continue
cleaned = _sanitize_value(item, depth + 1)
cleaned = _bounded_value(item, depth + 1)
if cleaned is None or cleaned in ({}, []):
continue
out[key[:DETAILS_MAX_STRING]] = cleaned
@ -391,19 +338,18 @@ def _sanitize_value(value: object, depth: int) -> object: # noqa: PLR0911
return out
if isinstance(value, Sequence | set | frozenset):
items = list(cast("Sequence[object]", value))[:DETAILS_MAX_ITEMS]
return [_sanitize_value(item, depth + 1) for item in items]
return [_bounded_value(item, depth + 1) for item in items]
return str(value)[:DETAILS_MAX_STRING]
def sanitize_details(value: object) -> dict[str, Any] | None:
"""A bounded, content-free copy of ``value`` (a mapping), or None.
def bound_details(value: object) -> dict[str, Any] | None:
"""A bounded copy of ``value`` (a mapping), or None.
Content keys and the caller's credential keys are dropped by name, strings
are capped, nesting, item counts and total size are bounded. When
Strings are capped; nesting, item counts and total size are bounded. When
the copy is still too large the biggest top-level entries go first and
their names are listed under ``_dropped``.
"""
cleaned = _sanitize_value(value, 0)
cleaned = _bounded_value(value, 0)
if not isinstance(cleaned, dict):
return None
details = cast("dict[str, Any]", cleaned)
@ -425,13 +371,13 @@ def sanitize_details(value: object) -> dict[str, Any] | None:
def merge_details(*parts: tuple[str, object]) -> dict[str, Any] | None:
"""``{name: sanitize(value)}`` for every non-empty part, bounded as one object."""
"""``{name: value}`` for every non-empty part, bounded as one object."""
merged: dict[str, Any] = {}
for name, value in parts:
if value is None:
continue
merged[name] = value
return sanitize_details(merged)
return bound_details(merged)
def json_text(value: object) -> str | None:
@ -1446,9 +1392,9 @@ __all__ = [
"RequestLoggingModel",
"api_host",
"bind_call_context",
"bound_details",
"clean_error_message",
"current_call_context",
"edge_request_id",
"emit",
"event_from_litellm",
"failure_text",
@ -1462,7 +1408,6 @@ __all__ = [
"request_id_from_reply",
"request_id_from_text",
"reset_call_context",
"sanitize_details",
"set_retry_attempt",
"unregister_sink",
]

37
tests/test_llm_request_log.py
View file

@ -447,16 +447,14 @@ def test_response_headers_are_bounded() -> None:
assert all(len(v) <= request_log.HEADER_VALUE_MAX_CHARS for v in picked.values())
def test_details_drop_content_and_the_callers_key_keep_metadata() -> None:
details = request_log.sanitize_details(
def test_details_keep_everything_as_written_and_bound_it() -> None:
details = request_log.bound_details(
{
"max_tokens": 4096,
"temperature": 0,
"thinking": {"type": "enabled", "budget_tokens": 1024},
"messages": [{"role": "user", "content": "SECRET PROMPT"}],
"tools": [{"name": "t", "input_schema": {}}],
"api_key": "sk-ant-secret",
"extra_headers": {"authorization": "Bearer x"},
"extra_headers": {"x-title": "strix"},
"api_base": "https://gw.example/v1?tenant=abc",
"usage": {
"input_tokens": 10,
@ -474,6 +472,8 @@ def test_details_drop_content_and_the_callers_key_keep_metadata() -> None:
"max_tokens": 4096,
"temperature": 0,
"thinking": {"type": "enabled", "budget_tokens": 1024},
"tools": [{"name": "t"}],
"extra_headers": {"x-title": "strix"},
"api_base": "https://gw.example/v1?tenant=abc",
"usage": {
"input_tokens": 10,
@ -484,16 +484,16 @@ def test_details_drop_content_and_the_callers_key_keep_metadata() -> None:
"when": "2026-09-19T00:00:00+00:00",
"nested": {"a": {"b": {"c": {"d": {"e": "…"}}}}},
}
assert request_log.sanitize_details({}) is None
assert request_log.sanitize_details("not a mapping") is None
assert request_log.sanitize_details({"messages": []}) is None
assert request_log.bound_details({}) is None
assert request_log.bound_details("not a mapping") is None
assert request_log.bound_details({"messages": []}) is None
def test_details_are_bounded_by_size_and_name_the_dropped_keys() -> None:
big = {"small": 1, "huge": ["x" * 200] * 32, "medium": {"k": "y" * 200}}
request_log.DETAILS_MAX_BYTES, saved = 1024, request_log.DETAILS_MAX_BYTES
try:
details = request_log.sanitize_details(big)
details = request_log.bound_details(big)
finally:
request_log.DETAILS_MAX_BYTES = saved
assert details is not None
@ -504,7 +504,7 @@ def test_details_are_bounded_by_size_and_name_the_dropped_keys() -> None:
def test_details_list_and_string_bounds() -> None:
details = request_log.sanitize_details(
details = request_log.bound_details(
{"items": list(range(100)), "long": "z" * 1000, "keys": {str(i): i for i in range(100)}}
)
assert details is not None
@ -572,6 +572,8 @@ def test_litellm_success_carries_sizes_finish_reason_headers_and_details() -> No
assert event.details["request"] == {
"max_tokens": 4096,
"temperature": 0,
"tools": [{"name": "a"}, {"name": "b"}],
"extra_headers": {"authorization": "Bearer x"},
"message_count": 1,
"tool_count": 2,
}
@ -586,10 +588,8 @@ def test_litellm_success_carries_sizes_finish_reason_headers_and_details() -> No
"cache_hit": False,
}
assert event.time_to_first_token_ms is None
assert "sk-ant-secret" not in str(event.to_dict())
assert "SECRET PROMPT" not in str(event.to_dict())
assert "SECRET COMPLETION" not in str(event.to_dict())
assert "Bearer" not in str(event.to_dict())
def test_litellm_request_size_prefers_the_provider_payload_litellm_built() -> None:
@ -612,14 +612,14 @@ def test_litellm_request_size_accepts_the_serialized_payload_streaming_adapters_
assert event.request_bytes == len(payload.encode())
def test_openrouter_generation_id_beats_the_cloudflare_ray() -> None:
def test_openrouter_generation_id_is_the_request_id_and_the_cloudflare_ray_is_not() -> None:
headers = {"cf-ray": "a3f5d8f23fde88dc-PDX", "x-generation-id": "gen-1790127690-M8vfqPTAr"}
assert request_log.request_id_from_reply(headers) == "gen-1790127690-M8vfqPTAr"
assert request_log.request_id_from_headers({"cf-ray": "a3f5-PDX"}) is None
assert request_log.request_id_from_reply({"cf-ray": "a3f5-PDX"}) == "a3f5-PDX"
assert request_log.request_id_from_reply({"cf-ray": "a3f5-PDX"}) is None
def test_request_id_in_the_error_body_beats_the_cloudflare_ray() -> None:
def test_request_id_in_the_error_body_when_the_gateway_strips_the_header() -> None:
"""A gateway that strips ``request-id`` still forwards Anthropic's body."""
body = '{"type":"error","error":{"type":"not_found_error"},"request_id":"req_body1"}'
exc = _anthropic_error(404, body, {"cf-ray": "a3f5-PDX", "content-type": "application/json"})
@ -628,7 +628,7 @@ def test_request_id_in_the_error_body_beats_the_cloudflare_ray() -> None:
)
assert event.provider_request_id == "req_body1"
assert request_log.request_id_from_reply({"cf-ray": "a3f5-PDX"}, body) == "req_body1"
assert request_log.request_id_from_reply({"cf-ray": "a3f5-PDX"}, "no id here") == "a3f5-PDX"
assert request_log.request_id_from_reply({"cf-ray": "a3f5-PDX"}, "no id here") is None
def test_litellm_streaming_time_to_first_token_from_completion_start() -> None:
@ -733,7 +733,7 @@ def test_litellm_api_error_with_headers_keeps_its_status() -> None:
assert event.details is not None
assert event.details["error"]["body"] == {
"type": "error",
"error": {"type": "overloaded_error"},
"error": {"type": "overloaded_error", "message": "Overloaded"},
}
@ -750,7 +750,8 @@ def test_litellm_mapped_error_without_a_json_body_has_no_size_but_keeps_its_stat
)
assert event.status_code == 502
assert event.response_bytes is None
assert event.provider_request_id == "a3f5-PDX"
assert event.provider_request_id is None
assert event.response_headers == {"cf-ray": "a3f5-PDX"}
assert event.details is not None
assert "body" not in event.details["error"]