From 657bb777fa557f7089c56fedb42f3056d7b55847 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 30 Sep 2026 13:12:28 -0700 Subject: [PATCH 01/28] fix(hosted_vllm): keep reasoning_content on replayed assistant messages (#43599) * fix(hosted_vllm): keep reasoning_content on assistant messages in _transform_messages vLLM accepts reasoning_content (200 on the wire) and qwen/deepseek/glm chat templates consume it, so popping it made reasoning models lose earlier reasoning across tool loops. thinking_blocks is still removed for vLLM compatibility. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(hosted_vllm): forward replayed reasoning_content only when it is a string * test(integration): cover hosted_vllm reasoning_content replay across endpoints * test(integration): require the surviving worker to serve its held requests in the sigkill chaos cell --------- Co-authored-by: mateo Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Co-authored-by: yassin --- .../llms/hosted_vllm/chat/transformation.py | 7 +- ...pic_messages_hosted_vllm_reasoning_wire.py | 223 +++++++ ...est_hosted_vllm_reasoning_content_chaos.py | 393 ++++++++++++ ...test_hosted_vllm_reasoning_content_wire.py | 565 ++++++++++++++++++ .../test_hosted_vllm_chat_transformation.py | 39 +- 5 files changed, 1220 insertions(+), 7 deletions(-) create mode 100644 tests/integration/messages_endpoint/chat_bridge/test_anthropic_messages_hosted_vllm_reasoning_wire.py create mode 100644 tests/integration/providers/test_hosted_vllm_reasoning_content_chaos.py create mode 100644 tests/integration/providers/test_hosted_vllm_reasoning_content_wire.py diff --git a/litellm/llms/hosted_vllm/chat/transformation.py b/litellm/llms/hosted_vllm/chat/transformation.py index 32c60bd01b5..43eb2af171e 100644 --- a/litellm/llms/hosted_vllm/chat/transformation.py +++ b/litellm/llms/hosted_vllm/chat/transformation.py @@ -161,13 +161,14 @@ class HostedVLLMChatConfig(OpenAIGPTConfig): """ Support translating: - video files from file_id or file_data to video_url - - thinking_blocks and reasoning_content on assistant messages are removed, - and content lists are converted to strings for vLLM compatibility + - thinking_blocks and non-string reasoning_content on assistant messages + are removed, and content lists are converted to strings for vLLM compatibility """ for message in messages: if message["role"] == "assistant": message.pop("thinking_blocks", None) - message.pop("reasoning_content", None) + if not isinstance(message.get("reasoning_content"), str): + message.pop("reasoning_content", None) existing_content = message.get("content") if isinstance(existing_content, list): text_parts = [] diff --git a/tests/integration/messages_endpoint/chat_bridge/test_anthropic_messages_hosted_vllm_reasoning_wire.py b/tests/integration/messages_endpoint/chat_bridge/test_anthropic_messages_hosted_vllm_reasoning_wire.py new file mode 100644 index 00000000000..2359a8f768f --- /dev/null +++ b/tests/integration/messages_endpoint/chat_bridge/test_anthropic_messages_hosted_vllm_reasoning_wire.py @@ -0,0 +1,223 @@ +import json +import uuid +from typing import Final + +import anthropic +from integration._support.client import Gateway, eventually +from integration._support.database import read_rows +from integration._support.wire import Reply, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +_BACKEND: Final = "glm-reasoning" +_API_KEY: Final = "synthetic-hosted-vllm-key" +_TOOL_USE_ID: Final = "toolu_weather_1" +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_MESSAGES: Final = TypeAdapter(list[dict[str, JsonValue]]) +_TOOLS: Final[list[dict[str, JsonValue]]] = [ + { + "name": "get_weather", + "description": "Get the current weather for a city", + "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, + } +] + + +def _completion(identity: str) -> bytes: + return json.dumps( + { + "id": identity, + "object": "chat.completion", + "created": 1, + "model": _BACKEND, + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "It is raining."}, "finish_reason": "stop"} + ], + "usage": {"prompt_tokens": 30, "completion_tokens": 5, "total_tokens": 35}, + } + ).encode() + + +def _streamed_completion(identity: str) -> Reply: + chunk: Final = {"id": identity, "object": "chat.completion.chunk", "created": 1, "model": _BACKEND} + frames: Final = ( + {**chunk, "choices": [{"index": 0, "delta": {"role": "assistant", "content": "It is raining."}}]}, + { + **chunk, + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 30, "completion_tokens": 5, "total_tokens": 35}, + }, + ) + return Reply( + content_type="text/event-stream", + chunks=(*(b"data: " + json.dumps(frame).encode() + b"\n\n" for frame in frames), b"data: [DONE]\n\n"), + ) + + +def _tool_loop(thinking: str, marker: str) -> list[dict[str, JsonValue]]: + return [ + {"role": "user", "content": f"What is the weather in Paris? {marker}"}, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": thinking, "signature": "opaque-signature"}, + {"type": "text", "text": "Let me check."}, + {"type": "tool_use", "id": _TOOL_USE_ID, "name": "get_weather", "input": {"city": "Paris"}}, + ], + }, + { + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": _TOOL_USE_ID, "content": "light rain, 14C"}], + }, + ] + + +def _expected_upstream(thinking: str, marker: str) -> list[dict[str, JsonValue]]: + return [ + {"role": "user", "content": f"What is the weather in Paris? {marker}"}, + { + "role": "assistant", + "content": "Let me check.", + "reasoning_content": thinking, + "tool_calls": [ + { + "id": _TOOL_USE_ID, + "type": "function", + "function": {"name": "get_weather", "arguments": json.dumps({"city": "Paris"})}, + } + ], + }, + {"role": "tool", "tool_call_id": _TOOL_USE_ID, "content": "light rain, 14C"}, + ] + + +def _only_body(wire: Wire) -> dict[str, JsonValue]: + received: Final = wire.drain() + assert [(request.method, request.target) for request in received] == [("POST", "/v1/chat/completions")] + return _JSON_OBJECT.validate_json(received[0].body) + + +def _sent_messages(body: dict[str, JsonValue]) -> list[dict[str, JsonValue]]: + return _MESSAGES.validate_python(body["messages"]) + + +def _spend_status(identity: str) -> JsonValue: + rows: Final = eventually( + lambda: read_rows('SELECT status FROM "LiteLLM_SpendLogs" WHERE request_id=%s', (identity,)), + lambda found: len(found) == 1, + seconds=70, + ) + return rows[0]["status"] + + +def _post_messages(gateway: Gateway, model: str, messages: list[dict[str, JsonValue]]) -> dict[str, JsonValue]: + response: Final = gateway.request( + "POST", + "/v1/messages", + {"model": model, "max_tokens": 256, "messages": messages, "cache": {"no-cache": True}}, + headers={"anthropic-version": "2023-06-01"}, + ) + assert response.status_code == 200, response.text + return _JSON_OBJECT.validate_json(response.content) + + +def test_anthropic_sdk_thinking_block_reaches_hosted_vllm_as_reasoning_content(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + identity: Final = f"chatcmpl-messages-{marker}" + thinking: Final = f"The user wants Paris weather, codeword mango{marker[:4]}." + with wire_server(lambda _: Reply(body=_completion(identity))) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"hosted_vllm/{_BACKEND}", api_base=wire.url + "/v1", api_key=_API_KEY) + client: Final = anthropic.Anthropic(base_url=str(gateway.client.base_url), api_key=gateway.key, max_retries=0) + message: Final = client.messages.create( + model=model, + max_tokens=256, + tools=_TOOLS, # pyright: ignore[reportArgumentType] # plain JSON tool definitions + messages=_tool_loop(thinking, marker), # pyright: ignore[reportArgumentType] # plain JSON content blocks + ) + assert message.id == identity + assert [(block.type, getattr(block, "text", None)) for block in message.content] == [("text", "It is raining.")] + body: Final = _only_body(wire) + assert _sent_messages(body) == _expected_upstream(thinking, marker) + assert "thinking_blocks" not in json.dumps(body) and "opaque-signature" not in json.dumps(body), body + assert _spend_status(identity) == "success" + + +async def test_async_anthropic_sdk_stream_forwards_thinking_to_hosted_vllm(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + identity: Final = f"chatcmpl-messages-stream-{marker}" + thinking: Final = f"Streaming thought {marker}." + with wire_server(lambda _: _streamed_completion(identity)) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"hosted_vllm/{_BACKEND}", api_base=wire.url + "/v1", api_key=_API_KEY) + client: Final = anthropic.AsyncAnthropic( + base_url=str(gateway.client.base_url), api_key=gateway.key, max_retries=0 + ) + stream: Final = await client.messages.create( + model=model, + max_tokens=256, + tools=_TOOLS, # pyright: ignore[reportArgumentType] # plain JSON tool definitions + messages=_tool_loop(thinking, marker), # pyright: ignore[reportArgumentType] # plain JSON content blocks + stream=True, + ) + events: Final = [event async for event in stream] + assert events[0].type == "message_start" and events[-1].type == "message_stop" + message_id: Final = events[0].message.id + assert "".join( + event.delta.text + for event in events + if event.type == "content_block_delta" and event.delta.type == "text_delta" + ) == ("It is raining.") + body: Final = _only_body(wire) + assert body["stream"] is True + assert _sent_messages(body) == _expected_upstream(thinking, marker) + assert _spend_status(message_id) == "success" + + +def test_redacted_thinking_alone_sends_no_reasoning_content(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + with ( + wire_server(lambda _: Reply(body=_completion(f"chatcmpl-{marker}"))) as wire, + gateway.scenario() as scenario, + ): + model: Final = scenario.model(model=f"hosted_vllm/{_BACKEND}", api_base=wire.url + "/v1", api_key=_API_KEY) + _post_messages( + gateway, + model, + [ + {"role": "user", "content": f"Hello {marker}"}, + { + "role": "assistant", + "content": [ + {"type": "redacted_thinking", "data": "opaque-redacted"}, + {"type": "text", "text": "Hi."}, + ], + }, + {"role": "user", "content": "Again"}, + ], + ) + assert _sent_messages(_only_body(wire)) == [ + {"role": "user", "content": f"Hello {marker}"}, + {"role": "assistant", "content": "Hi."}, + {"role": "user", "content": "Again"}, + ] + + +def test_assistant_turn_without_thinking_sends_no_reasoning_content(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + with ( + wire_server(lambda _: Reply(body=_completion(f"chatcmpl-{marker}"))) as wire, + gateway.scenario() as scenario, + ): + model: Final = scenario.model(model=f"hosted_vllm/{_BACKEND}", api_base=wire.url + "/v1", api_key=_API_KEY) + _post_messages( + gateway, + model, + [ + {"role": "user", "content": f"Hello {marker}"}, + {"role": "assistant", "content": [{"type": "text", "text": "Hi."}]}, + {"role": "user", "content": "Again"}, + ], + ) + assert _sent_messages(_only_body(wire)) == [ + {"role": "user", "content": f"Hello {marker}"}, + {"role": "assistant", "content": "Hi."}, + {"role": "user", "content": "Again"}, + ] diff --git a/tests/integration/providers/test_hosted_vllm_reasoning_content_chaos.py b/tests/integration/providers/test_hosted_vllm_reasoning_content_chaos.py new file mode 100644 index 00000000000..84a5d71e8b8 --- /dev/null +++ b/tests/integration/providers/test_hosted_vllm_reasoning_content_chaos.py @@ -0,0 +1,393 @@ +import asyncio +import json +import re +import signal +import threading +import uuid +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path +from queue import SimpleQueue +from types import MappingProxyType +from typing import Final, Literal +from urllib.parse import urlsplit + +import httpx +import psutil +import pytest +import yaml +from integration._support.client import Gateway, eventually +from integration._support.database import read_rows +from integration._support.process import owned_proxy_process +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +_BACKEND: Final = "qwen3-reasoning-chaos" +_API_KEY: Final = "synthetic-hosted-vllm-key" +_CONFIG_MODEL: Final = "hosted-vllm-reasoning-chaos" +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_MESSAGES: Final = TypeAdapter(list[dict[str, JsonValue]]) +_MARKER: Final = re.compile(r"marker-([0-9a-f]{32})") +_STARTED_WORKER: Final = re.compile(r"Started server process \[(\d+)\]") +_MODEL_LIST: Final = json.dumps( + {"object": "list", "data": [{"id": _BACKEND, "object": "model", "owned_by": "vllm"}]} +).encode() + +Endpoint = Literal["chat", "messages", "responses"] + + +@dataclass(frozen=True, slots=True) +class _Call: + endpoint: Endpoint + stream: bool + marker: str + + +@dataclass(frozen=True, slots=True) +class _Served: + call: _Call + status: int + text: str + + +def _thought(marker: str) -> str: + return f"private thought for {marker}" + + +def _answer(marker: str) -> str: + return f"answer marker-{marker}" + + +def _path(endpoint: Endpoint) -> str: + match endpoint: + case "chat": + return "/v1/chat/completions" + case "messages": + return "/v1/messages" + case "responses": + return "/v1/responses" + + +def _body(model: str, call: _Call) -> dict[str, JsonValue]: + question: Final = f"Question marker-{call.marker}" + common: Final[dict[str, JsonValue]] = {"model": model, "stream": call.stream, "num_retries": 0} + match call.endpoint: + case "chat": + return { + **common, + "messages": [ + {"role": "user", "content": question}, + {"role": "assistant", "content": "Working on it.", "reasoning_content": _thought(call.marker)}, + {"role": "user", "content": "Go on."}, + ], + } + case "messages": + return { + **common, + "max_tokens": 64, + "messages": [ + {"role": "user", "content": question}, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": _thought(call.marker), "signature": "sig"}, + {"type": "text", "text": "Working on it."}, + ], + }, + {"role": "user", "content": "Go on."}, + ], + } + case "responses": + return { + **common, + "input": [ + {"role": "user", "content": question}, + { + "id": f"rs_{call.marker}", + "type": "reasoning", + "summary": [{"type": "summary_text", "text": _thought(call.marker)}], + }, + {"role": "user", "content": "Go on."}, + ], + } + + +def _chat_reply(marker: str, stream: bool, abort_after: int | None = None, pause: float = 0) -> Reply: + usage: Final = {"prompt_tokens": 30, "completion_tokens": 5, "total_tokens": 35} + if not stream: + return Reply( + body=json.dumps( + { + "id": f"chatcmpl-{marker}", + "object": "chat.completion", + "created": 1, + "model": _BACKEND, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": _answer(marker)}, + "finish_reason": "stop", + } + ], + "usage": usage, + } + ).encode() + ) + chunk: Final = {"id": f"chatcmpl-{marker}", "object": "chat.completion.chunk", "created": 1, "model": _BACKEND} + frames: Final = ( + {**chunk, "choices": [{"index": 0, "delta": {"role": "assistant", "content": "answer "}}]}, + {**chunk, "choices": [{"index": 0, "delta": {"content": f"marker-{marker}"}}]}, + {**chunk, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], "usage": usage}, + ) + return Reply( + content_type="text/event-stream", + chunks=(*(b"data: " + json.dumps(frame).encode() + b"\n\n" for frame in frames), b"data: [DONE]\n\n"), + abort_after=abort_after, + pause_between_chunks=pause, + ) + + +def _responses_reply(marker: str, stream: bool) -> Reply: + identity: Final = f"resp_upstream_{marker}" + response: Final = { + "id": identity, + "object": "response", + "created_at": 1, + "status": "completed", + "model": _BACKEND, + "output": [ + { + "id": f"msg_{marker}", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": _answer(marker), "annotations": []}], + } + ], + "usage": {"input_tokens": 30, "output_tokens": 5, "total_tokens": 35}, + } + if not stream: + return Reply(body=json.dumps(response).encode()) + events: Final = ( + { + "type": "response.created", + "sequence_number": 0, + "response": {**response, "status": "in_progress", "output": []}, + }, + { + "type": "response.output_text.delta", + "sequence_number": 1, + "item_id": f"msg_{marker}", + "output_index": 0, + "content_index": 0, + "delta": _answer(marker), + }, + {"type": "response.completed", "sequence_number": 2, "response": response}, + ) + return Reply( + content_type="text/event-stream", + chunks=tuple(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode() for event in events), + ) + + +def _marker_of(request: Request) -> str: + found: Final = _MARKER.search(request.body.decode()) + assert found is not None, request.body + return found.group(1) + + +def _echo(request: Request) -> Reply: + marker: Final = _marker_of(request) + stream: Final = _JSON_OBJECT.validate_json(request.body).get("stream") is True + if request.target == "/v1/responses": + return _responses_reply(marker, stream) + return _chat_reply(marker, stream) + + +def _forwarded_reasoning(request: Request) -> tuple[str, JsonValue]: + body: Final = _JSON_OBJECT.validate_json(request.body) + if request.target == "/v1/responses": + reasoning_item: Final = _MESSAGES.validate_python(body["input"])[1] + return _marker_of(request), _MESSAGES.validate_python(reasoning_item["summary"])[0]["text"] + assert request.target == "/v1/chat/completions", request.target + return _marker_of(request), _MESSAGES.validate_python(body["messages"])[1].get("reasoning_content") + + +def _assert_no_bleed(received: tuple[Request, ...], markers: frozenset[str]) -> None: + forwarded: Final = [_forwarded_reasoning(request) for request in received] + assert sorted(marker for marker, _ in forwarded) == sorted(markers) + assert all(reasoning == _thought(marker) for marker, reasoning in forwarded), forwarded + + +def _spend_statuses(model: str, expected: int) -> list[JsonValue]: + rows: Final = eventually( + lambda: read_rows('SELECT request_id, status FROM "LiteLLM_SpendLogs" WHERE model_group=%s', (model,)), + lambda found: len(found) >= expected, + seconds=60, + ) + assert len({row["request_id"] for row in rows}) == len(rows), rows + return [row["status"] for row in rows] + + +async def _send(client: httpx.AsyncClient, key: str, model: str, call: _Call) -> _Served: + async with client.stream( + "POST", + _path(call.endpoint), + json=_body(model, call), + headers={"Authorization": f"Bearer {key}", "anthropic-version": "2023-06-01"}, + ) as response: + raw: Final = await response.aread() + return _Served(call=call, status=response.status_code, text=raw.decode()) + + +async def _burst( + base_url: str, key: str, model: str, calls: tuple[_Call, ...], *, tolerate_transport_errors: bool = False +) -> tuple[_Served, ...]: + async with httpx.AsyncClient(base_url=base_url, timeout=60, trust_env=False) as client: + results: Final = await asyncio.gather( + *(_send(client, key, model, call) for call in calls), return_exceptions=tolerate_transport_errors + ) + for result in results: + assert not isinstance(result, BaseException) or isinstance(result, httpx.TransportError), repr(result) + return tuple(result for result in results if isinstance(result, _Served)) + + +def _calls(count: int, endpoints: tuple[Endpoint, ...], stream: Callable[[int], bool]) -> tuple[_Call, ...]: + return tuple( + _Call(endpoint=endpoints[index % len(endpoints)], stream=stream(index), marker=uuid.uuid4().hex) + for index in range(count) + ) + + +def _assert_answered_with_its_own_marker(served: _Served) -> None: + assert served.status == 200, served.text + assert set(_MARKER.findall(served.text)) == {served.call.marker}, served.text + + +async def test_concurrent_replays_across_endpoints_keep_each_reasoning_with_its_request(gateway: Gateway) -> None: + calls: Final = _calls(30, ("chat", "messages", "responses"), lambda index: index % 2 == 0) + with wire_server(_echo) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"hosted_vllm/{_BACKEND}", api_base=wire.url + "/v1", api_key=_API_KEY) + served: Final = await _burst(str(gateway.client.base_url), gateway.key, model, calls) + assert len(served) == 30 + for item in served: + _assert_answered_with_its_own_marker(item) + _assert_no_bleed(wire.drain(), frozenset(call.marker for call in calls)) + assert _spend_statuses(model, 30) == ["success"] * 30 + + +async def test_upstream_stream_aborts_reach_callers_and_later_replays_still_forward_reasoning( + gateway: Gateway, +) -> None: + calls: Final = _calls(12, ("chat",), lambda _: True) + aborted: Final = frozenset(call.marker for index, call in enumerate(calls) if index % 3 == 0) + + def respond(request: Request) -> Reply: + marker: Final = _marker_of(request) + return _chat_reply(marker, stream=True, abort_after=0 if marker in aborted else None) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"hosted_vllm/{_BACKEND}", api_base=wire.url + "/v1", api_key=_API_KEY) + served: Final = await _burst(str(gateway.client.base_url), gateway.key, model, calls) + assert len(served) == 12 + for item in served: + if item.call.marker in aborted: + assert item.status == 500, item.text + assert "APIConnectionError" in item.text and "marker-" not in item.text, item.text + else: + _assert_answered_with_its_own_marker(item) + assert item.text.rstrip().endswith("data: [DONE]"), item.text + recovery: Final = _Call(endpoint="chat", stream=True, marker=uuid.uuid4().hex) + (recovered,) = await _burst(str(gateway.client.base_url), gateway.key, model, (recovery,)) + _assert_answered_with_its_own_marker(recovered) + _assert_no_bleed(wire.drain(), frozenset(call.marker for call in (*calls, recovery))) + + +async def test_slow_upstream_streams_are_forwarded_once_with_their_own_reasoning(gateway: Gateway) -> None: + calls: Final = _calls(10, ("chat",), lambda _: True) + with ( + wire_server(lambda request: _chat_reply(_marker_of(request), stream=True, pause=0.3)) as wire, + gateway.scenario() as scenario, + ): + model: Final = scenario.model(model=f"hosted_vllm/{_BACKEND}", api_base=wire.url + "/v1", api_key=_API_KEY) + served: Final = await _burst(str(gateway.client.base_url), gateway.key, model, calls) + assert len(served) == 10 + for item in served: + _assert_answered_with_its_own_marker(item) + assert item.text.rstrip().endswith("data: [DONE]"), item.text + _assert_no_bleed(wire.drain(), frozenset(call.marker for call in calls)) + assert _spend_statuses(model, 10) == ["success"] * 10 + + +def _chaos_config(wire: Wire, tmp_path: Path) -> Path: + config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + config["model_list"] = [ + { + "model_name": _CONFIG_MODEL, + "litellm_params": {"model": f"hosted_vllm/{_BACKEND}", "api_base": wire.url + "/v1", "api_key": _API_KEY}, + } + ] + path: Final = tmp_path / "hosted-vllm-reasoning-chaos.yaml" + path.write_text(yaml.safe_dump(config)) + return path + + +def _open_upstream_connections(pid: int, upstream: str) -> int: + port: Final = urlsplit(upstream).port + return sum( + 1 + for connection in psutil.Process(pid).net_connections(kind="tcp") + if connection.status == psutil.CONN_ESTABLISHED and connection.raddr and connection.raddr.port == port + ) + + +@pytest.mark.timeout(180) +async def test_worker_sigkill_mid_burst_leaves_the_sibling_forwarding_reasoning( + gateway: Gateway, tmp_path: Path +) -> None: + calls: Final = _calls(20, ("chat",), lambda _: False) + release: Final = threading.Event() + held_markers: Final[SimpleQueue[str]] = SimpleQueue() + + def held(request: Request) -> Reply: + if (request.method, request.target) == ("GET", "/v1/models"): + return Reply(body=_MODEL_LIST) + held_markers.put(_marker_of(request)) + assert release.wait(timeout=60), "The burst was never released" + return _echo(request) + + with wire_server(held) as wire: + path: Final = _chaos_config(wire, tmp_path) + with owned_proxy_process(gateway, tmp_path, {}, config=path, workers=2) as owned: + candidate: Final = owned.gateway + workers: Final = eventually( + lambda: tuple(int(pid) for pid in _STARTED_WORKER.findall(owned.log.read_text())), + lambda pids: len(pids) == 2, + seconds=30, + ) + burst: Final = asyncio.create_task( + _burst( + str(candidate.client.base_url), candidate.key, _CONFIG_MODEL, calls, tolerate_transport_errors=True + ) + ) + await asyncio.to_thread(eventually, held_markers.qsize, lambda size: size == 20, 60) + held_by: Final = MappingProxyType({pid: _open_upstream_connections(pid, wire.url) for pid in workers}) + assert sum(held_by.values()) == 20, held_by + victim_pid, survivor_pid = sorted(workers, key=held_by.__getitem__) + victim: Final = psutil.Process(victim_pid) + victim.suspend() + victim.send_signal(signal.SIGKILL) + release.set() + served: Final = await burst + assert held_by[survivor_pid] >= 10, held_by + assert len(served) == held_by[survivor_pid], (held_by, len(served)) + for item in served: + _assert_answered_with_its_own_marker(item) + follow_up: Final = _Call(endpoint="chat", stream=False, marker=uuid.uuid4().hex) + (answered,) = await _burst(str(candidate.client.base_url), candidate.key, _CONFIG_MODEL, (follow_up,)) + _assert_answered_with_its_own_marker(answered) + received: Final = wire.drain() + chats: Final = tuple(request for request in received if request.method == "POST") + probes: Final = [(request.method, request.target) for request in received if request.method != "POST"] + assert set(probes) <= {("GET", "/v1/models")}, probes + _assert_no_bleed(chats, frozenset(call.marker for call in (*calls, follow_up))) diff --git a/tests/integration/providers/test_hosted_vllm_reasoning_content_wire.py b/tests/integration/providers/test_hosted_vllm_reasoning_content_wire.py new file mode 100644 index 00000000000..858ec1af242 --- /dev/null +++ b/tests/integration/providers/test_hosted_vllm_reasoning_content_wire.py @@ -0,0 +1,565 @@ +import json +import uuid +from collections.abc import Sequence +from typing import Final + +import openai +import pytest +from integration._support.client import Gateway, eventually +from integration._support.database import read_rows +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +_BACKEND: Final = "qwen3-reasoning" +_FALLBACK_BACKEND: Final = "qwen3-reasoning-fallback" +_API_KEY: Final = "synthetic-hosted-vllm-key" +_REASONING: Final = "I compared the two invoices and the totals differ by 42." +_ANSWER_REASONING: Final = "The user wants the difference, which is 42." +_TOOL_CALL_ID: Final = "call_reasoning_wire_1" +_NO_CACHE: Final[dict[str, JsonValue]] = {"no-cache": True} +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_MESSAGES: Final = TypeAdapter(list[dict[str, JsonValue]]) + + +def _completion(identity: str, content: str) -> bytes: + return json.dumps( + { + "id": identity, + "object": "chat.completion", + "created": 1, + "model": _BACKEND, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": content, "reasoning_content": _ANSWER_REASONING}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 30, "completion_tokens": 5, "total_tokens": 35}, + } + ).encode() + + +def _streamed_completion(identity: str, content: str) -> Reply: + chunk: Final = {"id": identity, "object": "chat.completion.chunk", "created": 1, "model": _BACKEND} + frames: Final = ( + {**chunk, "choices": [{"index": 0, "delta": {"role": "assistant", "reasoning_content": _ANSWER_REASONING}}]}, + {**chunk, "choices": [{"index": 0, "delta": {"content": content}}]}, + { + **chunk, + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 30, "completion_tokens": 5, "total_tokens": 35}, + }, + ) + return Reply( + content_type="text/event-stream", + chunks=(*(b"data: " + json.dumps(frame).encode() + b"\n\n" for frame in frames), b"data: [DONE]\n\n"), + ) + + +def _replayed_conversation(reasoning: JsonValue, marker: str) -> list[dict[str, JsonValue]]: + return [ + {"role": "user", "content": f"Compare these invoices {marker}."}, + {"role": "assistant", "content": "Checking the totals.", "reasoning_content": reasoning}, + {"role": "user", "content": "What is the difference?"}, + ] + + +def _sent_messages(request: Request) -> list[dict[str, JsonValue]]: + return _MESSAGES.validate_python(_JSON_OBJECT.validate_json(request.body)["messages"]) + + +def _only_request(wire: Wire) -> Request: + received: Final = wire.drain() + assert [(request.method, request.target) for request in received] == [("POST", "/v1/chat/completions")] + return received[0] + + +def _spend_row(identity: str) -> dict[str, JsonValue]: + rows: Final = eventually( + lambda: read_rows( + 'SELECT model_group, status, prompt_tokens, completion_tokens FROM "LiteLLM_SpendLogs" WHERE request_id=%s', + (identity,), + ), + lambda found: len(found) == 1, + seconds=70, + ) + return rows[0] + + +def _model_spend_statuses(model: str) -> list[JsonValue]: + rows: Final = eventually( + lambda: read_rows('SELECT status FROM "LiteLLM_SpendLogs" WHERE model_group=%s', (model,)), + lambda found: len(found) >= 1, + seconds=70, + ) + return [row["status"] for row in rows] + + +def _openai_client(gateway: Gateway) -> openai.OpenAI: + return openai.OpenAI(base_url=str(gateway.client.base_url) + "/v1", api_key=gateway.key, max_retries=0) + + +def _async_openai_client(gateway: Gateway) -> openai.AsyncOpenAI: + return openai.AsyncOpenAI(base_url=str(gateway.client.base_url) + "/v1", api_key=gateway.key, max_retries=0) + + +def _post_chat(gateway: Gateway, model: str, messages: Sequence[dict[str, JsonValue]]) -> dict[str, JsonValue]: + response: Final = gateway.request( + "POST", "/v1/chat/completions", {"model": model, "messages": list(messages), "cache": _NO_CACHE} + ) + assert response.status_code == 200, response.text + return _JSON_OBJECT.validate_json(response.content) + + +def test_hosted_vllm_assistant_reasoning_content_reaches_the_wire(gateway: Gateway) -> None: + identity: Final = f"hosted-vllm-reasoning-{uuid.uuid4().hex}" + + def respond(request: Request) -> Reply: + assert request.method == "POST" + assert request.target == "/v1/chat/completions" + assert request.headers["authorization"] == f"Bearer {_API_KEY}" + body: Final = _JSON_OBJECT.validate_json(request.body) + assert body["model"] == _BACKEND + assert body["messages"] == [ + {"role": "user", "content": "Compare these invoices."}, + { + "role": "assistant", + "content": "Checking the totals.", + "reasoning_content": _REASONING, + "tool_calls": [ + { + "id": _TOOL_CALL_ID, + "type": "function", + "function": {"name": "lookup_invoice", "arguments": json.dumps({"id": "inv-7"})}, + } + ], + }, + {"role": "tool", "tool_call_id": _TOOL_CALL_ID, "content": "invoice total is 1042"}, + ], body["messages"] + return Reply(body=_completion(identity, "The totals differ by 42.")) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"hosted_vllm/{_BACKEND}", api_base=wire.url + "/v1", api_key=_API_KEY) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "messages": [ + {"role": "user", "content": "Compare these invoices."}, + { + "role": "assistant", + "content": "Checking the totals.", + "reasoning_content": _REASONING, + "tool_calls": [ + { + "id": _TOOL_CALL_ID, + "type": "function", + "function": {"name": "lookup_invoice", "arguments": json.dumps({"id": "inv-7"})}, + } + ], + }, + {"role": "tool", "tool_call_id": _TOOL_CALL_ID, "content": "invoice total is 1042"}, + ], + }, + ) + assert response.status_code == 200, response.text + payload: Final = _JSON_OBJECT.validate_json(response.content) + assert payload["id"] == identity + assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/v1/chat/completions")] + + +def test_openai_sdk_replayed_reasoning_reaches_hosted_vllm_and_is_billed_once(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + identity: Final = f"chatcmpl-sdk-{marker}" + with wire_server(lambda _: Reply(body=_completion(identity, "They differ by 42."))) as wire: + with gateway.scenario() as scenario: + model: Final = scenario.model(model=f"hosted_vllm/{_BACKEND}", api_base=wire.url + "/v1", api_key=_API_KEY) + completion: Final = _openai_client(gateway).chat.completions.create( + model=model, + messages=_replayed_conversation(_REASONING, marker), # pyright: ignore[reportArgumentType] # reasoning_content is a provider extension the SDK types omit + ) + assert completion.id == identity + assert completion.choices[0].message.content == "They differ by 42." + assert (completion.choices[0].message.model_extra or {})["reasoning_content"] == _ANSWER_REASONING + assert _sent_messages(_only_request(wire)) == _replayed_conversation(_REASONING, marker) + assert _spend_row(identity) == { + "model_group": model, + "status": "success", + "prompt_tokens": 30, + "completion_tokens": 5, + } + + +async def test_async_openai_sdk_stream_forwards_replayed_reasoning_to_hosted_vllm(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + identity: Final = f"chatcmpl-stream-{marker}" + with wire_server(lambda _: _streamed_completion(identity, "They differ by 42.")) as wire: + with gateway.scenario() as scenario: + model: Final = scenario.model(model=f"hosted_vllm/{_BACKEND}", api_base=wire.url + "/v1", api_key=_API_KEY) + stream: Final = await _async_openai_client(gateway).chat.completions.create( + model=model, + messages=_replayed_conversation(_REASONING, marker), # pyright: ignore[reportArgumentType] # reasoning_content is a provider extension the SDK types omit + stream=True, + stream_options={"include_usage": True}, + ) + chunks: Final = [chunk async for chunk in stream] + assert {chunk.id for chunk in chunks} == {identity} + assert "".join(choice.delta.content or "" for chunk in chunks for choice in chunk.choices) == ( + "They differ by 42." + ) + sent: Final = _only_request(wire) + assert _JSON_OBJECT.validate_json(sent.body)["stream"] is True + assert _sent_messages(sent) == _replayed_conversation(_REASONING, marker) + assert _spend_row(identity)["status"] == "success" + + +def test_each_replayed_turn_keeps_its_own_reasoning_in_order(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + conversation: Final[list[dict[str, JsonValue]]] = [ + {"role": "user", "content": f"Plan the migration {marker}."}, + {"role": "assistant", "content": "Step one.", "reasoning_content": f"first thought {marker}"}, + {"role": "user", "content": "Continue."}, + {"role": "assistant", "content": "Step two.", "reasoning_content": f"second thought {marker}"}, + {"role": "user", "content": "Summarize."}, + ] + with wire_server(lambda _: Reply(body=_completion(f"chatcmpl-{marker}", "Done."))) as wire: + with gateway.scenario() as scenario: + model: Final = scenario.model(model=f"hosted_vllm/{_BACKEND}", api_base=wire.url + "/v1", api_key=_API_KEY) + assert _post_chat(gateway, model, conversation)["id"] == f"chatcmpl-{marker}" + assert _sent_messages(_only_request(wire)) == conversation + + +@pytest.mark.parametrize( + ("reasoning", "forwarded"), + [ + pytest.param("", "", id="empty-string-forwarded"), + pytest.param("x" * 5120, "x" * 5120, id="5kb-string-forwarded-intact"), + pytest.param(None, None, id="null-dropped"), + pytest.param(42, None, id="int-dropped"), + pytest.param(["step one", "step two"], None, id="list-dropped"), + pytest.param({"text": "step one"}, None, id="object-dropped"), + ], +) +def test_only_string_reasoning_content_is_forwarded_to_hosted_vllm( + gateway: Gateway, reasoning: JsonValue, forwarded: str | None +) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(lambda _: Reply(body=_completion(f"chatcmpl-{marker}", "Done."))) as wire: + with gateway.scenario() as scenario: + model: Final = scenario.model(model=f"hosted_vllm/{_BACKEND}", api_base=wire.url + "/v1", api_key=_API_KEY) + assert _post_chat(gateway, model, _replayed_conversation(reasoning, marker))["id"] == f"chatcmpl-{marker}" + sent_assistant: Final = _sent_messages(_only_request(wire))[1] + expected_assistant: Final[dict[str, JsonValue]] = {"role": "assistant", "content": "Checking the totals."} + assert sent_assistant == ( + expected_assistant if forwarded is None else {**expected_assistant, "reasoning_content": forwarded} + ) + + +def test_assistant_turn_without_reasoning_gets_no_reasoning_key(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + conversation: Final[list[dict[str, JsonValue]]] = [ + {"role": "user", "content": f"Hello {marker}"}, + {"role": "assistant", "content": "Hi there."}, + {"role": "user", "content": "Again"}, + ] + with wire_server(lambda _: Reply(body=_completion(f"chatcmpl-{marker}", "Hello again."))) as wire: + with gateway.scenario() as scenario: + model: Final = scenario.model(model=f"hosted_vllm/{_BACKEND}", api_base=wire.url + "/v1", api_key=_API_KEY) + _post_chat(gateway, model, conversation) + assert _sent_messages(_only_request(wire)) == conversation + + +def test_same_reasoning_on_two_turns_is_forwarded_on_both(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + reasoning: Final = f"repeated thought {marker}" + conversation: Final[list[dict[str, JsonValue]]] = [ + {"role": "user", "content": "One"}, + {"role": "assistant", "content": "First.", "reasoning_content": reasoning}, + {"role": "user", "content": "Two"}, + {"role": "assistant", "content": "Second.", "reasoning_content": reasoning}, + {"role": "user", "content": "Three"}, + ] + with wire_server(lambda _: Reply(body=_completion(f"chatcmpl-{marker}", "Third."))) as wire: + with gateway.scenario() as scenario: + model: Final = scenario.model(model=f"hosted_vllm/{_BACKEND}", api_base=wire.url + "/v1", api_key=_API_KEY) + _post_chat(gateway, model, conversation) + assert _sent_messages(_only_request(wire)) == conversation + + +def test_thinking_blocks_are_stripped_while_reasoning_content_is_kept(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(lambda _: Reply(body=_completion(f"chatcmpl-{marker}", "Done."))) as wire: + with gateway.scenario() as scenario: + model: Final = scenario.model(model=f"hosted_vllm/{_BACKEND}", api_base=wire.url + "/v1", api_key=_API_KEY) + _post_chat( + gateway, + model, + [ + {"role": "user", "content": f"Hello {marker}"}, + { + "role": "assistant", + "content": "Hi.", + "reasoning_content": _REASONING, + "thinking_blocks": [{"type": "thinking", "thinking": _REASONING, "signature": "sig"}], + }, + {"role": "user", "content": "Again"}, + ], + ) + assert _sent_messages(_only_request(wire))[1] == { + "role": "assistant", + "content": "Hi.", + "reasoning_content": _REASONING, + } + + +def test_list_content_is_flattened_while_reasoning_content_is_kept(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(lambda _: Reply(body=_completion(f"chatcmpl-{marker}", "Done."))) as wire: + with gateway.scenario() as scenario: + model: Final = scenario.model(model=f"hosted_vllm/{_BACKEND}", api_base=wire.url + "/v1", api_key=_API_KEY) + _post_chat( + gateway, + model, + [ + {"role": "user", "content": f"Hello {marker}"}, + { + "role": "assistant", + "content": [{"type": "text", "text": "Part one."}, {"type": "text", "text": "Part two."}], + "reasoning_content": _REASONING, + }, + {"role": "user", "content": "Again"}, + ], + ) + assert _sent_messages(_only_request(wire))[1] == { + "role": "assistant", + "content": "Part one.\nPart two.", + "reasoning_content": _REASONING, + } + + +def test_unauthenticated_replay_is_rejected_before_hosted_vllm(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(lambda _: Reply(body=_completion(f"chatcmpl-{marker}", "Done."))) as wire: + with gateway.scenario() as scenario: + model: Final = scenario.model(model=f"hosted_vllm/{_BACKEND}", api_base=wire.url + "/v1", api_key=_API_KEY) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": _replayed_conversation(_REASONING, marker)}, + key=f"sk-not-a-key-{marker}", + ) + assert response.status_code == 401, response.text + assert wire.drain() == () + + +def test_hosted_vllm_auth_error_reaches_the_caller_after_one_attempt_with_reasoning(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + error_message: Final = f"invalid api key for deployment {marker}" + reply: Final = Reply( + status=401, + body=json.dumps({"error": {"message": error_message, "type": "authentication_error"}}).encode(), + ) + with wire_server(lambda _: reply) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"hosted_vllm/{_BACKEND}", api_base=wire.url + "/v1", api_key=_API_KEY) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": _replayed_conversation(_REASONING, marker), "cache": _NO_CACHE}, + ) + assert response.status_code == 401, response.text + assert error_message in response.text, response.text + assert _sent_messages(_only_request(wire)) == _replayed_conversation(_REASONING, marker) + + +def test_fallback_attempt_replays_reasoning_to_the_second_deployment(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + + def respond(request: Request) -> Reply: + if _JSON_OBJECT.validate_json(request.body)["model"] == _BACKEND: + return Reply(status=500, body=b'{"error": {"message": "primary deployment is down"}}') + return Reply(body=_completion(f"chatcmpl-fallback-{marker}", "Recovered.")) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + primary: Final = scenario.model(model=f"hosted_vllm/{_BACKEND}", api_base=wire.url + "/v1", api_key=_API_KEY) + fallback: Final = scenario.model( + model=f"hosted_vllm/{_FALLBACK_BACKEND}", api_base=wire.url + "/v1", api_key=_API_KEY + ) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + { + "model": primary, + "messages": _replayed_conversation(_REASONING, marker), + "fallbacks": [fallback], + "num_retries": 0, + "cache": _NO_CACHE, + }, + ) + assert response.status_code == 200, response.text + assert _JSON_OBJECT.validate_json(response.content)["id"] == f"chatcmpl-fallback-{marker}" + attempts: Final = wire.drain() + assert [_JSON_OBJECT.validate_json(attempt.body)["model"] for attempt in attempts] == [ + _BACKEND, + _FALLBACK_BACKEND, + ] + assert [_sent_messages(attempt) for attempt in attempts] == [ + _replayed_conversation(_REASONING, marker), + _replayed_conversation(_REASONING, marker), + ] + + +def test_identical_uncached_replays_are_each_forwarded_and_billed_once(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + identities: Final = iter((f"chatcmpl-first-{marker}", f"chatcmpl-second-{marker}")) + with wire_server(lambda _: Reply(body=_completion(next(identities), "Done."))) as wire: + with gateway.scenario() as scenario: + model: Final = scenario.model(model=f"hosted_vllm/{_BACKEND}", api_base=wire.url + "/v1", api_key=_API_KEY) + first: Final = _post_chat(gateway, model, _replayed_conversation(_REASONING, marker)) + second: Final = _post_chat(gateway, model, _replayed_conversation(_REASONING, marker)) + assert (first["id"], second["id"]) == (f"chatcmpl-first-{marker}", f"chatcmpl-second-{marker}") + assert [_sent_messages(request) for request in wire.drain()] == [ + _replayed_conversation(_REASONING, marker), + _replayed_conversation(_REASONING, marker), + ] + assert _spend_row(f"chatcmpl-first-{marker}")["status"] == "success" + assert _spend_row(f"chatcmpl-second-{marker}")["status"] == "success" + + +def test_cached_replay_hits_only_for_the_same_reasoning(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + identities: Final = iter((f"chatcmpl-cached-{marker}", f"chatcmpl-other-{marker}")) + with wire_server(lambda _: Reply(body=_completion(next(identities), "Done."))) as wire: + with gateway.scenario() as scenario: + model: Final = scenario.model(model=f"hosted_vllm/{_BACKEND}", api_base=wire.url + "/v1", api_key=_API_KEY) + + def ask(reasoning: str) -> dict[str, JsonValue]: + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": _replayed_conversation(reasoning, marker)}, + ) + assert response.status_code == 200, response.text + return _JSON_OBJECT.validate_json(response.content) + + assert ask(_REASONING)["id"] == f"chatcmpl-cached-{marker}" + assert ask(_REASONING)["id"] == f"chatcmpl-cached-{marker}" + assert ask(f"a different thought {marker}")["id"] == f"chatcmpl-other-{marker}" + assert [_sent_messages(request)[1].get("reasoning_content") for request in wire.drain()] == [ + _REASONING, + f"a different thought {marker}", + ] + + +def _responses_input(marker: str) -> list[dict[str, JsonValue]]: + return [ + {"role": "user", "content": f"Compare these invoices {marker}."}, + { + "id": f"rs_{marker}", + "type": "reasoning", + "summary": [{"type": "summary_text", "text": _REASONING}], + }, + { + "id": f"msg_prior_{marker}", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "Checking the totals.", "annotations": []}], + }, + {"role": "user", "content": "What is the difference?"}, + ] + + +def _responses_reply(identity: str, stream: bool) -> Reply: + response: Final = { + "id": identity, + "object": "response", + "created_at": 1, + "status": "completed", + "model": _BACKEND, + "output": [ + { + "id": "msg_" + identity, + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "They differ by 42.", "annotations": []}], + } + ], + "usage": {"input_tokens": 30, "output_tokens": 5, "total_tokens": 35}, + } + if not stream: + return Reply(body=json.dumps(response).encode()) + events: Final = ( + { + "type": "response.created", + "sequence_number": 0, + "response": {**response, "status": "in_progress", "output": []}, + }, + { + "type": "response.output_text.delta", + "sequence_number": 1, + "item_id": "msg_" + identity, + "output_index": 0, + "content_index": 0, + "delta": "They differ by 42.", + }, + {"type": "response.completed", "sequence_number": 2, "response": response}, + ) + return Reply( + content_type="text/event-stream", + chunks=tuple(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode() for event in events), + ) + + +def _only_responses_body(wire: Wire) -> dict[str, JsonValue]: + received: Final = wire.drain() + assert [(request.method, request.target) for request in received] == [("POST", "/v1/responses")] + return _JSON_OBJECT.validate_json(received[0].body) + + +def test_openai_sdk_responses_replay_reaches_hosted_vllm_with_its_reasoning_item(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(lambda _: _responses_reply(f"resp_upstream_{marker}", stream=False)) as wire: + with gateway.scenario() as scenario: + model: Final = scenario.model(model=f"hosted_vllm/{_BACKEND}", api_base=wire.url + "/v1", api_key=_API_KEY) + response: Final = _openai_client(gateway).responses.create( + model=model, + input=_responses_input(marker), # pyright: ignore[reportArgumentType] # plain JSON input items + ) + assert response.output_text == "They differ by 42." + assert _only_responses_body(wire)["input"] == _responses_input(marker) + assert _spend_row(response.id) == { + "model_group": model, + "status": "success", + "prompt_tokens": 30, + "completion_tokens": 5, + } + + +async def test_async_openai_sdk_responses_stream_reaches_hosted_vllm_with_its_reasoning_item( + gateway: Gateway, +) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(lambda _: _responses_reply(f"resp_upstream_{marker}", stream=True)) as wire: + with gateway.scenario() as scenario: + model: Final = scenario.model(model=f"hosted_vllm/{_BACKEND}", api_base=wire.url + "/v1", api_key=_API_KEY) + stream: Final = await _async_openai_client(gateway).responses.create( + model=model, + input=_responses_input(marker), # pyright: ignore[reportArgumentType] # plain JSON input items + stream=True, + ) + events: Final = [event async for event in stream] + assert [event.type for event in events] == [ + "response.created", + "response.output_text.delta", + "response.completed", + ] + completed: Final = events[-1] + assert completed.type == "response.completed" + body: Final = _only_responses_body(wire) + assert body["stream"] is True + assert body["input"] == _responses_input(marker) + assert completed.response.output_text == "They differ by 42." + assert _model_spend_statuses(model) == ["success"] diff --git a/tests/unit/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py b/tests/unit/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py index 1cc6a1457fc..c792d5dffcd 100644 --- a/tests/unit/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py +++ b/tests/unit/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py @@ -1,6 +1,8 @@ import json from unittest.mock import MagicMock, patch +import pytest + from litellm.constants import ( DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET, @@ -112,10 +114,10 @@ def test_hosted_vllm_supports_thinking(): assert optional_params["reasoning_effort"] == "low" -def test_hosted_vllm_thinking_blocks_prepended_to_assistant_content(): +def test_hosted_vllm_reasoning_content_kept_and_thinking_blocks_removed(): """ - Test that thinking_blocks on assistant messages are removed and content - stays a string for vLLM compatibility. + Test that reasoning_content on assistant messages is forwarded to vLLM + while thinking_blocks are removed and content stays a string. """ config = HostedVLLMChatConfig() messages = [ @@ -152,7 +154,36 @@ def test_hosted_vllm_thinking_blocks_prepended_to_assistant_content(): assert isinstance(assistant_msg["content"], str) assert assistant_msg["content"] == "Here is my answer." assert "thinking_blocks" not in assistant_msg - assert "reasoning_content" not in assistant_msg + assert assistant_msg["reasoning_content"] == "Let me reason about this..." + + +@pytest.mark.parametrize( + ("reasoning_content", "expected"), + [ + ("step one, then step two", "step one, then step two"), + ("", ""), + (None, "absent"), + (42, "absent"), + (["step one", "step two"], "absent"), + ({"text": "step one"}, "absent"), + ], +) +def test_hosted_vllm_forwards_only_string_reasoning_content(reasoning_content, expected): + config = HostedVLLMChatConfig() + transformed = config.transform_request( + model="hosted_vllm/qwen3", + messages=[ + {"role": "user", "content": "Hello"}, + {"role": "assistant", "content": "Hi", "reasoning_content": reasoning_content}, + {"role": "user", "content": "Again"}, + ], + optional_params={}, + litellm_params={}, + headers={}, + ) + assistant_msg = transformed["messages"][1] + assert assistant_msg.get("reasoning_content", "absent") == expected + assert assistant_msg["content"] == "Hi" def test_hosted_vllm_thinking_blocks_with_list_content(): From f285229b511390f124aa92706101fe6437863dda Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 30 Sep 2026 13:49:18 -0700 Subject: [PATCH 02/28] fix(proxy): delete large teams without per-member transaction fan-out (#42998) * fix(proxy): delete large teams without per-member transaction fan-out Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): evict email-only member caches and reset team members metric on delete Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): keep new delete-team literals within the LIT002 ceiling Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): resolve deleted-team member ids before the locked delete Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): resolve email-only deleted-team members with one case-insensitive lookup `_deleted_team_member_user_ids` looked each email-only roster entry up with its own `find_users_by_email` call inside an unbounded `asyncio.gather`: one exact-match query per email, so a large roster fanned out against the pool again and a roster email that differed in case from its user row was missed. Add `UserRepository.find_by_emails`, a single case-insensitive `in` query, and call it once before the locked delete. `management_helpers/utils.py` goes back to its main-branch shape since the single-email helper no longer needs exporting. * fix(repositories): slice find_by_emails into bounded IN statements The unbounded-IN lint flagged the case-insensitive email lookup added for /team/delete cache eviction. chunked_in.find_many_in cannot carry Prisma's insensitive mode, so the repository slices the deduplicated list into IN_LIST_CHUNK_SIZE statements itself and concatenates the pages. Empty input still returns () without a query. * fix(proxy): delete a team once when /team/delete repeats its id The audit sent {"team_ids": [T, T]}: main answered 400 "User not found in team" after deleting the keys and memberships and writing two tombstones, leaving the team row behind; this branch answered 200 but still wrote the tombstone, audit row and eviction twice. DeleteTeamRequest now collapses repeated ids in order, so every later step sees each team once and the response lists each deleted team once. * test(integration): audit cells for /team/delete on large, legacy and concurrent teams Thirty-eight deterministic cells in tests/integration/management/ (the CircleCI integration-management group) covering the /team/delete happy, sad, edge and chaos rows: 250 members against a pool limit of five on two workers, the advisory-lock wait, email-only legacy roster entries in every casing, member and team cache eviction on both proxies for every client and endpoint, the Prometheus gauge, audit rows, malformed and duplicate input, the route gate, and a worker kill, a Redis outage and a proxy restart mid-burst. Every cell runs against the real proxy, Postgres and Redis with the scripted upstream; no component is mocked. On the merge base the rows this fix changes are red (P2028 on the 250-member team, two lock waiters, case-mismatched email lookups, duplicate ids, orphaned LiteLLM_UserTable.teams references under a concurrent burst); on the tip every cell is green twice with identical selections. Two pre-existing behaviours are pinned as observed rather than fixed here: a roster entry with neither user_id nor user_email answers 500, and the LiteLLM_DeletedTeamTable row is committed before the locked transaction, so a delete that dies in between leaves a tombstone for a live team and the retry adds a second. * test(integration): pin each chaos outage to a live /team/delete The three chaos cells applied the outage once three deletes had answered, which on a fast run let the whole burst finish before the worker kill, Redis stop or SIGTERM landed, so the cells passed without exercising the failure. Each cell now holds the first team's advisory lock from a test-owned transaction, waits until that team's delete is queued behind it in Postgres with its request unanswered, applies the outage, and only then releases the lock, so an in-flight delete meets the failure on every run and both legs. The pinned team's outcome and the number of deletes answered before the outage are recorded as junit properties (pinned_delete, answered_before_outage). --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: ryan-crabbe-berri --- litellm/proxy/_types.py | 8 +- .../management_endpoints/team_endpoints.py | 88 ++- litellm/repositories/user_repository.py | 29 +- tests/e2e/coverage_registry/mgmt.yaml | 1 + tests/e2e/management/management_client.py | 21 + tests/e2e/management/test_management_e2e.py | 39 ++ tests/e2e/models.py | 2 +- .../management/test_team_delete_chaos.py | 499 ++++++++++++++ .../management/test_team_delete_inputs.py | 330 +++++++++ .../test_team_delete_large_membership.py | 634 ++++++++++++++++++ .../test_team_delete_member_cache_eviction.py | 446 ++++++++++++ .../management/test_team_delete_prometheus.py | 122 ++++ .../test_team_endpoints.py | 117 +++- tests/unit/repositories/test_repositories.py | 27 + 14 files changed, 2331 insertions(+), 32 deletions(-) create mode 100644 tests/integration/management/test_team_delete_chaos.py create mode 100644 tests/integration/management/test_team_delete_inputs.py create mode 100644 tests/integration/management/test_team_delete_large_membership.py create mode 100644 tests/integration/management/test_team_delete_member_cache_eviction.py create mode 100644 tests/integration/management/test_team_delete_prometheus.py diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 6c9e1511b00..9c7dece6894 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1,7 +1,7 @@ import enum import json import os -from collections.abc import Callable, Mapping +from collections.abc import Callable, Mapping, Sequence from datetime import datetime from types import MappingProxyType from typing import TYPE_CHECKING, Annotated, Any, Final, Literal, NamedTuple, TypeAlias @@ -2241,6 +2241,12 @@ class ResetTeamBudgetRequest(LiteLLMPydanticObjectBase): class DeleteTeamRequest(LiteLLMPydanticObjectBase): team_ids: list[str] # required + @field_validator("team_ids") + @classmethod + def distinct_team_ids(cls, team_ids: Sequence[str]) -> list[str]: + """One delete per team: a repeated id would otherwise write its tombstone and audit row twice.""" + return list(dict.fromkeys(team_ids)) + class BlockTeamRequest(LiteLLMPydanticObjectBase): team_id: str # required diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 6cec3e714ec..f0e59389d48 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -4517,27 +4517,13 @@ async def delete_team( llm_router=llm_router, ) - # ## DELETE TEAM MEMBERSHIPS - for team_row in team_rows: - ### get all team members - team_members = team_row.members_with_roles - ### call team_member_delete for each team member - tasks = [] - for team_member in team_members: - tasks.append( - _team_member_delete( - data=TeamMemberDeleteRequest( - team_id=team_row.team_id, - user_id=team_member.user_id, - user_email=team_member.user_email, - ), - user_api_key_dict=user_api_key_dict, - ) - ) - await asyncio.gather(*tasks) - await _sweep_deleted_team_references(team_ids=data.team_ids, prisma_client=prisma_client) + member_ids_per_team: Final = await _resolve_deleted_team_member_user_ids( + teams=team_rows, + prisma_client=prisma_client, + ) + ## DELETE TEAMS # Both the delete and the reconcile sweep run under every team's advisory lock # (TEAM_ADVISORY_LOCK_SQL, the same one /team/member_add takes before its own writes), @@ -4565,8 +4551,15 @@ async def delete_team( user_api_key_cache=user_api_key_cache, proxy_logging_obj=proxy_logging_obj, ) + await _invalidate_deleted_team_member_cache( + member_ids_per_team=member_ids_per_team, + user_api_key_cache=user_api_key_cache, + ) for deleted_team in team_rows: + _emit_team_members_metric( + deleted_team.model_copy(update={"members_with_roles": ()}) # mutable-ok: pydantic update payload + ) await sync_team_access_group_membership(prisma_client=prisma_client, team_id=deleted_team.team_id) return deleted_teams @@ -4641,6 +4634,63 @@ async def _invalidate_deleted_team_cache( ) +async def _invalidate_deleted_team_member_cache( + member_ids_per_team: Sequence[tuple[str, Sequence[str]]], + user_api_key_cache: UserApiKeyCache, +) -> None: + for team_id, member_user_ids in member_ids_per_team: + await _evict_deleted_team_member_cache( + team_id=team_id, + member_user_ids=member_user_ids, + user_api_key_cache=user_api_key_cache, + ) + + +async def _evict_deleted_team_member_cache( + team_id: str, + member_user_ids: Sequence[str], + user_api_key_cache: UserApiKeyCache, +) -> None: + await evict_and_broadcast(cache_keys=tuple(member_user_ids), user_api_key_cache=user_api_key_cache) + await asyncio.gather( + *( + invalidate_team_member_spend_state( + user_id=user_id, + team_id=team_id, + user_api_key_cache=user_api_key_cache, + ) + for user_id in member_user_ids + ) + ) + + +async def _resolve_deleted_team_member_user_ids( + teams: Sequence[LiteLLM_TeamTable], + prisma_client: PrismaClient, +) -> tuple[tuple[str, tuple[str, ...]], ...]: + resolved: Final = await asyncio.gather( + *(_deleted_team_member_user_ids(team=team, prisma_client=prisma_client) for team in teams) + ) + return tuple(zip((team.team_id for team in teams), resolved)) + + +async def _deleted_team_member_user_ids(team: LiteLLM_TeamTable, prisma_client: PrismaClient) -> tuple[str, ...]: + roster_user_ids: Final = frozenset( + member.user_id for member in team.members_with_roles if member.user_id is not None + ) + email_only_member_emails: Final = frozenset( + member.user_email + for member in team.members_with_roles + if member.user_id is None and member.user_email is not None + ) + if not email_only_member_emails: + return tuple(sorted(roster_user_ids)) + # One case-insensitive lookup for the whole roster. A per-email fan-out would size the + # query count by team membership, the same shape as the P2028 fan-out this path removed. + email_only_users: Final = await UserRepository(prisma_client).find_by_emails(sorted(email_only_member_emails)) + return tuple(sorted(roster_user_ids.union(user.user_id for user in email_only_users))) + + def _transform_teams_to_deleted_records( teams: list[LiteLLM_TeamTable], user_api_key_dict: UserAPIKeyAuth, diff --git a/litellm/repositories/user_repository.py b/litellm/repositories/user_repository.py index 4a2aea46197..7bf516a2bc1 100644 --- a/litellm/repositories/user_repository.py +++ b/litellm/repositories/user_repository.py @@ -3,13 +3,15 @@ User repository for database operations on LiteLLM_UserTable. """ import json -from collections.abc import Mapping +from collections.abc import Mapping, Sequence +from itertools import chain from typing import TYPE_CHECKING, Final from pydantic import TypeAdapter from litellm.models.user import LiteLLM_UserTable, SCIMPlaceholder from litellm.repositories.base_repository import BaseRepository, DbRecord, record_to_dict +from litellm.repositories.chunked_in import IN_LIST_CHUNK_SIZE from litellm.repositories.prisma_protocols import TableActions if TYPE_CHECKING: @@ -71,6 +73,31 @@ class UserRepository(BaseRepository[LiteLLM_UserTable]): records: Final = await self.find_many(where={"user_email": user_email}) return records[0] if records else None + async def find_by_emails(self, user_emails: Sequence[str]) -> Sequence[LiteLLM_UserTable]: + """Every user whose email matches one of ``user_emails``, ignoring case. + + A roster entry stored by email can differ in case from its user row (member_add + resolves emails case-insensitively), so an exact match would miss it. The list goes + out in slices of ``IN_LIST_CHUNK_SIZE`` so one statement stays under Postgres's + bind-parameter cap; ``chunked_in.find_many_in`` cannot carry the insensitive mode. + """ + unique: Final = sorted(frozenset(user_emails)) + pages: Final = tuple( + [ + await self.find_many( + where={ # mutable-ok: Prisma query filters are dict-shaped + "user_email": { # mutable-ok: Prisma query filters are dict-shaped + # bounded-ok: sliced to IN_LIST_CHUNK_SIZE values per statement + "in": unique[start : start + IN_LIST_CHUNK_SIZE], + "mode": "insensitive", + } + } + ) + for start in range(0, len(unique), IN_LIST_CHUNK_SIZE) + ] + ) + return tuple(chain.from_iterable(pages)) + async def find_by_sso_id(self, sso_user_id: str) -> LiteLLM_UserTable | None: """Find a user by SSO ID.""" return await self.find_by_id(sso_user_id, id_field="sso_user_id") diff --git a/tests/e2e/coverage_registry/mgmt.yaml b/tests/e2e/coverage_registry/mgmt.yaml index e1a840b1239..bc728efacd4 100644 --- a/tests/e2e/coverage_registry/mgmt.yaml +++ b/tests/e2e/coverage_registry/mgmt.yaml @@ -35,6 +35,7 @@ - {id: mgmt.team.update.team_admin_cannot_grow_budget, module: mgmt, tier: P0, surface: api, assertions: [team_admin_cannot_grow_budget], source: "team_endpoints.py:1203", fail_before_fix: proven, rationale: "With max_budget enabled, a team admin may keep or lower its team's budget; raising or removing it is 403 and writes nothing, also under an organization's larger cap"} - {id: mgmt.team.update.team_admin_resend_keeps_budget_reset, module: mgmt, tier: P1, surface: api, assertions: [team_admin_resend_keeps_budget_reset], source: "team_admin_field_permissions.py:147", fail_before_fix: proven, rationale: "A team admin resending unchanged budget settings with an enabled field must not push the team's budget reset times back"} - {id: mgmt.team.delete.persists, module: mgmt, tier: P1, surface: api, assertions: [persists], source: "team_endpoints.py:1750", rationale: "Deletion prevents key access"} +- {id: mgmt.team.delete.membership_larger_than_db_pool, module: mgmt, tier: P0, surface: api, assertions: [membership_larger_than_db_pool], source: "team_endpoints.py:4362", rationale: "Deleting a team with more members than the Prisma connection pool still completes instead of exhausting the pool and answering 500", fail_before_fix: proven} - {id: mgmt.team.block.persists, module: mgmt, tier: P1, surface: api, assertions: [persists], source: "team_endpoints.py", rationale: "Block suspends all members"} - {id: mgmt.team.info.happy_path, module: mgmt, tier: P1, surface: api, assertions: [happy_path], source: "team_endpoints.py:2244", rationale: "Metadata+members+budgets"} - {id: mgmt.team.daily_activity.happy_path, module: mgmt, tier: P1, surface: api, assertions: [happy_path], source: "vendor testing strategy §9.20 / LIT-4778", rationale: "GET /team/daily/activity returns results+metadata for a valid date range"} diff --git a/tests/e2e/management/management_client.py b/tests/e2e/management/management_client.py index 7366695c0d1..3da9bea12a3 100644 --- a/tests/e2e/management/management_client.py +++ b/tests/e2e/management/management_client.py @@ -411,6 +411,27 @@ class ManagementClient: assert last is not None raise AssertionError(last) + def add_team_members(self, team_id: str, members: list[TeamMemberEntry]) -> None: + """Bulk form of /team/member_add: `member` accepts a list, so one call + seeds a whole roster the way an admin import does.""" + _ = unwrap( + self.proxy.transport.post( + "/team/member_add", + headers=self.proxy.management_headers(), + json=TeamMemberAddBody(team_id=team_id, member=members), + response_type=NoBody, + ) + ) + + def delete_team_status(self, team_id: str) -> StreamingResponse: + """POST /team/delete judged by HTTP outcome: the raw status and body, so a + test can assert on what a caller actually sees when the delete fails.""" + return self.proxy.transport.send( + "/team/delete", + headers=self.proxy.management_headers(), + json=TeamDeleteBody(team_ids=[team_id]), + ) + def delete_team_member(self, team_id: str, user_id: str) -> None: _ = unwrap( self.proxy.transport.post( diff --git a/tests/e2e/management/test_management_e2e.py b/tests/e2e/management/test_management_e2e.py index da0fc37aff8..908eb752611 100644 --- a/tests/e2e/management/test_management_e2e.py +++ b/tests/e2e/management/test_management_e2e.py @@ -37,6 +37,7 @@ from models import ( OrgUpdateBody, TagListEntry, TagNewBody, + TeamMemberEntry, TeamNewBody, TeamUpdateBody, UserNewBody, @@ -48,6 +49,7 @@ pytestmark = pytest.mark.e2e REGENERATE_GRACE_PERIOD = "15s" REGENERATE_GRACE_SECONDS = 15.0 +TEAM_DELETE_POOL_OVERFLOW_MEMBERS = 250 def _poll[T](client: ManagementClient, attempt: Callable[[], T | None], failure: str) -> T: @@ -479,6 +481,43 @@ class TestTeamRoutes: client, rejected, "team-bound key was still accepted on chat (never rejected 401) after team deletion" ) + @pytest.mark.covers("mgmt.team.delete.membership_larger_than_db_pool") + def test_team_delete_succeeds_for_team_larger_than_db_pool( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + """Customer repro: /team/delete fans one transaction per member out over a + Prisma pool of 10 connections, each queued on the team's advisory lock, + so a team bigger than the pool must still delete cleanly instead of + answering 500 P2028.""" + team_id = _create_team(client, resources, f"e2e-mgmt-team-{unique_marker()}", []) + user_ids = tuple( + _create_user( + client, + resources, + UserNewBody( + user_email=f"e2e-mgmt-bulk-{i}-{unique_marker()}@example.com", + user_role="internal_user", + ), + ) + for i in range(TEAM_DELETE_POOL_OVERFLOW_MEMBERS) + ) + client.add_team_members(team_id, [TeamMemberEntry(role="user", user_id=user_id) for user_id in user_ids]) + seated = len(client.team_info(team_id).members_with_roles) + assert seated >= len(user_ids), ( + f"/team/info lists {seated} members after the bulk /team/member_add, expected at least {len(user_ids)}" + ) + + outcome = client.delete_team_status(team_id) + + assert outcome.status_code == 200, ( + f"/team/delete on a {len(user_ids)}-member team must succeed, got " + f"{outcome.status_code}: {outcome.body[:500]}" + ) + probe = client.team_info_status(team_id) + assert probe.status_code == 404, ( + f"deleted team {team_id} still resolves: /team/info returned {probe.status_code}: {probe.body[:300]}" + ) + @pytest.mark.covers("mgmt.team.member_add.persists") def test_member_add_and_delete_persist_to_team_info( self, client: ManagementClient, resources: ResourceManager diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 0383da48c43..4f8f6926095 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -1485,7 +1485,7 @@ class TeamInfoResponse(BaseModel): class TeamMemberAddBody(BaseModel): team_id: str - member: TeamMemberEntry + member: TeamMemberEntry | list[TeamMemberEntry] class TeamMemberDeleteBody(BaseModel): diff --git a/tests/integration/management/test_team_delete_chaos.py b/tests/integration/management/test_team_delete_chaos.py new file mode 100644 index 00000000000..ebe515f59ec --- /dev/null +++ b/tests/integration/management/test_team_delete_chaos.py @@ -0,0 +1,499 @@ +"""Chaos rows for ``/team/delete`` on an owned two-worker proxy: C1 worker kill, C2 Redis outage, C3 proxy restart. + +Each leg creates 24 teams through the owned proxy (two internal users per team in one bulk +``/team/member_add``, plus one team key), then deletes all 24 in a 24-thread burst and breaks the +infrastructure while a delete is provably in flight: the test holds the first team's advisory lock +from its own transaction, waits until that team's delete is queued behind it inside Postgres with +its request unanswered, applies the failure once the third of the other deletes has answered, and +only then releases the lock. The outage therefore overlaps a live delete on every run and both legs, +and the pinned delete finishes, or is dropped, under the failure: + +- C1 SIGKILLs one uvicorn worker child; the survivor still answers ``/health/readiness`` and uvicorn + respawns the worker. +- C2 shuts the owned Redis down; ``/cache/ping`` reports it, the deletes keep answering 200 because + cache eviction and the invalidation broadcast are best-effort, then Redis comes back. +- C3 SIGTERMs the owned proxy root and a fresh proxy starts on the same database. + +After recovery the burst outcomes (status or transport error per team) are recorded, every team whose +row survived is deleted once more, and the invariants must hold for every team: no ``LiteLLM_TeamTable`` +row, no ``LiteLLM_TeamMembership`` row, no ``LiteLLM_UserTable.teams`` entry naming it, its key gone +from ``LiteLLM_VerificationToken``, and one ``LiteLLM_DeletedTeamTable`` row per attempt that reached +the tombstone write. Both legs commit that tombstone before the locked transaction that removes the +team, so an attempt that died in between leaves a tombstone for a live team and the retry adds a +second; that count is pinned as observed (pre-existing, outside this PR's diff, recorded in the audit +report) and the affected teams are recorded as ``double_tombstones``. Teams found half-deleted before +the retry are recorded as ``partial_states_before_retry`` and named in any failure; the pinned team's +outcome is recorded as ``pinned_delete`` and the answers the outage interrupted as +``answered_before_outage``. + +Nothing sleeps, and only processes the test started are signalled. +""" + +from __future__ import annotations + +import os +import threading +import uuid +from collections import Counter +from collections.abc import Callable, Iterator, Sequence +from concurrent.futures import Future, ThreadPoolExecutor +from contextlib import contextmanager +from dataclasses import dataclass +from hashlib import sha256 +from pathlib import Path +from typing import Final + +import httpx +import psutil +import psycopg +import pytest + +from tests.integration._support.client import ( + JSON_OBJECT, + Gateway, + Scenario, + delete_key_if_present, + eventually, + string_value, +) +from tests.integration._support.database import read_rows +from tests.integration._support.process import owned_proxy_process +from tests.integration._support.redis_process import owned_redis + +RecordProperty = Callable[[str, object], None] + +TEAMS: Final = 24 +MEMBERS_PER_TEAM: Final = 2 +CHAOS_AFTER_ANSWERS: Final = 3 +WORKERS: Final = 2 +DELETE_TIMEOUT_SECONDS: Final = 60 +REMOVE_FROM_ENVIRONMENT: Final = ("DATABASE_URL_READ_REPLICA",) + +TEAM_SQL: Final = 'SELECT team_id FROM "LiteLLM_TeamTable" WHERE team_id = %s' +TOMBSTONE_SQL: Final = 'SELECT id FROM "LiteLLM_DeletedTeamTable" WHERE team_id = %s' +MEMBERSHIP_SQL: Final = 'SELECT user_id FROM "LiteLLM_TeamMembership" WHERE team_id = %s' +REFERENCING_USERS_SQL: Final = 'SELECT user_id FROM "LiteLLM_UserTable" WHERE %s = ANY(teams)' +TOKEN_SQL: Final = 'SELECT token FROM "LiteLLM_VerificationToken" WHERE token = %s' +TAKE_TEAM_LOCK_SQL: Final = "SELECT pg_advisory_xact_lock(hashtext(%s))" +# Sessions blocked on an advisory lock the given backend holds: the pinned team's delete, on either leg. +WAITERS_ON_HELD_LOCK_SQL: Final = """ +SELECT count(*)::int AS waiting +FROM pg_locks waiter +JOIN pg_stat_activity session ON session.pid = waiter.pid +WHERE waiter.locktype = 'advisory' + AND NOT waiter.granted + AND session.wait_event_type = 'Lock' + AND session.query ILIKE %s + AND (waiter.classid, waiter.objid, waiter.objsubid) IN ( + SELECT held.classid, held.objid, held.objsubid + FROM pg_locks held + WHERE held.locktype = 'advisory' AND held.granted AND held.pid = %s::int + ) +""" + + +@dataclass(frozen=True, slots=True) +class Team: + team_id: str + members: tuple[str, ...] + hashed_key: str + + +@dataclass(frozen=True, slots=True) +class Outcome: + """One burst delete: the HTTP status, or ``None`` with the transport error's class and message.""" + + team_id: str + status: int | None + detail: str + + @property + def label(self) -> str: + return str(self.status) if self.status is not None else self.detail.split(":", 1)[0] + + @property + def answered_or_dropped(self) -> bool: + """200, a 5xx from a dying process, or a transport error; a 4xx would mean a wrong delete.""" + return self.status is None or self.status == 200 or self.status >= 500 + + +@dataclass(frozen=True, slots=True) +class TeamState: + team_id: str + row_present: bool + tombstones: int + memberships: tuple[str, ...] + referencing_users: tuple[str, ...] + key_present: bool + + @property + def clean(self) -> bool: + """Row, memberships, ``teams`` references and key all gone; tombstones are counted per attempt.""" + return not self.row_present and not self.memberships and not self.referencing_users and not self.key_present + + @property + def untouched(self) -> bool: + return self.row_present and self.tombstones == 0 and self.key_present + + @property + def partial(self) -> bool: + return not (self.clean and self.tombstones == 1) and not self.untouched + + def describe(self) -> str: + return ( + f"{self.team_id}: row={'present' if self.row_present else 'gone'} tombstones={self.tombstones} " + f"memberships={len(self.memberships)} referencing_users={len(self.referencing_users)} " + f"key={'present' if self.key_present else 'gone'}" + ) + + +def _state(team: Team) -> TeamState: + return TeamState( + team.team_id, + row_present=bool(read_rows(TEAM_SQL, (team.team_id,))), + tombstones=len(read_rows(TOMBSTONE_SQL, (team.team_id,))), + memberships=tuple(string_value(row["user_id"]) for row in read_rows(MEMBERSHIP_SQL, (team.team_id,))), + referencing_users=tuple( + string_value(row["user_id"]) for row in read_rows(REFERENCING_USERS_SQL, (team.team_id,)) + ), + key_present=bool(read_rows(TOKEN_SQL, (team.hashed_key,))), + ) + + +def _states(fleet: Sequence[Team]) -> tuple[TeamState, ...]: + return tuple(_state(team) for team in fleet) + + +def _overrides() -> dict[str, str]: + return {"DATABASE_URL": os.environ["DATABASE_URL"]} + + +def _user(candidate: Gateway, scenario: Scenario) -> str: + """An internal user created through ``candidate``; its removal is registered on the shared rig.""" + user_id: Final = f"integration-chaos-{uuid.uuid4().hex}" + candidate.post("/user/new", {"user_id": user_id, "auto_create_key": False, "user_role": "internal_user"}) + scenario.cleanups.callback(scenario.delete_user, user_id) + return user_id + + +def _delete_team_if_present(candidate: Gateway, team_id: str) -> None: + if read_rows(TEAM_SQL, (team_id,)): + candidate.post("/team/delete", {"team_ids": [team_id]}) + assert read_rows(TEAM_SQL, (team_id,)) == [] + + +def _team(candidate: Gateway, scenario: Scenario, index: int) -> Team: + alias: Final = f"integration-chaos-{index:02d}-{uuid.uuid4().hex}" + team_id: Final = string_value(candidate.post("/team/new", {"team_alias": alias})["team_id"]) + scenario.cleanups.callback(_delete_team_if_present, scenario.gateway, team_id) + members: Final = tuple(_user(candidate, scenario) for _ in range(MEMBERS_PER_TEAM)) + candidate.post( + "/team/member_add", + {"team_id": team_id, "member": [{"role": "user", "user_id": user_id} for user_id in members]}, + ) + key: Final = string_value(candidate.post("/key/generate", {"team_id": team_id, "key_alias": alias})["key"]) + scenario.cleanups.callback(delete_key_if_present, scenario.gateway, key) + return Team(team_id, members, sha256(key.encode()).hexdigest()) + + +def _fleet(candidate: Gateway, scenario: Scenario) -> tuple[Team, ...]: + """24 teams created through ``candidate``, each verified intact: row, key, both members' membership + rows and ``teams`` entries present, so the invariants after the burst have something to remove. + + A master-key ``/team/new`` also seats ``default_user_id`` as an admin (roster entry, membership row and + ``teams`` entry), so the checks are supersets. Cleanup is registered on the shared rig; the team + callback only acts when a run fails before its delete. + """ + fleet: Final = tuple(_team(candidate, scenario, index) for index in range(TEAMS)) + for team, state in zip(fleet, _states(fleet)): + assert state.untouched, state.describe() + assert set(state.memberships) >= set(team.members), state.describe() + assert set(state.referencing_users) >= set(team.members), state.describe() + return fleet + + +class Burst: + """One ``/team/delete`` per team on ``target``, all submitted at once; ``chaos_point`` is set once the + third delete has answered (or failed), so the leg breaks the infrastructure mid-burst.""" + + def __init__(self, target: Gateway) -> None: + self._target: Final = target + self._lock: Final = threading.Lock() + self._answers = 0 # rebind-ok: counter behind _lock + self._futures: dict[str, Future[Outcome]] = {} + self.chaos_point: Final = threading.Event() + + def start(self, pool: ThreadPoolExecutor, fleet: Sequence[Team]) -> None: + assert not self._futures, "burst already started" + self._futures.update((team.team_id, pool.submit(self._delete, team)) for team in fleet) + assert self.chaos_point.wait(DELETE_TIMEOUT_SECONDS), ( + f"fewer than {CHAOS_AFTER_ANSWERS} deletes answered within {DELETE_TIMEOUT_SECONDS}s" + ) + + def _delete(self, team: Team) -> Outcome: + try: + response: Final = self._target.client.request( + "POST", + "/team/delete", + json={"team_ids": [team.team_id]}, + headers={"Authorization": f"Bearer {self._target.key}"}, + timeout=DELETE_TIMEOUT_SECONDS, + ) + outcome = Outcome(team.team_id, response.status_code, response.text[:200]) + except httpx.HTTPError as error: # a killed worker or a stopped proxy drops the in-flight request + outcome = Outcome(team.team_id, None, f"{type(error).__name__}: {error}"[:200]) + with self._lock: + self._answers += 1 + if self._answers >= CHAOS_AFTER_ANSWERS: + self.chaos_point.set() + return outcome + + def answered(self) -> int: + with self._lock: + return self._answers + + def pending(self, team_id: str) -> bool: + return not self._futures[team_id].done() + + def outcomes(self) -> tuple[Outcome, ...]: + return tuple(future.result(timeout=DELETE_TIMEOUT_SECONDS + 30) for future in self._futures.values()) + + +def _waiters_on_lock_held_by(backend_pid: int) -> int: + rows: Final = read_rows(WAITERS_ON_HELD_LOCK_SQL, ("%pg_advisory_xact_lock%", str(backend_pid))) + waiting: Final = rows[0]["waiting"] + assert isinstance(waiting, int) + return waiting + + +@contextmanager +def _holding_team_lock(team_id: str) -> Iterator[int]: + """Hold ``team_id``'s advisory lock in a test-owned transaction and yield the holder's backend pid; + leaving the block commits, which releases the lock.""" + with psycopg.connect(os.environ["DATABASE_URL"]) as holder: + holder.execute(TAKE_TEAM_LOCK_SQL, (team_id,)) + yield holder.info.backend_pid + + +def _await_pinned_delete_blocked(burst: Burst, pinned: Team, holder_pid: int, record_property: RecordProperty) -> None: + """The pinned team's delete is queued behind the held lock inside Postgres with its request unanswered, + so the failure applied next lands on a live delete; records how many other deletes had answered.""" + eventually(lambda: _waiters_on_lock_held_by(holder_pid), lambda waiting: waiting >= 1, seconds=20) + assert burst.pending(pinned.team_id), f"{pinned.team_id}: delete answered while its team lock was held" + record_property("answered_before_outage", burst.answered()) + + +def _record_burst( + record_property: RecordProperty, outcomes: Sequence[Outcome], observed: Sequence[TeamState], pinned: Team +) -> None: + """Record the status split, the pinned team's outcome and the half-deleted teams seen before the retry.""" + split: Final = Counter(outcome.label for outcome in outcomes) + record_property("status_split", dict(sorted(split.items()))) + pinned_outcome: Final = next(outcome for outcome in outcomes if outcome.team_id == pinned.team_id) + record_property( + "pinned_delete", + {"team_id": pinned.team_id, "status": pinned_outcome.status, "detail": pinned_outcome.detail}, + ) + record_property("partial_states_before_retry", [state.describe() for state in observed if state.partial]) + record_property("rows_present_before_retry", sum(state.row_present for state in observed)) + + +def _retry_survivors(target: Gateway, fleet: Sequence[Team], observed: Sequence[TeamState]) -> tuple[str, ...]: + """Delete once more, through ``target``, every team whose row survived the burst; each must answer 200.""" + survivors: Final = tuple(team.team_id for team, state in zip(fleet, observed) if state.row_present) + for team_id in survivors: + assert target.post("/team/delete", {"team_ids": [team_id]}) == {"deleted_teams": [team_id]} + return survivors + + +def _expected_tombstones(before: TeamState, retried: bool) -> int: + """One ``LiteLLM_DeletedTeamTable`` row per attempt that reached the tombstone write. + + Both legs commit the tombstone before the locked transaction that removes the team, so a burst + attempt that died in between left one (``before.tombstones``, 0 or 1) for a team whose row + survived, and the retry adds one more. Pinned as observed: pre-existing on the merge base, + outside this PR's diff, recorded in the audit report. + """ + assert before.tombstones <= 1, before.describe() + return before.tombstones + (1 if retried else 0) + + +def _assert_every_team_fully_deleted( + record_property: RecordProperty, + before_retry: Sequence[TeamState], + final: Sequence[TeamState], + retried: Sequence[str], +) -> None: + """Every team: row, memberships, ``teams`` references and key gone; tombstones one per attempt.""" + expected: Final = {state.team_id: _expected_tombstones(state, state.team_id in retried) for state in before_retry} + record_property("double_tombstones", sorted(team_id for team_id, count in expected.items() if count == 2)) + violations: Final = tuple( + f"{state.describe()} expected tombstones={expected[state.team_id]}" + for state in final + if not state.clean or state.tombstones != expected[state.team_id] or expected[state.team_id] == 0 + ) + assert not violations, ( + f"{len(violations)} of {len(final)} teams are not fully deleted after the retry:\n " + + "\n ".join(violations) + + f"\nhalf-deleted before the retry ({sum(state.partial for state in before_retry)}):\n " + + "\n ".join(state.describe() for state in before_retry if state.partial) + + f"\nretried ({len(retried)}): {sorted(retried)}" + ) + + +def _workers(root: psutil.Process) -> tuple[psutil.Process, ...]: + """uvicorn's worker children of the owned proxy root, spawned through ``multiprocessing.spawn``. + + The root's other child is the multiprocessing resource tracker; each worker's prisma query engine + is a grandchild. A worker that just died shows as a zombie whose cmdline raises, so it is left out. + """ + workers: Final = [] + for child in root.children(): + try: + cmdline = child.cmdline() + except (psutil.NoSuchProcess, psutil.AccessDenied): + continue + if any("multiprocessing.spawn" in part for part in cmdline): + workers.append(child) + return tuple(sorted(workers, key=lambda process: process.pid)) + + +def _cache_ping(target: Gateway) -> httpx.Response: + return target.request("GET", "/cache/ping") + + +def _cache_status(response: httpx.Response) -> str: + assert response.status_code == 200, f"/cache/ping: {response.status_code} {response.text}" + return string_value(JSON_OBJECT.validate_json(response.content)["status"]) + + +@pytest.mark.timeout(240) # owned two-worker proxy boot plus a 24-team fleet and its cleanup +def test_worker_killed_mid_burst_leaves_every_team_fully_deleted_after_retry( + gateway: Gateway, tmp_path: Path, record_property: RecordProperty +) -> None: + with ( + gateway.scenario() as scenario, + owned_proxy_process( + gateway, tmp_path, _overrides(), remove_environment=REMOVE_FROM_ENVIRONMENT, workers=WORKERS + ) as owned, + ThreadPoolExecutor(TEAMS) as pool, + ): + root: Final = psutil.Process(owned.process.pid) + fleet: Final = _fleet(owned.gateway, scenario) + pinned: Final = fleet[0] + burst: Final = Burst(owned.gateway) + with _holding_team_lock(pinned.team_id) as holder_pid: + burst.start(pool, fleet) + _await_pinned_delete_blocked(burst, pinned, holder_pid, record_property) + before: Final = _workers(root) + assert len(before) == WORKERS, [process.pid for process in before] + victim: Final = before[0] + victim.kill() # SIGKILL with the pinned delete blocked: the worker cannot finish its in-flight deletes + victim.wait(timeout=10) + with httpx.Client(base_url=str(owned.gateway.client.base_url), timeout=15, trust_env=False) as fresh: + readiness: Final = fresh.get("/health/readiness") + assert readiness.status_code == 200, ( + f"/health/readiness with worker {victim.pid} dead: {readiness.status_code} {readiness.text}" + ) + # The lock is released: the pinned delete finishes on the survivor, or was dropped with the victim. + outcomes: Final = burst.outcomes() + respawned: Final = eventually( + lambda: tuple(process.pid for process in _workers(root)), + lambda pids: len(pids) == WORKERS and victim.pid not in pids, + seconds=60, + ) + record_property( + "worker_pids", {"before": [process.pid for process in before], "killed": victim.pid, "after": respawned} + ) + observed: Final = _states(fleet) + _record_burst(record_property, outcomes, observed, pinned) + assert all(outcome.answered_or_dropped for outcome in outcomes), [ + (outcome.team_id, outcome.status, outcome.detail) for outcome in outcomes if not outcome.answered_or_dropped + ] + retried: Final = _retry_survivors(owned.gateway, fleet, observed) + _assert_every_team_fully_deleted(record_property, observed, _states(fleet), retried) + + +@pytest.mark.timeout(240) # owned Redis, owned two-worker proxy boot, 24-team fleet, Redis restart +def test_redis_stopped_mid_burst_keeps_deletes_answering_200( + gateway: Gateway, tmp_path: Path, record_property: RecordProperty +) -> None: + with ( + gateway.scenario() as scenario, + owned_redis(tmp_path) as coordination, + owned_proxy_process( + gateway, + tmp_path, + { + **_overrides(), + "REDIS_HOST": coordination.host, + "REDIS_PORT": str(coordination.port), + # The breaker opens during the outage; the default 60 s before it probes again would + # keep /cache/ping (whose set_cache runs under the breaker) at 503 long after restart. + "REDIS_CIRCUIT_BREAKER_RECOVERY_TIMEOUT": "5", + }, + remove_environment=REMOVE_FROM_ENVIRONMENT, + workers=WORKERS, + ) as owned, + ThreadPoolExecutor(TEAMS) as pool, + ): + fleet: Final = _fleet(owned.gateway, scenario) + pinned: Final = fleet[0] + assert _cache_status(_cache_ping(owned.gateway)) == "healthy" + burst: Final = Burst(owned.gateway) + with _holding_team_lock(pinned.team_id) as holder_pid: + burst.start(pool, fleet) + _await_pinned_delete_blocked(burst, pinned, holder_pid, record_property) + coordination.stop() + down: Final = _cache_ping(owned.gateway) + assert down.status_code == 503, f"/cache/ping with Redis stopped: {down.status_code} {down.text}" + assert "Service Unhealthy" in down.text, down.text + assert burst.pending(pinned.team_id), f"{pinned.team_id}: delete answered while its team lock was held" + # The lock is released with Redis down: the pinned delete's cache eviction runs against the outage. + outcomes: Final = burst.outcomes() + coordination.start() + recovered: Final = eventually(lambda: _cache_ping(owned.gateway), lambda r: r.status_code == 200, seconds=60) + assert _cache_status(recovered) == "healthy" + + observed: Final = _states(fleet) + _record_burst(record_property, outcomes, observed, pinned) + assert all(outcome.status == 200 for outcome in outcomes), ( + "deletes not answered 200 while Redis was down: " + + str([(outcome.team_id, outcome.status, outcome.detail) for outcome in outcomes if outcome.status != 200]) + + f"; split {dict(Counter(outcome.label for outcome in outcomes))}" + ) + retried: Final = _retry_survivors(owned.gateway, fleet, observed) + _assert_every_team_fully_deleted(record_property, observed, _states(fleet), retried) + + +@pytest.mark.timeout(240) # two owned two-worker proxy boots (before and after SIGTERM) plus a 24-team fleet +def test_proxy_terminated_mid_burst_then_restarted_leaves_every_team_fully_deleted( + gateway: Gateway, tmp_path: Path, record_property: RecordProperty +) -> None: + with gateway.scenario() as scenario, ThreadPoolExecutor(TEAMS) as pool: + with owned_proxy_process( + gateway, tmp_path, _overrides(), remove_environment=REMOVE_FROM_ENVIRONMENT, workers=WORKERS + ) as doomed: + fleet: Final = _fleet(doomed.gateway, scenario) + pinned: Final = fleet[0] + burst: Final = Burst(doomed.gateway) + with _holding_team_lock(pinned.team_id) as holder_pid: + burst.start(pool, fleet) + _await_pinned_delete_blocked(burst, pinned, holder_pid, record_property) + doomed.process.terminate() # SIGTERM with the pinned delete blocked: uvicorn stops accepting and drains + assert burst.pending(pinned.team_id), f"{pinned.team_id}: delete answered while its team lock was held" + # The lock is released: the drain lets the pinned delete finish before the proxy exits. + doomed.process.wait(timeout=120) + outcomes: Final = burst.outcomes() + + at_restart: Final = _states(fleet) + _record_burst(record_property, outcomes, at_restart, pinned) + assert all(outcome.answered_or_dropped for outcome in outcomes), [ + (outcome.team_id, outcome.status, outcome.detail) for outcome in outcomes if not outcome.answered_or_dropped + ] + with owned_proxy_process( + gateway, tmp_path, _overrides(), remove_environment=REMOVE_FROM_ENVIRONMENT, workers=WORKERS + ) as fresh: + retried: Final = _retry_survivors(fresh.gateway, fleet, at_restart) + final: Final = _states(fleet) + _assert_every_team_fully_deleted(record_property, at_restart, final, retried) diff --git a/tests/integration/management/test_team_delete_inputs.py b/tests/integration/management/test_team_delete_inputs.py new file mode 100644 index 00000000000..d385c915696 --- /dev/null +++ b/tests/integration/management/test_team_delete_inputs.py @@ -0,0 +1,330 @@ +"""Sad inputs for /team/delete: malformed ids, callers without access, and rosters the API can no longer produce. + +Legacy roster shapes (email-only entries, entries with neither id nor email) are seeded straight into +``members_with_roles`` because ``/team/member_add`` backfills ``user_id`` and will not write them any more. +""" + +from __future__ import annotations + +import json +import os +import uuid +from collections.abc import Callable, Mapping +from hashlib import sha256 +from typing import Final + +import httpx +import pytest +from pydantic import JsonValue +from redis import Redis + +from tests.integration._support.client import ( + Gateway, + Scenario, + delete_key_if_present, + eventually, + object_value, + string_value, +) +from tests.integration._support.database import read_rows, write_rows + +RecordProperty = Callable[[str, object], None] + +TEAM_SQL: Final = 'SELECT team_id FROM "LiteLLM_TeamTable" WHERE team_id = %s' +ROSTER_READ_SQL: Final = 'SELECT members_with_roles FROM "LiteLLM_TeamTable" WHERE team_id = %s' +ROSTER_SQL: Final = 'UPDATE "LiteLLM_TeamTable" SET members_with_roles = %s::jsonb WHERE team_id = %s' +MEMBERSHIP_SQL: Final = 'SELECT user_id FROM "LiteLLM_TeamMembership" WHERE team_id = %s' +USER_SQL: Final = 'SELECT user_id FROM "LiteLLM_UserTable" WHERE user_id = %s' +USER_EMAIL_SQL: Final = 'UPDATE "LiteLLM_UserTable" SET user_email = %s WHERE user_id = %s' +TOKEN_SQL: Final = 'SELECT token FROM "LiteLLM_VerificationToken" WHERE token = %s' +TOMBSTONE_SQL: Final = 'SELECT id FROM "LiteLLM_DeletedTeamTable" WHERE team_id = %s' +AUDIT_SQL: Final = 'SELECT id, table_name, action FROM "LiteLLM_AuditLog" WHERE object_id = %s' + +NOT_FOUND: Final = "Team not found, passed team_id=" +# /team/delete sits on management_routes but on no internal-user route list, so the route gate in +# RouteChecks.non_proxy_admin_allowed_routes_check answers 401 before _verify_team_access ever runs +# (pinned by tests/integration/authorization/test_team_admin_gate.py as team_admin=401, others=401). +ROUTE_GATE_MESSAGE: Final = "Only proxy admin can be used" +UNKNOWN_TEAM: Final = f"integration-missing-{uuid.uuid4().hex}" +FIVE_KB_TEAM: Final = "t" * 5120 + + +def _team_rows(team_id: str) -> list[dict[str, JsonValue]]: + return read_rows(TEAM_SQL, (team_id,)) + + +def _delete_team_if_present(gateway: Gateway, team_id: str) -> None: + """Cleanup for a team the test deletes itself: a no-op once the row is gone.""" + if _team_rows(team_id): + gateway.post("/team/delete", {"team_ids": [team_id]}) + assert _team_rows(team_id) == [] + + +def _reset_roster_if_present(team_id: str) -> None: + """Cleanup for a seeded roster: put back a shape the delete path always accepts.""" + if _team_rows(team_id): + write_rows(ROSTER_SQL, ("[]", team_id)) + + +def _delete_user_if_present(gateway: Gateway, user_id: str) -> None: + if read_rows(USER_SQL, (user_id,)): + response: Final = gateway.request("POST", "/user/delete", {"user_ids": [user_id]}) + assert response.status_code == 200, response.text + assert read_rows(USER_SQL, (user_id,)) == [] + + +def _own_team(scenario: Scenario, **fields: JsonValue) -> str: + """A team the test deletes itself, so cleanup tolerates the row already being gone.""" + created: Final = scenario.gateway.post("/team/new", {"team_alias": f"integration-{uuid.uuid4().hex}", **fields}) + team_id: Final = string_value(created["team_id"]) + scenario.cleanups.callback(_delete_team_if_present, scenario.gateway, team_id) + return team_id + + +def _own_key(scenario: Scenario, **fields: JsonValue) -> str: + """A key the team delete is expected to remove, so cleanup tolerates it already being gone.""" + created: Final = scenario.gateway.post("/key/generate", fields) + token: Final = string_value(created["key"]) + scenario.cleanups.callback(delete_key_if_present, scenario.gateway, token) + return token + + +def _own_user(scenario: Scenario) -> str: + """An internal user the test may remove by SQL, so cleanup tolerates the row already being gone.""" + created: Final = scenario.gateway.post( + "/user/new", + {"user_id": f"integration-{uuid.uuid4().hex}", "auto_create_key": False, "user_role": "internal_user"}, + ) + user_id: Final = string_value(created["user_id"]) + scenario.cleanups.callback(_delete_user_if_present, scenario.gateway, user_id) + return user_id + + +def _seed_roster(scenario: Scenario, team_id: str, entries: list[dict[str, JsonValue]]) -> None: + write_rows(ROSTER_SQL, (json.dumps(entries), team_id)) + scenario.cleanups.callback(_reset_roster_if_present, team_id) + + +def _team_admin(scenario: Scenario, team_id: str) -> str: + """Add a member and flip their roster role to admin by SQL: the API gates that role behind a license.""" + user_id: Final = scenario.member(team_id) + rows: Final = read_rows(ROSTER_READ_SQL, (team_id,)) + assert len(rows) == 1, rows + roster: Final = rows[0]["members_with_roles"] + assert isinstance(roster, list), roster + promoted: Final = [ + {**object_value(entry), "role": "admin"} if object_value(entry).get("user_id") == user_id else entry + for entry in roster + ] + assert any(object_value(entry).get("user_id") == user_id for entry in promoted), promoted + write_rows(ROSTER_SQL, (json.dumps(promoted), team_id)) + return user_id + + +def _membership_user_ids(team_id: str) -> frozenset[str]: + return frozenset(string_value(row["user_id"]) for row in read_rows(MEMBERSHIP_SQL, (team_id,))) + + +def _delete(gateway: Gateway, team_ids: JsonValue, *, key: str | None = None) -> httpx.Response: + return gateway.request("POST", "/team/delete", {"team_ids": team_ids}, key=key) + + +def _hashed(token: str) -> str: + return sha256(token.encode()).hexdigest() + + +@pytest.mark.parametrize( + ("body", "status", "needle"), + [ + pytest.param({"team_ids": [UNKNOWN_TEAM]}, 404, f"{NOT_FOUND}{UNKNOWN_TEAM}", id="S1-unknown-id"), + pytest.param({"team_ids": "abc"}, 422, "list_type", id="S3-string-not-list"), + pytest.param({"team_ids": [123]}, 422, "string_type", id="S4-integer-item"), + pytest.param({"team_ids": [""]}, 404, NOT_FOUND, id="S5-empty-id"), + pytest.param({"team_ids": [FIVE_KB_TEAM]}, 404, NOT_FOUND, id="S6-5kb-id"), + ], +) +def test_rejects_malformed_team_ids(gateway: Gateway, body: Mapping[str, JsonValue], status: int, needle: str) -> None: + response: Final = gateway.request("POST", "/team/delete", body) + assert response.status_code == status, f"{response.status_code} {response.text}" + assert needle in response.text, response.text + + +def test_empty_list_deletes_nothing(gateway: Gateway) -> None: + response: Final = _delete(gateway, []) + assert response.status_code == 200, f"{response.status_code} {response.text}" + assert response.json() == {"deleted_teams": []}, response.text + + +def test_duplicate_ids_delete_once(gateway: Gateway, record_property: RecordProperty) -> None: + """Repeated ids collapse to one delete: the body names the team once and exactly one tombstone row lands.""" + with gateway.scenario() as scenario: + team: Final = _own_team(scenario) + first: Final = scenario.member(team) + second: Final = scenario.member(team) + key: Final = _own_key(scenario, team_id=team) + # The master key's /team/new also seats the proxy admin, so the table holds more than these two. + assert {first, second} <= _membership_user_ids(team), _membership_user_ids(team) + response: Final = _delete(gateway, [team, team]) + # Read every table before the first assert so a red cell carries the partial state with it. + present: Final = _team_rows(team) + memberships: Final = _membership_user_ids(team) + key_rows: Final = read_rows(TOKEN_SQL, (_hashed(key),)) + tombstones: Final = read_rows(TOMBSTONE_SQL, (team,)) + audit: Final = read_rows(AUDIT_SQL, (team,)) + state: Final = ( + f"team_present={bool(present)} membership_rows={len(memberships)} key_present={bool(key_rows)} " + f"tombstone_rows={len(tombstones)} audit_rows={len(audit)}" + ) + record_property("status", response.status_code) + record_property("body", response.text) + record_property("state_after", state) + record_property("audit_rows", len(audit)) # recorded only: the shared rigs cannot enable audit logging + assert response.status_code == 200, f"{response.status_code} {response.text}; {state}" + assert response.json() == {"deleted_teams": [team]}, response.text + assert present == [], state + assert memberships == frozenset(), state + assert key_rows == [], state + assert len(tombstones) == 1, f"tombstone rows for {team}: {len(tombstones)}; {state}" + + +def test_missing_authorization_is_401(gateway: Gateway) -> None: + response: Final = gateway.client.post("/team/delete", json={"team_ids": [UNKNOWN_TEAM]}) + assert response.status_code == 401, f"{response.status_code} {response.text}" + assert "error" in response.text.lower(), response.text + + +def test_internal_user_outside_team_is_refused_by_the_route_gate(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + team: Final = scenario.team() + outsider: Final = scenario.user(user_role="internal_user") + key: Final = scenario.key(user_id=outsider) + response: Final = _delete(gateway, [team], key=key) + assert response.status_code == 401, f"{response.status_code} {response.text}" + assert ROUTE_GATE_MESSAGE in response.text, response.text + assert len(_team_rows(team)) == 1 + + +def test_admin_of_another_team_is_refused_by_the_route_gate(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + target: Final = scenario.team() + other: Final = scenario.team() + admin: Final = _team_admin(scenario, other) + key: Final = scenario.key(team_id=other, user_id=admin) + response: Final = _delete(gateway, [target], key=key) + assert response.status_code == 401, f"{response.status_code} {response.text}" + assert ROUTE_GATE_MESSAGE in response.text, response.text + assert len(_team_rows(target)) == 1 + assert len(_team_rows(other)) == 1 + + +def test_team_admin_of_own_team_is_refused_by_the_route_gate(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + team: Final = scenario.team() + admin: Final = _team_admin(scenario, team) + key: Final = scenario.key(team_id=team, user_id=admin) + response: Final = _delete(gateway, [team], key=key) + assert response.status_code == 401, f"{response.status_code} {response.text}" + assert ROUTE_GATE_MESSAGE in response.text, response.text + assert len(_team_rows(team)) == 1 + + +def test_roster_user_whose_row_was_removed(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + team: Final = _own_team(scenario) + ghost: Final = _own_user(scenario) + gateway.post("/team/member_add", {"team_id": team, "member": {"role": "user", "user_id": ghost}}) + assert ghost in _membership_user_ids(team), _membership_user_ids(team) + write_rows('DELETE FROM "LiteLLM_UserTable" WHERE user_id = %s', (ghost,)) + assert read_rows(USER_SQL, (ghost,)) == [] + response: Final = _delete(gateway, [team]) + assert response.status_code == 200, f"{response.status_code} {response.text}" + assert _team_rows(team) == [] + assert read_rows(MEMBERSHIP_SQL, (team,)) == [] + + +def test_email_only_roster_entry_matching_no_user(gateway: Gateway, record_property: RecordProperty) -> None: + with gateway.scenario() as scenario: + team: Final = _own_team(scenario) + _seed_roster( + scenario, + team, + [{"role": "user", "user_id": None, "user_email": f"nobody-{uuid.uuid4().hex}@example.com"}], + ) + response: Final = _delete(gateway, [team]) + record_property("status", response.status_code) + record_property("body", response.text) + assert response.status_code == 200, f"{response.status_code} {response.text}" + assert _team_rows(team) == [] + + +def test_email_only_roster_entry_matching_two_case_variants(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + model: Final = scenario.model() + tag: Final = uuid.uuid4().hex + upper: Final = scenario.user(user_role="internal_user") + lower: Final = scenario.user(user_role="internal_user") + # /user/new rejects a second email that matches case-insensitively, so the pair is seeded by SQL. + write_rows(USER_EMAIL_SQL, (f"Case-{tag}@example.com", upper)) + write_rows(USER_EMAIL_SQL, (f"case-{tag}@example.com", lower)) + for user in (upper, lower): + gateway.chat(model, key=scenario.key(user_id=user, models=[model])) + with Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"])) as cache: + + def cached() -> dict[str, int]: + return {user: int(cache.exists(user)) for user in (upper, lower)} + + assert eventually(cached, lambda seen: seen == {upper: 1, lower: 1}, seconds=10) == {upper: 1, lower: 1} + team: Final = _own_team(scenario) + _seed_roster(scenario, team, [{"role": "user", "user_id": None, "user_email": f"CASE-{tag}@EXAMPLE.COM"}]) + response: Final = _delete(gateway, [team]) + assert response.status_code == 200, f"{response.status_code} {response.text}" + assert _team_rows(team) == [] + remaining: Final = eventually( + cached, lambda seen: seen == {upper: 0, lower: 0}, seconds=10, return_last_on_timeout=True + ) + assert remaining == {upper: 0, lower: 0}, ( + f"user cache entries still present after /team/delete: " + f"{upper} exists={remaining[upper]}, {lower} exists={remaining[lower]}" + ) + + +def test_roster_entry_without_id_or_email_pins_the_500(gateway: Gateway, record_property: RecordProperty) -> None: + """Pins a pre-existing defect outside this PR's diff until it gets its own ticket: for a roster entry with neither + id nor email, LiteLLM_TeamTable.model_validate raises outside delete_team's 404 try/except, so the call is a 500 + that writes nothing (team row intact, no tombstone, membership rows untouched).""" + with gateway.scenario() as scenario: + team: Final = _own_team(scenario) + before: Final = _membership_user_ids(team) + _seed_roster(scenario, team, [{"role": "user", "user_id": None, "user_email": None}]) + response: Final = _delete(gateway, [team]) + present: Final = _team_rows(team) + tombstones: Final = read_rows(TOMBSTONE_SQL, (team,)) + after: Final = _membership_user_ids(team) + state: Final = f"team_present={bool(present)} tombstone_rows={len(tombstones)} membership_rows={len(after)}" + record_property("status", response.status_code) + record_property("body", response.text) + record_property("state_after", state) + assert response.status_code == 500, f"{response.status_code} {response.text}; {state}" + assert "Internal server error" in response.text, response.text + assert len(present) == 1, state + assert tombstones == [], state + assert after == before, f"membership rows changed: before={sorted(before)} after={sorted(after)}" + + +def test_failed_delete_leaves_unrelated_key_serving(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + model: Final = scenario.model() + key: Final = scenario.key(models=[model]) + assert object_value(gateway.chat(model, key=key)["usage"])["total_tokens"] == 40 + missing: Final = f"integration-missing-{uuid.uuid4().hex}" + response: Final = _delete(gateway, [missing]) + assert response.status_code == 404, f"{response.status_code} {response.text}" + assert f"{NOT_FOUND}{missing}" in response.text, response.text + completion: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": "after failed delete"}]}, + key=key, + ) + assert completion.status_code == 200, f"{completion.status_code} {completion.text}" + assert object_value(object_value(completion.json())["usage"])["total_tokens"] == 40 diff --git a/tests/integration/management/test_team_delete_large_membership.py b/tests/integration/management/test_team_delete_large_membership.py new file mode 100644 index 00000000000..5912256e5cd --- /dev/null +++ b/tests/integration/management/test_team_delete_large_membership.py @@ -0,0 +1,634 @@ +"""`/team/delete` as one locked transaction, whatever the roster size. + +The delete removes the team row, its membership rows, every member's `teams` reference and every +team key in one pass, writes one tombstone per team, evicts the cached team object and takes the +team's advisory lock (the one `/team/member_add` takes) before it writes. A roster larger than the +Prisma pool used to fail with P2028 because each member got its own transaction. +""" + +from __future__ import annotations + +import json +import os +import uuid +from collections.abc import Callable, Mapping, Sequence +from concurrent.futures import Future, ThreadPoolExecutor +from hashlib import sha256 +from pathlib import Path +from typing import Final + +import httpx +import psycopg +import pytest +import yaml +from pydantic import JsonValue +from redis import Redis + +from tests.integration._support.client import ( + Gateway, + Scenario, + delete_key_if_present, + eventually, + object_value, + string_value, +) +from tests.integration._support.database import read_rows, write_rows +from tests.integration._support.process import owned_proxy_process + +LARGE_ROSTER: Final = 250 +POOL_LIMIT: Final = 5 +# One statement seeds the whole roster: 250 individual /user/new calls would dominate the runtime. +SEED_USERS_SQL: Final = """ +INSERT INTO "LiteLLM_UserTable" (user_id, user_role, teams, models) +SELECT %s || '-' || lpad(n::text, 3, '0'), 'internal_user', '{}'::text[], '{}'::text[] +FROM generate_series(1, %s::int) AS n +""" +# The master key's user id. `/team/new` appends the creator to the roster as an admin, so every team +# created here carries this member alongside the ones the test adds. +PROXY_ADMIN: Final = "default_user_id" +TAKE_TEAM_LOCK_SQL: Final = "SELECT pg_advisory_xact_lock(hashtext(%s))" +# Sessions blocked on the advisory lock the given backend holds, and nothing else on the shared rig. +WAITERS_ON_HELD_LOCK_SQL: Final = """ +SELECT count(*)::int AS waiting +FROM pg_locks waiter +JOIN pg_stat_activity session ON session.pid = waiter.pid +WHERE waiter.locktype = 'advisory' + AND NOT waiter.granted + AND session.wait_event_type = 'Lock' + AND session.query ILIKE %s + AND (waiter.classid, waiter.objid, waiter.objsubid) IN ( + SELECT held.classid, held.objid, held.objsubid + FROM pg_locks held + WHERE held.locktype = 'advisory' AND held.granted AND held.pid = %s::int + ) +""" + + +def _hashed(key: str) -> str: + return sha256(key.encode()).hexdigest() + + +def _team_rows(team_id: str) -> list[dict[str, JsonValue]]: + return read_rows('SELECT team_id FROM "LiteLLM_TeamTable" WHERE team_id = %s', (team_id,)) + + +def _membership_user_ids(team_id: str) -> list[JsonValue]: + rows: Final = read_rows( + 'SELECT user_id FROM "LiteLLM_TeamMembership" WHERE team_id = %s ORDER BY user_id', (team_id,) + ) + return [row["user_id"] for row in rows] + + +def _user_teams(user_id: str) -> JsonValue: + rows: Final = read_rows('SELECT teams FROM "LiteLLM_UserTable" WHERE user_id = %s', (user_id,)) + assert len(rows) == 1, f"user row for {user_id}: {rows}" + return rows[0]["teams"] + + +def _users_referencing(team_id: str) -> list[JsonValue]: + rows: Final = read_rows( + 'SELECT user_id FROM "LiteLLM_UserTable" WHERE %s = ANY(teams) ORDER BY user_id', (team_id,) + ) + return [row["user_id"] for row in rows] + + +def _user_ids_with_prefix(prefix: str) -> list[JsonValue]: + rows: Final = read_rows( + 'SELECT user_id FROM "LiteLLM_UserTable" WHERE user_id LIKE %s ORDER BY user_id', (f"{prefix}-%",) + ) + return [row["user_id"] for row in rows] + + +def _live_token(hashed: str) -> list[dict[str, JsonValue]]: + return read_rows('SELECT token, team_id FROM "LiteLLM_VerificationToken" WHERE token = %s', (hashed,)) + + +def _deleted_token(hashed: str) -> list[dict[str, JsonValue]]: + return read_rows('SELECT token, team_id FROM "LiteLLM_DeletedVerificationToken" WHERE token = %s', (hashed,)) + + +def _tombstones(team_id: str) -> list[dict[str, JsonValue]]: + return read_rows( + 'SELECT team_id, members_with_roles FROM "LiteLLM_DeletedTeamTable" WHERE team_id = %s', (team_id,) + ) + + +def _roster_user_ids(roster: JsonValue) -> list[str]: + assert isinstance(roster, list), f"roster is not a list: {roster!r}" + return sorted(string_value(object_value(member)["user_id"]) for member in roster) + + +def _waiters_on_lock_held_by(backend_pid: int) -> int: + rows: Final = read_rows(WAITERS_ON_HELD_LOCK_SQL, ("%pg_advisory_xact_lock%", str(backend_pid))) + waiting: Final = rows[0]["waiting"] + assert isinstance(waiting, int) + return waiting + + +def _remove_team_by_sql(team_id: str) -> None: + """Cleanup for a team the test expects to have deleted itself. Whatever a failed delete left behind + (row, memberships, `teams` references) goes by SQL so the shared rig stays clean without sending + another request through the proxy under test.""" + if not _team_rows(team_id): + return + write_rows('DELETE FROM "LiteLLM_TeamMembership" WHERE team_id = %s', (team_id,)) + write_rows( + 'UPDATE "LiteLLM_UserTable" SET teams = array_remove(teams, %s) WHERE %s = ANY(teams)', (team_id, team_id) + ) + write_rows('DELETE FROM "LiteLLM_TeamTable" WHERE team_id = %s', (team_id,)) + assert _team_rows(team_id) == [] + + +def _remove_team_if_present(gateway: Gateway, team_id: str) -> None: + """Cleanup for teams the test deletes itself: the API delete first, SQL for anything it leaves.""" + if not _team_rows(team_id): + return + gateway.request("POST", "/team/delete", {"team_ids": [team_id]}) + _remove_team_by_sql(team_id) + + +def _create_team(scenario: Scenario, **fields: JsonValue) -> str: + """A team the test deletes itself; cleanup only removes it if the test left it behind.""" + created: Final = scenario.gateway.post("/team/new", {"team_alias": f"integration-{uuid.uuid4().hex}", **fields}) + team_id: Final = string_value(created["team_id"]) + scenario.cleanups.callback(_remove_team_if_present, scenario.gateway, team_id) + return team_id + + +def _generate_key(scenario: Scenario, **fields: JsonValue) -> str: + """A key the team delete is expected to remove; cleanup only deletes it if it is still live.""" + key: Final = string_value(scenario.gateway.post("/key/generate", fields)["key"]) + scenario.cleanups.callback(delete_key_if_present, scenario.gateway, key) + return key + + +def _delete_seeded_users(prefix: str) -> None: + write_rows('DELETE FROM "LiteLLM_UserTable" WHERE user_id LIKE %s', (f"{prefix}-%",)) + assert _user_ids_with_prefix(prefix) == [] + + +def _seed_users(scenario: Scenario, prefix: str, count: int) -> tuple[str, ...]: + """Insert `count` user rows in one statement; ids are `-001` … `-`.""" + users: Final = tuple(f"{prefix}-{index:03d}" for index in range(1, count + 1)) + write_rows(SEED_USERS_SQL, (prefix, str(count))) + scenario.cleanups.callback(_delete_seeded_users, prefix) + assert _user_ids_with_prefix(prefix) == list(users) + return users + + +def _bulk_member_add(gateway: Gateway, team_id: str, users: Sequence[str]) -> None: + gateway.post( + "/team/member_add", + {"team_id": team_id, "member": [{"role": "user", "user_id": user_id} for user_id in users]}, + ) + + +def _delete_teams(gateway: Gateway, team_ids: Sequence[str]) -> httpx.Response: + return gateway.request("POST", "/team/delete", {"team_ids": list(team_ids)}) + + +def _team_info(gateway: Gateway, team_id: str) -> httpx.Response: + return gateway.request("GET", "/team/info", params={"team_id": team_id}) + + +def _team_not_found_body(team_id: str) -> dict[str, JsonValue]: + """The proxy's exception handler wraps the 404 detail as an `error` object with the detail stringified.""" + return { + "error": { + "message": f"{{'message': 'Team not found, passed team id: {team_id}.'}}", + "type": "auth_error", + "param": "None", + "code": "404", + } + } + + +def _chat(gateway: Gateway, model: str, key: str) -> httpx.Response: + return gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": f"team delete {uuid.uuid4().hex}"}]}, + key=key, + ) + + +def _post_with_timeout(gateway: Gateway, path: str, body: Mapping[str, JsonValue], timeout: float) -> httpx.Response: + """Like `Gateway.request` with a per-call timeout longer than the client's default 15 s.""" + return gateway.client.request( + "POST", path, json=body, headers={"Authorization": f"Bearer {gateway.key}"}, timeout=timeout + ) + + +def _post_in_background( + pool: ThreadPoolExecutor, gateway: Gateway, path: str, body: Mapping[str, JsonValue] +) -> Future[httpx.Response]: + return pool.submit(_post_with_timeout, gateway, path, body, 60) + + +def _config_with_pool_limit(tmp_path: Path, pool_limit: int) -> Path: + config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + config["general_settings"]["database_connection_pool_limit"] = pool_limit + config["general_settings"]["database_connection_pool_timeout"] = 60 + path: Final = tmp_path / f"pool-{pool_limit}.yaml" + path.write_text(yaml.safe_dump(config)) + return path + + +def _redis() -> Redis: + return Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"])) + + +def test_delete_small_team_removes_rows_keys_tombstone_and_cache( + gateway: Gateway, record_property: Callable[[str, object], None] +) -> None: + with gateway.scenario() as scenario, _redis() as cache: + model: Final = scenario.model() + users: Final = sorted(scenario.user() for _ in range(3)) + team: Final = _create_team(scenario) + _bulk_member_add(gateway, team, users) + keys: Final = tuple(_generate_key(scenario, team_id=team) for _ in range(2)) + hashed: Final = tuple(_hashed(key) for key in keys) + roster: Final = sorted([PROXY_ADMIN, *users]) + assert _membership_user_ids(team) == roster + assert _users_referencing(team) == roster + assert all(_user_teams(user) == [team] for user in users), [_user_teams(user) for user in users] + assert all(len(_live_token(digest)) == 1 for digest in hashed), hashed + + warm: Final = _chat(gateway, model, keys[0]) + assert warm.status_code == 200, warm.text + team_cache_key: Final = f"team_id:{team}" + eventually(lambda: cache.exists(team_cache_key), lambda present: present == 1, seconds=10) + record_property("redis_keys_before_delete", sorted(entry.decode() for entry in cache.keys(f"*{team}*"))) + + response: Final = _delete_teams(gateway, [team]) + assert response.status_code == 200, response.text + assert response.json() == {"deleted_teams": [team]} + + assert _team_rows(team) == [] + assert _membership_user_ids(team) == [] + assert _users_referencing(team) == [] + assert [_user_teams(user) for user in users] == [[], [], []] + assert [_live_token(digest) for digest in hashed] == [[], []] + assert [_deleted_token(digest) for digest in hashed] == [ + [{"token": hashed[0], "team_id": team}], + [{"token": hashed[1], "team_id": team}], + ] + tombstones: Final = _tombstones(team) + assert len(tombstones) == 1, tombstones + assert tombstones[0]["team_id"] == team + assert _roster_user_ids(tombstones[0]["members_with_roles"]) == roster + + info: Final = _team_info(gateway, team) + assert info.status_code == 404, info.text + assert info.json() == _team_not_found_body(team) + + assert cache.exists(team_cache_key) == 0 + record_property("redis_keys_after_delete", sorted(entry.decode() for entry in cache.keys(f"*{team}*"))) + + +@pytest.mark.timeout(240) # owned two-worker proxy boot plus a 250-member roster +def test_delete_250_member_team_succeeds_with_pool_limit_five_on_two_workers( + gateway: Gateway, tmp_path: Path, record_property: Callable[[str, object], None] +) -> None: + prefix: Final = f"integration-roster-{uuid.uuid4().hex}" + # The scenario is bound to the shared gateway and its cleanups are SQL, so a failed delete on the + # owned proxy (and whatever it does to that proxy's workers) cannot mask the assertion below with a + # second failure during cleanup. The owned proxy is stopped before the cleanups run. + with ( + gateway.scenario() as scenario, + owned_proxy_process( + gateway, + tmp_path, + {"DATABASE_URL": os.environ["DATABASE_URL"]}, + config=_config_with_pool_limit(tmp_path, POOL_LIMIT), + remove_environment=("DATABASE_URL_READ_REPLICA",), + workers=2, + ) as owned, + ): + team: Final = string_value( + owned.gateway.post("/team/new", {"team_alias": f"integration-{uuid.uuid4().hex}"})["team_id"] + ) + scenario.cleanups.callback(_remove_team_by_sql, team) + users: Final = _seed_users(scenario, prefix, LARGE_ROSTER) + added: Final = _post_with_timeout( + owned.gateway, + "/team/member_add", + {"team_id": team, "member": [{"role": "user", "user_id": user_id} for user_id in users]}, + timeout=120, + ) + assert added.status_code == 200, added.text + roster: Final = sorted([PROXY_ADMIN, *users]) + assert _membership_user_ids(team) == roster + assert _users_referencing(team) == roster + + response: Final = _post_with_timeout(owned.gateway, "/team/delete", {"team_ids": [team]}, timeout=120) + record_property("h2_delete_response", f"{response.status_code} {response.text[:300]}") + assert response.status_code == 200, ( + f"/team/delete of a {LARGE_ROSTER}-member team with database_connection_pool_limit={POOL_LIMIT}: " + f"{response.status_code} {response.text}" + ) + assert response.json() == {"deleted_teams": [team]} + assert _team_rows(team) == [] + assert _membership_user_ids(team) == [] + assert _users_referencing(team) == [] + assert _user_ids_with_prefix(prefix) == list(users) + tombstones: Final = _tombstones(team) + assert len(tombstones) == 1, tombstones + assert _roster_user_ids(tombstones[0]["members_with_roles"]) == roster + + +def test_delete_waits_for_the_team_advisory_lock_and_completes_after_release( + gateway: Gateway, record_property: Callable[[str, object], None] +) -> None: + with gateway.scenario() as scenario: + user: Final = scenario.user() + team: Final = _create_team(scenario) + _bulk_member_add(gateway, team, [user]) + hashed: Final = _hashed(_generate_key(scenario, team_id=team)) + with ThreadPoolExecutor(max_workers=1) as pool: + with psycopg.connect(os.environ["DATABASE_URL"]) as holder: + holder.execute(TAKE_TEAM_LOCK_SQL, (team,)) + pending: Final = _post_in_background(pool, gateway, "/team/delete", {"team_ids": [team]}) + eventually( + lambda: _waiters_on_lock_held_by(holder.info.backend_pid), + lambda waiting: waiting >= 1, + seconds=20, + ) + assert not pending.done(), "delete returned while the team lock was still held" + assert _team_rows(team) == [{"team_id": team}], "team row deleted while the team lock was held" + # Recorded before the count assertion so both legs document what the delete had already + # written by the time it reached the lock. + record_property( + "state_while_blocked", + json.dumps( + { + "membership_user_ids": _membership_user_ids(team), + "user_teams": _user_teams(user), + "live_token_rows": len(_live_token(hashed)), + "tombstones": len(_tombstones(team)), + } + ), + ) + # One transaction per delete: a per-member fan-out would queue one waiter per roster entry. + eventually( + lambda: _waiters_on_lock_held_by(holder.info.backend_pid), + lambda waiting: waiting == 1, + seconds=10, + ) + # leaving the holder block commits its transaction, which releases the advisory lock + response: Final = pending.result(timeout=60) + assert response.status_code == 200, response.text + assert response.json() == {"deleted_teams": [team]} + assert _team_rows(team) == [] + assert _membership_user_ids(team) == [] + assert _user_teams(user) == [] + assert _live_token(hashed) == [] + assert len(_tombstones(team)) == 1, _tombstones(team) + + +def test_deleting_two_teams_sharing_a_member_in_one_call_clears_both_from_the_member(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + user: Final = scenario.user() + first: Final = _create_team(scenario) + second: Final = _create_team(scenario) + _bulk_member_add(gateway, first, [user]) + _bulk_member_add(gateway, second, [user]) + assert _user_teams(user) == [first, second] + + response: Final = _delete_teams(gateway, [first, second]) + assert response.status_code == 200, response.text + assert response.json() == {"deleted_teams": [first, second]} + + assert _team_rows(first) == [] + assert _team_rows(second) == [] + assert _membership_user_ids(first) == [] + assert _membership_user_ids(second) == [] + assert _user_teams(user) == [] + assert [row["team_id"] for row in _tombstones(first)] == [first] + assert [row["team_id"] for row in _tombstones(second)] == [second] + + +def test_deleting_one_team_leaves_the_members_other_team_and_key_intact(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + model: Final = scenario.model() + user: Final = scenario.user() + deleted: Final = _create_team(scenario) + kept: Final = scenario.team() + _bulk_member_add(gateway, deleted, [user]) + _bulk_member_add(gateway, kept, [user]) + kept_key: Final = scenario.key(team_id=kept, user_id=user) + before: Final = _chat(gateway, model, kept_key) + assert before.status_code == 200, before.text + + response: Final = _delete_teams(gateway, [deleted]) + assert response.status_code == 200, response.text + assert response.json() == {"deleted_teams": [deleted]} + + assert _team_rows(deleted) == [] + assert _team_rows(kept) == [{"team_id": kept}] + assert _membership_user_ids(deleted) == [] + assert _membership_user_ids(kept) == [PROXY_ADMIN, user] + assert _user_teams(user) == [kept] + assert _live_token(_hashed(kept_key)) == [{"token": _hashed(kept_key), "team_id": kept}] + after: Final = _chat(gateway, model, kept_key) + assert after.status_code == 200, after.text + + +@pytest.mark.timeout(240) # owned proxy boot +def test_delete_writes_one_audit_row_for_the_team_and_one_per_key(gateway: Gateway, tmp_path: Path) -> None: + with ( + owned_proxy_process( + gateway, + tmp_path, + {"DATABASE_URL": os.environ["DATABASE_URL"], "LITELLM_STORE_AUDIT_LOGS": "true"}, + remove_environment=("DATABASE_URL_READ_REPLICA",), + ) as owned, + owned.gateway.scenario() as scenario, + ): + user: Final = scenario.user() + team: Final = _create_team(scenario) + _bulk_member_add(owned.gateway, team, [user]) + hashed: Final = _hashed(_generate_key(scenario, team_id=team)) + + response: Final = _delete_teams(owned.gateway, [team]) + assert response.status_code == 200, response.text + assert response.json() == {"deleted_teams": [team]} + assert _team_rows(team) == [] + + def deleted_audit_rows() -> list[dict[str, JsonValue]]: + return read_rows( + 'SELECT table_name, action, object_id FROM "LiteLLM_AuditLog" ' + "WHERE object_id IN (%s, %s) AND action = 'deleted' ORDER BY table_name", + (team, hashed), + ) + + rows: Final = eventually(deleted_audit_rows, lambda found: len(found) >= 2, seconds=30) + assert rows == [ + {"table_name": "LiteLLM_TeamTable", "action": "deleted", "object_id": team}, + {"table_name": "LiteLLM_VerificationToken", "action": "deleted", "object_id": hashed}, + ] + + +def test_second_delete_of_the_same_team_is_404_with_one_tombstone(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + user: Final = scenario.user() + team: Final = _create_team(scenario) + _bulk_member_add(gateway, team, [user]) + + first: Final = _delete_teams(gateway, [team]) + assert first.status_code == 200, first.text + assert first.json() == {"deleted_teams": [team]} + + second: Final = _delete_teams(gateway, [team]) + assert second.status_code == 404, second.text + assert second.json() == {"detail": {"error": f"Team not found, passed team_id={team}"}} + + assert _team_rows(team) == [] + assert [row["team_id"] for row in _tombstones(team)] == [team] + assert _user_teams(user) == [] + + +def test_delete_empty_team_writes_tombstone_and_team_info_is_404(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + team: Final = _create_team(scenario) + present: Final = _team_info(gateway, team) + assert present.status_code == 200, present.text + + response: Final = _delete_teams(gateway, [team]) + assert response.status_code == 200, response.text + assert response.json() == {"deleted_teams": [team]} + + assert _team_rows(team) == [] + assert _membership_user_ids(team) == [] + assert _tombstones(team) == [ + {"team_id": team, "members_with_roles": [{"role": "admin", "user_id": PROXY_ADMIN, "user_email": None}]} + ] + info: Final = _team_info(gateway, team) + assert info.status_code == 404, info.text + assert info.json() == _team_not_found_body(team) + + +def test_delete_keys_only_team_removes_keys_and_revokes_them(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + model: Final = scenario.model() + team: Final = _create_team(scenario) + keys: Final = tuple(_generate_key(scenario, team_id=team) for _ in range(2)) + hashed: Final = tuple(_hashed(key) for key in keys) + assert _membership_user_ids(team) == [PROXY_ADMIN] + for key in keys: + warm = _chat(gateway, model, key) + assert warm.status_code == 200, warm.text + + response: Final = _delete_teams(gateway, [team]) + assert response.status_code == 200, response.text + assert response.json() == {"deleted_teams": [team]} + + assert _team_rows(team) == [] + assert [_live_token(digest) for digest in hashed] == [[], []] + assert [_deleted_token(digest) for digest in hashed] == [ + [{"token": hashed[0], "team_id": team}], + [{"token": hashed[1], "team_id": team}], + ] + for key in keys: + revoked = _chat(gateway, model, key) + assert revoked.status_code == 401, f"{revoked.status_code} {revoked.text}" + + +def test_delete_three_teams_in_one_call_lists_all_and_tombstones_each_once(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + teams: Final = tuple(_create_team(scenario) for _ in range(3)) + for team in teams: + _bulk_member_add(gateway, team, [scenario.user()]) + + response: Final = _delete_teams(gateway, teams) + assert response.status_code == 200, response.text + assert response.json() == {"deleted_teams": list(teams)} + + for team in teams: + assert _team_rows(team) == [] + assert _membership_user_ids(team) == [] + assert _users_referencing(team) == [] + assert [row["team_id"] for row in _tombstones(team)] == [team] + + +def test_recreating_the_same_team_id_after_delete_serves_the_fresh_team(gateway: Gateway) -> None: + with gateway.scenario() as scenario, _redis() as cache: + model: Final = scenario.model() + original_member: Final = scenario.user() + replacement_member: Final = scenario.user() + team: Final = _create_team(scenario) + _bulk_member_add(gateway, team, [original_member]) + original_key: Final = _generate_key(scenario, team_id=team) + warm: Final = _chat(gateway, model, original_key) + assert warm.status_code == 200, warm.text + team_cache_key: Final = f"team_id:{team}" + eventually(lambda: cache.exists(team_cache_key), lambda present: present == 1, seconds=10) + + response: Final = _delete_teams(gateway, [team]) + assert response.status_code == 200, response.text + assert _team_rows(team) == [] + assert cache.exists(team_cache_key) == 0 + + fresh_alias: Final = f"integration-recreated-{uuid.uuid4().hex}" + recreated: Final = gateway.request( + "POST", + "/team/new", + { + "team_id": team, + "team_alias": fresh_alias, + "members_with_roles": [{"role": "user", "user_id": replacement_member}], + }, + ) + assert recreated.status_code == 200, recreated.text + assert recreated.json()["team_id"] == team + + info: Final = _team_info(gateway, team) + assert info.status_code == 200, info.text + team_info: Final = object_value(info.json()["team_info"]) + assert team_info["team_alias"] == fresh_alias + assert _roster_user_ids(team_info["members_with_roles"]) == [PROXY_ADMIN, replacement_member] + assert _membership_user_ids(team) == [PROXY_ADMIN, replacement_member] + assert _user_teams(replacement_member) == [team] + assert _user_teams(original_member) == [] + + fresh_key: Final = scenario.key(team_id=team) + served: Final = _chat(gateway, model, fresh_key) + assert served.status_code == 200, served.text + cached: Final = eventually(lambda: cache.get(team_cache_key), lambda value: value is not None, seconds=10) + assert isinstance(cached, bytes), cached + assert json.loads(cached)["team_alias"] == fresh_alias, cached + revoked: Final = _chat(gateway, model, original_key) + assert revoked.status_code == 401, f"{revoked.status_code} {revoked.text}" + + +def test_member_add_and_delete_released_together_leave_no_team_reference(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + newcomer: Final = scenario.user() + team: Final = _create_team(scenario) + with ThreadPoolExecutor(max_workers=2) as pool: + with psycopg.connect(os.environ["DATABASE_URL"]) as holder: + holder.execute(TAKE_TEAM_LOCK_SQL, (team,)) + pending_delete: Final = _post_in_background(pool, gateway, "/team/delete", {"team_ids": [team]}) + pending_add: Final = _post_in_background( + pool, + gateway, + "/team/member_add", + {"team_id": team, "member": {"role": "user", "user_id": newcomer}}, + ) + eventually( + lambda: _waiters_on_lock_held_by(holder.info.backend_pid), + lambda waiting: waiting == 2, + seconds=20, + ) + assert not pending_delete.done() and not pending_add.done() + # leaving the holder block commits its transaction, which releases the advisory lock + deleted: Final = pending_delete.result(timeout=60) + added: Final = pending_add.result(timeout=60) + assert deleted.status_code == 200, deleted.text + assert deleted.json() == {"deleted_teams": [team]} + assert added.status_code in (200, 404), f"{added.status_code} {added.text}" + assert _team_rows(team) == [] + assert _membership_user_ids(team) == [] + assert _user_teams(newcomer) == [] + assert _users_referencing(team) == [] + assert [row["team_id"] for row in _tombstones(team)] == [team] diff --git a/tests/integration/management/test_team_delete_member_cache_eviction.py b/tests/integration/management/test_team_delete_member_cache_eviction.py new file mode 100644 index 00000000000..0b1daa1f527 --- /dev/null +++ b/tests/integration/management/test_team_delete_member_cache_eviction.py @@ -0,0 +1,446 @@ +""" +`/team/delete` cache eviction across both proxies: member user objects, the team object and the +team's keys must stop being served by every worker once the team rows are gone. + +Auth caches the user object under the Redis key ``, the team under `team_id:` +and the key under its sha256; `enable_redis_auth_cache` is on, so Redis is the observable and +the pubsub channel carries the in-memory eviction to the peer proxy. +""" + +import asyncio +import json +import os +import uuid +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from hashlib import sha256 +from typing import Final + +import anthropic +import httpx +import openai +import pytest +from pydantic import JsonValue +from redis import Redis + +from tests.integration._support.client import ( + JSON_OBJECT, + Gateway, + Scenario, + delete_key_if_present, + eventually, + string_value, +) +from tests.integration._support.database import read_rows, write_rows +from tests.integration._support.wire import Reply, Request, wire_server + +_USAGE: Final = {"prompt_tokens": 11, "completion_tokens": 4, "total_tokens": 15} +_CACHE_KEY_HEADER: Final = "x-litellm-cache-key" + + +def _redis() -> Redis: + return Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"])) + + +def _cached_user(cache: Redis, user_id: str) -> dict[str, JsonValue] | None: + raw: Final = cache.get(user_id) + if raw is None: + return None + assert isinstance(raw, bytes), raw + return JSON_OBJECT.validate_json(raw) + + +def _warmed_user(cache: Redis, user_id: str) -> dict[str, JsonValue]: + """The cached user once its Redis SET has landed: auth writes memory at once but sends the Redis + SET on the request's pipeline, so the entry can trail the response that warmed it.""" + cached: Final = eventually(lambda: _cached_user(cache, user_id), lambda value: value is not None, seconds=10) + assert cached is not None + return cached + + +def _delete_team_if_present(gateway: Gateway, team_id: str) -> None: + if read_rows('SELECT team_id FROM "LiteLLM_TeamTable" WHERE team_id = %s', (team_id,)): + gateway.post("/team/delete", {"team_ids": [team_id]}) + + +def _team(gateway: Gateway, scenario: Scenario) -> str: + """A team the test deletes itself; cleanup removes it only if the test failed before that delete.""" + created: Final = gateway.post("/team/new", {"team_alias": f"integration-{uuid.uuid4().hex}"}) + team_id: Final = string_value(created["team_id"]) + scenario.cleanups.callback(_delete_team_if_present, gateway, team_id) + return team_id + + +def _team_key(gateway: Gateway, scenario: Scenario, team_id: str, model: str) -> str: + """A key `/team/delete` removes; cleanup deletes it only if the team delete never ran.""" + created: Final = gateway.post("/key/generate", {"team_id": team_id, "models": [model]}) + token: Final = string_value(created["key"]) + scenario.cleanups.callback(delete_key_if_present, gateway, token) + return token + + +def _delete_team(gateway: Gateway, team_id: str) -> None: + deleted: Final = gateway.post("/team/delete", {"team_ids": [team_id]}) + assert deleted == {"deleted_teams": [team_id]}, deleted + assert read_rows('SELECT team_id FROM "LiteLLM_TeamTable" WHERE team_id = %s', (team_id,)) == [] + + +def _chat_body(model: str, text: str, stream: bool = False) -> dict[str, JsonValue]: + body: dict[str, JsonValue] = {"model": model, "messages": [{"role": "user", "content": text}]} + if stream: + body["stream"] = True + return body + + +def _chat(proxy: Gateway, model: str, key: str, text: str) -> httpx.Response: + return proxy.request("POST", "/v1/chat/completions", _chat_body(model, text), key=key) + + +def _team_info(proxy: Gateway, team_id: str) -> httpx.Response: + return proxy.request("GET", "/team/info", params={"team_id": team_id}) + + +@pytest.mark.parametrize("roster_case", ("exact", "lower"), ids=("exact-case", "different-case")) +def test_team_delete_evicts_legacy_email_only_member_from_redis(gateway: Gateway, roster_case: str) -> None: + """A roster entry carrying only an email (pre-backfill legacy shape) still names a cached user; the + delete has to resolve it, in whatever case the roster stored it, and drop that user's cache entry.""" + with gateway.scenario() as scenario, _redis() as cache: + model: Final = scenario.model() + email: Final = f"Legacy-{uuid.uuid4().hex[:12]}@Example.com" + user: Final = scenario.user(user_email=email) + key: Final = scenario.key(user_id=user, models=[model]) + team: Final = _team(gateway, scenario) + roster_email: Final = email if roster_case == "exact" else email.lower() + assert (roster_email == email) is (roster_case == "exact"), (email, roster_email) + write_rows( + 'UPDATE "LiteLLM_TeamTable" SET members_with_roles = %s::jsonb WHERE team_id = %s', + (json.dumps([{"role": "user", "user_id": None, "user_email": roster_email}]), team), + ) + write_rows('UPDATE "LiteLLM_UserTable" SET teams = array_append(teams, %s) WHERE user_id = %s', (team, user)) + warm: Final = _chat(gateway, model, key, "warm legacy member " + uuid.uuid4().hex) + assert warm.status_code == 200, warm.text + warmed: Final = _warmed_user(cache, user) + assert warmed["teams"] == [team], warmed + + _delete_team(gateway, team) + + eventually(lambda: _cached_user(cache, user), lambda cached: cached is None, seconds=10) + rows: Final = read_rows('SELECT teams FROM "LiteLLM_UserTable" WHERE user_id = %s', (user,)) + assert rows == [{"teams": []}], rows + + +def test_team_delete_evicts_member_cached_on_peer_and_peer_rehydrates_without_the_team( + gateway: Gateway, peer: Gateway +) -> None: + """The peer's in-memory copy of the member is evicted over pubsub: its next request misses locally + and re-caches the user from the db, whose `teams` no longer holds the deleted team.""" + with gateway.scenario() as scenario, _redis() as cache: + model: Final = scenario.model() + user: Final = scenario.user(user_role="internal_user") + team: Final = _team(gateway, scenario) + gateway.post("/team/member_add", {"team_id": team, "member": {"role": "user", "user_id": user}}) + key: Final = scenario.key(user_id=user, models=[model]) + warm: Final = _chat(peer, model, key, "warm member on peer " + uuid.uuid4().hex) + assert warm.status_code == 200, warm.text + warmed: Final = _warmed_user(cache, user) + assert warmed["teams"] == [team], warmed + + _delete_team(gateway, team) + + eventually(lambda: _cached_user(cache, user), lambda cached: cached is None, seconds=10) + + def rehydrate() -> dict[str, JsonValue] | None: + # A peer worker still holding the stale in-memory copy answers from it and never + # rewrites Redis, so each poll issues a fresh request rather than re-reading Redis alone. + response: Final = _chat(peer, model, key, "rehydrate member on peer " + uuid.uuid4().hex) + assert response.status_code == 200, response.text + return _cached_user(cache, user) + + rehydrated: Final = eventually(rehydrate, lambda cached: cached is not None, seconds=10) + assert rehydrated is not None and rehydrated["teams"] == [], rehydrated + + +def _sse(events: Sequence[object]) -> tuple[bytes, ...]: + return tuple(b"data: " + json.dumps(event).encode() + b"\n\n" for event in events) + (b"data: [DONE]\n\n",) + + +def _chat_reply(stream: bool) -> Reply: + identity: Final = "chatcmpl-" + uuid.uuid4().hex + if not stream: + return Reply( + body=json.dumps( + { + "id": identity, + "object": "chat.completion", + "created": 1, + "model": "gpt-4o-mini", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "team probe"}, "finish_reason": "stop"} + ], + "usage": _USAGE, + } + ).encode() + ) + head: Final = {"id": identity, "object": "chat.completion.chunk", "created": 1, "model": "gpt-4o-mini"} + return Reply( + content_type="text/event-stream", + chunks=_sse( + ( + {**head, "choices": [{"index": 0, "delta": {"role": "assistant", "content": "team "}}]}, + {**head, "choices": [{"index": 0, "delta": {"content": "probe"}}]}, + {**head, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}, + {**head, "choices": [], "usage": _USAGE}, + ) + ), + ) + + +def _responses_reply(stream: bool) -> Reply: + identity: Final = uuid.uuid4().hex + completed: Final = { + "id": "resp_" + identity, + "object": "response", + "created_at": 1, + "status": "completed", + "model": "gpt-4o-mini", + "output": [ + { + "type": "message", + "id": "msg_" + identity, + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "team probe", "annotations": []}], + } + ], + "parallel_tool_calls": False, + "tool_choice": "auto", + "tools": [], + "usage": {"input_tokens": 11, "output_tokens": 4, "total_tokens": 15}, + } + if not stream: + return Reply(body=json.dumps(completed).encode()) + events: Final = ( + {"type": "response.created", "response": {**completed, "status": "in_progress", "output": [], "usage": None}}, + { + "type": "response.output_text.delta", + "item_id": "msg_" + identity, + "output_index": 0, + "content_index": 0, + "delta": "team probe", + }, + {"type": "response.completed", "response": completed}, + ) + return Reply( + content_type="text/event-stream", + chunks=tuple(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode() for event in events), + ) + + +def _upstream(request: Request) -> Reply: + stream: Final = json.loads(request.body).get("stream") is True + if request.target.endswith("/responses"): + return _responses_reply(stream) + return _chat_reply(stream) + + +def _v1(proxy: Gateway) -> str: + return str(proxy.client.base_url).rstrip("/") + "/v1" + + +def _sdk_status(error: openai.APIStatusError | anthropic.APIStatusError) -> int | str: + if isinstance(error, (openai.AuthenticationError, anthropic.AuthenticationError)): + return error.status_code + return f"{type(error).__name__}:{error.status_code}" + + +def _httpx_chat(proxy: Gateway, model: str, key: str, stream: bool, text: str) -> int | str: + return proxy.request("POST", "/v1/chat/completions", _chat_body(model, text, stream), key=key).status_code + + +def _httpx_messages(proxy: Gateway, model: str, key: str, stream: bool, text: str) -> int | str: + body: Final = {"model": model, "max_tokens": 64, "messages": [{"role": "user", "content": text}], "stream": stream} + return proxy.request("POST", "/v1/messages", body, key=key).status_code + + +def _httpx_responses(proxy: Gateway, model: str, key: str, stream: bool, text: str) -> int | str: + return proxy.request( + "POST", "/v1/responses", {"model": model, "input": text, "stream": stream}, key=key + ).status_code + + +def _openai_sync(proxy: Gateway, model: str, key: str, stream: bool, text: str) -> int | str: + with openai.OpenAI( + api_key=key, base_url=_v1(proxy), max_retries=0, http_client=httpx.Client(timeout=15, trust_env=False) + ) as client: + try: + if stream: + for _ in client.chat.completions.create( + model=model, messages=[{"role": "user", "content": text}], stream=True + ): + pass + else: + client.chat.completions.create(model=model, messages=[{"role": "user", "content": text}]) + except openai.APIStatusError as error: + return _sdk_status(error) + return 200 + + +def _openai_async(proxy: Gateway, model: str, key: str, stream: bool, text: str) -> int | str: + async def call() -> int | str: + async with openai.AsyncOpenAI( + api_key=key, base_url=_v1(proxy), max_retries=0, http_client=httpx.AsyncClient(timeout=15, trust_env=False) + ) as client: + try: + if stream: + async for _ in await client.chat.completions.create( + model=model, messages=[{"role": "user", "content": text}], stream=True + ): + pass + else: + await client.chat.completions.create(model=model, messages=[{"role": "user", "content": text}]) + except openai.APIStatusError as error: + return _sdk_status(error) + return 200 + + return asyncio.run(call()) + + +def _anthropic_sync(proxy: Gateway, model: str, key: str, stream: bool, text: str) -> int | str: + with anthropic.Anthropic( + api_key=key, + base_url=str(proxy.client.base_url), + max_retries=0, + http_client=httpx.Client(timeout=15, trust_env=False), + ) as client: + try: + if stream: + for _ in client.messages.create( + model=model, max_tokens=64, messages=[{"role": "user", "content": text}], stream=True + ): + pass + else: + client.messages.create(model=model, max_tokens=64, messages=[{"role": "user", "content": text}]) + except anthropic.APIStatusError as error: + return _sdk_status(error) + return 200 + + +@dataclass(frozen=True, slots=True) +class _Client: + name: str + call: Callable[[Gateway, str, str, bool, str], int | str] + stream: bool + + +_CLIENTS: Final = ( + _Client("httpx-chat", _httpx_chat, False), + _Client("httpx-chat-stream", _httpx_chat, True), + _Client("httpx-messages", _httpx_messages, False), + _Client("httpx-messages-stream", _httpx_messages, True), + _Client("httpx-responses", _httpx_responses, False), + _Client("httpx-responses-stream", _httpx_responses, True), + _Client("openai-sync", _openai_sync, False), + _Client("openai-sync-stream", _openai_sync, True), + _Client("openai-async", _openai_async, False), + _Client("openai-async-stream", _openai_async, True), + _Client("anthropic-sync", _anthropic_sync, False), + _Client("anthropic-sync-stream", _anthropic_sync, True), +) + + +def _observe(proxies: Mapping[str, Gateway], model: str, key: str) -> dict[str, int | str]: + """One cell per proxy and client; unique text per cell keeps the response cache out of the picture.""" + return { + f"{proxy_name}/{client.name}": client.call( + proxy, model, key, client.stream, f"{client.name} {uuid.uuid4().hex}" + ) + for proxy_name, proxy in proxies.items() + for client in _CLIENTS + } + + +def _off(observed: Mapping[str, int | str], expected: int) -> dict[str, int | str]: + return {cell: status for cell, status in observed.items() if status != expected} + + +def test_team_delete_refuses_the_team_key_for_every_client_on_both_proxies(gateway: Gateway, peer: Gateway) -> None: + """Every surface a deleted team's key can reach, on the primary and on the peer, answers 401 + once the team is gone; every cell is checked and every failing cell is reported at once.""" + proxies: Final = {"primary": gateway, "peer": peer} + with wire_server(_upstream) as upstream, gateway.scenario() as scenario: + model: Final = scenario.model(api_base=upstream.url + "/v1") + team: Final = _team(gateway, scenario) + key: Final = _team_key(gateway, scenario, team, model) + before: Final = _observe(proxies, model, key) + assert _off(before, 200) == {}, _off(before, 200) + + _delete_team(gateway, team) + + eventually( + lambda: _httpx_chat(peer, model, key, False, "deleted team key on peer"), + lambda status: status == 401, + seconds=10, + ) + after: Final = _observe(proxies, model, key) + assert _off(after, 401) == {}, _off(after, 401) + + +def test_team_delete_rejects_the_deleted_key_before_the_response_cache(gateway: Gateway) -> None: + """A request the response cache already answers for this key is refused at auth after the delete: + 401, and the upstream never sees it, so the cache-hit path cannot outlive the key.""" + with wire_server(_upstream) as upstream, gateway.scenario() as scenario: + model: Final = scenario.model(api_base=upstream.url + "/v1") + team: Final = _team(gateway, scenario) + key: Final = _team_key(gateway, scenario, team, model) + marker: Final = "cache twin " + uuid.uuid4().hex + body: Final = _chat_body(model, marker) + first: Final = gateway.request("POST", "/v1/chat/completions", body, key=key) + assert first.status_code == 200, first.text + assert first.headers.get(_CACHE_KEY_HEADER) is None, dict(first.headers) + second: Final = gateway.request("POST", "/v1/chat/completions", body, key=key) + assert second.status_code == 200, second.text + assert second.headers.get(_CACHE_KEY_HEADER), dict(second.headers) + assert second.json()["id"] == first.json()["id"], (first.text, second.text) + received: Final = upstream.drain() + assert len(received) == 1 and marker.encode() in received[0].body, received + + _delete_team(gateway, team) + + third: Final = gateway.request("POST", "/v1/chat/completions", body, key=key) + assert third.status_code == 401, third.text + assert "token_not_found_in_db" in third.text, third.text + assert upstream.drain() == (), "upstream saw a request for the deleted key" + + +def test_team_delete_evicts_team_object_and_key_on_both_proxies(gateway: Gateway, peer: Gateway) -> None: + """Team object and key warm on both proxies before the delete: `/team/info` is 404 and the key is + 401 on both afterwards, and neither the team nor the key entry is left in Redis.""" + with gateway.scenario() as scenario, _redis() as cache: + model: Final = scenario.model() + team: Final = _team(gateway, scenario) + key: Final = _team_key(gateway, scenario, team, model) + hashed: Final = sha256(key.encode()).hexdigest() + for proxy in (gateway, peer): + info: httpx.Response = _team_info(proxy, team) + assert info.status_code == 200 and info.json()["team_id"] == team, info.text + warm: httpx.Response = _chat(proxy, model, key, "warm team key " + uuid.uuid4().hex) + assert warm.status_code == 200, warm.text + # Both SETs ride the warming request's Redis pipeline and can land after its response. + eventually(lambda: cache.exists(f"team_id:{team}"), lambda present: present == 1, seconds=10) + eventually(lambda: cache.exists(hashed), lambda present: present == 1, seconds=10) + + _delete_team(gateway, team) + + eventually(lambda: _team_info(peer, team).status_code, lambda status: status == 404, seconds=10) + eventually( + lambda: _chat(peer, model, key, "deleted team key on peer").status_code, + lambda status: status == 401, + seconds=10, + ) + for proxy in (gateway, peer): + gone: httpx.Response = _team_info(proxy, team) + assert gone.status_code == 404 and "Team not found" in gone.text, gone.text + refused: httpx.Response = _chat(proxy, model, key, "deleted team key " + uuid.uuid4().hex) + assert refused.status_code == 401 and "token_not_found_in_db" in refused.text, refused.text + assert cache.exists(f"team_id:{team}") == 0, cache.keys(f"*{team}*") + assert cache.exists(hashed) == 0, cache.keys(f"*{hashed}*") diff --git a/tests/integration/management/test_team_delete_prometheus.py b/tests/integration/management/test_team_delete_prometheus.py new file mode 100644 index 00000000000..c5e383131f2 --- /dev/null +++ b/tests/integration/management/test_team_delete_prometheus.py @@ -0,0 +1,122 @@ +"""H7: the Prometheus team members gauge follows ``/team/member_add`` and ``/team/delete``. + +An owned single-worker proxy registers the ``prometheus`` callback, so ``GET /metrics/`` serves the +in-process registry (one worker, so no ``PROMETHEUS_MULTIPROC_DIR``). A team with an alias takes three +users in one bulk ``/team/member_add``; the ``litellm_team_members_metric`` series carrying that team's +id then reads 3.0. ``/team/delete`` re-emits the gauge with an empty roster instead of dropping the +series, so the same series afterwards reads 0.0. + +``disable_auto_add_proxy_admin_to_teams`` is on for the owned proxy: a master-key ``/team/new`` +otherwise seeds the roster with ``default_user_id`` and the gauge would read 4.0 after three adds. +""" + +from __future__ import annotations + +import os +import uuid +from pathlib import Path +from typing import Final + +import pytest +import yaml + +from tests.integration._support.client import Gateway, Scenario, eventually, object_value, string_value +from tests.integration._support.database import read_rows +from tests.integration._support.process import owned_proxy_process + +METRIC: Final = "litellm_team_members_metric" +METRICS_ROUTE: Final = "/metrics/" +MEMBERS: Final = 3 +TEAM_SQL: Final = 'SELECT team_id, members_with_roles FROM "LiteLLM_TeamTable" WHERE team_id = %s' + + +def _prometheus_config(tmp_path: Path) -> Path: + config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + config["litellm_settings"]["callbacks"] = ["prometheus"] + config["general_settings"]["disable_auto_add_proxy_admin_to_teams"] = True + path: Final = tmp_path / "prometheus.yaml" + path.write_text(yaml.safe_dump(config)) + return path + + +def _labels(text: str) -> dict[str, str]: + """``team="a",team_alias="b"`` to ``{"team": "a", "team_alias": "b"}``; ids and aliases carry no commas or quotes.""" + return {name: value.strip('"') for name, _, value in (pair.partition("=") for pair in text.split(","))} + + +def _team_members_series(scrape: str, team_id: str) -> tuple[dict[str, str], float] | None: + """The one ``litellm_team_members_metric`` sample whose ``team`` label is ``team_id``, as (labels, value).""" + samples: Final = tuple( + (labels, float(value)) + for line in scrape.splitlines() + if line.startswith(METRIC + "{") + for label_text, _, value in (line[len(METRIC) + 1 :].partition("} "),) + for labels in (_labels(label_text),) + if labels.get("team") == team_id + ) + assert len(samples) <= 1, f"{METRIC} exported more than one series for team {team_id}: {samples}" + return samples[0] if samples else None + + +def _scrape(candidate: Gateway) -> str: + response: Final = candidate.request("GET", METRICS_ROUTE) + assert response.status_code == 200, f"GET {METRICS_ROUTE}: {response.status_code} {response.text}" + return response.text + + +def _user(candidate: Gateway, scenario: Scenario) -> str: + """An internal user created through ``candidate``; its removal is registered on the shared rig.""" + user_id: Final = f"integration-h7-{uuid.uuid4().hex}" + candidate.post("/user/new", {"user_id": user_id, "auto_create_key": False, "user_role": "internal_user"}) + scenario.cleanups.callback(scenario.delete_user, user_id) + return user_id + + +def _delete_team_if_present(candidate: Gateway, team_id: str) -> None: + if read_rows(TEAM_SQL, (team_id,)): + candidate.post("/team/delete", {"team_ids": [team_id]}) + assert read_rows(TEAM_SQL, (team_id,)) == [] + + +@pytest.mark.timeout(240) # owned proxy boot (prisma db push + readiness) takes 20-40 s +def test_team_members_gauge_reads_roster_size_then_zero_after_delete(gateway: Gateway, tmp_path: Path) -> None: + with ( + gateway.scenario() as scenario, + owned_proxy_process( + gateway, + tmp_path, + {"DATABASE_URL": os.environ["DATABASE_URL"]}, + config=_prometheus_config(tmp_path), + remove_environment=("DATABASE_URL_READ_REPLICA",), + ) as owned, + ): + candidate: Final = owned.gateway + alias: Final = f"integration-h7-{uuid.uuid4().hex}" + team_id: Final = string_value(candidate.post("/team/new", {"team_alias": alias})["team_id"]) + scenario.cleanups.callback(_delete_team_if_present, gateway, team_id) + users: Final = tuple(_user(candidate, scenario) for _ in range(MEMBERS)) + candidate.post( + "/team/member_add", + {"team_id": team_id, "member": [{"role": "user", "user_id": user_id} for user_id in users]}, + ) + rows: Final = read_rows(TEAM_SQL, (team_id,)) + assert len(rows) == 1, rows + roster: Final = rows[0]["members_with_roles"] + assert isinstance(roster, list), roster + assert sorted(string_value(object_value(member)["user_id"]) for member in roster) == sorted(users), roster + + before: Final = eventually( + lambda: _team_members_series(_scrape(candidate), team_id), + lambda sample: sample is not None, + seconds=30, + ) + assert before == ({"team": team_id, "team_alias": alias}, 3.0), before + + assert candidate.post("/team/delete", {"team_ids": [team_id]}) == {"deleted_teams": [team_id]} + assert read_rows(TEAM_SQL, (team_id,)) == [] + after: Final = eventually( + lambda: _team_members_series(_scrape(candidate), team_id), + lambda sample: sample is not None and sample[1] == 0.0, + seconds=30, + ) + assert after == ({"team": team_id, "team_alias": alias}, 0.0), after diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py index a53894fcd1b..f2ce01f899e 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py @@ -8869,10 +8869,6 @@ async def test_delete_team_persists_deleted_teams( "litellm.proxy.proxy_server.litellm_proxy_admin_name", "admin", ) - monkeypatch.setattr( - "litellm.proxy.management_endpoints.team_endpoints._team_member_delete", - AsyncMock(return_value=(team1, (), ())), - ) data = DeleteTeamRequest(team_ids=["team-1"]) @@ -9015,6 +9011,113 @@ async def test_delete_team_sweeps_references_outside_members_with_roles( assert cache_state_when_rows_deleted["doomed_still_cached"] is True +def test_delete_team_request_collapses_repeated_ids_in_order(): + """`[T, T, U]` deletes T once and U once: one tombstone, one audit row and one eviction per team.""" + from litellm.proxy._types import DeleteTeamRequest + + assert DeleteTeamRequest(team_ids=["team-a", "team-b", "team-a", "team-b", "team-c"]).team_ids == [ + "team-a", + "team-b", + "team-c", + ] + + +@pytest.mark.asyncio +async def test_delete_team_evicts_member_caches_with_one_transaction( + monkeypatch, + disable_audit_logging_for_mocked_team, +): + """ + Regression pin for LIT-8533: `delete_team` used to fan out one + `_team_member_delete` per roster entry via `asyncio.gather`, and each opened + its own `prisma_client.tx()` and queued on the team's advisory lock, so a + team larger than the Prisma pool exhausted it and the late transactions died + on P2028. Every member-side db effect is already covered by the key delete + and the locked sweep, so the only work left is evicting each member's cache + entries, which needs no transaction at all. + """ + from litellm.proxy._types import DeleteTeamRequest + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + + member_user_ids = tuple(f"member-{i}" for i in range(3)) + team = LiteLLM_TeamTable( + team_id="team-doomed", + team_alias="doomed-team", + members_with_roles=[Member(user_id=user_id, role="user") for user_id in member_user_ids] + + [ + Member(user_id=None, user_email="invitee@example.com", role="user"), + Member(user_id=None, user_email="Second.Invitee@Example.com", role="user"), + ], + metadata={}, + model_max_budget={}, + model_spend={}, + ) + + mock_prisma_client = AsyncMock() + mock_prisma_client.db.litellm_teamtable.find_unique = AsyncMock(return_value=team) + mock_prisma_client.delete_data = AsyncMock(return_value={"deleted_keys": 0}) + mock_prisma_client.db.litellm_deletedteamtable.create_many = AsyncMock() + mock_prisma_client.db.litellm_deletedverificationtoken.create_many = AsyncMock() + mock_prisma_client.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) + mock_prisma_client.db.execute_raw = AsyncMock() + mock_prisma_client.db.litellm_teammembership.delete_many = AsyncMock() + mock_prisma_client.db.litellm_usertable.find_many = AsyncMock( + return_value=[ + LiteLLM_UserTable(user_id="invited-user", user_email="invitee@example.com"), + LiteLLM_UserTable(user_id="second-invited-user", user_email="second.invitee@example.com"), + ] + ) + + mock_tx = AsyncMock() + mock_tx.litellm_proxymodeltable.find_many = AsyncMock(return_value=[]) + mock_tx_cm = MagicMock() + mock_tx_cm.__aenter__ = AsyncMock(return_value=mock_tx) + mock_tx_cm.__aexit__ = AsyncMock(return_value=False) + mock_prisma_client.db.tx = MagicMock(return_value=mock_tx_cm) + _wire_team_delete_tx(mock_prisma_client) + + fresh_cache = UserApiKeyCache() + for user_id in member_user_ids: + fresh_cache.set_cache(key=user_id, value=UserAPIKeyAuth(user_id=user_id)) + fresh_cache.set_cache(key="invited-user", value=UserAPIKeyAuth(user_id="invited-user")) + fresh_cache.set_cache(key="second-invited-user", value=UserAPIKeyAuth(user_id="second-invited-user")) + fresh_cache.set_cache(key="bystander-user", value=UserAPIKeyAuth(user_id="bystander-user")) + + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + monkeypatch.setattr("litellm.proxy.proxy_server.user_api_key_cache", fresh_cache) + monkeypatch.setattr("litellm.proxy.proxy_server.create_audit_log_for_update", AsyncMock()) + monkeypatch.setattr("litellm.proxy.proxy_server.litellm_proxy_admin_name", "admin") + + await delete_team( + data=DeleteTeamRequest(team_ids=["team-doomed"]), + http_request=MagicMock(), + user_api_key_dict=UserAPIKeyAuth( + user_id="admin-user", + api_key="sk-admin", + user_role=LitellmUserRoles.PROXY_ADMIN.value, + ), + litellm_changed_by="admin-user", + ) + + assert mock_prisma_client.tx.call_count == 1, ( + f"delete_team must run a single locked transaction for the whole delete, not one per member; " + f"prisma_client.tx() was entered {mock_prisma_client.tx.call_count} times for " + f"{len(member_user_ids)} members" + ) + for user_id in member_user_ids: + assert fresh_cache.get_cache(key=user_id) is None, ( + f"member {user_id}'s cached user object survived the team delete" + ) + for user_id in ("invited-user", "second-invited-user"): + assert fresh_cache.get_cache(key=user_id) is None, ( + f"the email-only roster entry resolving to {user_id} must have its cached user object evicted too" + ) + assert fresh_cache.get_cache(key="bystander-user") is not None + assert mock_prisma_client.db.litellm_usertable.find_many.await_count == 1, ( + "email-only roster entries must resolve in one lookup, not one query per email" + ) + + @pytest.mark.asyncio async def test_delete_team_evicts_the_auth_cache_of_the_keys_it_deletes( monkeypatch, @@ -14146,12 +14249,6 @@ async def test_delete_team_emits_only_the_deleted_audit_event(monkeypatch): monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) monkeypatch.setattr("litellm.proxy.proxy_server.litellm_proxy_admin_name", "admin") - removals = [(team, members, members[1:]), (team, members[1:], ())] - monkeypatch.setattr( - "litellm.proxy.management_endpoints.team_endpoints._team_member_delete", - AsyncMock(side_effect=lambda **_kwargs: removals.pop(0)), - ) - await delete_team( data=DeleteTeamRequest(team_ids=["team-gone"]), http_request=MagicMock(), diff --git a/tests/unit/repositories/test_repositories.py b/tests/unit/repositories/test_repositories.py index e185d95ffb8..bd0f194b326 100644 --- a/tests/unit/repositories/test_repositories.py +++ b/tests/unit/repositories/test_repositories.py @@ -18,6 +18,7 @@ from litellm.models.credentials import CredentialItem from litellm.models.team import LiteLLM_TeamTable from litellm.repositories.base_repository import BaseRepository from litellm.repositories.budget_repository import BudgetRepository +from litellm.repositories.chunked_in import IN_LIST_CHUNK_SIZE from litellm.repositories.config_repository import ConfigRepository from litellm.repositories.credentials_repository import CredentialsRepository from litellm.repositories.model_repository import ModelRepository @@ -891,6 +892,32 @@ class TestUserRepository: user = await repo.find_by_email("test@example.com") assert user is not None + @pytest.mark.asyncio + async def test_find_by_emails_is_one_case_insensitive_query(self, repo): + repo._prisma_client.db.litellm_usertable.find_many = AsyncMock(return_value=[]) + await repo.find_by_emails(["B@Example.com", "a@example.com", "B@Example.com"]) + repo._prisma_client.db.litellm_usertable.find_many.assert_awaited_once() + where = repo._prisma_client.db.litellm_usertable.find_many.await_args.kwargs["where"] + assert where["user_email"] == {"in": ["B@Example.com", "a@example.com"], "mode": "insensitive"} + + @pytest.mark.asyncio + async def test_find_by_emails_slices_the_list_into_bounded_statements(self, repo): + repo._prisma_client.db.litellm_usertable.find_many = AsyncMock(return_value=[]) + emails = [f"user{index}@example.com" for index in range(IN_LIST_CHUNK_SIZE + 1)] + await repo.find_by_emails(emails) + assert repo._prisma_client.db.litellm_usertable.find_many.await_count == 2 + sizes = [ + len(call.kwargs["where"]["user_email"]["in"]) + for call in repo._prisma_client.db.litellm_usertable.find_many.await_args_list + ] + assert sizes == [IN_LIST_CHUNK_SIZE, 1] + + @pytest.mark.asyncio + async def test_find_by_emails_skips_the_query_for_no_emails(self, repo): + repo._prisma_client.db.litellm_usertable.find_many = AsyncMock() + assert await repo.find_by_emails(()) == () + repo._prisma_client.db.litellm_usertable.find_many.assert_not_awaited() + @pytest.mark.asyncio async def test_find_by_sso_id(self, repo): repo._prisma_client.db.litellm_usertable._records["sso-123"] = { From 268eb4d6e62d56bdf4cc08f055b61a5c8f4c4b5e Mon Sep 17 00:00:00 2001 From: yujonglee Date: Wed, 30 Sep 2026 14:12:29 -0700 Subject: [PATCH 03/28] feat(tracing): add OTLP trace ingestion and reads (#43915) --- .github/workflows/test-unit.yml | 4 +- litellm-rust/Cargo.lock | 67 +- litellm-rust/crates/python-bridge/src/lib.rs | 7 +- .../crates/python-bridge/src/routes/traces.rs | 166 ++-- litellm-rust/crates/traces/AGENTS.md | 1 + litellm-rust/crates/traces/Cargo.toml | 4 + litellm-rust/crates/traces/config/reader.xml | 2 +- .../traces/migrations/0001_otel_traces.sql | 2 +- .../traces/migrations/0002_agent_traces.sql | 6 +- .../migrations/0003_agent_traces_mv.sql | 7 +- .../migrations/0006_agent_traces_ttl.sql | 2 +- litellm-rust/crates/traces/src/error.rs | 14 + litellm-rust/crates/traces/src/insert.rs | 138 ++- litellm-rust/crates/traces/src/lib.rs | 37 +- litellm-rust/crates/traces/src/otlp.rs | 221 +++++ litellm-rust/crates/traces/src/schema.rs | 23 +- litellm-rust/crates/traces/tests/admin_sql.rs | 14 + .../crates/traces/tests/migrations.rs | 112 ++- litellm-rust/crates/traces/tests/otlp.rs | 47 + litellm/constants.py | 13 + .../clickhouse/clickhouse_batch_logger.py | 100 ++ litellm/integrations/clickhouse/schema.py | 11 + litellm/integrations/custom_batch_logger.py | 2 +- litellm/proxy/_types.py | 4 + .../proxy/common_utils/http_parsing_utils.py | 22 +- litellm/proxy/proxy_server.py | 28 + litellm/proxy/tracing_endpoints.py | 141 +++ litellm/rust_bridge/_native.pyi | 28 +- litellm/rust_bridge/traces.py | 107 +- litellm/tracing/AGENTS.md | 6 + litellm/tracing/__init__.py | 16 + litellm/tracing/decode.py | 284 ++++++ litellm/tracing/receiver.py | 120 +++ litellm/tracing/store.py | 286 ++++++ litellm/tracing/types.py | 120 +++ .../test_clickhouse_batch_logger.py | 67 ++ .../common_utils/test_http_parsing_utils.py | 29 +- .../proxy/test_tracing_endpoints.py | 180 ++++ .../fixtures/langsmith_deep_agent_export.json | 924 ++++++++++++++++++ tests/test_litellm/tracing/test_decode.py | 313 ++++++ tests/test_litellm/tracing/test_receiver.py | 117 +++ tests/test_litellm/tracing/test_store.py | 311 ++++++ tests/test_litellm_rust/test_traces.py | 51 +- ui/litellm-dashboard/package-lock.json | 82 +- ui/litellm-dashboard/package.json | 2 +- .../src/components/networking.tsx | 15 +- .../TraceView/AgentTracesSection.tsx | 19 +- .../view_logs/TraceView/AgentTracesTable.tsx | 6 +- .../view_logs/TraceView/DetailContent.tsx | 11 +- .../view_logs/TraceView/DetailPane.test.tsx | 2 +- .../view_logs/TraceView/DetailPane.tsx | 17 +- .../view_logs/TraceView/TraceDrawer.tsx | 19 +- .../view_logs/TraceView/traceTypes.ts | 1 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 170 ++++ uv.lock | 12 +- 55 files changed, 4240 insertions(+), 270 deletions(-) create mode 100644 litellm-rust/crates/traces/src/otlp.rs create mode 100644 litellm-rust/crates/traces/tests/otlp.rs create mode 100644 litellm/integrations/clickhouse/clickhouse_batch_logger.py create mode 100644 litellm/integrations/clickhouse/schema.py create mode 100644 litellm/proxy/tracing_endpoints.py create mode 100644 litellm/tracing/AGENTS.md create mode 100644 litellm/tracing/__init__.py create mode 100644 litellm/tracing/decode.py create mode 100644 litellm/tracing/receiver.py create mode 100644 litellm/tracing/store.py create mode 100644 litellm/tracing/types.py create mode 100644 tests/test_litellm/integrations/clickhouse/test_clickhouse_batch_logger.py create mode 100644 tests/test_litellm/proxy/test_tracing_endpoints.py create mode 100644 tests/test_litellm/tracing/fixtures/langsmith_deep_agent_export.json create mode 100644 tests/test_litellm/tracing/test_decode.py create mode 100644 tests/test_litellm/tracing/test_receiver.py create mode 100644 tests/test_litellm/tracing/test_store.py diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index f55e186e3b2..d3604a81e8f 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -79,7 +79,9 @@ jobs: - shard: integrations artifact-name: integrations - test-path: "" + test-path: >- + tests/test_litellm/integrations + tests/test_litellm/tracing unit-flag: integrations workers: 2 reruns: 3 diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 5fdf0266e8c..474f45ac0cd 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -1274,6 +1274,18 @@ version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e8ccc4ea9f6acc32d102c0f6d471d11d913ad15f20c04de743374861fa1d414" +[[package]] +name = "const-hex" +version = "1.19.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e59eef12462b0f9b0a3620219be5d639afd79fe39dff0a42c3997061f9298b4" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "proptest", + "serde_core", +] + [[package]] name = "const-oid" version = "0.9.6" @@ -2372,9 +2384,9 @@ dependencies = [ "http-body-util", "hyper 1.10.1", "lazy_static", - "opentelemetry", + "opentelemetry 0.32.0", "opentelemetry-semantic-conventions", - "opentelemetry_sdk", + "opentelemetry_sdk 0.32.1", "percent-encoding", "pin-project", "prost", @@ -4356,7 +4368,11 @@ dependencies = [ name = "litellm-traces" version = "0.1.0" dependencies = [ + "base64 0.22.1", + "flate2", "litellm-http", + "opentelemetry-proto", + "prost", "rstest", "serde", "serde_json", @@ -4776,6 +4792,33 @@ dependencies = [ "tracing", ] +[[package]] +name = "opentelemetry" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cdb0b1b267eb9db3331b434ed9ddab10d50e280a9adf9d13e5233e2002b61b5" +dependencies = [ + "futures-core", + "futures-sink", + "js-sys", + "pin-project-lite", + "thiserror 2.0.19", +] + +[[package]] +name = "opentelemetry-proto" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25da1ac11a0aeccf38d7f77ee0348715adaf8340f65ad46c94a02c6b20e2f65d" +dependencies = [ + "base64 0.22.1", + "const-hex", + "opentelemetry 0.33.0", + "opentelemetry_sdk 0.33.0", + "prost", + "serde", +] + [[package]] name = "opentelemetry-semantic-conventions" version = "0.32.1" @@ -4791,7 +4834,23 @@ dependencies = [ "futures-channel", "futures-executor", "futures-util", - "opentelemetry", + "opentelemetry 0.32.0", + "percent-encoding", + "portable-atomic", + "rand 0.9.5", + "thiserror 2.0.19", +] + +[[package]] +name = "opentelemetry_sdk" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb39533d9d1c912123efd7d41d7e0c29d16917b60ce15b4c8d87cb1af7f67520" +dependencies = [ + "futures-channel", + "futures-executor", + "futures-util", + "opentelemetry 0.33.0", "percent-encoding", "portable-atomic", "rand 0.9.5", @@ -7524,7 +7583,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adbc64cba7137545b8044cb1fe9814f7aacf3c6b5f9b45be8bb5db538befdb26" dependencies = [ "js-sys", - "opentelemetry", + "opentelemetry 0.32.0", "tracing", "tracing-core", "tracing-subscriber", diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index f65cb350cfe..0d4df996552 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -44,7 +44,7 @@ mod _native { #[pymodule_export] use crate::routes::token_counter::TokenCounter; #[pymodule_export] - use crate::routes::traces::{trace_encode_rows, trace_ensure_schema, trace_query}; + use crate::routes::traces::{NativeTraceStorage, trace_decode_otlp}; #[cfg(feature = "huggingface")] #[pymodule_export] use crate::tokenizer::HuggingFaceEncoding; @@ -109,9 +109,8 @@ mod tests { "aresponses", "ResponsesWebSocketConnection", "NativeDiagnosticProcessor", - "trace_encode_rows", - "trace_ensure_schema", - "trace_query", + "NativeTraceStorage", + "trace_decode_otlp", "TokenCounter", "Tokenizer", "gil_stats", diff --git a/litellm-rust/crates/python-bridge/src/routes/traces.rs b/litellm-rust/crates/python-bridge/src/routes/traces.rs index fdd2a7eb93d..6a18273ed4c 100644 --- a/litellm-rust/crates/python-bridge/src/routes/traces.rs +++ b/litellm-rust/crates/python-bridge/src/routes/traces.rs @@ -1,19 +1,21 @@ use std::collections::BTreeMap; use litellm_http::ClientVariant; -use litellm_traces::{Connection, Error, Parameter}; +use litellm_traces::{Connection, Error, InsertTable, Parameter}; use pyo3::{ - exceptions::{PyRuntimeError, PyValueError}, + exceptions::{PyOverflowError, PyRuntimeError, PyValueError}, prelude::*, }; fn map_error(error: Error) -> PyErr { match error { - Error::InvalidRow | Error::InvalidSchema | Error::EmptySql => { + Error::InvalidRow | Error::InvalidTable | Error::InvalidSchema | Error::EmptySql => { PyValueError::new_err(error.to_string()) } + Error::InsertTooLarge => PyOverflowError::new_err(error.to_string()), Error::InvalidUrl | Error::QueryFailed(_) + | Error::InsertFailed(_) | Error::SchemaFailed(_) | Error::ResponseTooLarge | Error::InvalidResponse @@ -21,63 +23,115 @@ fn map_error(error: Error) -> PyErr { } } -#[pyfunction] -pub fn trace_ensure_schema<'py>( - py: Python<'py>, - url: &str, +#[pyclass] +pub struct NativeTraceStorage { database: String, - user: &str, - password: &str, - trace_retention_days: u32, - spend_log_retention_days: u32, -) -> PyResult> { - let connection = Connection::writer(url, user, password).map_err(map_error)?; - let client = crate::http::host_client(py, ClientVariant::NoRedirect)?; - crate::execution::run_async( - py, - async move { - litellm_traces::ensure_schema( - &client, - &connection, - &database, - trace_retention_days, - spend_log_retention_days, - ) - .await - }, - map_error, - ) + writer: Connection, + reader: Option, +} + +#[pymethods] +impl NativeTraceStorage { + #[new] + fn new(database: String, url: &str, reader_url: Option<&str>) -> PyResult { + litellm_traces::schema_statements(&database, 1, 1).map_err(map_error)?; + Ok(Self { + writer: Connection::writer(url).map_err(map_error)?, + reader: reader_url + .map(|value| Connection::reader(value, &database)) + .transpose() + .map_err(map_error)?, + database, + }) + } + + fn ensure_schema<'py>( + &self, + py: Python<'py>, + trace_retention_days: u32, + spend_log_retention_days: u32, + ) -> PyResult> { + let client = crate::http::host_client(py, ClientVariant::NoRedirect)?; + let connection = self.writer.clone(); + let database = self.database.clone(); + crate::execution::run_async( + py, + async move { + litellm_traces::ensure_schema( + &client, + &connection, + &database, + trace_retention_days, + spend_log_retention_days, + ) + .await + }, + map_error, + ) + } + + fn insert_rows<'py>( + &self, + py: Python<'py>, + table: &str, + #[pyo3(from_py_with = litellm_host_python::from_py_argument)] rows: Vec< + BTreeMap, + >, + ) -> PyResult> { + let table = InsertTable::parse(table).map_err(map_error)?; + let client = crate::http::host_client(py, ClientVariant::NoRedirect)?; + let connection = self.writer.clone(); + let database = self.database.clone(); + crate::execution::run_async( + py, + async move { + litellm_traces::insert_rows(&client, &connection, &database, table, rows).await + }, + map_error, + ) + } + + fn query<'py>( + &self, + py: Python<'py>, + sql: String, + #[pyo3(from_py_with = litellm_host_python::from_py_argument)] parameters: BTreeMap< + String, + Parameter, + >, + ) -> PyResult> { + let connection = self.reader.clone().ok_or_else(|| { + PyRuntimeError::new_err("Trace reads require a separate ClickHouse reader URL") + })?; + let client = crate::http::host_client(py, ClientVariant::NoRedirect)?; + crate::execution::run_async( + py, + async move { litellm_traces::execute_read(&client, &connection, &sql, ¶meters).await }, + map_error, + ) + } } #[pyfunction] -pub fn trace_query<'py>( +pub fn trace_decode_otlp<'py>( py: Python<'py>, - url: &str, - database: &str, - user: &str, - password: &str, - sql: String, - #[pyo3(from_py_with = litellm_host_python::from_py_argument)] parameters: BTreeMap< - String, - Parameter, - >, + body: &[u8], + content_type: Option<&str>, + content_encoding: Option<&str>, + max_decompressed_bytes: usize, ) -> PyResult> { - let connection = Connection::configured(url, database, user, password).map_err(map_error)?; - let client = crate::http::host_client(py, ClientVariant::NoRedirect)?; - crate::execution::run_async( - py, - async move { litellm_traces::execute_read(&client, &connection, &sql, ¶meters).await }, - map_error, - ) -} - -#[pyfunction] -pub fn trace_encode_rows( - py: Python<'_>, - #[pyo3(from_py_with = litellm_host_python::from_py_argument)] rows: Vec< - BTreeMap, - >, -) -> PyResult { - py.detach(|| litellm_traces::encode_rows(rows)) - .map_err(map_error) + let spans = py + .detach(|| { + litellm_traces::decode_otlp( + body, + content_type, + content_encoding, + max_decompressed_bytes, + ) + }) + .map_err(|error| match error { + litellm_traces::DecodeError::TooLarge => PyOverflowError::new_err(error.to_string()), + _ => PyValueError::new_err(error.to_string()), + })?; + litellm_host_python::Pythonized(spans).into_pyobject(py) } diff --git a/litellm-rust/crates/traces/AGENTS.md b/litellm-rust/crates/traces/AGENTS.md index d330429b315..a5e2d4be53a 100644 --- a/litellm-rust/crates/traces/AGENTS.md +++ b/litellm-rust/crates/traces/AGENTS.md @@ -2,5 +2,6 @@ - Keep this crate independent of Python; PyO3 conversion and public Python exceptions belong in `python-bridge` - Keep the SQL migrations here as the only ClickHouse schema definition - Use typed query parameters and a dedicated SELECT-only reader with server-side limits +- Keep `config/reader.xml` grants on the database the schema is created in (CLICKHOUSE_DATABASE, default `litellm`) - Bound insert time and encoded bytes; make retry deduplication behavior explicit for supported ClickHouse versions - Test storage behavior through the crate's public API against ClickHouse diff --git a/litellm-rust/crates/traces/Cargo.toml b/litellm-rust/crates/traces/Cargo.toml index a8091cf9ec8..0aeec4c8276 100644 --- a/litellm-rust/crates/traces/Cargo.toml +++ b/litellm-rust/crates/traces/Cargo.toml @@ -6,6 +6,10 @@ license.workspace = true repository.workspace = true [dependencies] +base64.workspace = true +flate2.workspace = true +opentelemetry-proto = { version = "0.33.0", default-features = false, features = ["gen-tonic-messages", "trace", "with-serde"] } +prost = "0.14.4" time = { workspace = true, features = ["formatting"] } litellm-http.workspace = true serde.workspace = true diff --git a/litellm-rust/crates/traces/config/reader.xml b/litellm-rust/crates/traces/config/reader.xml index 73a63035e2e..3ab337a13fc 100644 --- a/litellm-rust/crates/traces/config/reader.xml +++ b/litellm-rust/crates/traces/config/reader.xml @@ -24,7 +24,7 @@ litellm_traces_reader GRANT SELECT ON litellm.otel_traces - GRANT SELECT ON litellm.agent_traces + GRANT SELECT ON litellm.agent_traces_by_key GRANT SELECT ON litellm.spend_logs diff --git a/litellm-rust/crates/traces/migrations/0001_otel_traces.sql b/litellm-rust/crates/traces/migrations/0001_otel_traces.sql index aed869e6ee0..d8e0184b5a3 100644 --- a/litellm-rust/crates/traces/migrations/0001_otel_traces.sql +++ b/litellm-rust/crates/traces/migrations/0001_otel_traces.sql @@ -44,4 +44,4 @@ CREATE TABLE IF NOT EXISTS {database}.otel_traces ENGINE = MergeTree PARTITION BY toDate(Timestamp) ORDER BY (TeamId, ServiceName, toDateTime(Timestamp), TraceId) -SETTINGS ttl_only_drop_parts = 1 +SETTINGS ttl_only_drop_parts = 1, non_replicated_deduplication_window = 1000 diff --git a/litellm-rust/crates/traces/migrations/0002_agent_traces.sql b/litellm-rust/crates/traces/migrations/0002_agent_traces.sql index d994829f9c0..0c3547872bb 100644 --- a/litellm-rust/crates/traces/migrations/0002_agent_traces.sql +++ b/litellm-rust/crates/traces/migrations/0002_agent_traces.sql @@ -1,6 +1,7 @@ -CREATE TABLE IF NOT EXISTS {database}.agent_traces +CREATE TABLE IF NOT EXISTS {database}.agent_traces_by_key ( TeamId LowCardinality(String), + ApiKeyHash String, TraceId String, StartTs SimpleAggregateFunction(min, DateTime64(9)), EndTs SimpleAggregateFunction(max, DateTime64(9)), @@ -20,4 +21,5 @@ CREATE TABLE IF NOT EXISTS {database}.agent_traces RequestIds SimpleAggregateFunction(groupArrayArray, Array(String)) ) ENGINE = AggregatingMergeTree -ORDER BY (TeamId, TraceId) +ORDER BY (TeamId, ApiKeyHash, TraceId) +SETTINGS non_replicated_deduplication_window = 1000 diff --git a/litellm-rust/crates/traces/migrations/0003_agent_traces_mv.sql b/litellm-rust/crates/traces/migrations/0003_agent_traces_mv.sql index e162d4da569..94dad81f998 100644 --- a/litellm-rust/crates/traces/migrations/0003_agent_traces_mv.sql +++ b/litellm-rust/crates/traces/migrations/0003_agent_traces_mv.sql @@ -1,6 +1,7 @@ -CREATE MATERIALIZED VIEW IF NOT EXISTS {database}.agent_traces_mv TO {database}.agent_traces AS +CREATE MATERIALIZED VIEW IF NOT EXISTS {database}.agent_traces_by_key_mv +TO {database}.agent_traces_by_key AS SELECT - TeamId, TraceId, + TeamId, ApiKeyHash, TraceId, min(Timestamp) AS StartTs, max(Timestamp + toIntervalNanosecond(Duration)) AS EndTs, any(ServiceName) AS ServiceName, @@ -18,4 +19,4 @@ SELECT groupUniqArrayIf(SpanName, ObservationType = 'agent') AS AgentNames, groupArrayIf(LiteLLMRequestId, LiteLLMRequestId != '') AS RequestIds FROM {database}.otel_traces -GROUP BY TeamId, TraceId +GROUP BY TeamId, ApiKeyHash, TraceId diff --git a/litellm-rust/crates/traces/migrations/0006_agent_traces_ttl.sql b/litellm-rust/crates/traces/migrations/0006_agent_traces_ttl.sql index d4c4a113329..8681f0622a4 100644 --- a/litellm-rust/crates/traces/migrations/0006_agent_traces_ttl.sql +++ b/litellm-rust/crates/traces/migrations/0006_agent_traces_ttl.sql @@ -1 +1 @@ -ALTER TABLE {database}.agent_traces MODIFY TTL toDateTime(StartTs) + INTERVAL {trace_retention_days} DAY +ALTER TABLE {database}.agent_traces_by_key MODIFY TTL toDateTime(StartTs) + INTERVAL {trace_retention_days} DAY diff --git a/litellm-rust/crates/traces/src/error.rs b/litellm-rust/crates/traces/src/error.rs index a06eb27397a..125edc35422 100644 --- a/litellm-rust/crates/traces/src/error.rs +++ b/litellm-rust/crates/traces/src/error.rs @@ -2,6 +2,8 @@ pub enum Error { #[error("invalid ClickHouse insert row")] InvalidRow, + #[error("invalid ClickHouse insert table")] + InvalidTable, #[error("invalid ClickHouse HTTP URL")] InvalidUrl, #[error("database must be a nonempty SQL identifier and retention must be positive")] @@ -10,6 +12,10 @@ pub enum Error { EmptySql, #[error("ClickHouse query failed with HTTP status {0}")] QueryFailed(u16), + #[error("ClickHouse insert failed with HTTP status {0}")] + InsertFailed(u16), + #[error("ClickHouse insert exceeds the encoded size limit")] + InsertTooLarge, #[error("ClickHouse schema setup failed with HTTP status {0}")] SchemaFailed(u16), #[error("ClickHouse query exceeded the response size limit")] @@ -19,3 +25,11 @@ pub enum Error { #[error("ClickHouse query transport failed")] Transport, } + +#[derive(Debug, thiserror::Error)] +pub enum DecodeError { + #[error("invalid OTLP trace payload")] + InvalidPayload, + #[error("OTLP trace payload exceeds the decompressed size limit")] + TooLarge, +} diff --git a/litellm-rust/crates/traces/src/insert.rs b/litellm-rust/crates/traces/src/insert.rs index 05b14651ce0..6f2d6acb023 100644 --- a/litellm-rust/crates/traces/src/insert.rs +++ b/litellm-rust/crates/traces/src/insert.rs @@ -1,21 +1,109 @@ -use std::collections::BTreeMap; +use std::{collections::BTreeMap, io::Write, time::Duration}; +use flate2::{Compression, write::GzEncoder}; +use litellm_http::Client; use serde_json::Value; use time::{OffsetDateTime, format_description::well_known::Rfc3339}; -use crate::Error; +use crate::{Connection, Error}; + +const MAX_INSERT_BYTES: usize = 64 * 1024 * 1024; +const INSERT_TIMEOUT: Duration = Duration::from_secs(30); + +pub enum InsertTable { + OtelTraces, + SpendLogs, +} + +impl InsertTable { + pub fn parse(value: &str) -> Result { + match value { + "otel_traces" => Ok(Self::OtelTraces), + "spend_logs" => Ok(Self::SpendLogs), + _ => Err(Error::InvalidTable), + } + } + + fn name(&self) -> &'static str { + match self { + Self::OtelTraces => "otel_traces", + Self::SpendLogs => "spend_logs", + } + } +} + +pub async fn insert_rows( + client: &Client, + connection: &Connection, + database: &str, + table: InsertTable, + rows: Vec>, +) -> Result<(), Error> { + if rows.is_empty() { + return Ok(()); + } + let encoded = encode_rows_with_limit(rows, MAX_INSERT_BYTES)?; + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder + .write_all(encoded.as_bytes()) + .map_err(|_| Error::InvalidRow)?; + let body = encoder.finish().map_err(|_| Error::InvalidRow)?; + let mut url = connection.url().clone(); + url.query_pairs_mut() + .append_pair( + "query", + &format!( + "INSERT INTO `{database}`.{} FORMAT JSONEachRow", + table.name() + ), + ) + .append_pair("async_insert", "1") + .append_pair("async_insert_deduplicate", "1") + .append_pair("wait_for_async_insert", "1") + .append_pair("date_time_input_format", "best_effort"); + let response = client + .post(url) + .timeout(INSERT_TIMEOUT) + .header("Content-Encoding", "gzip") + .body(body) + .send() + .await + .map_err(|_| Error::Transport)?; + if !response.status().is_success() { + return Err(Error::InsertFailed(response.status().as_u16())); + } + Ok(()) +} pub fn encode_rows(rows: Vec>) -> Result { - rows.into_iter() - .map(|row| { - let encoded = row - .into_iter() - .map(|(name, value)| insert_value(&name, value).map(|value| (name, value))) - .collect::, _>>()?; - serde_json::to_string(&encoded).map_err(|_| Error::InvalidRow) - }) - .collect::, _>>() - .map(|rows| rows.join("\n")) + encode_rows_with_limit(rows, usize::MAX) +} + +fn encode_rows_with_limit( + rows: Vec>, + limit: usize, +) -> Result { + let mut body = Vec::new(); + for row in rows { + let encoded = row + .into_iter() + .map(|(name, value)| insert_value(&name, value).map(|value| (name, value))) + .collect::, _>>()?; + let record = serde_json::to_vec(&encoded).map_err(|_| Error::InvalidRow)?; + let size = body + .len() + .checked_add(record.len()) + .and_then(|size| size.checked_add(usize::from(!body.is_empty()))) + .ok_or(Error::InsertTooLarge)?; + if size > limit { + return Err(Error::InsertTooLarge); + } + if !body.is_empty() { + body.push(b'\n'); + } + body.extend_from_slice(&record); + } + String::from_utf8(body).map_err(|_| Error::InvalidRow) } fn insert_value(name: &str, value: Value) -> Result { @@ -35,3 +123,29 @@ fn insert_value(name: &str, value: Value) -> Result { .map(Value::String) .map_err(|_| Error::InvalidRow) } + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use rstest::rstest; + use serde_json::json; + + use super::encode_rows_with_limit; + use crate::Error; + + #[rstest] + fn encoded_limit_counts_utf8_bytes_across_rows() { + let rows = vec![ + BTreeMap::from([("Input".to_owned(), json!("雪"))]), + BTreeMap::from([("Input".to_owned(), json!("雪"))]), + ]; + let encoded = encode_rows_with_limit(rows.clone(), usize::MAX).expect("valid rows"); + + assert!(encode_rows_with_limit(rows.clone(), encoded.len()).is_ok()); + assert!(matches!( + encode_rows_with_limit(rows, encoded.len() - 1), + Err(Error::InsertTooLarge) + )); + } +} diff --git a/litellm-rust/crates/traces/src/lib.rs b/litellm-rust/crates/traces/src/lib.rs index 1f56bb5b3a6..279afb20e9b 100644 --- a/litellm-rust/crates/traces/src/lib.rs +++ b/litellm-rust/crates/traces/src/lib.rs @@ -1,10 +1,12 @@ mod error; mod insert; +mod otlp; mod schema; mod sql; -pub use error::Error; -pub use insert::encode_rows; +pub use error::{DecodeError, Error}; +pub use insert::{InsertTable, encode_rows, insert_rows}; +pub use otlp::{DecodedSpan, decode_otlp}; pub use schema::{ensure_schema, schema_statements}; pub use sql::{Parameter, execute_read}; use url::Url; @@ -53,17 +55,32 @@ impl Connection { Ok(connection) } - pub fn writer(url: &str, user: &str, password: &str) -> Result { + pub fn writer(url: &str) -> Result { let mut connection = Self::parse(url)?; + let pairs: Vec<_> = connection + .url + .query_pairs() + .filter(|(key, _)| !matches!(key.as_ref(), "database" | "readonly" | "query")) + .map(|(key, value)| (key.into_owned(), value.into_owned())) + .collect(); + connection.url.query_pairs_mut().clear().extend_pairs(pairs); + Ok(connection) + } + + pub fn reader(url: &str, database: &str) -> Result { + let mut connection = Self::parse(url)?; + let pairs: Vec<_> = connection + .url + .query_pairs() + .filter(|(key, _)| key != "database") + .map(|(key, value)| (key.into_owned(), value.into_owned())) + .collect(); connection .url - .set_username(user) - .map_err(|_| Error::InvalidUrl)?; - connection - .url - .set_password(Some(password)) - .map_err(|_| Error::InvalidUrl)?; - connection.url.set_query(None); + .query_pairs_mut() + .clear() + .extend_pairs(pairs) + .append_pair("database", database); Ok(connection) } diff --git a/litellm-rust/crates/traces/src/otlp.rs b/litellm-rust/crates/traces/src/otlp.rs new file mode 100644 index 00000000000..f162256ef1f --- /dev/null +++ b/litellm-rust/crates/traces/src/otlp.rs @@ -0,0 +1,221 @@ +use std::{collections::BTreeMap, io::Read}; + +use base64::Engine; +use flate2::read::GzDecoder; +use opentelemetry_proto::tonic::{ + collector::trace::v1::ExportTraceServiceRequest, + common::v1::{AnyValue, KeyValue, any_value::Value as AttributeValue}, + trace::v1::{Span, span::SpanKind, status::StatusCode}, +}; +use prost::Message; +use serde::Serialize; +use serde_json::Value; + +use crate::DecodeError; + +#[derive(Serialize)] +pub struct DecodedEvent { + pub name: String, + pub attributes: BTreeMap, +} + +#[derive(Serialize)] +pub struct DecodedSpan { + pub trace_id: String, + pub span_id: String, + pub parent_span_id: String, + pub trace_state: String, + pub name: String, + pub kind: String, + pub resource_attributes: BTreeMap, + pub scope_name: String, + pub scope_version: String, + pub attributes: BTreeMap, + pub start_ns: u64, + pub end_ns: u64, + pub status_code: String, + pub status_message: String, + pub events: Vec, +} + +pub fn decode_otlp( + body: &[u8], + content_type: Option<&str>, + content_encoding: Option<&str>, + max_decompressed_bytes: usize, +) -> Result, DecodeError> { + let payload = if content_encoding == Some("gzip") || body.starts_with(&[0x1f, 0x8b]) { + let limit = u64::try_from(max_decompressed_bytes).map_err(|_| DecodeError::TooLarge)?; + let mut decoded = Vec::new(); + GzDecoder::new(body) + .take(limit + 1) + .read_to_end(&mut decoded) + .map_err(|_| DecodeError::InvalidPayload)?; + decoded + } else { + body.to_vec() + }; + if payload.len() > max_decompressed_bytes { + return Err(DecodeError::TooLarge); + } + let request = if content_type.is_some_and(|value| value.contains("json")) { + let value: Value = + serde_json::from_slice(&payload).map_err(|_| DecodeError::InvalidPayload)?; + serde_json::from_value(normalize_json_ids(value)?) + .map_err(|_| DecodeError::InvalidPayload)? + } else { + ExportTraceServiceRequest::decode(payload.as_slice()) + .map_err(|_| DecodeError::InvalidPayload)? + }; + Ok(request + .resource_spans + .into_iter() + .flat_map(|resource_spans| { + let resource_attributes = attributes( + resource_spans + .resource + .map(|resource| resource.attributes) + .unwrap_or_default(), + ); + resource_spans + .scope_spans + .into_iter() + .flat_map(move |scope_spans| { + let scope = scope_spans.scope.unwrap_or_default(); + let resource_attributes = resource_attributes.clone(); + scope_spans.spans.into_iter().map(move |span| { + decoded_span(span, &resource_attributes, &scope.name, &scope.version) + }) + }) + }) + .collect()) +} + +fn normalize_json_ids(value: Value) -> Result { + match value { + Value::Object(fields) => fields + .into_iter() + .map(|(name, value)| { + let normalized = if matches!(name.as_str(), "traceId" | "spanId" | "parentSpanId") { + let encoded = value.as_str().ok_or(DecodeError::InvalidPayload)?; + let bytes = base64::engine::general_purpose::STANDARD + .decode(encoded) + .map_err(|_| DecodeError::InvalidPayload)?; + Value::String(hex_bytes(&bytes)) + } else if name == "kind" && value.is_string() { + let kind = SpanKind::from_str_name(value.as_str().unwrap_or_default()) + .ok_or(DecodeError::InvalidPayload)?; + Value::from(kind as i32) + } else if name == "code" && value.is_string() { + let code = StatusCode::from_str_name(value.as_str().unwrap_or_default()) + .ok_or(DecodeError::InvalidPayload)?; + Value::from(code as i32) + } else { + normalize_json_ids(value)? + }; + Ok((name, normalized)) + }) + .collect::, _>>() + .map(Value::Object), + Value::Array(values) => values + .into_iter() + .map(normalize_json_ids) + .collect::, _>>() + .map(Value::Array), + value => Ok(value), + } +} + +fn hex_bytes(bytes: &[u8]) -> String { + bytes.iter().map(|byte| format!("{byte:02x}")).collect() +} + +fn decoded_span( + span: Span, + resource_attributes: &BTreeMap, + scope_name: &str, + scope_version: &str, +) -> DecodedSpan { + let status = span.status.unwrap_or_default(); + DecodedSpan { + trace_id: hex_bytes(&span.trace_id), + span_id: hex_bytes(&span.span_id), + parent_span_id: hex_bytes(&span.parent_span_id), + trace_state: span.trace_state, + name: span.name, + kind: SpanKind::try_from(span.kind) + .unwrap_or(SpanKind::Unspecified) + .as_str_name() + .to_owned(), + resource_attributes: resource_attributes.clone(), + scope_name: scope_name.to_owned(), + scope_version: scope_version.to_owned(), + attributes: attributes(span.attributes), + start_ns: span.start_time_unix_nano, + end_ns: span.end_time_unix_nano, + status_code: StatusCode::try_from(status.code) + .unwrap_or(StatusCode::Unset) + .as_str_name() + .to_owned(), + status_message: status.message, + events: span + .events + .into_iter() + .map(|event| DecodedEvent { + name: event.name, + attributes: attributes(event.attributes), + }) + .collect(), + } +} + +fn attributes(values: Vec) -> BTreeMap { + values + .into_iter() + .map(|entry| { + ( + entry.key, + entry.value.as_ref().map(attribute_text).unwrap_or_default(), + ) + }) + .collect() +} + +fn attribute_text(value: &AnyValue) -> String { + match value.value.as_ref() { + Some(AttributeValue::StringValue(value)) => value.clone(), + Some(AttributeValue::BoolValue(value)) => value.to_string(), + Some(AttributeValue::IntValue(value)) => value.to_string(), + Some(AttributeValue::DoubleValue(value)) => { + serde_json::to_string(value).unwrap_or_default() + } + Some(AttributeValue::BytesValue(value)) => String::from_utf8_lossy(value).into_owned(), + Some(AttributeValue::ArrayValue(value)) => format!( + "[{}]", + value + .values + .iter() + .map(|value| serde_json::to_string(&attribute_text(value)).unwrap_or_default()) + .collect::>() + .join(", ") + ), + Some(AttributeValue::KvlistValue(value)) => format!( + "{{{}}}", + value + .values + .iter() + .map(|entry| format!( + "{}: {}", + serde_json::to_string(&entry.key).unwrap_or_default(), + serde_json::to_string( + &entry.value.as_ref().map(attribute_text).unwrap_or_default() + ) + .unwrap_or_default() + )) + .collect::>() + .join(", ") + ), + Some(AttributeValue::StringValueStrindex(value)) => value.to_string(), + None => String::new(), + } +} diff --git a/litellm-rust/crates/traces/src/schema.rs b/litellm-rust/crates/traces/src/schema.rs index 5258c86ca78..5a154eb87c3 100644 --- a/litellm-rust/crates/traces/src/schema.rs +++ b/litellm-rust/crates/traces/src/schema.rs @@ -4,6 +4,8 @@ use std::time::Duration; use crate::Connection; use crate::Error; +const SCHEMA_REQUEST_TIMEOUT: Duration = Duration::from_secs(30); + const MIGRATIONS: [&str; 7] = [ include_str!("../migrations/0001_otel_traces.sql"), include_str!("../migrations/0002_agent_traces.sql"), @@ -49,11 +51,30 @@ pub async fn ensure_schema( database: &str, trace_retention_days: u32, spend_log_retention_days: u32, +) -> Result<(), Error> { + ensure_schema_with_timeout( + client, + connection, + database, + trace_retention_days, + spend_log_retention_days, + SCHEMA_REQUEST_TIMEOUT, + ) + .await +} + +async fn ensure_schema_with_timeout( + client: &Client, + connection: &Connection, + database: &str, + trace_retention_days: u32, + spend_log_retention_days: u32, + request_timeout: Duration, ) -> Result<(), Error> { for statement in schema_statements(database, trace_retention_days, spend_log_retention_days)? { let response = client .post(connection.url().clone()) - .timeout(Duration::from_secs(10)) + .timeout(request_timeout) .body(statement) .send() .await diff --git a/litellm-rust/crates/traces/tests/admin_sql.rs b/litellm-rust/crates/traces/tests/admin_sql.rs index 20a7a229fdc..ab0eb873a28 100644 --- a/litellm-rust/crates/traces/tests/admin_sql.rs +++ b/litellm-rust/crates/traces/tests/admin_sql.rs @@ -40,7 +40,12 @@ async fn database() -> Result> { "CREATE DATABASE litellm", "CREATE TABLE litellm.otel_traces (n UInt8) ENGINE = Memory", "INSERT INTO litellm.otel_traces VALUES (1)", + "CREATE TABLE litellm.agent_traces_by_key (n UInt8) ENGINE = Memory", + "INSERT INTO litellm.agent_traces_by_key VALUES (4)", + "CREATE TABLE litellm.spend_logs (n UInt8) ENGINE = Memory", + "INSERT INTO litellm.spend_logs VALUES (3)", "CREATE TABLE litellm.private_traces (n UInt8) ENGINE = Memory", + "CREATE TABLE private_traces (n UInt8) ENGINE = Memory", ] { client .post(&admin_url) @@ -81,6 +86,15 @@ async fn admin_sql_reads_rows_with_enforced_settings( let json: Value = serde_json::from_str(&result)?; assert_eq!(json["data"][0]["answer"], 1); + let result = read( + &database.client, + &connection, + "SELECT n AS answer FROM agent_traces_by_key", + ) + .await?; + let json: Value = serde_json::from_str(&result)?; + assert_eq!(json["data"][0]["answer"], 4); + Ok(()) } diff --git a/litellm-rust/crates/traces/tests/migrations.rs b/litellm-rust/crates/traces/tests/migrations.rs index 48baaee0828..ac0266409fa 100644 --- a/litellm-rust/crates/traces/tests/migrations.rs +++ b/litellm-rust/crates/traces/tests/migrations.rs @@ -2,7 +2,7 @@ use std::{collections::BTreeMap, time::Duration}; use litellm_http::Client; use litellm_traces::{ - Connection, Error, encode_rows, ensure_schema, execute_read, schema_statements, + Connection, Error, InsertTable, encode_rows, ensure_schema, execute_read, schema_statements, }; use rstest::{fixture, rstest}; use testcontainers_modules::{ @@ -107,7 +107,7 @@ async fn schema_supports_span_rollups_and_spend_joins( #[future(awt)] database: TestResult, ) -> TestResult { let database = database?; - let writer = Connection::writer(&database.url, "default", "")?; + let writer = Connection::writer(&database.url)?; ensure_schema(&database.client, &writer, "trace_test", 7, 14).await?; ensure_schema(&database.client, &writer, "trace_test", 7, 14).await?; let timestamp = time::OffsetDateTime::now_utc().unix_timestamp_nanos() as i64; @@ -144,7 +144,7 @@ async fn schema_supports_span_rollups_and_spend_joins( let body = read_json( &database, "SELECT toUInt32(sum(SpanCount)) AS spans, toUInt32(sum(InputTokens)) AS tokens \ - FROM trace_test.agent_traces WHERE TeamId = 'team-1' AND TraceId = 'trace-1'", + FROM trace_test.agent_traces_by_key WHERE TeamId = 'team-1' AND TraceId = 'trace-1'", ) .await?; assert_eq!( @@ -154,13 +154,92 @@ async fn schema_supports_span_rollups_and_spend_joins( Ok(()) } +#[rstest] +#[tokio::test] +async fn retried_trace_insert_does_not_inflate_rollup( + #[future(awt)] database: TestResult, +) -> TestResult { + let database = database?; + let writer = Connection::writer(&database.url)?; + ensure_schema(&database.client, &writer, "trace_test", 7, 14).await?; + let row: BTreeMap = serde_json::from_value(serde_json::json!({ + "Timestamp": time::OffsetDateTime::now_utc().unix_timestamp_nanos() as i64, + "TraceId": "retried-trace", "SpanId": "span-1", "ParentSpanId": "", + "TeamId": "team-1", "ApiKeyHash": "key-1", "SpanName": "root", "InputTokens": 7 + }))?; + for _ in 0..2 { + litellm_traces::insert_rows( + &database.client, + &writer, + "trace_test", + InsertTable::OtelTraces, + vec![row.clone()], + ) + .await?; + } + let counts = read_json( + &database, + "SELECT toUInt32(sum(SpanCount)) AS spans, toUInt32(sum(InputTokens)) AS tokens \ + FROM trace_test.agent_traces_by_key WHERE TraceId = 'retried-trace'", + ) + .await?; + assert_eq!(table_rows(&database, "otel_traces").await?, 1); + assert_eq!(counts["data"][0]["spans"], 1); + assert_eq!(counts["data"][0]["tokens"], 7); + Ok(()) +} + +#[rstest] +#[tokio::test] +async fn keyed_rollup_keeps_same_trace_ids_separate_by_api_key( + #[future(awt)] database: TestResult, +) -> TestResult { + let database = database?; + let writer = Connection::writer(&database.url)?; + ensure_schema(&database.client, &writer, "trace_test", 7, 14).await?; + let timestamp = time::OffsetDateTime::now_utc().unix_timestamp_nanos() as i64; + let rows = vec![ + serde_json::from_value(serde_json::json!({ + "Timestamp": timestamp, "TraceId": "shared-id", "SpanId": "root-one", + "ParentSpanId": "", "SpanName": "root-one", "Input": "private-one", + "ResourceAttributes": {"litellm.api_key_hash": "key-one"} + }))?, + serde_json::from_value(serde_json::json!({ + "Timestamp": timestamp, "TraceId": "shared-id", "SpanId": "root-two", + "ParentSpanId": "", "SpanName": "root-two", "Input": "private-two", + "ResourceAttributes": {"litellm.api_key_hash": "key-two"} + }))?, + ]; + insert_rows(&database, "otel_traces", rows).await?; + execute_write( + &database, + "OPTIMIZE TABLE trace_test.agent_traces_by_key FINAL", + ) + .await?; + let rows = read_json( + &database, + "SELECT ApiKeyHash, any(RootInput) AS RootInput \ + FROM trace_test.agent_traces_by_key WHERE TraceId = 'shared-id' \ + GROUP BY ApiKeyHash ORDER BY ApiKeyHash", + ) + .await?; + assert_eq!( + rows["data"], + serde_json::json!([ + {"ApiKeyHash": "key-one", "RootInput": "private-one"}, + {"ApiKeyHash": "key-two", "RootInput": "private-two"} + ]) + ); + Ok(()) +} + #[rstest] #[tokio::test] async fn rollup_merges_spans_across_days_without_losing_root_fields( #[future(awt)] database: TestResult, ) -> TestResult { let database = database?; - let writer = Connection::writer(&database.url, "default", "")?; + let writer = Connection::writer(&database.url)?; ensure_schema(&database.client, &writer, "trace_test", 7, 14).await?; let day_start = time::OffsetDateTime::now_utc() .replace_time(time::Time::MIDNIGHT) @@ -179,12 +258,16 @@ async fn rollup_merges_spans_across_days_without_losing_root_fields( "ResourceAttributes": {"litellm.team_id": "team-1"} }))?; insert_rows(&database, "otel_traces", vec![child]).await?; - execute_write(&database, "OPTIMIZE TABLE trace_test.agent_traces FINAL").await?; + execute_write( + &database, + "OPTIMIZE TABLE trace_test.agent_traces_by_key FINAL", + ) + .await?; let response = read_json( &database, "SELECT count() AS rows, any(RootName) AS RootName, any(RootInput) AS RootInput, \ any(RootStatus) AS RootStatus, sum(SpanCount) AS SpanCount \ - FROM trace_test.agent_traces", + FROM trace_test.agent_traces_by_key", ) .await?; assert_eq!( @@ -203,7 +286,7 @@ async fn spend_deduplication_preserves_subsecond_requests_and_retries( #[future(awt)] database: TestResult, ) -> TestResult { let database = database?; - let writer = Connection::writer(&database.url, "default", "")?; + let writer = Connection::writer(&database.url)?; ensure_schema(&database.client, &writer, "trace_test", 7, 14).await?; let now_ms = time::OffsetDateTime::now_utc().unix_timestamp_nanos() as i64 / 1_000_000; let base_start_time = now_ms / 1000 * 1000; @@ -255,7 +338,7 @@ async fn retention_changes_materialize_existing_rows_and_remain_idempotent( #[future(awt)] database: TestResult, ) -> TestResult { let database = database?; - let writer = Connection::writer(&database.url, "default", "")?; + let writer = Connection::writer(&database.url)?; ensure_schema(&database.client, &writer, "trace_test", 30, 30).await?; let old_time = time::OffsetDateTime::now_utc() - time::Duration::days(20); let old_timestamp_ns = old_time.unix_timestamp_nanos() as i64; @@ -271,6 +354,7 @@ async fn retention_changes_materialize_existing_rows_and_remain_idempotent( }))?; insert_rows(&database, "otel_traces", vec![span]).await?; insert_rows(&database, "spend_logs", vec![spend]).await?; + assert_eq!(table_rows(&database, "agent_traces_by_key").await?, 1); ensure_schema(&database.client, &writer, "trace_test", 14, 14).await?; let deadline = tokio::time::Instant::now() + Duration::from_secs(60); loop { @@ -293,10 +377,14 @@ async fn retention_changes_materialize_existing_rows_and_remain_idempotent( tokio::time::sleep(Duration::from_millis(100)).await; } execute_write(&database, "OPTIMIZE TABLE trace_test.otel_traces FINAL").await?; - execute_write(&database, "OPTIMIZE TABLE trace_test.agent_traces FINAL").await?; + execute_write( + &database, + "OPTIMIZE TABLE trace_test.agent_traces_by_key FINAL", + ) + .await?; execute_write(&database, "OPTIMIZE TABLE trace_test.spend_logs FINAL").await?; assert_eq!(table_rows(&database, "otel_traces").await?, 0); - assert_eq!(table_rows(&database, "agent_traces").await?, 0); + assert_eq!(table_rows(&database, "agent_traces_by_key").await?, 0); assert_eq!(table_rows(&database, "spend_logs").await?, 0); let mutation_count = mutation_rows(&database).await?; ensure_schema(&database.client, &writer, "trace_test", 14, 14).await?; @@ -315,9 +403,9 @@ async fn schema_statement_timeout_maps_to_transport_error() -> TestResult { }); let client = Client::no_redirect_for_test(); let url = format!("http://{address}"); - let writer = Connection::writer(&url, "default", "")?; + let writer = Connection::writer(&url)?; let result = tokio::time::timeout( - Duration::from_secs(12), + Duration::from_secs(35), ensure_schema(&client, &writer, "trace_test", 7, 14), ) .await; diff --git a/litellm-rust/crates/traces/tests/otlp.rs b/litellm-rust/crates/traces/tests/otlp.rs new file mode 100644 index 00000000000..002ba159ef9 --- /dev/null +++ b/litellm-rust/crates/traces/tests/otlp.rs @@ -0,0 +1,47 @@ +use flate2::{Compression, write::GzEncoder}; +use litellm_traces::decode_otlp; +use rstest::rstest; +use std::io::Write; + +const FIXTURE: &[u8] = include_bytes!( + "../../../../tests/test_litellm/tracing/fixtures/langsmith_deep_agent_export.json" +); + +#[rstest] +#[case::json(FIXTURE, Some("application/json"), None)] +#[case::gzip_json(FIXTURE, Some("application/json"), Some("gzip"))] +fn decodes_neutral_spans( + #[case] body: &[u8], + #[case] content_type: Option<&str>, + #[case] content_encoding: Option<&str>, +) { + let payload = if content_encoding == Some("gzip") { + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(body).expect("gzip input"); + encoder.finish().expect("gzip payload") + } else { + body.to_vec() + }; + let spans = decode_otlp(&payload, content_type, content_encoding, 8 * 1024 * 1024) + .expect("valid OTLP export"); + assert_eq!(spans.len(), 6); + assert_eq!(spans[0].trace_id, "4bad42b84e9de3ba46fc870185f8f023"); + assert_eq!(spans[0].resource_attributes["service.name"], "agent-demo"); + assert_eq!(spans[0].scope_name, "langsmith"); + assert!( + spans + .iter() + .any(|span| span.attributes.contains_key("gen_ai.prompt")) + ); +} + +#[rstest] +#[case::invalid(b"not protobuf", None, 8 * 1024 * 1024)] +#[case::too_large(FIXTURE, Some("application/json"), 1)] +fn rejects_invalid_or_oversized_payload( + #[case] body: &[u8], + #[case] content_type: Option<&str>, + #[case] limit: usize, +) { + assert!(decode_otlp(body, content_type, None, limit).is_err()); +} diff --git a/litellm/constants.py b/litellm/constants.py index 530d678457d..9af40744896 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -46,6 +46,19 @@ ROUTER_SETTINGS_MANAGED_OUTSIDE_CONFIG: Final[frozenset[str]] = frozenset( ) DEFAULT_BATCH_SIZE: Final = int(os.getenv("DEFAULT_BATCH_SIZE", 512)) DEFAULT_FLUSH_INTERVAL_SECONDS: Final = int(os.getenv("DEFAULT_FLUSH_INTERVAL_SECONDS", 5)) +# Agent tracing / ClickHouse +CLICKHOUSE_BATCH_SIZE: Final = get_env_int("CLICKHOUSE_BATCH_SIZE", 10_000) +CLICKHOUSE_FLUSH_INTERVAL_SECONDS: Final = float(os.getenv("CLICKHOUSE_FLUSH_INTERVAL_SECONDS", "1.0")) +CLICKHOUSE_MAX_BUFFERED_ROWS: Final = get_env_int("CLICKHOUSE_MAX_BUFFERED_ROWS", 200_000) +CLICKHOUSE_MAX_RETRIES: Final = get_env_int("CLICKHOUSE_MAX_RETRIES", 3) +AGENT_TRACING_RETENTION_DAYS: Final = get_env_int("AGENT_TRACING_RETENTION_DAYS", 30) +AGENT_TRACING_SPEND_LOG_RETENTION_DAYS: Final = get_env_int("AGENT_TRACING_SPEND_LOG_RETENTION_DAYS", 90) +OTLP_MAX_BODY_BYTES: Final = get_env_int("OTLP_MAX_BODY_BYTES", 8 * 1024 * 1024) +OTLP_MAX_ATTRIBUTE_VALUE_BYTES: Final = get_env_int("OTLP_MAX_ATTRIBUTE_VALUE_BYTES", 64 * 1024) +OTLP_RETRY_AFTER_SECONDS: Final = get_env_int("OTLP_RETRY_AFTER_SECONDS", 2) +OTLP_OFFLOAD_DECODE_BYTES: Final = get_env_int("OTLP_OFFLOAD_DECODE_BYTES", 256 * 1024) +AGENT_TRACING_INPUT_PREVIEW_CHARS: Final = get_env_int("AGENT_TRACING_INPUT_PREVIEW_CHARS", 240) +AGENT_TRACING_LIST_PAGE_SIZE: Final = get_env_int("AGENT_TRACING_LIST_PAGE_SIZE", 50) DEFAULT_S3_FLUSH_INTERVAL_SECONDS: Final = int(os.getenv("DEFAULT_S3_FLUSH_INTERVAL_SECONDS", 10)) DEFAULT_S3_BATCH_SIZE: Final = int(os.getenv("DEFAULT_S3_BATCH_SIZE", 512)) DEFAULT_S3_MAX_CONCURRENT_UPLOADS: Final = int(os.getenv("DEFAULT_S3_MAX_CONCURRENT_UPLOADS", "16")) diff --git a/litellm/integrations/clickhouse/clickhouse_batch_logger.py b/litellm/integrations/clickhouse/clickhouse_batch_logger.py new file mode 100644 index 00000000000..fb9088f44ff --- /dev/null +++ b/litellm/integrations/clickhouse/clickhouse_batch_logger.py @@ -0,0 +1,100 @@ +""" +Shared base for everything LiteLLM writes to ClickHouse. + +Built on `CustomBatchLogger`: rows accumulate in `log_queue` and are flushed as one +gzip JSONEachRow insert, either every `CLICKHOUSE_FLUSH_INTERVAL_SECONDS` or as soon as +`batch_size` rows are queued. Subclasses only pick the table and build rows: + +- `ClickHouseSpendLogger` -> spend_logs (LiteLLM requests, via the `clickhouse` callback) +""" + +import asyncio +import os +from typing import Any, ClassVar + +from litellm._logging import verbose_logger +from litellm.constants import ( + CLICKHOUSE_BATCH_SIZE, + CLICKHOUSE_FLUSH_INTERVAL_SECONDS, + CLICKHOUSE_MAX_BUFFERED_ROWS, + CLICKHOUSE_MAX_RETRIES, +) +from litellm.integrations.custom_batch_logger import CustomBatchLogger +from litellm.rust_bridge.traces import TraceStorage + + +def clickhouse_storage_from_env() -> TraceStorage: + return TraceStorage( + database=os.getenv("CLICKHOUSE_DATABASE", "litellm"), + url=os.getenv("CLICKHOUSE_URL", ""), + ) + + +class ClickHouseBatchLogger(CustomBatchLogger): + table: ClassVar[str] + + def __init__(self, storage: TraceStorage | None = None) -> None: + self.storage = storage or clickhouse_storage_from_env() + self.rows_written = 0 + self.rows_dropped = 0 + self._failed_attempts = 0 + super().__init__( + flush_lock=asyncio.Lock(), + batch_size=CLICKHOUSE_BATCH_SIZE, + flush_interval=CLICKHOUSE_FLUSH_INTERVAL_SECONDS, + ) + try: + asyncio.get_running_loop().create_task(self.periodic_flush()) + except RuntimeError: # no loop yet (e.g. sync config load); proxy startup calls start() + pass + + def start(self) -> None: + asyncio.get_running_loop().create_task(self.periodic_flush()) + + def is_full(self) -> bool: + """Backpressure signal: producers should reject (429) instead of enqueueing.""" + return len(self.log_queue) >= CLICKHOUSE_MAX_BUFFERED_ROWS + + def enqueue(self, rows: list[dict[str, Any]]) -> None: + """Never awaits ClickHouse. Kicks off an early flush once a full batch is queued.""" + self.log_queue.extend(rows) + if len(self.log_queue) >= self.batch_size: + asyncio.get_running_loop().create_task(self.flush_queue()) + + async def flush_queue(self) -> None: + # Swap the queue under the lock so rows enqueued during the insert are kept. + if self.flush_lock is None: + return + async with self.flush_lock: + while self.log_queue: + batch = self.log_queue[: self.batch_size] + self.log_queue = self.log_queue[len(batch) :] + if not await self._insert(batch): + break + + async def async_send_batch(self) -> None: + await self.flush_queue() + + async def _insert(self, batch: list[dict[str, Any]]) -> bool: + try: + await self.storage.insert_rows(self.table, batch) + self.rows_written += len(batch) + self._failed_attempts = 0 + return True + except Exception as e: + self._failed_attempts += 1 + if self._failed_attempts >= CLICKHOUSE_MAX_RETRIES: + self.rows_dropped += len(batch) + self._failed_attempts = 0 + verbose_logger.error( + "ClickHouse: dropped %s rows for %s after %s attempts: %s", + len(batch), + self.table, + CLICKHOUSE_MAX_RETRIES, + e, + ) + else: + # put it back; the next periodic flush retries it + self.log_queue = batch + self.log_queue + verbose_logger.warning("ClickHouse: insert into %s failed, will retry: %s", self.table, e) + return False diff --git a/litellm/integrations/clickhouse/schema.py b/litellm/integrations/clickhouse/schema.py new file mode 100644 index 00000000000..6bec35c5630 --- /dev/null +++ b/litellm/integrations/clickhouse/schema.py @@ -0,0 +1,11 @@ +from typing import Final + +from litellm.rust_bridge.traces import TraceStorage + +OTEL_TRACES_TABLE: Final = "otel_traces" +AGENT_TRACES_BY_KEY_TABLE: Final = "agent_traces_by_key" +SPEND_LOGS_TABLE: Final = "spend_logs" + + +async def ensure_schema(storage: TraceStorage, trace_retention_days: int, spend_log_retention_days: int) -> None: + await storage.ensure_schema(trace_retention_days, spend_log_retention_days) diff --git a/litellm/integrations/custom_batch_logger.py b/litellm/integrations/custom_batch_logger.py index bfc78b93715..2e1cf291716 100644 --- a/litellm/integrations/custom_batch_logger.py +++ b/litellm/integrations/custom_batch_logger.py @@ -27,7 +27,7 @@ class CustomBatchLogger(CustomLogger): self, flush_lock: asyncio.Lock | None = None, batch_size: int | None = None, - flush_interval: int | None = None, + flush_interval: float | None = None, max_queue_size: int | None = None, **kwargs, ) -> None: diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 9c7dece6894..a68462313f1 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -520,6 +520,10 @@ class LiteLLMRoutes(enum.Enum): "/v1/rag/ingest", "/rag/query", "/v1/rag/query", + # agent tracing: OTLP ingest + reads (scoped to the caller's team in the handler) + "/v1/traces", + "/v1/traces/{trace_id}", + "/v1/traces/{trace_id}/spans/{span_id}", ] anthropic_routes = [ diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.py index f436bc96dbd..ac757f5f6a7 100644 --- a/litellm/proxy/common_utils/http_parsing_utils.py +++ b/litellm/proxy/common_utils/http_parsing_utils.py @@ -21,6 +21,9 @@ from litellm.proxy.common_utils.callback_utils import ( from litellm.types.router import Deployment _FORM_CONTENT_TYPES: Final[frozenset[str]] = frozenset({"application/x-www-form-urlencoded", "multipart/form-data"}) +# Binary bodies (e.g. OTLP trace exports on POST /v1/traces) are not JSON: arbitrary bytes used to +# hit the JSON surrogate-repair path and fail auth with a 400. JSON under these types still parses. +_BINARY_CONTENT_TYPES: Final[frozenset[str]] = frozenset({"application/x-protobuf", "application/protobuf"}) _ANNOTATION_QUALIFIERS: Final[frozenset[object]] = frozenset({Annotated, NotRequired, ReadOnly, Required}) @@ -153,6 +156,17 @@ def coerce_numeric_form_fields( } +def _parse_binary_body(body: bytes) -> dict: + """JSON sent under a binary content type still parses; real binary (protobuf) carries no params -> {}.""" + try: + parsed: Final = orjson.loads(body) + if isinstance(parsed, dict): + return parsed + except orjson.JSONDecodeError: + pass + return {} # mutable-ok: auth parser returns a fresh dict per request + + async def _read_request_body(request: Request | None) -> dict: """ Safely read the request body and parse it as JSON. @@ -175,7 +189,13 @@ async def _read_request_body(request: Request | None) -> dict: _request_headers: Final[dict] = _safe_get_request_headers(request=request) content_type: Final = _request_headers.get("content-type", "") - if _is_form_content_type(content_type): + if _normalize_media_type(content_type) in _BINARY_CONTENT_TYPES or ( + request.scope.get("path") == "/v1/traces" + and request.scope.get("method") == "POST" + and _request_headers.get("content-encoding", "").lower() == "gzip" + ): + parsed_body = _parse_binary_body(await request.body()) + elif _is_form_content_type(content_type): try: form_data: Final = await request.form() except Exception as e: diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 7200407b64c..6ae5cb6a0c0 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -714,6 +714,7 @@ try: except ImportError: build_billing_metrics_recorder = None shutdown_billing_metrics_recorder = None +from litellm.proxy import tracing_endpoints from litellm.proxy.middleware.admission_control_middleware import ( AdmissionControlMiddleware, admission_control_state, @@ -845,6 +846,7 @@ from litellm.secret_managers.main import ( secret_manager_would_be_consulted, str_to_bool, ) +from litellm.tracing import TraceReceiver from litellm.types.integrations.slack_alerting import AlertType, SlackAlertingArgs from litellm.types.llms.anthropic import ( AnthropicMessagesRequest, @@ -1521,6 +1523,9 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[None, None]: _tagged.strategy._state_loaded = True asyncio.create_task(_adaptive_router_flusher_loop()) + ## [Optional] Initialize agent tracing + asyncio.create_task(ProxyStartupEvent.init_tracing(general_settings)) + ## [Optional] Initialize dd tracer ProxyStartupEvent._init_dd_tracer() @@ -11310,6 +11315,28 @@ class ProxyStartupEvent: ) return connected_client + @classmethod + async def init_tracing(cls, general_settings: dict) -> None: + """ + Enable agent tracing (`POST/GET /v1/traces`) when configured: + + general_settings: + tracing: + store: clickhouse # CLICKHOUSE_URL / _USER / _PASSWORD / _DATABASE + """ + settings: Final = general_settings.get("tracing") + if not isinstance(settings, dict) or settings.get("store") != "clickhouse": + return + try: + tracing: Final = TraceReceiver.from_env() + await tracing.start() + except (KeyError, OSError, RuntimeError, ValueError) as error: + tracing_endpoints.receiver = None + verbose_proxy_logger.warning("Agent tracing unavailable: %s", error) + return + tracing_endpoints.receiver = tracing + verbose_proxy_logger.info("Agent tracing enabled (store=clickhouse)") + @classmethod def _init_dd_tracer(cls): """ @@ -19849,6 +19876,7 @@ app.include_router(rag_router) app.include_router(video_router) app.include_router(container_router) app.include_router(search_router) +app.include_router(tracing_endpoints.router) app.include_router(image_router) app.include_router(fine_tuning_router) app.include_router(credential_router) diff --git a/litellm/proxy/tracing_endpoints.py b/litellm/proxy/tracing_endpoints.py new file mode 100644 index 00000000000..06dbf359ba9 --- /dev/null +++ b/litellm/proxy/tracing_endpoints.py @@ -0,0 +1,141 @@ +""" +Agent tracing endpoints. Thin wrappers over `TraceReceiver`: auth -> tenant/scope -> one call. + +POST /v1/traces OTLP/HTTP trace export (protobuf or JSON) +GET /v1/traces TracePage +GET /v1/traces/{trace_id} Trace +GET /v1/traces/{trace_id}/spans/{span_id} SpanDetail +""" + +import time +from typing import Annotated, Final + +from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response + +from litellm.constants import OTLP_MAX_BODY_BYTES, OTLP_RETRY_AFTER_SECONDS +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.tracing import ( + Tenant, + TraceReceiver, + TracingPayloadTooLargeError, +) +from litellm.tracing.decode import InvalidOTLPPayloadError, encode_otlp_response +from litellm.tracing.types import SpanDetail, Trace, TracePage, TraceScope + +router = APIRouter(tags=["agent tracing"]) # mutable-ok: FastAPI copies the mutable tags list + +MS_PER_DAY: Final = 24 * 60 * 60 * 1000 +_ADMIN_ROLES: Final = (LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY) + +receiver: TraceReceiver | None = None + + +def get_receiver() -> TraceReceiver: + if receiver is None: + raise HTTPException( + status_code=501, + detail="Agent tracing is not enabled. Set `tracing:` in general_settings and CLICKHOUSE_URL.", + ) + return receiver + + +def tenant_for(user_api_key_dict: UserAPIKeyAuth) -> Tenant: + return Tenant( + team_id=user_api_key_dict.team_id or "", + api_key_hash=user_api_key_dict.token or "", + org_id=user_api_key_dict.org_id or "", + ) + + +def scope_for(user_api_key_dict: UserAPIKeyAuth) -> TraceScope: + """Admins see everything; team members see their team; team-less keys see their own traces.""" + if user_api_key_dict.user_role in _ADMIN_ROLES: + return TraceScope(team_ids=(), api_key_hash="") + if user_api_key_dict.team_id: + return TraceScope(team_ids=(user_api_key_dict.team_id,), api_key_hash="") + if not user_api_key_dict.token: + raise HTTPException(status_code=403, detail="Not allowed to view agent traces") + return TraceScope(team_ids=("",), api_key_hash=user_api_key_dict.token) + + +async def _read_otlp_body(request: Request) -> bytes: + body: Final = bytearray() + async for chunk in request.stream(): + if len(body) + len(chunk) > OTLP_MAX_BODY_BYTES: + raise TracingPayloadTooLargeError(f"OTLP body exceeds {OTLP_MAX_BODY_BYTES} bytes") + body.extend(chunk) + return bytes(body) + + +@router.post("/v1/traces", include_in_schema=False) +async def ingest_otlp_traces( + request: Request, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> Response: + if user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY: + raise HTTPException(status_code=403, detail="Not allowed to ingest agent traces") + tracing: Final = get_receiver() + content_type: Final = request.headers.get("content-type") + try: + await tracing.ingest( + body=await _read_otlp_body(request), + content_type=content_type, + content_encoding=request.headers.get("content-encoding"), + tenant=tenant_for(user_api_key_dict), + ) + except TracingPayloadTooLargeError as e: + raise HTTPException(status_code=413, detail=str(e)) + except InvalidOTLPPayloadError as error: + raise HTTPException(status_code=400, detail=str(error)) from error + except RuntimeError: + raise HTTPException( + status_code=503, + headers={"Retry-After": str(OTLP_RETRY_AFTER_SECONDS)}, # mutable-ok: FastAPI requires dict headers + ) + body, media_type = encode_otlp_response(content_type) + return Response(content=body, media_type=media_type) + + +@router.get("/v1/traces", response_model=None) +async def list_agent_traces( + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], + start_ms: Annotated[int | None, Query(description="Window start, unix ms. Default: 24h ago")] = None, + end_ms: Annotated[int | None, Query(description="Window end, unix ms. Default: now")] = None, + cursor: Annotated[str | None, Query()] = None, +) -> TracePage: + now_ms: Final = int(time.time() * 1000) + try: + return await get_receiver().list_traces( + scope=scope_for(user_api_key_dict), + start_ms=start_ms if start_ms is not None else now_ms - MS_PER_DAY, + end_ms=end_ms if end_ms is not None else now_ms, + cursor=cursor, + ) + except ValueError as error: + raise HTTPException(status_code=400, detail=str(error)) from error + + +@router.get("/v1/traces/{trace_id}", response_model=None) +async def get_agent_trace( + trace_id: str, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], + trace_ref: Annotated[str, Query()] = "", +) -> Trace: + trace: Final = await get_receiver().get_trace(trace_id, scope_for(user_api_key_dict), trace_ref) + if trace is None: + raise HTTPException(status_code=404, detail=f"Trace {trace_id} not found") + return trace + + +@router.get("/v1/traces/{trace_id}/spans/{span_id}", response_model=None) +async def get_agent_trace_span( + trace_id: str, + span_id: str, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], + trace_ref: Annotated[str, Query()] = "", +) -> SpanDetail: + span: Final = await get_receiver().get_span(trace_id, span_id, scope_for(user_api_key_dict), trace_ref) + if span is None: + raise HTTPException(status_code=404, detail=f"Span {span_id} not found") + return span diff --git a/litellm/rust_bridge/_native.pyi b/litellm/rust_bridge/_native.pyi index 5d2af93a9bd..30d4bbfb68e 100644 --- a/litellm/rust_bridge/_native.pyi +++ b/litellm/rust_bridge/_native.pyi @@ -11,6 +11,7 @@ from litellm.rust_bridge.embeddings.entrypoints import LiteLLMEmbeddingRequest from litellm.rust_bridge.messages.entrypoints import LiteLLMMessagesRequest from litellm.rust_bridge.ocr.entrypoints import LiteLLMOcrRequest from litellm.rust_bridge.responses.entrypoints import LiteLLMResponsesRequest +from litellm.rust_bridge.traces import DecodedSpan from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.utils import EmbeddingResponse, ModelResponse @@ -20,18 +21,16 @@ class RustUpstreamError(Exception): ... class ForkedAfterNativeRuntimeStarted(RuntimeError): ... class ProcessReservedForForking(RuntimeError): ... -def trace_encode_rows(rows: Sequence[Mapping[str, JsonValue]]) -> str: ... -def trace_ensure_schema( - url: str, database: str, user: str, password: str, trace_retention_days: int, spend_log_retention_days: int -) -> Future[None]: ... -def trace_query( - url: str, - database: str, - user: str, - password: str, - sql: str, - parameters: Mapping[str, str | int | Sequence[str]], -) -> Future[str]: ... +def trace_decode_otlp( + body: bytes, content_type: str | None, content_encoding: str | None, max_decompressed_bytes: int +) -> list[DecodedSpan]: ... + +@final +class NativeTraceStorage: + def __new__(cls, database: str, url: str, reader_url: str | None = None) -> NativeTraceStorage: ... + def ensure_schema(self, trace_retention_days: int, spend_log_retention_days: int) -> Future[None]: ... + def insert_rows(self, table: str, rows: Sequence[Mapping[str, JsonValue]]) -> Future[None]: ... + def query(self, sql: str, parameters: Mapping[str, str | int | Sequence[str]]) -> Future[str]: ... @final class NativeDiagnosticProcessor: @@ -327,6 +326,7 @@ __all__ = [ "ForkedAfterNativeRuntimeStarted", "HuggingFaceEncoding", "NativeDiagnosticProcessor", + "NativeTraceStorage", "ProcessReservedForForking", "ResponsesWebSocketConnection", "RustBridgeDeclined", @@ -351,9 +351,7 @@ __all__ = [ "process_state_started", "reserve_process_for_forking", "responses", - "trace_encode_rows", - "trace_ensure_schema", - "trace_query", + "trace_decode_otlp", "transcription", ] diff --git a/litellm/rust_bridge/traces.py b/litellm/rust_bridge/traces.py index cb4fd4a5bd3..a0b010caea0 100644 --- a/litellm/rust_bridge/traces.py +++ b/litellm/rust_bridge/traces.py @@ -1,33 +1,56 @@ from collections.abc import Awaitable, Mapping, Sequence -from typing import Final, Protocol, cast +from types import MappingProxyType +from typing import Final, Protocol, TypedDict, cast -from pydantic import BaseModel, ConfigDict, JsonValue +from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter +from typing_extensions import ReadOnly from litellm.rust_bridge.loader import get_native_bridge +class DecodedEvent(TypedDict): + name: ReadOnly[str] + attributes: ReadOnly[dict[str, str]] + + +class DecodedSpan(TypedDict): + trace_id: ReadOnly[str] + span_id: ReadOnly[str] + parent_span_id: ReadOnly[str] + trace_state: ReadOnly[str] + name: ReadOnly[str] + kind: ReadOnly[str] + resource_attributes: ReadOnly[dict[str, str]] + scope_name: ReadOnly[str] + scope_version: ReadOnly[str] + attributes: ReadOnly[dict[str, str]] + start_ns: ReadOnly[int] + end_ns: ReadOnly[int] + status_code: ReadOnly[str] + status_message: ReadOnly[str] + events: ReadOnly[list[DecodedEvent]] + + +class NativeStore(Protocol): + def __init__(self, database: str, url: str, reader_url: str | None = None) -> None: ... + + def ensure_schema(self, trace_retention_days: int, spend_log_retention_days: int) -> Awaitable[None]: ... + + def insert_rows(self, table: str, rows: Sequence[Mapping[str, JsonValue]]) -> Awaitable[None]: ... + + def query(self, sql: str, parameters: Mapping[str, str | int | Sequence[str]]) -> Awaitable[str]: ... + + class NativeTraces(Protocol): - def trace_encode_rows(self, rows: Sequence[Mapping[str, JsonValue]]) -> str: ... + NativeTraceStorage: type[NativeStore] - def trace_ensure_schema( + def trace_decode_otlp( self, - url: str, - database: str, - user: str, - password: str, - trace_retention_days: int, - spend_log_retention_days: int, - ) -> Awaitable[None]: ... - - def trace_query( - self, - url: str, - database: str, - user: str, - password: str, - sql: str, - parameters: Mapping[str, str | int | Sequence[str]], - ) -> Awaitable[str]: ... + body: bytes, + content_type: str | None, + content_encoding: str | None, + max_decompressed_bytes: int, + ) -> list[DecodedSpan]: ... class QueryResponse(BaseModel): @@ -35,6 +58,10 @@ class QueryResponse(BaseModel): data: list[dict[str, JsonValue]] +INSERT_ROWS: Final = TypeAdapter(list[dict[str, JsonValue]]) +QUERY_PARAMETERS: Final = TypeAdapter(dict[str, str | int | list[str]]) + + def _native() -> NativeTraces: native: Final = get_native_bridge() if native is None: @@ -42,28 +69,24 @@ def _native() -> NativeTraces: return cast(NativeTraces, native) # cast-ok: the native extension is validated against this protocol at call sites -async def ensure_schema( - url: str, - database: str, - user: str, - password: str, - trace_retention_days: int, - spend_log_retention_days: int, -) -> None: - await _native().trace_ensure_schema(url, database, user, password, trace_retention_days, spend_log_retention_days) +def decode_otlp( + body: bytes, content_type: str | None, content_encoding: str | None, max_decompressed_bytes: int +) -> list[DecodedSpan]: + return _native().trace_decode_otlp(body, content_type, content_encoding, max_decompressed_bytes) -async def query( - url: str, - database: str, - user: str, - password: str, - sql: str, - parameters: Mapping[str, str | int | Sequence[str]], -) -> list[dict[str, JsonValue]]: - result: Final = await _native().trace_query(url, database, user, password, sql, parameters) - return QueryResponse.model_validate_json(result).data +class TraceStorage: + def __init__(self, database: str, url: str, reader_url: str | None = None) -> None: + self._native: Final = _native().NativeTraceStorage(database, url, reader_url) + async def ensure_schema(self, trace_retention_days: int, spend_log_retention_days: int) -> None: + await self._native.ensure_schema(trace_retention_days, spend_log_retention_days) -def encode_rows(rows: Sequence[Mapping[str, JsonValue]]) -> bytes: - return _native().trace_encode_rows(rows).encode("utf-8") + async def insert_rows(self, table: str, rows: Sequence[Mapping[str, object]]) -> None: + await self._native.insert_rows(table, INSERT_ROWS.validate_python(rows)) + + async def query(self, sql: str, parameters: Mapping[str, object] | None = None) -> list[dict[str, JsonValue]]: + result: Final = await self._native.query( + sql, QUERY_PARAMETERS.validate_python(parameters or MappingProxyType({})) + ) + return QueryResponse.model_validate_json(result).data diff --git a/litellm/tracing/AGENTS.md b/litellm/tracing/AGENTS.md new file mode 100644 index 00000000000..f69866c0419 --- /dev/null +++ b/litellm/tracing/AGENTS.md @@ -0,0 +1,6 @@ +- Python owns tracing endpoints, authenticated tenant scope, framework normalization and API response shaping +- Trace ingestion awaits `TraceStorage.insert_rows` before returning success; propagate storage failures so OTLP exporters can retry +- Spend logging keeps its separate batch queue in `litellm/integrations/clickhouse` +- Use `litellm.rust_bridge.traces.TraceStorage` for ClickHouse; keep schema, SQL, encoding and transport in `litellm-traces` +- Derive tenant fields from authentication and overwrite matching fields supplied by the exporter +- Test confirmed writes, failures, tenant isolation and read behavior through public functions diff --git a/litellm/tracing/__init__.py b/litellm/tracing/__init__.py new file mode 100644 index 00000000000..681100ed76a --- /dev/null +++ b/litellm/tracing/__init__.py @@ -0,0 +1,16 @@ +""" +LiteLLM agent tracing: OTLP traces from agents, joined to LiteLLM spend logs, in ClickHouse. + +""" + +from litellm.tracing.receiver import ( + Tenant, + TraceReceiver, + TracingPayloadTooLargeError, +) + +__all__ = ( + "Tenant", + "TraceReceiver", + "TracingPayloadTooLargeError", +) diff --git a/litellm/tracing/decode.py b/litellm/tracing/decode.py new file mode 100644 index 00000000000..60ae7732444 --- /dev/null +++ b/litellm/tracing/decode.py @@ -0,0 +1,284 @@ +""" +OTLP/HTTP trace export -> `SpanRow`s. + +Pure functions, no I/O. Two steps: +1. `decode_otlp()` protobuf / JSON / gzip `ExportTraceServiceRequest` -> flat spans +2. `normalize()` framework conventions -> LiteLLM columns (type, agent, input/output, + LiteLLM request id). Supported: LangSmith (LangChain, LangGraph, + Deep Agents), OTEL GenAI semconv, OpenInference. +""" + +import json +from collections.abc import Callable, Mapping +from types import MappingProxyType +from typing import Any, Final + +from litellm.constants import OTLP_MAX_ATTRIBUTE_VALUE_BYTES, OTLP_MAX_BODY_BYTES +from litellm.rust_bridge.traces import DecodedSpan +from litellm.rust_bridge.traces import decode_otlp as native_decode_otlp +from litellm.tracing.types import SpanRow, SpanType + +# attributes whose content we lift into Input/Output and drop from SpanAttributes +_HEAVY_ATTRIBUTES: Final = frozenset( + { + "gen_ai.prompt", + "gen_ai.completion", + "gen_ai.tool.definitions", + "gen_ai.input.messages", + "gen_ai.output.messages", + "input.value", + "output.value", + } +) +# LangChain / Deep Agents middleware wrappers: real spans, but noise in the UI +_FRAMEWORK_SUFFIXES: Final = ( + ".wrap_model_call", + ".wrap_tool_call", + ".before_agent", + ".after_agent", + ".before_model", + ".after_model", +) +_LLM_OPERATIONS: Final = frozenset({"chat", "text_completion", "generate_content"}) +_LC_ROLES: Final = MappingProxyType({"human": "user", "ai": "assistant", "system": "system", "tool": "tool"}) +_OPENINFERENCE_TYPES: Final[Mapping[str, SpanType]] = MappingProxyType({"AGENT": "agent", "LLM": "llm", "TOOL": "tool"}) + + +class InvalidOTLPPayloadError(ValueError): + pass + + +class OTLPPayloadTooLargeError(OverflowError): + pass + + +# ---------------------------------------------------------------- decode + + +def _truncate(value: str) -> str: + size = len(value.encode("utf-8")) + if size <= OTLP_MAX_ATTRIBUTE_VALUE_BYTES: + return value + kept = value.encode("utf-8")[:OTLP_MAX_ATTRIBUTE_VALUE_BYTES].decode("utf-8", "ignore") + return f"{kept}…[truncated {size - OTLP_MAX_ATTRIBUTE_VALUE_BYTES} bytes]" + + +def decode_otlp( + body: bytes, content_type: str | None = None, content_encoding: str | None = None +) -> tuple[SpanRow, ...]: + """Decode an OTLP trace export and normalize every span.""" + try: + spans: Final = native_decode_otlp(body, content_type, content_encoding, OTLP_MAX_BODY_BYTES) + except OverflowError as error: + raise OTLPPayloadTooLargeError(str(error)) from error + except ValueError as error: + raise InvalidOTLPPayloadError(str(error)) from error + return tuple(_span_row(span) for span in spans) + + +def _exception_message(span: DecodedSpan) -> str: + """`span.record_exception()` writes an `exception` event; surface it when status.message is empty.""" + for event in span["events"]: + if event["name"] == "exception": + attributes = event["attributes"] + return attributes.get("exception.message") or attributes.get("exception.type", "") + return "" + + +def _span_row(span: DecodedSpan) -> SpanRow: + attributes = span["attributes"] + resource = span["resource_attributes"] + row = SpanRow( + Timestamp=span["start_ns"], + TraceId=span["trace_id"], + SpanId=span["span_id"], + ParentSpanId=span["parent_span_id"], + TraceState=span["trace_state"], + SpanName=span["name"], + SpanKind=span["kind"], + ServiceName=resource.get("service.name", ""), + ResourceAttributes=resource, + ScopeName=span["scope_name"], + ScopeVersion=span["scope_version"], + SpanAttributes=attributes, + Duration=max(span["end_ns"] - span["start_ns"], 0), + StatusCode=span["status_code"], + StatusMessage=span["status_message"] or _exception_message(span), + TeamId="", + ApiKeyHash="", + ObservationType="chain", + AgentName="", + LiteLLMRequestId="", + Model="", + InputTokens=0, + OutputTokens=0, + Input="", + Output="", + ) + normalize(row, attributes) + row["SpanAttributes"] = { # mutable-ok: the Rust JSON bridge requires a plain dict for span attributes + k: _truncate(v) for k, v in attributes.items() if k not in _HEAVY_ATTRIBUTES + } + row["Input"], row["Output"] = _truncate(row["Input"]), _truncate(row["Output"]) + return row + + +# ---------------------------------------------------------------- normalize + + +def _loads(value: str) -> object: + try: + return json.loads(value) + except (ValueError, TypeError): + return None + + +def _lc_message(message: Mapping[str, Any]) -> dict[str, Any]: + """LangChain serialized message (or plain {role, content}) -> {role, content, tool_calls?}.""" + kwargs = message.get("kwargs", message) + role = _LC_ROLES.get(kwargs.get("type") or kwargs.get("role"), kwargs.get("role") or kwargs.get("type") or "") + content = kwargs.get("content", "") + out: dict[str, Any] = { # mutable-ok: the framework message is built for JSON serialization + "role": role, + "content": content if isinstance(content, str) else json.dumps(content), + } + if kwargs.get("tool_calls"): + out["tool_calls"] = tuple( + {"name": t.get("name"), "args": t.get("args")} # mutable-ok: JSON tool calls need object payloads + for t in kwargs["tool_calls"] + ) + if role == "tool" and kwargs.get("name"): + out["name"] = kwargs["name"] + return out + + +def _langsmith_type(row: SpanRow, attributes: Mapping[str, str]) -> SpanType: + kind = attributes.get("langsmith.span.kind", "chain") + name = row["SpanName"] + if not row["ParentSpanId"] or name == attributes.get("langsmith.metadata.lc_agent_name"): + return "agent" + if kind in ("llm", "tool"): + return kind + if name.endswith(_FRAMEWORK_SUFFIXES): + return "framework" + return "chain" + + +def _langsmith_io(row: SpanRow, attributes: Mapping[str, str]) -> None: + prompt = _loads(attributes.get("gen_ai.prompt", "")) + completion = _loads(attributes.get("gen_ai.completion", "")) + prompt_payload = prompt if isinstance(prompt, dict) else MappingProxyType({}) + if row["ObservationType"] == "llm" and isinstance(completion, dict): + messages = prompt_payload.get("messages") or ((),) + batch = messages[0] if messages and isinstance(messages[0], list) else messages + row["Input"] = ( + json.dumps(tuple(_lc_message(m) for m in batch if isinstance(m, dict))) + if isinstance(batch, (list, tuple)) + else "" + ) + generations: Final = completion.get("generations") + first: Final = generations[0] if isinstance(generations, list) and generations else None + item: Final = first[0] if isinstance(first, list) and first else None + message: Final = item.get("message") if isinstance(item, dict) else None + generation: Final = message.get("kwargs") if isinstance(message, dict) else None + if isinstance(generation, dict): + row["Output"] = json.dumps(_lc_message(generation)) + metadata: Final = generation.get("response_metadata") + row["LiteLLMRequestId"] = metadata.get("id", "") if isinstance(metadata, dict) else "" + else: + row["Output"] = attributes.get("gen_ai.completion", "") + return + if row["ObservationType"] == "tool": + output = completion.get("output", completion) if isinstance(completion, dict) else completion + if isinstance(output, dict) and "update" in output: # LangGraph Command, e.g. Deep Agents `task` + update: Final = output.get("update") + update_messages = update.get("messages") or () if isinstance(update, dict) else () + output = update_messages[-1] if update_messages else output + if isinstance(output, dict): + output = output.get("content", output) + row["Input"] = attributes.get("gen_ai.prompt", "") + row["Output"] = output if isinstance(output, str) else json.dumps(output) + return + if row["ObservationType"] == "agent": + input_messages = prompt.get("messages") if isinstance(prompt, dict) else None + output_messages = completion.get("messages") if isinstance(completion, dict) else None + # agents built with @traceable take arbitrary args, not a message list: keep the raw payload then + row["Input"] = ( + json.dumps(tuple(_lc_message(m) for m in input_messages if isinstance(m, dict))) + if input_messages + else attributes.get("gen_ai.prompt", "") + ) + row["Output"] = ( + json.dumps(_lc_message(output_messages[-1])) + if output_messages and isinstance(output_messages[-1], dict) + else attributes.get("gen_ai.completion", "") + ) + return + row["Input"] = attributes.get("gen_ai.prompt", "") + row["Output"] = attributes.get("gen_ai.completion", "") + + +def normalize_langsmith(row: SpanRow, attributes: Mapping[str, str]) -> None: + row["ObservationType"] = _langsmith_type(row, attributes) + row["AgentName"] = attributes.get("langsmith.metadata.lc_agent_name", "") + row["Model"] = attributes.get("gen_ai.request.model", "") + _langsmith_io(row, attributes) + + +def normalize_genai(row: SpanRow, attributes: Mapping[str, str]) -> None: + operation = attributes.get("gen_ai.operation.name", "") + if operation == "invoke_agent" or not row["ParentSpanId"]: + row["ObservationType"] = "agent" + elif operation in _LLM_OPERATIONS: + row["ObservationType"] = "llm" + elif operation == "execute_tool": + row["ObservationType"] = "tool" + row["AgentName"] = attributes.get("gen_ai.agent.name", "") + row["Model"] = attributes.get("gen_ai.request.model") or attributes.get("gen_ai.response.model", "") + row["LiteLLMRequestId"] = attributes.get("gen_ai.response.id", "") + row["Input"] = attributes.get("gen_ai.input.messages") or attributes.get("gen_ai.tool.call.arguments", "") + row["Output"] = attributes.get("gen_ai.output.messages") or attributes.get("gen_ai.tool.call.result", "") + + +def normalize_openinference(row: SpanRow, attributes: Mapping[str, str]) -> None: + kind = attributes.get("openinference.span.kind", "").upper() + row["ObservationType"] = _OPENINFERENCE_TYPES.get(kind, "agent" if not row["ParentSpanId"] else "chain") + row["AgentName"] = attributes.get("agent.name", "") + row["Model"] = attributes.get("llm.model_name", "") + row["Input"] = attributes.get("input.value", "") + row["Output"] = attributes.get("output.value", "") + row["InputTokens"] = _to_int(attributes.get("llm.token_count.prompt")) + row["OutputTokens"] = _to_int(attributes.get("llm.token_count.completion")) + + +def _set_tokens(row: SpanRow, attributes: Mapping[str, str]) -> None: + row["InputTokens"] = _to_int(attributes.get("gen_ai.usage.input_tokens")) + row["OutputTokens"] = _to_int(attributes.get("gen_ai.usage.output_tokens")) + + +def _to_int(value: str | None) -> int: + try: + return int(value) if value else 0 + except ValueError: + return 0 + + +def select_normalizer(scope_name: str, attributes: Mapping[str, str]) -> Callable[[SpanRow, Mapping[str, str]], None]: + if scope_name == "langsmith" or "langsmith.span.kind" in attributes: + return normalize_langsmith + if "openinference.span.kind" in attributes: + return normalize_openinference + return normalize_genai + + +def normalize(row: SpanRow, attributes: Mapping[str, str]) -> None: + select_normalizer(row["ScopeName"], attributes)(row, attributes) + if not row["InputTokens"] and not row["OutputTokens"]: + _set_tokens(row, attributes) + + +def encode_otlp_response(content_type: str | None) -> tuple[bytes, str]: + """Empty ExportTraceServiceResponse in the caller's encoding.""" + if content_type and "json" in content_type: + return b"{}", "application/json" + return b"", "application/x-protobuf" diff --git a/litellm/tracing/receiver.py b/litellm/tracing/receiver.py new file mode 100644 index 00000000000..be9641a602b --- /dev/null +++ b/litellm/tracing/receiver.py @@ -0,0 +1,120 @@ +""" +`TraceReceiver`: the one entry point for agent tracing. + + tracing = TraceReceiver.from_env() # or TraceReceiver(store=...) + await tracing.start() # create tables if missing + + tracing.ingest(otlp_body, content_type, content_encoding, tenant) # POST /v1/traces + await tracing.list_traces(scope, start_ms, end_ms, cursor) # GET /v1/traces + await tracing.get_trace(trace_id, scope) # GET /v1/traces/{id} + await tracing.get_span(trace_id, span_id, scope) # GET /v1/traces/{id}/spans/{span_id} + +The proxy endpoints are thin wrappers: auth -> build tenant/scope -> call one method. +""" + +import asyncio +import os +from typing import Final + +from litellm.constants import ( + AGENT_TRACING_RETENTION_DAYS, + AGENT_TRACING_SPEND_LOG_RETENTION_DAYS, + OTLP_MAX_BODY_BYTES, + OTLP_OFFLOAD_DECODE_BYTES, +) +from litellm.integrations.clickhouse.schema import ensure_schema +from litellm.rust_bridge.traces import TraceStorage +from litellm.tracing.decode import OTLPPayloadTooLargeError, decode_otlp +from litellm.tracing.store import ClickHouseTraceStore +from litellm.tracing.types import ( + SpanDetail, + SpanRow, + Trace, + TracePage, + TraceScope, +) + + +class TracingPayloadTooLargeError(Exception): + pass + + +class Tenant: + """Who sent the spans. Always taken from auth, never from span attributes.""" + + def __init__(self, team_id: str, api_key_hash: str, org_id: str = "") -> None: + self.team_id = team_id + self.api_key_hash = api_key_hash + self.org_id = org_id + + def stamp(self, row: SpanRow) -> SpanRow: + row["TeamId"] = self.team_id + row["ApiKeyHash"] = self.api_key_hash + row["ResourceAttributes"] = { # mutable-ok: the Rust JSON bridge requires a plain dict + **row["ResourceAttributes"], + "litellm.team_id": self.team_id, + "litellm.api_key_hash": self.api_key_hash, + "litellm.org_id": self.org_id, + } + return row + + +class TraceReceiver: + def __init__(self, store: ClickHouseTraceStore) -> None: + self.store = store + + @classmethod + def from_env(cls) -> "TraceReceiver": + return cls( + store=ClickHouseTraceStore( + TraceStorage( + database=os.getenv("CLICKHOUSE_DATABASE", "litellm"), + url=os.environ["CLICKHOUSE_URL"], + reader_url=os.environ["CLICKHOUSE_READER_URL"], + ) + ) + ) + + async def start(self) -> None: + await ensure_schema( + self.store.storage, + trace_retention_days=AGENT_TRACING_RETENTION_DAYS, + spend_log_retention_days=AGENT_TRACING_SPEND_LOG_RETENTION_DAYS, + ) + + # ------------------------------------------------------------ write + + async def ingest( + self, + body: bytes, + content_type: str | None, + content_encoding: str | None, + tenant: Tenant, + ) -> int: + """Decode an OTLP trace export and store its authenticated spans.""" + if len(body) > OTLP_MAX_BODY_BYTES: + raise TracingPayloadTooLargeError(f"OTLP body exceeds {OTLP_MAX_BODY_BYTES} bytes") + try: + rows: Final = ( + await asyncio.to_thread(decode_otlp, body, content_type, content_encoding) + if len(body) > OTLP_OFFLOAD_DECODE_BYTES + else decode_otlp(body, content_type, content_encoding) + ) + except OTLPPayloadTooLargeError as error: + raise TracingPayloadTooLargeError(str(error)) from error + try: + await self.store.insert_spans(tuple(tenant.stamp(r) for r in rows)) + except OverflowError as error: + raise TracingPayloadTooLargeError(str(error)) from error + return len(rows) + + # ------------------------------------------------------------ read + + async def list_traces(self, scope: TraceScope, start_ms: int, end_ms: int, cursor: str | None = None) -> TracePage: + return await self.store.list_traces(scope, start_ms, end_ms, cursor) + + async def get_trace(self, trace_id: str, scope: TraceScope, trace_ref: str = "") -> Trace | None: + return await self.store.get_trace(trace_id, scope, trace_ref) + + async def get_span(self, trace_id: str, span_id: str, scope: TraceScope, trace_ref: str = "") -> SpanDetail | None: + return await self.store.get_span(trace_id, span_id, scope, trace_ref) diff --git a/litellm/tracing/store.py b/litellm/tracing/store.py new file mode 100644 index 00000000000..244eddd3def --- /dev/null +++ b/litellm/tracing/store.py @@ -0,0 +1,286 @@ +"""ClickHouse-backed trace store: batched span writes and scoped reads.""" + +import base64 +import binascii +import json +from collections.abc import Mapping, Sequence +from datetime import datetime, timezone +from types import MappingProxyType +from typing import Any, Final + +from litellm.constants import AGENT_TRACING_LIST_PAGE_SIZE +from litellm.integrations.clickhouse.schema import ( + AGENT_TRACES_BY_KEY_TABLE, + OTEL_TRACES_TABLE, +) +from litellm.rust_bridge.traces import TraceStorage +from litellm.tracing.types import ( + AgentNode, + Span, + SpanDetail, + SpanRow, + SpanStatus, + Trace, + TracePage, + TraceScope, + TraceSummary, +) + +NANOS_PER_MS: Final = 1_000_000 +_STATUS: Final = MappingProxyType({"STATUS_CODE_OK": "ok", "STATUS_CODE_ERROR": "error"}) + +_SCOPE_OTEL: Final = ( + "(empty({team_ids:Array(String)}) OR TeamId IN {team_ids:Array(String)})" + " AND ({api_key_hash:String} = '' OR ApiKeyHash = {api_key_hash:String})" +) +_TRACE_REF_SQL: Final = "hex(SHA256(concat(TeamId, char(0), ApiKeyHash, char(0), TraceId)))" +LIST_TRACES_SQL: Final = f""" +SELECT TraceId AS trace_id, {_TRACE_REF_SQL} AS trace_ref, + ifNull(any(RootName), '') AS name, any(ServiceName) AS service, + ifNull(any(RootInput), '') AS input_preview, ifNull(any(RootStatus), '') AS status, + toUnixTimestamp64Milli(min(StartTs)) AS start_ms, + dateDiff('millisecond', min(StartTs), max(EndTs)) AS duration_ms, + sum(SpanCount) AS span_count, length(groupUniqArrayArray(AgentNames)) AS agent_count, + sum(AgentCount) AS agent_invocations, + sum(LlmCount) AS llm_calls, sum(ToolCount) AS tool_calls, + sum(InputTokens) AS input_tokens, sum(OutputTokens) AS output_tokens, + groupUniqArrayArray(Models) AS models, sum(ErrorCount) AS error_count +FROM {AGENT_TRACES_BY_KEY_TABLE} +WHERE (empty({{team_ids:Array(String)}}) OR TeamId IN {{team_ids:Array(String)}}) + AND ({{api_key_hash:String}} = '' OR ApiKeyHash = {{api_key_hash:String}}) +GROUP BY TeamId, ApiKeyHash, TraceId +HAVING min(StartTs) >= fromUnixTimestamp64Milli({{start_ms:Int64}}) + AND min(StartTs) < fromUnixTimestamp64Milli({{end_ms:Int64}}) + AND ({{cursor_ms:Int64}} = 0 OR (toUnixTimestamp64Milli(min(StartTs)), trace_ref) + < ({{cursor_ms:Int64}}, {{cursor_trace_id:String}})) +ORDER BY start_ms DESC, trace_ref DESC +LIMIT {{limit:UInt32}} +""" + +TRACE_SPANS_SQL: Final = f""" +SELECT o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name, + o.ObservationType AS type, o.AgentName AS agent, o.StatusCode AS status, + o.StatusMessage AS status_message, + toUnixTimestamp64Nano(o.Timestamp) AS start_ns, o.Duration AS duration_ns, + o.ServiceName AS service, o.InputPreview AS input_preview, o.Model AS model, + o.InputTokens AS input_tokens, o.OutputTokens AS output_tokens, + o.LiteLLMRequestId AS litellm_request_id +FROM {OTEL_TRACES_TABLE} AS o +WHERE o.TraceId = {{trace_id:String}} AND {_SCOPE_OTEL} + AND ({{trace_ref:String}} = '' OR {_TRACE_REF_SQL} = {{trace_ref:String}}) +ORDER BY o.Timestamp +LIMIT 1 BY o.SpanId +""" + +SPAN_DETAIL_SQL: Final = f""" +SELECT SpanId AS span_id, Input AS input, Output AS output, SpanAttributes AS attributes +FROM {OTEL_TRACES_TABLE} +WHERE TraceId = {{trace_id:String}} AND SpanId = {{span_id:String}} AND {_SCOPE_OTEL} + AND ({{trace_ref:String}} = '' OR {_TRACE_REF_SQL} = {{trace_ref:String}}) +LIMIT 1 +""" + + +def encode_cursor(start_ms: int, trace_id: str) -> str: + return base64.urlsafe_b64encode(json.dumps((start_ms, trace_id)).encode()).decode() + + +def decode_cursor(cursor: str | None) -> tuple[int, str]: + if not cursor: + return 0, "" + try: + value: Final = json.loads(base64.b64decode(cursor, altchars=b"-_", validate=True)) + if ( + not isinstance(value, list) + or len(value) != 2 + or not isinstance(value[0], int) + or isinstance(value[0], bool) + or value[0] <= 0 + or not isinstance(value[1], str) + or not value[1] + ): + raise ValueError("Invalid trace cursor") + return value[0], value[1] + except (ValueError, UnicodeError, binascii.Error) as error: + raise ValueError("Invalid trace cursor") from error + + +def _iso(ms: int) -> str: + return datetime.fromtimestamp(ms / 1000, tz=timezone.utc).isoformat() + + +def _status(code: str) -> SpanStatus: + return _STATUS.get(code, "unset") + + +def trace_summary_from_row(row: dict[str, Any]) -> TraceSummary: + return TraceSummary( + trace_id=row["trace_id"], + trace_ref=row.get("trace_ref", ""), + name=row["name"], + service=row["service"], + input_preview=row["input_preview"], + start_time=_iso(int(row["start_ms"])), + duration_ms=float(row["duration_ms"]), + status=_status(row["status"]), + span_count=int(row["span_count"]), + agent_count=int(row["agent_count"]), + agent_invocations=int(row.get("agent_invocations") or row["agent_count"]), + llm_calls=int(row["llm_calls"]), + tool_calls=int(row["tool_calls"]), + error_count=int(row.get("error_count") or 0), + input_tokens=int(row["input_tokens"]), + output_tokens=int(row["output_tokens"]), + models=tuple(row["models"]), + ) + + +def span_from_row(row: dict[str, Any], trace_start_ns: int) -> Span: + return Span( + span_id=row["span_id"], + parent_span_id=row["parent_span_id"] or None, + name=row["name"], + type=row["type"], + agent=row["agent"], + start_offset_ms=(int(row["start_ns"]) - trace_start_ns) / NANOS_PER_MS, + duration_ms=int(row["duration_ns"]) / NANOS_PER_MS, + status=_status(row["status"]), + error=row.get("status_message") or None, + input_preview=row["input_preview"], + model=row["model"] or None, + input_tokens=int(row["input_tokens"]), + output_tokens=int(row["output_tokens"]), + litellm_request_id=row["litellm_request_id"] or None, + ) + + +def _parent_agent_of(span: Span, by_id: Mapping[str, Span]) -> str | None: + parent_id = span["parent_span_id"] + for _ in by_id: + if parent_id is None or parent_id not in by_id or parent_id == span["span_id"]: + return None + parent = by_id[parent_id] + if parent["type"] == "agent" and parent["name"] != span["name"]: + return parent["name"] + parent_id = parent["parent_span_id"] + return None + + +def agent_nodes(spans: Sequence[Span]) -> tuple[AgentNode, ...]: + """One node per distinct agent name (200 `researcher` invocations = 1 node), with who invoked it.""" + by_id: Final = MappingProxyType({s["span_id"]: s for s in spans}) + agents: dict[str, AgentNode] = {} # mutable-ok: linear-time aggregation updates counters per agent + for span in spans: + if span["type"] != "agent": + continue + node = agents.setdefault( + span["name"], + AgentNode( + name=span["name"], + parent_agent=_parent_agent_of(span, by_id), + invocations=0, + llm_calls=0, + tool_calls=0, + duration_ms=0.0, + ), + ) + node["invocations"] += 1 + node["duration_ms"] += span["duration_ms"] + for span in spans: + owner = agents.get(span["agent"]) + if owner is None: + continue + if span["type"] == "llm": + owner["llm_calls"] += 1 + elif span["type"] == "tool": + owner["tool_calls"] += 1 + return tuple(agents.values()) + + +def trace_from_rows(trace_id: str, rows: list[dict[str, Any]], trace_ref: str = "") -> Trace | None: + if not rows: + return None + trace_start_ns: Final = min(int(r["start_ns"]) for r in rows) + trace_end_ns: Final = max(int(r["start_ns"]) + int(r["duration_ns"]) for r in rows) + spans: Final = tuple(span_from_row(r, trace_start_ns) for r in rows) + root: Final = next((s for s in spans if s["parent_span_id"] is None), spans[0]) + agents: Final = agent_nodes(spans) + llm_spans: Final = tuple(s for s in spans if s["type"] == "llm") + return Trace( + summary=TraceSummary( + trace_id=trace_id, + trace_ref=trace_ref, + name=root["name"], + service=rows[0]["service"], + input_preview=root["input_preview"], + start_time=_iso(trace_start_ns // NANOS_PER_MS), + duration_ms=(trace_end_ns - trace_start_ns) / NANOS_PER_MS, + status=root["status"], + span_count=len(spans), + agent_count=len(agents), + agent_invocations=sum(a["invocations"] for a in agents), + llm_calls=len(llm_spans), + tool_calls=sum(1 for s in spans if s["type"] == "tool"), + error_count=sum(1 for s in spans if s["status"] == "error"), + input_tokens=sum(s["input_tokens"] for s in spans), + output_tokens=sum(s["output_tokens"] for s in spans), + models=tuple(sorted(frozenset(s["model"] for s in llm_spans if s["model"]))), + ), + agents=agents, + spans=spans, + ) + + +class ClickHouseTraceStore: + """Stores spans and runs scoped trace reads.""" + + def __init__(self, storage: TraceStorage) -> None: + self.storage = storage + + async def insert_spans(self, rows: Sequence[SpanRow]) -> None: + await self.storage.insert_rows(OTEL_TRACES_TABLE, tuple(rows)) + + async def list_traces( + self, + scope: TraceScope, + start_ms: int, + end_ms: int, + cursor: str | None = None, + limit: int = AGENT_TRACING_LIST_PAGE_SIZE, + ) -> TracePage: + cursor_ms, cursor_trace_id = decode_cursor(cursor) + rows = await self.storage.query( + LIST_TRACES_SQL, + MappingProxyType( + { + **scope, + "start_ms": start_ms, + "end_ms": end_ms, + "cursor_ms": cursor_ms, + "cursor_trace_id": cursor_trace_id, + "limit": limit, + } + ), + ) + next_cursor = encode_cursor(int(rows[-1]["start_ms"]), rows[-1]["trace_ref"]) if len(rows) == limit else None + return TracePage(data=tuple(trace_summary_from_row(r) for r in rows), next_cursor=next_cursor) + + async def get_trace(self, trace_id: str, scope: TraceScope, trace_ref: str = "") -> Trace | None: + rows = await self.storage.query( + TRACE_SPANS_SQL, MappingProxyType({**scope, "trace_id": trace_id, "trace_ref": trace_ref}) + ) + return trace_from_rows(trace_id, rows, trace_ref) + + async def get_span(self, trace_id: str, span_id: str, scope: TraceScope, trace_ref: str = "") -> SpanDetail | None: + rows = await self.storage.query( + SPAN_DETAIL_SQL, + MappingProxyType({**scope, "trace_id": trace_id, "span_id": span_id, "trace_ref": trace_ref}), + ) + if not rows: + return None + return SpanDetail( + span_id=rows[0]["span_id"], + input=rows[0]["input"], + output=rows[0]["output"], + attributes=rows[0]["attributes"], + ) diff --git a/litellm/tracing/types.py b/litellm/tracing/types.py new file mode 100644 index 00000000000..b8b6f646111 --- /dev/null +++ b/litellm/tracing/types.py @@ -0,0 +1,120 @@ +""" +Agent tracing types. + +A trace is one agent run. It's made of spans (agent / llm / tool / chain / framework). + Trace + ├── summary: TraceSummary + ├── agents: list[AgentNode] one per distinct agent name (for the agent graph) + └── spans: list[Span] flat, linked by parent_span_id + +""" + +from typing import Literal + +from typing_extensions import NotRequired, ReadOnly, TypedDict + +SpanType = Literal["agent", "llm", "tool", "chain", "framework"] +SpanStatus = Literal["ok", "error", "unset"] + + +class Span(TypedDict): + span_id: ReadOnly[str] + parent_span_id: ReadOnly[str | None] + name: ReadOnly[str] + type: ReadOnly[SpanType] + agent: ReadOnly[str] # the agent this span runs inside, e.g. "researcher" + start_offset_ms: ReadOnly[float] # relative to trace start + duration_ms: ReadOnly[float] + status: ReadOnly[SpanStatus] + error: ReadOnly[str | None] # exception message when status == "error" + input_preview: ReadOnly[str] + model: ReadOnly[str | None] + input_tokens: ReadOnly[int] + output_tokens: ReadOnly[int] + litellm_request_id: ReadOnly[str | None] + + +class AgentNode(TypedDict): + """One distinct agent in a trace. 200 invocations of `researcher` = one node.""" + + name: ReadOnly[str] + parent_agent: ReadOnly[str | None] + invocations: int + llm_calls: int + tool_calls: int + duration_ms: float + + +class TraceSummary(TypedDict): + trace_id: ReadOnly[str] + trace_ref: ReadOnly[NotRequired[str]] + name: ReadOnly[str] + service: ReadOnly[str] + input_preview: ReadOnly[str] + start_time: ReadOnly[str] # ISO 8601 + duration_ms: ReadOnly[float] + status: ReadOnly[SpanStatus] + span_count: ReadOnly[int] + agent_count: ReadOnly[int] # distinct agent names (researcher x200 counts once) + agent_invocations: ReadOnly[int] # agent spans (researcher x200 counts 200) + llm_calls: ReadOnly[int] + tool_calls: ReadOnly[int] + error_count: ReadOnly[int] # spans with an error status; > 0 means the run shows as failed + input_tokens: ReadOnly[int] + output_tokens: ReadOnly[int] + models: ReadOnly[tuple[str, ...]] + + +class Trace(TypedDict): + summary: ReadOnly[TraceSummary] + agents: ReadOnly[tuple[AgentNode, ...]] + spans: ReadOnly[tuple[Span, ...]] + + +class TracePage(TypedDict): + data: ReadOnly[tuple[TraceSummary, ...]] + next_cursor: ReadOnly[str | None] + + +class SpanDetail(TypedDict): + span_id: ReadOnly[str] + input: ReadOnly[str] + output: ReadOnly[str] + attributes: ReadOnly[dict[str, str]] + + +class TraceScope(TypedDict): + """Who is asking. Empty team_ids = all teams (admins only).""" + + team_ids: ReadOnly[tuple[str, ...]] + api_key_hash: ReadOnly[str] + + +class SpanRow(TypedDict): + """One stored span (ClickHouse `otel_traces` row). Produced by `litellm.tracing.decode`.""" + + Timestamp: ReadOnly[int] # unix ns + TraceId: ReadOnly[str] + SpanId: ReadOnly[str] + ParentSpanId: ReadOnly[str] + TraceState: ReadOnly[str] + SpanName: ReadOnly[str] + SpanKind: ReadOnly[str] + ServiceName: ReadOnly[str] + ResourceAttributes: dict[str, str] + ScopeName: ReadOnly[str] + ScopeVersion: ReadOnly[str] + SpanAttributes: dict[str, str] + Duration: ReadOnly[int] # ns + StatusCode: ReadOnly[str] + StatusMessage: ReadOnly[str] + TeamId: str + ApiKeyHash: str + ObservationType: SpanType + AgentName: str + LiteLLMRequestId: str + Model: str + InputTokens: int + OutputTokens: int + Input: str + Output: str diff --git a/tests/test_litellm/integrations/clickhouse/test_clickhouse_batch_logger.py b/tests/test_litellm/integrations/clickhouse/test_clickhouse_batch_logger.py new file mode 100644 index 00000000000..e5b00b3c783 --- /dev/null +++ b/tests/test_litellm/integrations/clickhouse/test_clickhouse_batch_logger.py @@ -0,0 +1,67 @@ +""" +Tests for the CustomBatchLogger-based ClickHouse base logger. +""" + +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from litellm.integrations.clickhouse import clickhouse_batch_logger as module +from litellm.integrations.clickhouse.clickhouse_batch_logger import ClickHouseBatchLogger +from litellm.integrations.custom_batch_logger import CustomBatchLogger + + +class _TestLogger(ClickHouseBatchLogger): + table = "test_table" + + +def _logger(insert: AsyncMock) -> _TestLogger: + storage = MagicMock() + storage.insert_rows = insert + return _TestLogger(storage=storage) + + +def test_is_a_custom_batch_logger(): + assert issubclass(ClickHouseBatchLogger, CustomBatchLogger) + + +@pytest.mark.asyncio +async def test_flush_splits_into_batches_and_empties_queue(): + insert = AsyncMock() + logger = _logger(insert) + logger.batch_size = 2 + logger.log_queue.extend([{"i": i} for i in range(5)]) + + await logger.flush_queue() + + assert [len(c.args[1]) for c in insert.await_args_list] == [2, 2, 1] + assert all(c.args[0] == "test_table" for c in insert.await_args_list) + assert logger.log_queue == [] + assert logger.rows_written == 5 + + +@pytest.mark.asyncio +async def test_is_full_signals_backpressure(): + logger = _logger(AsyncMock()) + with patch.object(module, "CLICKHOUSE_MAX_BUFFERED_ROWS", 3): + logger.log_queue.extend([{}, {}]) + assert logger.is_full() is False + logger.log_queue.append({}) + assert logger.is_full() is True + + +@pytest.mark.asyncio +async def test_failed_insert_is_requeued_then_dropped(): + insert = AsyncMock(side_effect=RuntimeError("clickhouse down")) + logger = _logger(insert) + logger.log_queue.extend([{"request_id": "a"}, {"request_id": "b"}]) + + with patch.object(module, "CLICKHOUSE_MAX_RETRIES", 2): + await logger.flush_queue() + assert len(logger.log_queue) == 2 # kept for retry + await logger.flush_queue() + + assert insert.await_count == 2 + assert logger.rows_dropped == 2 + assert logger.rows_written == 0 + assert logger.log_queue == [] diff --git a/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py b/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py index 6087aa1f570..e3851f6c21a 100644 --- a/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py +++ b/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py @@ -1,3 +1,4 @@ +import gzip import io import json from collections.abc import Mapping @@ -31,12 +32,14 @@ from litellm.proxy.common_utils.http_parsing_utils import ( ) -def _starlette_request(body: bytes, content_type: str) -> Request: +def _starlette_request( + body: bytes, content_type: str, path: str = "/v1/messages", content_encoding: str = "" +) -> Request: scope = { "type": "http", "method": "POST", - "path": "/v1/messages", - "headers": [(b"content-type", content_type.encode())], + "path": path, + "headers": [(b"content-type", content_type.encode()), (b"content-encoding", content_encoding.encode())], "query_string": b"", } chunks = iter((body,)) @@ -72,6 +75,26 @@ async def test_read_raw_json_body_is_none_for_form_bodies(): assert await read_raw_json_body(request) is None +@pytest.mark.asyncio +@pytest.mark.parametrize("content_type", ["application/x-protobuf", "application/protobuf; charset=binary"]) +async def test_protobuf_body_is_not_parsed_as_json(content_type): + # OTLP trace exports (POST /v1/traces) are binary protobuf; arbitrary bytes like these + # used to hit the JSON surrogate-repair path and fail auth with a 400. + body = b"\n\xa2\x01\n\x1c\n\x0cservice.name\x12\x0c\n\nswarm\xed\xa0\x80\xff" + request = _starlette_request(body, content_type) + + assert await _read_request_body(request) == {} + assert await request.body() == body # body is still readable by the endpoint + + +@pytest.mark.asyncio +async def test_gzipped_json_trace_body_survives_auth_pre_read(): + body = gzip.compress(b'{"resourceSpans": []}') + request = _starlette_request(body, "application/json", "/v1/traces", "gzip") + assert await _read_request_body(request) == {} + assert await request.body() == body + + @pytest.mark.asyncio async def test_read_raw_json_body_is_none_for_a_request_that_only_mocks_the_parsed_body_path(): mock_request = MagicMock() diff --git a/tests/test_litellm/proxy/test_tracing_endpoints.py b/tests/test_litellm/proxy/test_tracing_endpoints.py new file mode 100644 index 00000000000..4c7c70a39f3 --- /dev/null +++ b/tests/test_litellm/proxy/test_tracing_endpoints.py @@ -0,0 +1,180 @@ +""" +Tests for the agent tracing endpoints (litellm/proxy/tracing_endpoints.py). +""" + +from unittest.mock import AsyncMock, MagicMock + +import pytest +from fastapi import FastAPI, HTTPException +from fastapi.testclient import TestClient + +from litellm.proxy import tracing_endpoints +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.tracing import TracingPayloadTooLargeError + +TEAM_KEY = UserAPIKeyAuth( + token="hashed-key", team_id="team-research", org_id="org-1", user_role=LitellmUserRoles.INTERNAL_USER +) + + +# ---------------------------------------------------------------- scope / tenant + + +def test_scope_for_admin_sees_everything(): + for role in (LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY): + auth = UserAPIKeyAuth(token="k", team_id="team-a", user_role=role) + assert tracing_endpoints.scope_for(auth) == {"team_ids": (), "api_key_hash": ""} + + +def test_scope_for_team_key_sees_its_team(): + assert tracing_endpoints.scope_for(TEAM_KEY) == {"team_ids": ("team-research",), "api_key_hash": ""} + + +def test_scope_for_teamless_key_sees_only_its_own_traces(): + auth = UserAPIKeyAuth(token="hashed-key", user_role=LitellmUserRoles.INTERNAL_USER) + assert tracing_endpoints.scope_for(auth) == {"team_ids": ("",), "api_key_hash": "hashed-key"} + + +def test_scope_for_no_team_no_token_is_forbidden(): + with pytest.raises(HTTPException) as e: + tracing_endpoints.scope_for(UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER)) + assert e.value.status_code == 403 + + +def test_tenant_for_comes_from_auth(): + tenant = tracing_endpoints.tenant_for(TEAM_KEY) + assert (tenant.team_id, tenant.api_key_hash, tenant.org_id) == ("team-research", "hashed-key", "org-1") + blank = tracing_endpoints.tenant_for(UserAPIKeyAuth()) + assert (blank.team_id, blank.api_key_hash, blank.org_id) == ("", "", "") + + +# ---------------------------------------------------------------- endpoints + + +@pytest.fixture +def receiver(monkeypatch) -> MagicMock: + fake = MagicMock() + fake.ingest = AsyncMock(return_value=1) + fake.list_traces = AsyncMock(return_value={"data": [], "next_cursor": None}) + fake.get_trace = AsyncMock(return_value=None) + fake.get_span = AsyncMock(return_value=None) + monkeypatch.setattr(tracing_endpoints, "receiver", fake) + return fake + + +@pytest.fixture +def client() -> TestClient: + app = FastAPI() + app.include_router(tracing_endpoints.router) + app.dependency_overrides[user_api_key_auth] = lambda: TEAM_KEY + return TestClient(app) + + +def test_501_when_tracing_not_enabled(client, monkeypatch): + monkeypatch.setattr(tracing_endpoints, "receiver", None) + assert client.post("/v1/traces", content=b"").status_code == 501 + assert client.get("/v1/traces").status_code == 501 + + +def test_post_protobuf_returns_empty_protobuf(client, receiver): + response = client.post( + "/v1/traces", + content=b"\x0a\x00", + headers={"content-type": "application/x-protobuf", "content-encoding": "gzip"}, + ) + assert response.status_code == 200 + assert response.content == b"" + assert response.headers["content-type"] == "application/x-protobuf" + kwargs = receiver.ingest.call_args.kwargs + assert kwargs["body"] == b"\x0a\x00" + assert kwargs["content_type"] == "application/x-protobuf" + assert kwargs["content_encoding"] == "gzip" + assert kwargs["tenant"].team_id == "team-research" + + +def test_post_json_returns_empty_json(client, receiver): + response = client.post("/v1/traces", content=b"{}", headers={"content-type": "application/json"}) + assert response.status_code == 200 + assert response.json() == {} + + +def test_post_clickhouse_failure_is_503_with_retry_after(client, receiver): + receiver.ingest.side_effect = RuntimeError("ClickHouse unavailable") + response = client.post("/v1/traces", content=b"", headers={"content-type": "application/x-protobuf"}) + assert response.status_code == 503 + assert response.headers["retry-after"] == str(tracing_endpoints.OTLP_RETRY_AFTER_SECONDS) + + +def test_post_too_large_is_413(client, receiver): + receiver.ingest.side_effect = TracingPayloadTooLargeError("OTLP body exceeds 10 bytes") + response = client.post("/v1/traces", content=b"x" * 20) + assert response.status_code == 413 + assert "exceeds" in response.json()["detail"] + + +def test_list_traces_passes_scope_window_and_cursor(client, receiver): + response = client.get("/v1/traces", params={"start_ms": 1, "end_ms": 2, "cursor": "abc"}) + assert response.status_code == 200 + assert response.json() == {"data": [], "next_cursor": None} + receiver.list_traces.assert_awaited_once_with( + scope={"team_ids": ("team-research",), "api_key_hash": ""}, start_ms=1, end_ms=2, cursor="abc" + ) + + +def test_list_traces_defaults_to_last_24h(client, receiver): + client.get("/v1/traces") + kwargs = receiver.list_traces.call_args.kwargs + assert kwargs["end_ms"] - kwargs["start_ms"] == tracing_endpoints.MS_PER_DAY + assert kwargs["cursor"] is None + + +def test_get_trace_404_and_200(client, receiver): + assert client.get("/v1/traces/missing").status_code == 404 + trace = {"summary": {"trace_id": "t1"}, "agents": [], "spans": []} + receiver.get_trace.return_value = trace + response = client.get("/v1/traces/t1") + assert response.status_code == 200 + assert response.json() == trace + receiver.get_trace.assert_awaited_with("t1", {"team_ids": ("team-research",), "api_key_hash": ""}, "") + + +def test_get_span_404_and_200(client, receiver): + assert client.get("/v1/traces/t1/spans/s1").status_code == 404 + receiver.get_span.return_value = {"span_id": "s1", "input": "", "output": "", "attributes": {}} + response = client.get("/v1/traces/t1/spans/s1") + assert response.status_code == 200 + assert response.json()["span_id"] == "s1" + receiver.get_span.assert_awaited_with("t1", "s1", {"team_ids": ("team-research",), "api_key_hash": ""}, "") + + +def test_trace_detail_passes_scoped_reference(client, receiver): + receiver.get_trace.return_value = {"summary": {"trace_id": "t1"}, "agents": [], "spans": []} + assert client.get("/v1/traces/t1?trace_ref=run-one").status_code == 200 + receiver.get_trace.assert_awaited_with("t1", {"team_ids": ("team-research",), "api_key_hash": ""}, "run-one") + + +def test_invalid_export_and_cursor_are_client_errors(client, receiver): + from litellm.tracing.decode import InvalidOTLPPayloadError + + receiver.ingest.side_effect = InvalidOTLPPayloadError("invalid OTLP trace payload") + assert client.post("/v1/traces", content=b"broken").status_code == 400 + receiver.list_traces.side_effect = ValueError("Invalid trace cursor") + assert client.get("/v1/traces?cursor=broken").status_code == 400 + + +def test_teamless_key_without_token_gets_403_on_reads(client, receiver): + client.app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER + ) + assert client.get("/v1/traces").status_code == 403 + receiver.list_traces.assert_not_called() + + +def test_view_only_admin_cannot_ingest_traces(client, receiver): + client.app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( + token="admin-key", user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY + ) + response = client.post("/v1/traces", content=b"{}") + assert response.status_code == 403 + receiver.ingest.assert_not_called() diff --git a/tests/test_litellm/tracing/fixtures/langsmith_deep_agent_export.json b/tests/test_litellm/tracing/fixtures/langsmith_deep_agent_export.json new file mode 100644 index 00000000000..9bd8e67633b --- /dev/null +++ b/tests/test_litellm/tracing/fixtures/langsmith_deep_agent_export.json @@ -0,0 +1,924 @@ +{ + "resourceSpans": [ + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.45.0" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "86db1687-77ed-422d-a6f7-0319594d9158" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "agent-demo" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "langsmith" + }, + "spans": [ + { + "traceId": "S61CuE6d47pG/IcBhfjwIw==", + "spanId": "XnnztbUEmF4=", + "name": "deep_research_agent", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1790742989377137920", + "endTimeUnixNano": "1790743040762587136", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "chain" + } + }, + { + "key": "langsmith.span.kind", + "value": { + "stringValue": "chain" + } + }, + { + "key": "langsmith.trace.name", + "value": { + "stringValue": "deep_research_agent" + } + }, + { + "key": "langsmith.trace.session_name", + "value": { + "stringValue": "default" + } + }, + { + "key": "gen_ai.system", + "value": { + "stringValue": "langchain" + } + }, + { + "key": "langsmith.metadata.ls_integration", + "value": { + "stringValue": "deepagents" + } + }, + { + "key": "langsmith.metadata.lc_agent_name", + "value": { + "stringValue": "deep_research_agent" + } + }, + { + "key": "langsmith.metadata.lc_versions", + "value": { + "stringValue": "{\"deepagents\":\"0.7.20\"}" + } + }, + { + "key": "langsmith.metadata.LANGSMITH_TRACING", + "value": { + "stringValue": "true" + } + }, + { + "key": "langsmith.metadata.LANGSMITH_TRACING_MODE", + "value": { + "stringValue": "otel" + } + }, + { + "key": "gen_ai.prompt", + "value": { + "bytesValue": "eyJtZXNzYWdlcyI6W3siY29udGVudCI6IlNob3VsZCB3ZSBzdG9yZSBPVEVMIGFnZW50IHNwYW5zIGluIENsaWNrSG91c2Ugb3IgUG9zdGdyZXMgYXQgNTBrIHNwYW5zL3NlYz8iLCJhZGRpdGlvbmFsX2t3YXJncyI6e30sInJlc3BvbnNlX21ldGFkYXRhIjp7fSwidHlwZSI6Imh1bWFuIiwiaWQiOiJiMTljODgzMS0wOWIwLTQ5ZjYtYjdlYS05YzQ3ZTM4OWNjMDAifV19" + } + }, + { + "key": "gen_ai.completion", + "value": { + "bytesValue": "eyJtZXNzYWdlcyI6W3siY29udGVudCI6IlNob3VsZCB3ZSBzdG9yZSBPVEVMIGFnZW50IHNwYW5zIGluIENsaWNrSG91c2Ugb3IgUG9zdGdyZXMgYXQgNTBrIHNwYW5zL3NlYz8iLCJhZGRpdGlvbmFsX2t3YXJncyI6e30sInJlc3BvbnNlX21ldGFkYXRhIjp7fSwidHlwZSI6Imh1bWFuIiwiaWQiOiJiMTljODgzMS0wOWIwLTQ5ZjYtYjdlYS05YzQ3ZTM4OWNjMDAifSx7ImNvbnRlbnQiOiJJJ2xsIGhlbHAgeW91IGRlY2lkZSBiZXR3ZWVuIENsaWNrSG91c2UgYW5kIFBvc3RncmVzIGZvciBzdG9yaW5nIE9wZW5UZWxlbWV0cnkgc3BhbnMgYXQgNTBrIHNwYW5zL3NlYy4gTGV0IG1lIHJlc2VhcmNoIHRoaXMgc3lzdGVtYXRpY2FsbHkuIiwiYWRkaXRpb25hbF9rd2FyZ3MiOnsicmVmdXNhbCI6bnVsbH0sInJlc3BvbnNlX21ldGFkYXRhIjp7InRva2VuX3VzYWdlIjp7ImNvbXBsZXRpb25fdG9rZW5zIjo0NjcsInByb21wdF90b2tlbnMiOjMzMzIsInRvdGFsX3Rva2VucyI6Mzc5OSwiY29tcGxldGlvbl90b2tlbnNfZGV0YWlscyI6eyJhY2NlcHRlZF9wcmVkaWN0aW9uX3Rva2VucyI6bnVsbCwiYXVkaW9fdG9rZW5zIjpudWxsLCJyZWFzb25pbmdfdG9rZW5zIjowLCJyZWplY3RlZF9wcmVkaWN0aW9uX3Rva2VucyI6bnVsbCwidGV4dF90b2tlbnMiOjQ2N30sInByb21wdF90b2tlbnNfZGV0YWlscyI6eyJhdWRpb190b2tlbnMiOm51bGwsImNhY2hlX3dyaXRlX3Rva2VucyI6MzMyOSwiY2FjaGVkX3Rva2VucyI6MCwiaW1hZ2VfdG9rZW5zIjpudWxsLCJ0ZXh0X3Rva2VucyI6MywiY2FjaGVfY3JlYXRpb25fdG9rZW5zIjozMzI5LCJjYWNoZV9jcmVhdGlvbl90b2tlbl9kZXRhaWxzIjp7ImVwaGVtZXJhbF81bV9pbnB1dF90b2tlbnMiOjMzMjksImVwaGVtZXJhbF8xaF9pbnB1dF90b2tlbnMiOjB9fSwiY2FjaGVfY3JlYXRpb25faW5wdXRfdG9rZW5zIjozMzI5LCJjYWNoZV9yZWFkX2lucHV0X3Rva2VucyI6MCwiaW5mZXJlbmNlX2dlbyI6Im5vdF9hdmFpbGFibGUiLCJzZXJ2aWNlX3RpZXIiOiJzdGFuZGFyZCJ9LCJtb2RlbF9wcm92aWRlciI6Im9wZW5haSIsIm1vZGVsX25hbWUiOiJjbGF1ZGUtc29ubmV0LTQtNSIsInN5c3RlbV9maW5nZXJwcmludCI6bnVsbCwiaWQiOiJjaGF0Y21wbC00MDc3YmIzNi05MzgwLTRhM2ItOTQ4MS0yNDU3MDBjZWYwOWEiLCJmaW5pc2hfcmVhc29uIjoidG9vbF9jYWxscyIsImxvZ3Byb2JzIjpudWxsfSwidHlwZSI6ImFpIiwibmFtZSI6ImRlZXBfcmVzZWFyY2hfYWdlbnQiLCJpZCI6ImxjX3J1bi0tMDFhMGYwOTktOGE0Ny03ZTQyLWE1ZjQtNWM0N2RlM2QxY2VjLTAiLCJ0b29sX2NhbGxzIjpbeyJuYW1lIjoid3JpdGVfZmlsZSIsImFyZ3MiOnsiZmlsZV9wYXRoIjoiL3RtcC9yZXNlYXJjaF90b2Rvcy5tZCIsImNvbnRlbnQiOiIjIFJlc2VhcmNoIFBsYW46IENsaWNrSG91c2UgdnMgUG9zdGdyZXMgZm9yIE9URUwgU3BhbnMgKDUway9zZWMpXG5cbiMjIFRhc2tzXG4tIFsgXSBSZXNlYXJjaCBDbGlja0hvdXNlIGFuZCBQb3N0Z3JlcyBjYXBhYmlsaXRpZXMgZm9yIGhpZ2gtdm9sdW1lIHRpbWUtc2VyaWVzIGRhdGFcbi4uLiJ9LCJpZCI6InRvb2x1XzAxNjFYaFlQM0I1Zmc0VTFwc1QzcGNpUiIsInR5cGUiOiJ0b29sX2NhbGwifSx7Im5hbWUiOiJ0YXNrIiwiYXJncyI6eyJzdWJhZ2VudF90eXBlIjoicmVzZWFyY2hlciIsImRlc2NyaXB0aW9uIjoiUmVzZWFyY2ggYW5kIGNvbXBhcmUgQ2xpY2tIb3VzZSB2cyBQb3N0Z3JlcyBmb3Igc3RvcmluZyBPcGVuVGVsZW1ldHJ5IChPVEVMKSBhZ2VudCBzcGFucyBhdCA1MCwwMDAgc3BhbnMgcGVyIHNlY29uZC5cblxuRm9jdXMgb246XG4xLiBXcml0ZSB0aHJvdWdocHV0IGNhcGFiaWxpdGllcy4uLiJ9LCJpZCI6InRvb2x1XzAxUEx5bzhUS0tUcFhSNGZwOTZEbjkzVyIsInR5cGUiOiJ0b29sX2NhbGwifV0sImludmFsaWRfdG9vbF9jYWxscyI6W10sInVzYWdlX21ldGFkYXRhIjp7ImlucHV0X3Rva2VucyI6MzMzMiwib3V0cHV0X3Rva2VucyI6NDY3LCJ0b3RhbF90b2tlbnMiOjM3OTksImlucHV0X3Rva2VuX2RldGFpbHMiOnsiY2FjaGVfcmVhZCI6MCwiY2FjaGVfY3JlYXRpb24iOjMzMjl9LCJvdXRwdXRfdG9rZW5fZGV0YWlscyI6eyJyZWFzb25pbmciOjB9fX0seyJjb250ZW50IjoiQmFzZWQgb24gbXkgcmVzZWFyY2gsIGhlcmUncyBhIGNvbXByZWhlbnNpdmUgY29tcGFyaXNvbiBvZiAqKkNsaWNrSG91c2UgdnMgUG9zdGdyZXMqKiBmb3Igc3RvcmluZyBPcGVuVGVsZW1ldHJ5IHNwYW5zIGF0IDUwLDAwMCBzcGFucy9zZWNvbmQ6XG5cbiMjICoqMS4gV3JpdGUgVGhyLi4uIiwiYWRkaXRpb25hbF9rd2FyZ3MiOnt9LCJyZXNwb25zZV9tZXRhZGF0YSI6e30sInR5cGUiOiJ0b29sIiwibmFtZSI6InRhc2siLCJpZCI6IjE0MzVkZTNjLWI4NzktNDQ2YS04MDU0LTFiMGI4MjQ1YmZhZSIsInRvb2xfY2FsbF9pZCI6InRvb2x1XzAxUEx5bzhUS0tUcFhSNGZwOTZEbjkzVyIsInN0YXR1cyI6InN1Y2Nlc3MifSx7ImNvbnRlbnQiOiJCYXNlZCBvbiB0aGUgcmVzZWFyY2ggZmluZGluZ3MsIGhlcmUncyBteSByZWNvbW1lbmRhdGlvbjpcblxuIyMgUmVjb21tZW5kYXRpb246ICoqVXNlIENsaWNrSG91c2UqKlxuXG4qKkNsaWNrSG91c2UgaXMgdGhlIGNsZWFyIGNob2ljZSoqIGZvciBzdG9yaW5nIDUwayBPVEVMIHNwYW5zLy4uLiIsImFkZGl0aW9uYWxfa3dhcmdzIjp7InJlZnVzYWwiOm51bGx9LCJyZXNwb25zZV9tZXRhZGF0YSI6eyJ0b2tlbl91c2FnZSI6eyJjb21wbGV0aW9uX3Rva2VucyI6MjQxLCJwcm9tcHRfdG9rZW5zIjo0NTcwLCJ0b3RhbF90b2tlbnMiOjQ4MTEsImNvbXBsZXRpb25fdG9rZW5zX2RldGFpbHMiOnsiYWNjZXB0ZWRfcHJlZGljdGlvbl90b2tlbnMiOm51bGwsImF1ZGlvX3Rva2VucyI6bnVsbCwicmVhc29uaW5nX3Rva2VucyI6MCwicmVqZWN0ZWRfcHJlZGljdGlvbl90b2tlbnMiOm51bGwsInRleHRfdG9rZW5zIjoyNDF9LCJwcm9tcHRfdG9rZW5zX2RldGFpbHMiOnsiYXVkaW9fdG9rZW5zIjpudWxsLCJjYWNoZV93cml0ZV90b2tlbnMiOjEyMzQsImNhY2hlZF90b2tlbnMiOjMzMjksImltYWdlX3Rva2VucyI6bnVsbCwidGV4dF90b2tlbnMiOjcsImNhY2hlX2NyZWF0aW9uX3Rva2VucyI6MTIzNCwiY2FjaGVfY3JlYXRpb25fdG9rZW5fZGV0YWlscyI6eyJlcGhlbWVyYWxfNW1faW5wdXRfdG9rZW5zIjoxMjM0LCJlcGhlbWVyYWxfMWhfaW5wdXRfdG9rZW5zIjowfX0sImNhY2hlX2NyZWF0aW9uX2lucHV0X3Rva2VucyI6MTIzNCwiY2FjaGVfcmVhZF9pbnB1dF90b2tlbnMiOjMzMjksImluZmVyZW5jZV9nZW8iOiJub3RfYXZhaWxhYmxlIiwic2VydmljZV90aWVyIjoic3RhbmRhcmQifSwibW9kZWxfcHJvdmlkZXIiOiJvcGVuYWkiLCJtb2RlbF9uYW1lIjoiY2xhdWRlLXNvbm5ldC00LTUiLCJzeXN0ZW1fZmluZ2VycHJpbnQiOm51bGwsImlkIjoiY2hhdGNtcGwtZjI2Y2NiNDUtYWIxYi00NGM2LWJkOWUtNDFhMDJjYTVmMTRkIiwiZmluaXNoX3JlYXNvbiI6InN0b3AiLCJsb2dwcm9icyI6bnVsbH0sInR5cGUiOiJhaSIsIm5hbWUiOiJkZWVwX3Jlc2VhcmNoX2FnZW50IiwiaWQiOiJsY19ydW4tLTAxYTBmMDlhLTM4ZTEtNzc0My04ZGIwLTNjNjU5YjdlMGY2MC0wIiwidG9vbF9jYWxscyI6W10sImludmFsaWRfdG9vbF9jYWxscyI6W10sInVzYWdlX21ldGFkYXRhIjp7ImlucHV0X3Rva2VucyI6NDU3MCwib3V0cHV0X3Rva2VucyI6MjQxLCJ0b3RhbF90b2tlbnMiOjQ4MTEsImlucHV0X3Rva2VuX2RldGFpbHMiOnsiY2FjaGVfcmVhZCI6MzMyOSwiY2FjaGVfY3JlYXRpb24iOjEyMzR9LCJvdXRwdXRfdG9rZW5fZGV0YWlscyI6eyJyZWFzb25pbmciOjB9fX1dLCJmaWxlcyI6eyIvdG1wL3Jlc2VhcmNoX3RvZG9zLm1kIjp7ImNvbnRlbnQiOiIjIFJlc2VhcmNoIFBsYW46IENsaWNrSG91c2UgdnMgUG9zdGdyZXMgZm9yIE9URUwgU3BhbnMgKDUway9zZWMpXG5cbiMjIFRhc2tzXG4tIFsgXSBSZXNlYXJjaCBDbGlja0hvdXNlIGFuZCBQb3N0Z3JlcyBjYXBhYmlsaXRpZXMgZm9yIGhpZ2gtdm9sdW1lIHRpbWUtc2VyaWVzIGRhdGFcbi4uLiIsImVuY29kaW5nIjoidXRmLTgiLCJjcmVhdGVkX2F0IjoiMjAyNi0wOS0zMFQwNDozNjozOC44OTkwNTArMDA6MDAiLCJtb2RpZmllZF9hdCI6IjIwMjYtMDktMzBUMDQ6MzY6MzguODk5MDUwKzAwOjAwIn19fQ==" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + }, + "flags": 256 + }, + { + "traceId": "S61CuE6d47pG/IcBhfjwIw==", + "spanId": "imocMZQNB68=", + "parentSpanId": "Hfr3D90RhPI=", + "name": "ChatOpenAI", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1790742989383207936", + "endTimeUnixNano": "1790742998893985024", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "chat" + } + }, + { + "key": "gen_ai.serialized.name", + "value": { + "stringValue": "ChatOpenAI" + } + }, + { + "key": "langsmith.span.kind", + "value": { + "stringValue": "llm" + } + }, + { + "key": "langsmith.trace.name", + "value": { + "stringValue": "ChatOpenAI" + } + }, + { + "key": "langsmith.trace.session_name", + "value": { + "stringValue": "default" + } + }, + { + "key": "gen_ai.system", + "value": { + "stringValue": "anthropic" + } + }, + { + "key": "gen_ai.request.model", + "value": { + "stringValue": "claude-sonnet-4-5" + } + }, + { + "key": "gen_ai.tool.definitions", + "value": { + "stringValue": "[{\"type\":\"function\",\"function\":{\"name\":\"ls\",\"description\":\"Lists all files in a directory.\\n\\nThis is useful for exploring the filesystem and finding the right file to read or edit.\\nYou should almost ALWAYS use this tool before using the read_file or edit_file tools.\",\"parameters\":{\"properties\":{\"path\":{\"description\":\"Absolute path to the directory to list. Must be absolute, not relative.\",\"type\":\"string\"}},\"required\":[\"path\"],\"type\":\"object\"}}}]" + } + }, + { + "key": "langsmith.metadata.ls_integration", + "value": { + "stringValue": "langchain_chat_model" + } + }, + { + "key": "langsmith.metadata.lc_agent_name", + "value": { + "stringValue": "deep_research_agent" + } + }, + { + "key": "langsmith.metadata.lc_versions", + "value": { + "stringValue": "{\"deepagents\":\"0.7.20\",\"langchain-core\":\"1.6.6\",\"langchain\":\"1.4.3\",\"langchain-openai\":\"1.6.6\"}" + } + }, + { + "key": "langsmith.metadata.langgraph_step", + "value": { + "intValue": "2" + } + }, + { + "key": "langsmith.metadata.langgraph_node", + "value": { + "stringValue": "model" + } + }, + { + "key": "langsmith.metadata.langgraph_triggers", + "value": { + "stringValue": "[\"branch:to:model\"]" + } + }, + { + "key": "langsmith.metadata.langgraph_path", + "value": { + "stringValue": "[\"__pregel_pull\",\"model\"]" + } + }, + { + "key": "langsmith.metadata.langgraph_checkpoint_ns", + "value": { + "stringValue": "model:9abb6d12-32f9-4289-15b6-36ac41ba926c" + } + }, + { + "key": "langsmith.metadata.checkpoint_ns", + "value": { + "stringValue": "model:9abb6d12-32f9-4289-15b6-36ac41ba926c" + } + }, + { + "key": "langsmith.metadata.ls_provider", + "value": { + "stringValue": "openai" + } + }, + { + "key": "langsmith.metadata.ls_model_name", + "value": { + "stringValue": "claude-sonnet-4-5" + } + }, + { + "key": "langsmith.metadata.ls_model_type", + "value": { + "stringValue": "chat" + } + }, + { + "key": "langsmith.metadata.ls_max_tokens", + "value": { + "intValue": "700" + } + }, + { + "key": "langsmith.metadata.ls_method", + "value": { + "stringValue": "traceable" + } + }, + { + "key": "langsmith.metadata.model", + "value": { + "stringValue": "claude-sonnet-4-5" + } + }, + { + "key": "langsmith.metadata.model_name", + "value": { + "stringValue": "claude-sonnet-4-5" + } + }, + { + "key": "langsmith.metadata.stream", + "value": { + "boolValue": false + } + }, + { + "key": "langsmith.metadata.max_completion_tokens", + "value": { + "intValue": "700" + } + }, + { + "key": "langsmith.metadata._type", + "value": { + "stringValue": "openai-chat" + } + }, + { + "key": "langsmith.metadata.usage_metadata", + "value": { + "stringValue": "{\"input_tokens\":3332,\"output_tokens\":467,\"total_tokens\":3799,\"input_token_details\":{\"cache_read\":0,\"cache_creation\":3329},\"output_token_details\":{\"reasoning\":0}}" + } + }, + { + "key": "langsmith.metadata.LANGSMITH_TRACING", + "value": { + "stringValue": "true" + } + }, + { + "key": "langsmith.metadata.LANGSMITH_TRACING_MODE", + "value": { + "stringValue": "otel" + } + }, + { + "key": "langsmith.span.tags", + "value": { + "stringValue": "seq:step:1" + } + }, + { + "key": "gen_ai.prompt", + "value": { + "bytesValue": "eyJtZXNzYWdlcyI6W1t7ImxjIjoxLCJ0eXBlIjoiY29uc3RydWN0b3IiLCJpZCI6WyJsYW5nY2hhaW4iLCJzY2hlbWEiLCJtZXNzYWdlcyIsIlN5c3RlbU1lc3NhZ2UiXSwia3dhcmdzIjp7ImNvbnRlbnQiOiJZb3UgYXJlIGEgcmVzZWFyY2ggbGVhZC4gUGxhbiB3aXRoIHdyaXRlX3RvZG9zLCBkZWxlZ2F0ZSBvbmUgcXVlc3Rpb24gdG8gdGhlIHJlc2VhcmNoZXIgc3ViYWdlbnQgdmlhIHRhc2ssIHRoZW4gd3JpdGUgYSBzaG9ydCByZWNvbW1lbmRhdGlvbiAoPD01IHNlbnRlbmNlcykuIiwidHlwZSI6InN5c3RlbSJ9fSx7ImxjIjoxLCJ0eXBlIjoiY29uc3RydWN0b3IiLCJpZCI6WyJsYW5nY2hhaW4iLCJzY2hlbWEiLCJtZXNzYWdlcyIsIkh1bWFuTWVzc2FnZSJdLCJrd2FyZ3MiOnsiY29udGVudCI6IlNob3VsZCB3ZSBzdG9yZSBPVEVMIGFnZW50IHNwYW5zIGluIENsaWNrSG91c2Ugb3IgUG9zdGdyZXMgYXQgNTBrIHNwYW5zL3NlYz8iLCJ0eXBlIjoiaHVtYW4iLCJpZCI6ImIxOWM4ODMxLTA5YjAtNDlmNi1iN2VhLTljNDdlMzg5Y2MwMCJ9fV1dfQ==" + } + }, + { + "key": "gen_ai.usage.input_tokens", + "value": { + "intValue": "3332" + } + }, + { + "key": "gen_ai.usage.output_tokens", + "value": { + "intValue": "467" + } + }, + { + "key": "gen_ai.usage.total_tokens", + "value": { + "intValue": "3799" + } + }, + { + "key": "gen_ai.completion", + "value": { + "bytesValue": "eyJnZW5lcmF0aW9ucyI6W1t7InRleHQiOiJJJ2xsIGhlbHAgeW91IGRlY2lkZSBiZXR3ZWVuIENsaWNrSG91c2UgYW5kIFBvc3RncmVzIGZvciBzdG9yaW5nIE9wZW5UZWxlbWV0cnkgc3BhbnMgYXQgNTBrIHNwYW5zL3NlYy4gTGV0IG1lIHJlc2VhcmNoIHRoaXMgc3lzdGVtYXRpY2FsbHkuIiwiZ2VuZXJhdGlvbl9pbmZvIjp7ImZpbmlzaF9yZWFzb24iOiJ0b29sX2NhbGxzIiwibG9ncHJvYnMiOm51bGx9LCJ0eXBlIjoiQ2hhdEdlbmVyYXRpb24iLCJtZXNzYWdlIjp7ImxjIjoxLCJ0eXBlIjoiY29uc3RydWN0b3IiLCJpZCI6WyJsYW5nY2hhaW4iLCJzY2hlbWEiLCJtZXNzYWdlcyIsIkFJTWVzc2FnZSJdLCJrd2FyZ3MiOnsiY29udGVudCI6IkknbGwgaGVscCB5b3UgZGVjaWRlIGJldHdlZW4gQ2xpY2tIb3VzZSBhbmQgUG9zdGdyZXMgZm9yIHN0b3JpbmcgT3BlblRlbGVtZXRyeSBzcGFucyBhdCA1MGsgc3BhbnMvc2VjLiBMZXQgbWUgcmVzZWFyY2ggdGhpcyBzeXN0ZW1hdGljYWxseS4iLCJhZGRpdGlvbmFsX2t3YXJncyI6eyJyZWZ1c2FsIjpudWxsfSwicmVzcG9uc2VfbWV0YWRhdGEiOnsidG9rZW5fdXNhZ2UiOnsiY29tcGxldGlvbl90b2tlbnMiOjQ2NywicHJvbXB0X3Rva2VucyI6MzMzMiwidG90YWxfdG9rZW5zIjozNzk5LCJjb21wbGV0aW9uX3Rva2Vuc19kZXRhaWxzIjp7ImFjY2VwdGVkX3ByZWRpY3Rpb25fdG9rZW5zIjpudWxsLCJhdWRpb190b2tlbnMiOm51bGwsInJlYXNvbmluZ190b2tlbnMiOjAsInJlamVjdGVkX3ByZWRpY3Rpb25fdG9rZW5zIjpudWxsLCJ0ZXh0X3Rva2VucyI6NDY3fSwicHJvbXB0X3Rva2Vuc19kZXRhaWxzIjp7ImF1ZGlvX3Rva2VucyI6bnVsbCwiY2FjaGVfd3JpdGVfdG9rZW5zIjozMzI5LCJjYWNoZWRfdG9rZW5zIjowLCJpbWFnZV90b2tlbnMiOm51bGwsInRleHRfdG9rZW5zIjozLCJjYWNoZV9jcmVhdGlvbl90b2tlbnMiOjMzMjksImNhY2hlX2NyZWF0aW9uX3Rva2VuX2RldGFpbHMiOnsiZXBoZW1lcmFsXzVtX2lucHV0X3Rva2VucyI6MzMyOSwiZXBoZW1lcmFsXzFoX2lucHV0X3Rva2VucyI6MH19LCJjYWNoZV9jcmVhdGlvbl9pbnB1dF90b2tlbnMiOjMzMjksImNhY2hlX3JlYWRfaW5wdXRfdG9rZW5zIjowLCJpbmZlcmVuY2VfZ2VvIjoibm90X2F2YWlsYWJsZSIsInNlcnZpY2VfdGllciI6InN0YW5kYXJkIn0sIm1vZGVsX3Byb3ZpZGVyIjoib3BlbmFpIiwibW9kZWxfbmFtZSI6ImNsYXVkZS1zb25uZXQtNC01Iiwic3lzdGVtX2ZpbmdlcnByaW50IjpudWxsLCJpZCI6ImNoYXRjbXBsLTQwNzdiYjM2LTkzODAtNGEzYi05NDgxLTI0NTcwMGNlZjA5YSIsImZpbmlzaF9yZWFzb24iOiJ0b29sX2NhbGxzIiwibG9ncHJvYnMiOm51bGx9LCJ0eXBlIjoiYWkiLCJpZCI6ImxjX3J1bi0tMDFhMGYwOTktOGE0Ny03ZTQyLWE1ZjQtNWM0N2RlM2QxY2VjLTAiLCJ0b29sX2NhbGxzIjpbeyJuYW1lIjoid3JpdGVfZmlsZSIsImFyZ3MiOnsiZmlsZV9wYXRoIjoiL3RtcC9yZXNlYXJjaF90b2Rvcy5tZCIsImNvbnRlbnQiOiIjIFJlc2VhcmNoIFBsYW46IENsaWNrSG91c2UgdnMgUG9zdGdyZXMgZm9yIE9URUwgU3BhbnMgKDUway9zZWMpXG5cbiMjIFRhc2tzXG4tIFsgXSBSZXNlYXJjaCBDbGlja0hvdXNlIGFuZCBQb3N0Z3JlcyBjYXBhYmlsaXRpZXMgZm9yIGhpZ2gtdm9sdW1lIHRpbWUtc2VyaWVzIGRhdGFcbi4uLiJ9LCJpZCI6InRvb2x1XzAxNjFYaFlQM0I1Zmc0VTFwc1QzcGNpUiIsInR5cGUiOiJ0b29sX2NhbGwifSx7Im5hbWUiOiJ0YXNrIiwiYXJncyI6eyJzdWJhZ2VudF90eXBlIjoicmVzZWFyY2hlciIsImRlc2NyaXB0aW9uIjoiUmVzZWFyY2ggYW5kIGNvbXBhcmUgQ2xpY2tIb3VzZSB2cyBQb3N0Z3JlcyBmb3Igc3RvcmluZyBPcGVuVGVsZW1ldHJ5IChPVEVMKSBhZ2VudCBzcGFucyBhdCA1MCwwMDAgc3BhbnMgcGVyIHNlY29uZC5cblxuRm9jdXMgb246XG4xLiBXcml0ZSB0aHJvdWdocHV0IGNhcGFiaWxpdGllcy4uLiJ9LCJpZCI6InRvb2x1XzAxUEx5bzhUS0tUcFhSNGZwOTZEbjkzVyIsInR5cGUiOiJ0b29sX2NhbGwifV0sInVzYWdlX21ldGFkYXRhIjp7ImlucHV0X3Rva2VucyI6MzMzMiwib3V0cHV0X3Rva2VucyI6NDY3LCJ0b3RhbF90b2tlbnMiOjM3OTksImlucHV0X3Rva2VuX2RldGFpbHMiOnsiY2FjaGVfcmVhZCI6MCwiY2FjaGVfY3JlYXRpb24iOjMzMjl9LCJvdXRwdXRfdG9rZW5fZGV0YWlscyI6eyJyZWFzb25pbmciOjB9fSwiaW52YWxpZF90b29sX2NhbGxzIjpbXX19fV1dLCJsbG1fb3V0cHV0Ijp7InRva2VuX3VzYWdlIjp7ImNvbXBsZXRpb25fdG9rZW5zIjo0NjcsInByb21wdF90b2tlbnMiOjMzMzIsInRvdGFsX3Rva2VucyI6Mzc5OSwiY29tcGxldGlvbl90b2tlbnNfZGV0YWlscyI6eyJhY2NlcHRlZF9wcmVkaWN0aW9uX3Rva2VucyI6bnVsbCwiYXVkaW9fdG9rZW5zIjpudWxsLCJyZWFzb25pbmdfdG9rZW5zIjowLCJyZWplY3RlZF9wcmVkaWN0aW9uX3Rva2VucyI6bnVsbCwidGV4dF90b2tlbnMiOjQ2N30sInByb21wdF90b2tlbnNfZGV0YWlscyI6eyJhdWRpb190b2tlbnMiOm51bGwsImNhY2hlX3dyaXRlX3Rva2VucyI6MzMyOSwiY2FjaGVkX3Rva2VucyI6MCwiaW1hZ2VfdG9rZW5zIjpudWxsLCJ0ZXh0X3Rva2VucyI6MywiY2FjaGVfY3JlYXRpb25fdG9rZW5zIjozMzI5LCJjYWNoZV9jcmVhdGlvbl90b2tlbl9kZXRhaWxzIjp7ImVwaGVtZXJhbF81bV9pbnB1dF90b2tlbnMiOjMzMjksImVwaGVtZXJhbF8xaF9pbnB1dF90b2tlbnMiOjB9fSwiY2FjaGVfY3JlYXRpb25faW5wdXRfdG9rZW5zIjozMzI5LCJjYWNoZV9yZWFkX2lucHV0X3Rva2VucyI6MCwiaW5mZXJlbmNlX2dlbyI6Im5vdF9hdmFpbGFibGUiLCJzZXJ2aWNlX3RpZXIiOiJzdGFuZGFyZCJ9LCJtb2RlbF9wcm92aWRlciI6Im9wZW5haSIsIm1vZGVsX25hbWUiOiJjbGF1ZGUtc29ubmV0LTQtNSIsInN5c3RlbV9maW5nZXJwcmludCI6bnVsbCwiaWQiOiJjaGF0Y21wbC00MDc3YmIzNi05MzgwLTRhM2ItOTQ4MS0yNDU3MDBjZWYwOWEifSwicnVuIjpudWxsLCJ0eXBlIjoiTExNUmVzdWx0In0=" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + }, + "flags": 256 + }, + { + "traceId": "S61CuE6d47pG/IcBhfjwIw==", + "spanId": "zwThqgPzRPo=", + "parentSpanId": "g0UfMjWEf2w=", + "name": "FilesystemMiddleware.wrap_model_call", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1790742989379030016", + "endTimeUnixNano": "1790742998895730944", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "chain" + } + }, + { + "key": "gen_ai.prompt", + "value": { + "bytesValue": "e30=" + } + }, + { + "key": "langsmith.span.kind", + "value": { + "stringValue": "chain" + } + }, + { + "key": "langsmith.trace.name", + "value": { + "stringValue": "FilesystemMiddleware.wrap_model_call" + } + }, + { + "key": "langsmith.trace.session_name", + "value": { + "stringValue": "default" + } + }, + { + "key": "gen_ai.system", + "value": { + "stringValue": "langchain" + } + }, + { + "key": "langsmith.metadata.ls_integration", + "value": { + "stringValue": "deepagents" + } + }, + { + "key": "langsmith.metadata.lc_agent_name", + "value": { + "stringValue": "deep_research_agent" + } + }, + { + "key": "langsmith.metadata.lc_versions", + "value": { + "stringValue": "{\"deepagents\":\"0.7.20\"}" + } + }, + { + "key": "langsmith.metadata.langgraph_step", + "value": { + "intValue": "2" + } + }, + { + "key": "langsmith.metadata.langgraph_node", + "value": { + "stringValue": "model" + } + }, + { + "key": "langsmith.metadata.langgraph_triggers", + "value": { + "stringValue": "[\"branch:to:model\"]" + } + }, + { + "key": "langsmith.metadata.langgraph_path", + "value": { + "stringValue": "[\"__pregel_pull\",\"model\"]" + } + }, + { + "key": "langsmith.metadata.langgraph_checkpoint_ns", + "value": { + "stringValue": "model:9abb6d12-32f9-4289-15b6-36ac41ba926c" + } + }, + { + "key": "langsmith.metadata.ls_method", + "value": { + "stringValue": "traceable" + } + }, + { + "key": "langsmith.metadata.LANGSMITH_TRACING", + "value": { + "stringValue": "true" + } + }, + { + "key": "langsmith.metadata.LANGSMITH_TRACING_MODE", + "value": { + "stringValue": "otel" + } + }, + { + "key": "gen_ai.completion", + "value": { + "bytesValue": "eyJvdXRwdXQiOnsicmVzdWx0IjpbeyJjb250ZW50IjoiSSdsbCBoZWxwIHlvdSBkZWNpZGUgYmV0d2VlbiBDbGlja0hvdXNlIGFuZCBQb3N0Z3JlcyBmb3Igc3RvcmluZyBPcGVuVGVsZW1ldHJ5IHNwYW5zIGF0IDUwayBzcGFucy9zZWMuIExldCBtZSByZXNlYXJjaCB0aGlzIHN5c3RlbWF0aWNhbGx5LiIsImFkZGl0aW9uYWxfa3dhcmdzIjp7InJlZnVzYWwiOm51bGx9LCJyZXNwb25zZV9tZXRhZGF0YSI6eyJ0b2tlbl91c2FnZSI6eyJjb21wbGV0aW9uX3Rva2VucyI6NDY3LCJwcm9tcHRfdG9rZW5zIjozMzMyLCJ0b3RhbF90b2tlbnMiOjM3OTksImNvbXBsZXRpb25fdG9rZW5zX2RldGFpbHMiOnsiYWNjZXB0ZWRfcHJlZGljdGlvbl90b2tlbnMiOm51bGwsImF1ZGlvX3Rva2VucyI6bnVsbCwicmVhc29uaW5nX3Rva2VucyI6MCwicmVqZWN0ZWRfcHJlZGljdGlvbl90b2tlbnMiOm51bGwsInRleHRfdG9rZW5zIjo0Njd9LCJwcm9tcHRfdG9rZW5zX2RldGFpbHMiOnsiYXVkaW9fdG9rZW5zIjpudWxsLCJjYWNoZV93cml0ZV90b2tlbnMiOjMzMjksImNhY2hlZF90b2tlbnMiOjAsImltYWdlX3Rva2VucyI6bnVsbCwidGV4dF90b2tlbnMiOjMsImNhY2hlX2NyZWF0aW9uX3Rva2VucyI6MzMyOSwiY2FjaGVfY3JlYXRpb25fdG9rZW5fZGV0YWlscyI6eyJlcGhlbWVyYWxfNW1faW5wdXRfdG9rZW5zIjozMzI5LCJlcGhlbWVyYWxfMWhfaW5wdXRfdG9rZW5zIjowfX0sImNhY2hlX2NyZWF0aW9uX2lucHV0X3Rva2VucyI6MzMyOSwiY2FjaGVfcmVhZF9pbnB1dF90b2tlbnMiOjAsImluZmVyZW5jZV9nZW8iOiJub3RfYXZhaWxhYmxlIiwic2VydmljZV90aWVyIjoic3RhbmRhcmQifSwibW9kZWxfcHJvdmlkZXIiOiJvcGVuYWkiLCJtb2RlbF9uYW1lIjoiY2xhdWRlLXNvbm5ldC00LTUiLCJzeXN0ZW1fZmluZ2VycHJpbnQiOm51bGwsImlkIjoiY2hhdGNtcGwtNDA3N2JiMzYtOTM4MC00YTNiLTk0ODEtMjQ1NzAwY2VmMDlhIiwiZmluaXNoX3JlYXNvbiI6InRvb2xfY2FsbHMiLCJsb2dwcm9icyI6bnVsbH0sInR5cGUiOiJhaSIsIm5hbWUiOiJkZWVwX3Jlc2VhcmNoX2FnZW50IiwiaWQiOiJsY19ydW4tLTAxYTBmMDk5LThhNDctN2U0Mi1hNWY0LTVjNDdkZTNkMWNlYy0wIiwidG9vbF9jYWxscyI6W3sibmFtZSI6IndyaXRlX2ZpbGUiLCJhcmdzIjp7ImZpbGVfcGF0aCI6Ii90bXAvcmVzZWFyY2hfdG9kb3MubWQiLCJjb250ZW50IjoiIyBSZXNlYXJjaCBQbGFuOiBDbGlja0hvdXNlIHZzIFBvc3RncmVzIGZvciBPVEVMIFNwYW5zICg1MGsvc2VjKVxuXG4jIyBUYXNrc1xuLSBbIF0gUmVzZWFyY2ggQ2xpY2tIb3VzZSBhbmQgUG9zdGdyZXMgY2FwYWJpbGl0aWVzIGZvciBoaWdoLXZvbHVtZSB0aW1lLXNlcmllcyBkYXRhXG4uLi4ifSwiaWQiOiJ0b29sdV8wMTYxWGhZUDNCNWZnNFUxcHNUM3BjaVIiLCJ0eXBlIjoidG9vbF9jYWxsIn0seyJuYW1lIjoidGFzayIsImFyZ3MiOnsic3ViYWdlbnRfdHlwZSI6InJlc2VhcmNoZXIiLCJkZXNjcmlwdGlvbiI6IlJlc2VhcmNoIGFuZCBjb21wYXJlIENsaWNrSG91c2UgdnMgUG9zdGdyZXMgZm9yIHN0b3JpbmcgT3BlblRlbGVtZXRyeSAoT1RFTCkgYWdlbnQgc3BhbnMgYXQgNTAsMDAwIHNwYW5zIHBlciBzZWNvbmQuXG5cbkZvY3VzIG9uOlxuMS4gV3JpdGUgdGhyb3VnaHB1dCBjYXBhYmlsaXRpZXMuLi4ifSwiaWQiOiJ0b29sdV8wMVBMeW84VEtLVHBYUjRmcDk2RG45M1ciLCJ0eXBlIjoidG9vbF9jYWxsIn1dLCJpbnZhbGlkX3Rvb2xfY2FsbHMiOltdLCJ1c2FnZV9tZXRhZGF0YSI6eyJpbnB1dF90b2tlbnMiOjMzMzIsIm91dHB1dF90b2tlbnMiOjQ2NywidG90YWxfdG9rZW5zIjozNzk5LCJpbnB1dF90b2tlbl9kZXRhaWxzIjp7ImNhY2hlX3JlYWQiOjAsImNhY2hlX2NyZWF0aW9uIjozMzI5fSwib3V0cHV0X3Rva2VuX2RldGFpbHMiOnsicmVhc29uaW5nIjowfX19XSwic3RydWN0dXJlZF9yZXNwb25zZSI6bnVsbH19" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + }, + "flags": 256 + }, + { + "traceId": "S61CuE6d47pG/IcBhfjwIw==", + "spanId": "svs6j1ovzgE=", + "parentSpanId": "Vt73x+GSQ0o=", + "name": "task", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1790742998900896000", + "endTimeUnixNano": "1790743034076956160", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "execute_tool" + } + }, + { + "key": "langsmith.span.kind", + "value": { + "stringValue": "tool" + } + }, + { + "key": "langsmith.trace.name", + "value": { + "stringValue": "task" + } + }, + { + "key": "langsmith.trace.session_name", + "value": { + "stringValue": "default" + } + }, + { + "key": "gen_ai.system", + "value": { + "stringValue": "langchain" + } + }, + { + "key": "gen_ai.tool.name", + "value": { + "stringValue": "task" + } + }, + { + "key": "gen_ai.tool.call.id", + "value": { + "stringValue": "toolu_01PLyo8TKKTpXR4fp96Dn93W" + } + }, + { + "key": "langsmith.metadata.ls_integration", + "value": { + "stringValue": "deepagents" + } + }, + { + "key": "langsmith.metadata.lc_agent_name", + "value": { + "stringValue": "deep_research_agent" + } + }, + { + "key": "langsmith.metadata.lc_versions", + "value": { + "stringValue": "{\"deepagents\":\"0.7.20\"}" + } + }, + { + "key": "langsmith.metadata.langgraph_step", + "value": { + "intValue": "3" + } + }, + { + "key": "langsmith.metadata.langgraph_node", + "value": { + "stringValue": "tools" + } + }, + { + "key": "langsmith.metadata.langgraph_triggers", + "value": { + "stringValue": "[\"__pregel_push\"]" + } + }, + { + "key": "langsmith.metadata.langgraph_path", + "value": { + "stringValue": "[\"__pregel_push\",1,false]" + } + }, + { + "key": "langsmith.metadata.langgraph_checkpoint_ns", + "value": { + "stringValue": "tools:800ca7c3-441c-7ae3-0a5b-ea3fb69766fc" + } + }, + { + "key": "langsmith.metadata.checkpoint_ns", + "value": { + "stringValue": "tools:800ca7c3-441c-7ae3-0a5b-ea3fb69766fc" + } + }, + { + "key": "langsmith.metadata.ls_method", + "value": { + "stringValue": "traceable" + } + }, + { + "key": "langsmith.metadata.LANGSMITH_TRACING", + "value": { + "stringValue": "true" + } + }, + { + "key": "langsmith.metadata.LANGSMITH_TRACING_MODE", + "value": { + "stringValue": "otel" + } + }, + { + "key": "langsmith.span.tags", + "value": { + "stringValue": "seq:step:1" + } + }, + { + "key": "gen_ai.prompt", + "value": { + "bytesValue": "eyJzdWJhZ2VudF90eXBlIjoicmVzZWFyY2hlciIsImRlc2NyaXB0aW9uIjoiUmVzZWFyY2ggYW5kIGNvbXBhcmUgQ2xpY2tIb3VzZSB2cyBQb3N0Z3JlcyBmb3Igc3RvcmluZyBPcGVuVGVsZW1ldHJ5IChPVEVMKSBhZ2VudCBzcGFucyBhdCA1MCwwMDAgc3BhbnMgcGVyIHNlY29uZC5cblxuRm9jdXMgb246XG4xLiBXcml0ZSB0aHJvdWdocHV0IGNhcGFiaWxpdGllcy4uLiJ9" + } + }, + { + "key": "gen_ai.completion", + "value": { + "bytesValue": "eyJvdXRwdXQiOnsiZ3JhcGgiOm51bGwsInVwZGF0ZSI6eyJmaWxlcyI6e30sIm1lc3NhZ2VzIjpbeyJjb250ZW50IjoiQmFzZWQgb24gbXkgcmVzZWFyY2gsIGhlcmUncyBhIGNvbXByZWhlbnNpdmUgY29tcGFyaXNvbiBvZiAqKkNsaWNrSG91c2UgdnMgUG9zdGdyZXMqKiBmb3Igc3RvcmluZyBPcGVuVGVsZW1ldHJ5IHNwYW5zIGF0IDUwLDAwMCBzcGFucy9zZWNvbmQ6XG5cbiMjICoqMS4gV3JpdGUgVGhyLi4uIiwiYWRkaXRpb25hbF9rd2FyZ3MiOnt9LCJyZXNwb25zZV9tZXRhZGF0YSI6e30sInR5cGUiOiJ0b29sIiwidG9vbF9jYWxsX2lkIjoidG9vbHVfMDFQTHlvOFRLS1RwWFI0ZnA5NkRuOTNXIiwic3RhdHVzIjoic3VjY2VzcyJ9XX0sInJlc3VtZSI6bnVsbCwiZ290byI6W119fQ==" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + }, + "flags": 256 + }, + { + "traceId": "S61CuE6d47pG/IcBhfjwIw==", + "spanId": "gUmbSS/ZP4U=", + "parentSpanId": "svs6j1ovzgE=", + "name": "researcher", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1790742998901422080", + "endTimeUnixNano": "1790743034076699904", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "chain" + } + }, + { + "key": "langsmith.span.kind", + "value": { + "stringValue": "chain" + } + }, + { + "key": "langsmith.trace.name", + "value": { + "stringValue": "researcher" + } + }, + { + "key": "langsmith.trace.session_name", + "value": { + "stringValue": "default" + } + }, + { + "key": "gen_ai.system", + "value": { + "stringValue": "langchain" + } + }, + { + "key": "langsmith.metadata.ls_integration", + "value": { + "stringValue": "langchain_create_agent" + } + }, + { + "key": "langsmith.metadata.lc_agent_name", + "value": { + "stringValue": "researcher" + } + }, + { + "key": "langsmith.metadata.lc_versions", + "value": { + "stringValue": "{\"deepagents\":\"0.7.20\"}" + } + }, + { + "key": "langsmith.metadata.langgraph_step", + "value": { + "intValue": "3" + } + }, + { + "key": "langsmith.metadata.langgraph_node", + "value": { + "stringValue": "tools" + } + }, + { + "key": "langsmith.metadata.langgraph_triggers", + "value": { + "stringValue": "[\"__pregel_push\"]" + } + }, + { + "key": "langsmith.metadata.langgraph_path", + "value": { + "stringValue": "[\"__pregel_push\",1,false]" + } + }, + { + "key": "langsmith.metadata.langgraph_checkpoint_ns", + "value": { + "stringValue": "tools:800ca7c3-441c-7ae3-0a5b-ea3fb69766fc" + } + }, + { + "key": "langsmith.metadata.checkpoint_ns", + "value": { + "stringValue": "tools:800ca7c3-441c-7ae3-0a5b-ea3fb69766fc" + } + }, + { + "key": "langsmith.metadata.ls_method", + "value": { + "stringValue": "traceable" + } + }, + { + "key": "langsmith.metadata.ls_agent_type", + "value": { + "stringValue": "subagent" + } + }, + { + "key": "langsmith.metadata.LANGSMITH_TRACING", + "value": { + "stringValue": "true" + } + }, + { + "key": "langsmith.metadata.LANGSMITH_TRACING_MODE", + "value": { + "stringValue": "otel" + } + }, + { + "key": "gen_ai.prompt", + "value": { + "bytesValue": "eyJmaWxlcyI6e30sIm1lc3NhZ2VzIjpbeyJjb250ZW50IjoiUmVzZWFyY2ggYW5kIGNvbXBhcmUgQ2xpY2tIb3VzZSB2cyBQb3N0Z3JlcyBmb3Igc3RvcmluZyBPcGVuVGVsZW1ldHJ5IChPVEVMKSBhZ2VudCBzcGFucyBhdCA1MCwwMDAgc3BhbnMgcGVyIHNlY29uZC5cblxuRm9jdXMgb246XG4xLiBXcml0ZSB0aHJvdWdocHV0IGNhcGFiaWxpdGllcy4uLiIsImFkZGl0aW9uYWxfa3dhcmdzIjp7fSwicmVzcG9uc2VfbWV0YWRhdGEiOnt9LCJ0eXBlIjoiaHVtYW4iLCJpZCI6ImFmOGRiNzQ5LTBiNTYtNGEzMi1hZGZlLTdmYzViOTRmZDAwMyJ9XX0=" + } + }, + { + "key": "gen_ai.completion", + "value": { + "bytesValue": "eyJtZXNzYWdlcyI6W3siY29udGVudCI6IlJlc2VhcmNoIGFuZCBjb21wYXJlIENsaWNrSG91c2UgdnMgUG9zdGdyZXMgZm9yIHN0b3JpbmcgT3BlblRlbGVtZXRyeSAoT1RFTCkgYWdlbnQgc3BhbnMgYXQgNTAsMDAwIHNwYW5zIHBlciBzZWNvbmQuXG5cbkZvY3VzIG9uOlxuMS4gV3JpdGUgdGhyb3VnaHB1dCBjYXBhYmlsaXRpZXMuLi4iLCJhZGRpdGlvbmFsX2t3YXJncyI6e30sInJlc3BvbnNlX21ldGFkYXRhIjp7fSwidHlwZSI6Imh1bWFuIiwiaWQiOiJhZjhkYjc0OS0wYjU2LTRhMzItYWRmZS03ZmM1Yjk0ZmQwMDMifSx7ImNvbnRlbnQiOiJJJ2xsIHJlc2VhcmNoIHRoZSBjb21wYXJpc29uIGJldHdlZW4gQ2xpY2tIb3VzZSBhbmQgUG9zdGdyZXMgZm9yIHN0b3JpbmcgT3BlblRlbGVtZXRyeSBzcGFucyBhdCBoaWdoIHZvbHVtZS4iLCJhZGRpdGlvbmFsX2t3YXJncyI6eyJyZWZ1c2FsIjpudWxsfSwicmVzcG9uc2VfbWV0YWRhdGEiOnsidG9rZW5fdXNhZ2UiOnsiY29tcGxldGlvbl90b2tlbnMiOjQyNywicHJvbXB0X3Rva2VucyI6Mjk4NiwidG90YWxfdG9rZW5zIjozNDEzLCJjb21wbGV0aW9uX3Rva2Vuc19kZXRhaWxzIjp7ImFjY2VwdGVkX3ByZWRpY3Rpb25fdG9rZW5zIjpudWxsLCJhdWRpb190b2tlbnMiOm51bGwsInJlYXNvbmluZ190b2tlbnMiOjAsInJlamVjdGVkX3ByZWRpY3Rpb25fdG9rZW5zIjpudWxsLCJ0ZXh0X3Rva2VucyI6NDI3fSwicHJvbXB0X3Rva2Vuc19kZXRhaWxzIjp7ImF1ZGlvX3Rva2VucyI6bnVsbCwiY2FjaGVfd3JpdGVfdG9rZW5zIjoyOTgzLCJjYWNoZWRfdG9rZW5zIjowLCJpbWFnZV90b2tlbnMiOm51bGwsInRleHRfdG9rZW5zIjozLCJjYWNoZV9jcmVhdGlvbl90b2tlbnMiOjI5ODMsImNhY2hlX2NyZWF0aW9uX3Rva2VuX2RldGFpbHMiOnsiZXBoZW1lcmFsXzVtX2lucHV0X3Rva2VucyI6Mjk4MywiZXBoZW1lcmFsXzFoX2lucHV0X3Rva2VucyI6MH19LCJjYWNoZV9jcmVhdGlvbl9pbnB1dF90b2tlbnMiOjI5ODMsImNhY2hlX3JlYWRfaW5wdXRfdG9rZW5zIjowLCJpbmZlcmVuY2VfZ2VvIjoibm90X2F2YWlsYWJsZSIsInNlcnZpY2VfdGllciI6InN0YW5kYXJkIn0sIm1vZGVsX3Byb3ZpZGVyIjoib3BlbmFpIiwibW9kZWxfbmFtZSI6ImNsYXVkZS1zb25uZXQtNC01Iiwic3lzdGVtX2ZpbmdlcnByaW50IjpudWxsLCJpZCI6ImNoYXRjbXBsLWFhYWE0Yjc4LTE3ZGMtNDM2NC04ZmE1LTJkODMzNjlmMWRiYyIsImZpbmlzaF9yZWFzb24iOiJ0b29sX2NhbGxzIiwibG9ncHJvYnMiOm51bGx9LCJ0eXBlIjoiYWkiLCJuYW1lIjoicmVzZWFyY2hlciIsImlkIjoibGNfcnVuLS0wMWEwZjA5OS1hZjdlLTc5ZTAtYTMzMy03MDdjMzQ5N2M3MzAtMCIsInRvb2xfY2FsbHMiOlt7Im5hbWUiOiJzZWFyY2hfZG9jcyIsImFyZ3MiOnsicXVlcnkiOiJDbGlja0hvdXNlIFBvc3RncmVzIE9wZW5UZWxlbWV0cnkgT1RFTCBzcGFucyBwZXJmb3JtYW5jZSBjb21wYXJpc29uIn0sImlkIjoidG9vbHVfMDFKc2pIRkZmcHN3NG9wbUs5VVppOFZOIiwidHlwZSI6InRvb2xfY2FsbCJ9LHsibmFtZSI6InNlYXJjaF9kb2NzIiwiYXJncyI6eyJxdWVyeSI6IkNsaWNrSG91c2Ugd3JpdGUgdGhyb3VnaHB1dCA1MDAwMCBzcGFucyBwZXIgc2Vjb25kIHRlbGVtZXRyeSJ9LCJpZCI6InRvb2x1XzAxS05ZcUhKS2kzcExlZU1RaEc1VDl1ZSIsInR5cGUiOiJ0b29sX2NhbGwifSx7Im5hbWUiOiJzZWFyY2hfZG9jcyIsImFyZ3MiOnsicXVlcnkiOiJQb3N0Z3JlcyB2cyBDbGlja0hvdXNlIG9ic2VydmFiaWxpdHkgbWV0cmljcyB0cmFjZXMifSwiaWQiOiJ0b29sdV8wMUpoRjh6NDQ0U1dVM0VXM2hQUUtkMlciLCJ0eXBlIjoidG9vbF9jYWxsIn0seyJuYW1lIjoic2VhcmNoX2RvY3MiLCJhcmdzIjp7InF1ZXJ5IjoiQ2xpY2tIb3VzZSBpbnNlcnQgcGVyZm9ybWFuY2UgYmF0Y2ggd3JpdGVzIHN1c3RhaW5lZCB0aHJvdWdocHV0In0sImlkIjoidG9vbHVfMDFXdXFyNTZKVHhDSllRUFMxWkZQbkg2IiwidHlwZSI6InRvb2xfY2FsbCJ9XSwiaW52YWxpZF90b29sX2NhbGxzIjpbXSwidXNhZ2VfbWV0YWRhdGEiOnsiaW5wdXRfdG9rZW5zIjoyOTg2LCJvdXRwdXRfdG9rZW5zIjo0MjcsInRvdGFsX3Rva2VucyI6MzQxMywiaW5wdXRfdG9rZW5fZGV0YWlscyI6eyJjYWNoZV9yZWFkIjowLCJjYWNoZV9jcmVhdGlvbiI6Mjk4M30sIm91dHB1dF90b2tlbl9kZXRhaWxzIjp7InJlYXNvbmluZyI6MH19fSx7ImNvbnRlbnQiOiJObyByZXN1bHRzLiIsImFkZGl0aW9uYWxfa3dhcmdzIjp7fSwicmVzcG9uc2VfbWV0YWRhdGEiOnt9LCJ0eXBlIjoidG9vbCIsIm5hbWUiOiJzZWFyY2hfZG9jcyIsImlkIjoiYTNhMDQxMmUtMWMxNS00ODk3LThjMmQtZGM0NmMwYmRlYzM2IiwidG9vbF9jYWxsX2lkIjoidG9vbHVfMDFVQmFYd0JQTmRxUkhHYmJhbmdLTFpVIiwic3RhdHVzIjoic3VjY2VzcyJ9LHsiY29udGVudCI6IkJhc2VkIG9uIG15IHJlc2VhcmNoLCBoZXJlJ3MgYSBjb21wcmVoZW5zaXZlIGNvbXBhcmlzb24gb2YgKipDbGlja0hvdXNlIHZzIFBvc3RncmVzKiogZm9yIHN0b3JpbmcgT3BlblRlbGVtZXRyeSBzcGFucyBhdCA1MCwwMDAgc3BhbnMvc2Vjb25kOlxuXG4jIyAqKjEuIFdyaXRlIFRoci4uLiIsImFkZGl0aW9uYWxfa3dhcmdzIjp7InJlZnVzYWwiOm51bGx9LCJyZXNwb25zZV9tZXRhZGF0YSI6eyJ0b2tlbl91c2FnZSI6eyJjb21wbGV0aW9uX3Rva2VucyI6NzAwLCJwcm9tcHRfdG9rZW5zIjo1NTM2LCJ0b3RhbF90b2tlbnMiOjYyMzYsImNvbXBsZXRpb25fdG9rZW5zX2RldGFpbHMiOnsiYWNjZXB0ZWRfcHJlZGljdGlvbl90b2tlbnMiOm51bGwsImF1ZGlvX3Rva2VucyI6bnVsbCwicmVhc29uaW5nX3Rva2VucyI6MCwicmVqZWN0ZWRfcHJlZGljdGlvbl90b2tlbnMiOm51bGwsInRleHRfdG9rZW5zIjo3MDB9LCJwcm9tcHRfdG9rZW5zX2RldGFpbHMiOnsiYXVkaW9fdG9rZW5zIjpudWxsLCJjYWNoZV93cml0ZV90b2tlbnMiOjQzMiwiY2FjaGVkX3Rva2VucyI6NTA5NywiaW1hZ2VfdG9rZW5zIjpudWxsLCJ0ZXh0X3Rva2VucyI6NywiY2FjaGVfY3JlYXRpb25fdG9rZW5zIjo0MzIsImNhY2hlX2NyZWF0aW9uX3Rva2VuX2RldGFpbHMiOnsiZXBoZW1lcmFsXzVtX2lucHV0X3Rva2VucyI6NDMyLCJlcGhlbWVyYWxfMWhfaW5wdXRfdG9rZW5zIjowfX0sImNhY2hlX2NyZWF0aW9uX2lucHV0X3Rva2VucyI6NDMyLCJjYWNoZV9yZWFkX2lucHV0X3Rva2VucyI6NTA5NywiaW5mZXJlbmNlX2dlbyI6Im5vdF9hdmFpbGFibGUiLCJzZXJ2aWNlX3RpZXIiOiJzdGFuZGFyZCJ9LCJtb2RlbF9wcm92aWRlciI6Im9wZW5haSIsIm1vZGVsX25hbWUiOiJjbGF1ZGUtc29ubmV0LTQtNSIsInN5c3RlbV9maW5nZXJwcmludCI6bnVsbCwiaWQiOiJjaGF0Y21wbC0zYzIwZTgwOC05YjE2LTQ0MjctOTk0Zi01Y2U3ZThiMWI5NGQiLCJmaW5pc2hfcmVhc29uIjoibGVuZ3RoIiwibG9ncHJvYnMiOm51bGx9LCJ0eXBlIjoiYWkiLCJuYW1lIjoicmVzZWFyY2hlciIsImlkIjoibGNfcnVuLS0wMWEwZjA5OS1mYmMyLTc5NjMtOWViYy1kYWUzNzZkYmJhMzktMCIsInRvb2xfY2FsbHMiOltdLCJpbnZhbGlkX3Rvb2xfY2FsbHMiOltdLCJ1c2FnZV9tZXRhZGF0YSI6eyJpbnB1dF90b2tlbnMiOjU1MzYsIm91dHB1dF90b2tlbnMiOjcwMCwidG90YWxfdG9rZW5zIjo2MjM2LCJpbnB1dF90b2tlbl9kZXRhaWxzIjp7ImNhY2hlX3JlYWQiOjUwOTcsImNhY2hlX2NyZWF0aW9uIjo0MzJ9LCJvdXRwdXRfdG9rZW5fZGV0YWlscyI6eyJyZWFzb25pbmciOjB9fX1dLCJmaWxlcyI6e319" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + }, + "flags": 256 + }, + { + "traceId": "S61CuE6d47pG/IcBhfjwIw==", + "spanId": "/mLyrQOgEWw=", + "parentSpanId": "SUm+6tN4+TU=", + "name": "search_docs", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1790743004976721920", + "endTimeUnixNano": "1790743004977214208", + "attributes": [ + { + "key": "langsmith.span.kind", + "value": { + "stringValue": "tool" + } + }, + { + "key": "langsmith.trace.name", + "value": { + "stringValue": "search_docs" + } + }, + { + "key": "langsmith.trace.session_name", + "value": { + "stringValue": "default" + } + }, + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "execute_tool" + } + }, + { + "key": "gen_ai.system", + "value": { + "stringValue": "langchain" + } + }, + { + "key": "gen_ai.tool.name", + "value": { + "stringValue": "search_docs" + } + }, + { + "key": "gen_ai.tool.call.id", + "value": { + "stringValue": "toolu_01JsjHFFfpsw4opmK9UZi8VN" + } + }, + { + "key": "langsmith.metadata.ls_integration", + "value": { + "stringValue": "langchain_create_agent" + } + }, + { + "key": "langsmith.metadata.lc_agent_name", + "value": { + "stringValue": "researcher" + } + }, + { + "key": "langsmith.metadata.lc_versions", + "value": { + "stringValue": "{\"deepagents\":\"0.7.20\"}" + } + }, + { + "key": "langsmith.metadata.langgraph_step", + "value": { + "intValue": "3" + } + }, + { + "key": "langsmith.metadata.langgraph_node", + "value": { + "stringValue": "tools" + } + }, + { + "key": "langsmith.metadata.langgraph_triggers", + "value": { + "stringValue": "[\"__pregel_push\"]" + } + }, + { + "key": "langsmith.metadata.langgraph_path", + "value": { + "stringValue": "[\"__pregel_push\",0,false]" + } + }, + { + "key": "langsmith.metadata.langgraph_checkpoint_ns", + "value": { + "stringValue": "tools:800ca7c3-441c-7ae3-0a5b-ea3fb69766fc|tools:49218779-253b-df87-734a-cfd23327bc5d" + } + }, + { + "key": "langsmith.metadata.checkpoint_ns", + "value": { + "stringValue": "tools:800ca7c3-441c-7ae3-0a5b-ea3fb69766fc" + } + }, + { + "key": "langsmith.metadata.ls_method", + "value": { + "stringValue": "traceable" + } + }, + { + "key": "langsmith.metadata.ls_agent_type", + "value": { + "stringValue": "subagent" + } + }, + { + "key": "langsmith.metadata.LANGSMITH_TRACING", + "value": { + "stringValue": "true" + } + }, + { + "key": "langsmith.metadata.LANGSMITH_TRACING_MODE", + "value": { + "stringValue": "otel" + } + }, + { + "key": "langsmith.span.tags", + "value": { + "stringValue": "seq:step:1" + } + }, + { + "key": "gen_ai.prompt", + "value": { + "bytesValue": "eyJxdWVyeSI6IkNsaWNrSG91c2UgUG9zdGdyZXMgT3BlblRlbGVtZXRyeSBPVEVMIHNwYW5zIHBlcmZvcm1hbmNlIGNvbXBhcmlzb24ifQ==" + } + }, + { + "key": "gen_ai.completion", + "value": { + "bytesValue": "eyJvdXRwdXQiOnsiY29udGVudCI6IkNsaWNrSG91c2UgaW5nZXN0cyAxTSsgcm93cy9zIHBlciBub2RlIHdpdGggYmF0Y2hlZCBpbnNlcnRzOyB1c2UgTWVyZ2VUcmVlIG9yZGVyZWQgYnkgKHRlbmFudCwgc2VydmljZSwgdGltZSkgYW5kIGEgYmxvb20gZmlsdGVyIGluZGV4IG9uIFRyYWNlSWQuXG5Qb3N0Z3JlcyBoYW5kLi4uIiwiYWRkaXRpb25hbF9rd2FyZ3MiOnt9LCJyZXNwb25zZV9tZXRhZGF0YSI6e30sInR5cGUiOiJ0b29sIiwibmFtZSI6InNlYXJjaF9kb2NzIiwidG9vbF9jYWxsX2lkIjoidG9vbHVfMDFKc2pIRkZmcHN3NG9wbUs5VVppOFZOIiwic3RhdHVzIjoic3VjY2VzcyJ9fQ==" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + }, + "flags": 256 + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/test_litellm/tracing/test_decode.py b/tests/test_litellm/tracing/test_decode.py new file mode 100644 index 00000000000..168ff2bf7fb --- /dev/null +++ b/tests/test_litellm/tracing/test_decode.py @@ -0,0 +1,313 @@ +""" +Tests for OTLP decode + normalization (litellm/tracing/decode.py). + +The fixture is a trimmed real export from a Deep Agents run (LangSmith OTEL mode): +deep_research_agent -> task (tool) -> researcher (subagent) -> search_docs (tool). +""" + +import gzip +import json +from pathlib import Path +from unittest.mock import patch + +import pytest +from google.protobuf.json_format import Parse +from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest +from opentelemetry.proto.common.v1.common_pb2 import AnyValue, KeyValue +from opentelemetry.proto.trace.v1.trace_pb2 import ResourceSpans, ScopeSpans, Span, Status + +from litellm.tracing import decode +from litellm.tracing.decode import decode_otlp, encode_otlp_response + +pytestmark = pytest.mark.requires_rust_extension + +FIXTURE = Path(__file__).parent / "fixtures" / "langsmith_deep_agent_export.json" +TRACE_ID = "4bad42b84e9de3ba46fc870185f8f023" + + +def _fixture_json() -> bytes: + return FIXTURE.read_bytes() + + +def _fixture_protobuf() -> bytes: + request = ExportTraceServiceRequest() + Parse(_fixture_json().decode(), request) + return request.SerializeToString() + + +@pytest.fixture +def rows_by_name() -> dict: + rows = decode_otlp(_fixture_json(), "application/json") + return {r["SpanName"]: r for r in rows} + + +def _kv(key: str, value: str | int) -> KeyValue: + if isinstance(value, int): + return KeyValue(key=key, value=AnyValue(int_value=value)) + return KeyValue(key=key, value=AnyValue(string_value=value)) + + +def _export(*spans: Span, service: str = "svc", scope: str = "test") -> bytes: + resource_spans = ResourceSpans(scope_spans=[ScopeSpans(spans=list(spans))]) + resource_spans.resource.attributes.append(_kv("service.name", service)) + resource_spans.scope_spans[0].scope.name = scope + return ExportTraceServiceRequest(resource_spans=[resource_spans]).SerializeToString() + + +def _span(name: str, span_id: bytes, parent: bytes = b"", **attributes: str | int) -> Span: + return Span( + trace_id=bytes.fromhex(TRACE_ID), + span_id=span_id, + parent_span_id=parent, + name=name, + start_time_unix_nano=1_000, + end_time_unix_nano=5_000, + attributes=[_kv(k.replace("__", "."), v) for k, v in attributes.items()], + ) + + +# ---------------------------------------------------------------- LangSmith / Deep Agents fixture + + +def test_classifies_every_langsmith_span(rows_by_name): + assert {name: r["ObservationType"] for name, r in rows_by_name.items()} == { + "deep_research_agent": "agent", + "ChatOpenAI": "llm", + "FilesystemMiddleware.wrap_model_call": "framework", + "task": "tool", + "researcher": "agent", + "search_docs": "tool", + } + + +def test_agent_name_is_the_enclosing_agent(rows_by_name): + assert rows_by_name["task"]["AgentName"] == "deep_research_agent" + assert rows_by_name["ChatOpenAI"]["AgentName"] == "deep_research_agent" + assert rows_by_name["researcher"]["AgentName"] == "researcher" + assert rows_by_name["search_docs"]["AgentName"] == "researcher" + + +def test_subagent_is_nested_under_task_tool(rows_by_name): + assert rows_by_name["researcher"]["ParentSpanId"] == rows_by_name["task"]["SpanId"] + assert rows_by_name["deep_research_agent"]["ParentSpanId"] == "" + + +def test_llm_span_carries_litellm_request_id_model_and_tokens(rows_by_name): + llm = rows_by_name["ChatOpenAI"] + assert llm["LiteLLMRequestId"] == "chatcmpl-4077bb36-9380-4a3b-9481-245700cef09a" + assert llm["Model"] == "claude-sonnet-4-5" + assert (llm["InputTokens"], llm["OutputTokens"]) == (3332, 467) + + +def test_llm_input_output_are_normalized_messages(rows_by_name): + llm = rows_by_name["ChatOpenAI"] + messages = json.loads(llm["Input"]) + assert [m["role"] for m in messages][:2] == ["system", "user"] + assert "research lead" in messages[0]["content"] + output = json.loads(llm["Output"]) + assert output["role"] == "assistant" + assert output["tool_calls"][0]["name"] + + +@pytest.mark.parametrize("completion", ["{}", '{"generations": []}', '{"generations": [[{}]]}']) +def test_incomplete_langsmith_completion_preserves_the_export(completion): + span = _span( + "ChatOpenAI", + b"\x03" * 8, + b"\x02" * 8, + langsmith__span__kind="llm", + gen_ai__prompt='{"messages": [[{"kwargs": {"type": "human", "content": "hi"}}]]}', + gen_ai__completion=completion, + ) + rows = decode_otlp(_export(span, scope="langsmith"), "application/x-protobuf") + assert len(rows) == 1 + assert json.loads(rows[0]["Input"])[0]["content"] == "hi" + assert rows[0]["Output"] == completion + + +def test_task_tool_output_is_subagent_final_message_text(rows_by_name): + task = rows_by_name["task"] + assert json.loads(task["Input"])["subagent_type"] == "researcher" + assert task["Output"].startswith("Based on my research") + assert not task["Output"].startswith("{") + + +def test_agent_input_output(rows_by_name): + root = rows_by_name["deep_research_agent"] + assert json.loads(root["Input"]) == [ + {"role": "user", "content": "Should we store OTEL agent spans in ClickHouse or Postgres at 50k spans/sec?"} + ] + assert json.loads(root["Output"])["role"] == "assistant" + + +def test_plain_tool_input_output(rows_by_name): + tool = rows_by_name["search_docs"] + assert json.loads(tool["Input"]) == {"query": "ClickHouse Postgres OpenTelemetry OTEL spans performance comparison"} + assert tool["Output"].startswith("ClickHouse ingests") + + +def test_heavy_attributes_are_lifted_out_of_span_attributes(rows_by_name): + for row in rows_by_name.values(): + assert not set(row["SpanAttributes"]) & decode._HEAVY_ATTRIBUTES + assert rows_by_name["ChatOpenAI"]["SpanAttributes"]["langsmith.span.kind"] == "llm" + + +def test_ids_are_hex_and_resource_is_kept(rows_by_name): + root = rows_by_name["deep_research_agent"] + assert root["TraceId"] == TRACE_ID + assert root["SpanId"] == "5e79f3b5b504985e" + assert root["ServiceName"] == "agent-demo" + assert root["ScopeName"] == "langsmith" + assert root["SpanKind"] == "SPAN_KIND_INTERNAL" + assert root["StatusCode"] == "STATUS_CODE_OK" + assert root["Duration"] > 0 + + +def test_protobuf_and_json_decode_identically(): + from_json = decode_otlp(_fixture_json(), "application/json") + from_protobuf = decode_otlp(_fixture_protobuf(), "application/x-protobuf") + assert from_json == from_protobuf + assert len(from_json) == 6 + + +def test_content_type_defaults_to_protobuf(): + assert len(decode_otlp(_fixture_protobuf(), None)) == 6 + + +@pytest.mark.parametrize("content_encoding", ["gzip", None]) +def test_gzip_body_by_header_or_magic_bytes(content_encoding): + rows = decode_otlp(gzip.compress(_fixture_protobuf()), "application/x-protobuf", content_encoding) + assert len(rows) == 6 + + +def test_long_values_are_truncated_with_marker(): + with patch.object(decode, "OTLP_MAX_ATTRIBUTE_VALUE_BYTES", 100): + rows = {r["SpanName"]: r for r in decode_otlp(_fixture_json(), "application/json")} + task = rows["task"] + assert "…[truncated " in task["Input"] + assert task["Input"].encode().startswith(task["Input"].split("…")[0].encode()) + assert len(task["Input"].split("…")[0].encode()) <= 100 + + +# ---------------------------------------------------------------- status / exceptions + + +def test_exception_event_fills_status_message(): + span = _span("get_customer_plan", b"\x01" * 8, b"\x02" * 8) + span.status.CopyFrom(Status(code=Status.STATUS_CODE_ERROR)) + event = span.events.add() + event.name = "exception" + event.attributes.extend( + [_kv("exception.type", "KeyError"), _kv("exception.message", "customer acme-404 not found")] + ) + (row,) = decode_otlp(_export(span)) + assert row["StatusCode"] == "STATUS_CODE_ERROR" + assert row["StatusMessage"] == "customer acme-404 not found" + + +def test_status_message_wins_over_exception_event(): + span = _span("tool", b"\x01" * 8, b"\x02" * 8) + span.status.CopyFrom(Status(code=Status.STATUS_CODE_ERROR, message="boom")) + event = span.events.add() + event.name = "exception" + event.attributes.append(_kv("exception.message", "other")) + (row,) = decode_otlp(_export(span)) + assert row["StatusMessage"] == "boom" + + +# ---------------------------------------------------------------- GenAI semconv / OpenInference + + +def test_genai_semconv_spans(): + root = _span( + "invoke_agent planner", b"\x01" * 8, gen_ai__operation__name="invoke_agent", gen_ai__agent__name="planner" + ) + chat = _span( + "chat gpt-4o", + b"\x02" * 8, + b"\x01" * 8, + gen_ai__operation__name="chat", + gen_ai__agent__name="planner", + gen_ai__request__model="gpt-4o", + gen_ai__response__id="chatcmpl-abc", + gen_ai__usage__input_tokens=12, + gen_ai__usage__output_tokens=3, + gen_ai__input__messages='[{"role":"user","content":"hi"}]', + gen_ai__output__messages='[{"role":"assistant","content":"hello"}]', + ) + tool = _span( + "execute_tool search", + b"\x03" * 8, + b"\x01" * 8, + gen_ai__operation__name="execute_tool", + gen_ai__tool__call__arguments='{"q":"x"}', + gen_ai__tool__call__result="found", + ) + rows = {r["SpanName"]: r for r in decode_otlp(_export(root, chat, tool))} + assert rows["invoke_agent planner"]["ObservationType"] == "agent" + assert rows["invoke_agent planner"]["AgentName"] == "planner" + llm = rows["chat gpt-4o"] + assert (llm["ObservationType"], llm["Model"], llm["LiteLLMRequestId"]) == ("llm", "gpt-4o", "chatcmpl-abc") + assert (llm["InputTokens"], llm["OutputTokens"]) == (12, 3) + assert json.loads(llm["Input"])[0]["content"] == "hi" + assert "gen_ai.input.messages" not in llm["SpanAttributes"] + assert (rows["execute_tool search"]["ObservationType"], rows["execute_tool search"]["Output"]) == ("tool", "found") + + +def test_openinference_spans(): + root = _span("agent", b"\x01" * 8, openinference__span__kind="AGENT", agent__name="writer", input__value="task") + llm = _span( + "llm", + b"\x02" * 8, + b"\x01" * 8, + openinference__span__kind="LLM", + llm__model_name="claude-sonnet-4-5", + llm__token_count__prompt=40, + llm__token_count__completion=8, + input__value="prompt", + output__value="answer", + ) + chain = _span("retriever", b"\x03" * 8, b"\x01" * 8, openinference__span__kind="RETRIEVER") + rows = {r["SpanName"]: r for r in decode_otlp(_export(root, llm, chain))} + assert (rows["agent"]["ObservationType"], rows["agent"]["AgentName"], rows["agent"]["Input"]) == ( + "agent", + "writer", + "task", + ) + assert rows["llm"]["ObservationType"] == "llm" + assert (rows["llm"]["Model"], rows["llm"]["InputTokens"], rows["llm"]["OutputTokens"]) == ( + "claude-sonnet-4-5", + 40, + 8, + ) + assert (rows["llm"]["Input"], rows["llm"]["Output"]) == ("prompt", "answer") + assert "input.value" not in rows["llm"]["SpanAttributes"] + assert rows["retriever"]["ObservationType"] == "chain" + + +def test_non_string_attribute_values_are_stringified(): + span = _span("root", b"\x01" * 8) + span.attributes.extend( + [ + KeyValue(key="flag", value=AnyValue(bool_value=True)), + KeyValue(key="ratio", value=AnyValue(double_value=0.5)), + KeyValue(key="raw", value=AnyValue(bytes_value=b"abc")), + ] + ) + array = KeyValue(key="list") + array.value.array_value.values.extend([AnyValue(string_value="a"), AnyValue(int_value=1)]) + span.attributes.append(array) + (row,) = decode_otlp(_export(span)) + assert row["SpanAttributes"]["flag"] == "true" + assert row["SpanAttributes"]["ratio"] == "0.5" + assert row["SpanAttributes"]["raw"] == "abc" + assert json.loads(row["SpanAttributes"]["list"]) == ["a", "1"] + + +# ---------------------------------------------------------------- helpers + + +def test_encode_otlp_response_matches_request_encoding(): + assert encode_otlp_response("application/json") == (b"{}", "application/json") + assert encode_otlp_response("application/x-protobuf") == (b"", "application/x-protobuf") + assert encode_otlp_response(None) == (b"", "application/x-protobuf") diff --git a/tests/test_litellm/tracing/test_receiver.py b/tests/test_litellm/tracing/test_receiver.py new file mode 100644 index 00000000000..d492844db79 --- /dev/null +++ b/tests/test_litellm/tracing/test_receiver.py @@ -0,0 +1,117 @@ +""" +Tests for TraceReceiver.ingest (litellm/tracing/receiver.py) with a fake store. +""" + +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest +from opentelemetry.proto.common.v1.common_pb2 import AnyValue, KeyValue +from opentelemetry.proto.trace.v1.trace_pb2 import ResourceSpans, ScopeSpans, Span + +from litellm.tracing import Tenant, TraceReceiver, TracingPayloadTooLargeError +from litellm.tracing import receiver as receiver_module +from litellm.tracing.types import TraceScope + +pytestmark = pytest.mark.requires_rust_extension + +FIXTURE = Path(__file__).parent / "fixtures" / "langsmith_deep_agent_export.json" +TENANT = Tenant(team_id="team-research", api_key_hash="hashed-key", org_id="org-1") + + +def _fake_store() -> MagicMock: + store = MagicMock() + store.insert_spans = AsyncMock() + store.get_trace = AsyncMock(return_value=None) + return store + + +def _spoofed_export() -> bytes: + """A client that tries to claim another team via resource attributes.""" + resource_spans = ResourceSpans(scope_spans=[ScopeSpans(spans=[Span(trace_id=b"\x01" * 16, span_id=b"\x02" * 8)])]) + resource_spans.resource.attributes.extend( + [ + KeyValue(key="service.name", value=AnyValue(string_value="svc")), + KeyValue(key="litellm.team_id", value=AnyValue(string_value="someone-elses-team")), + KeyValue(key="litellm.api_key_hash", value=AnyValue(string_value="someone-elses-key")), + ] + ) + return ExportTraceServiceRequest(resource_spans=[resource_spans]).SerializeToString() + + +@pytest.mark.asyncio +async def test_ingest_returns_span_count_and_writes_stamped_rows(): + store = _fake_store() + count = await TraceReceiver(store).ingest(FIXTURE.read_bytes(), "application/json", None, TENANT) + assert count == 6 + (rows,) = store.insert_spans.await_args.args + assert len(rows) == 6 + for row in rows: + assert (row["TeamId"], row["ApiKeyHash"]) == ("team-research", "hashed-key") + assert row["ResourceAttributes"]["litellm.org_id"] == "org-1" + assert row["ResourceAttributes"]["service.name"] == "agent-demo" + + +@pytest.mark.asyncio +async def test_ingest_overwrites_client_supplied_tenant_attributes(): + store = _fake_store() + await TraceReceiver(store).ingest(_spoofed_export(), "application/x-protobuf", None, TENANT) + ((row,),) = store.insert_spans.await_args.args + assert row["TeamId"] == "team-research" + assert row["ResourceAttributes"]["litellm.team_id"] == "team-research" + assert row["ResourceAttributes"]["litellm.api_key_hash"] == "hashed-key" + + +@pytest.mark.asyncio +async def test_ingest_does_not_acknowledge_failed_clickhouse_write(): + store = _fake_store() + store.insert_spans.side_effect = RuntimeError("ClickHouse unavailable") + with pytest.raises(RuntimeError, match="ClickHouse unavailable"): + await TraceReceiver(store).ingest(FIXTURE.read_bytes(), "application/json", None, TENANT) + store.insert_spans.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_ingest_rejects_oversized_encoded_batch(): + store = _fake_store() + store.insert_spans.side_effect = OverflowError("ClickHouse insert exceeds the encoded size limit") + with pytest.raises(TracingPayloadTooLargeError, match="encoded size limit"): + await TraceReceiver(store).ingest(FIXTURE.read_bytes(), "application/json", None, TENANT) + + +@pytest.mark.asyncio +async def test_ingest_rejects_oversized_body(): + store = _fake_store() + with patch.object(receiver_module, "OTLP_MAX_BODY_BYTES", 10): + with pytest.raises(TracingPayloadTooLargeError): + await TraceReceiver(store).ingest(FIXTURE.read_bytes(), "application/json", None, TENANT) + store.insert_spans.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_large_body_is_decoded_off_the_event_loop(): + store = _fake_store() + with ( + patch.object(receiver_module, "OTLP_OFFLOAD_DECODE_BYTES", 0), + patch.object(receiver_module.asyncio, "to_thread", wraps=receiver_module.asyncio.to_thread) as to_thread, + ): + count = await TraceReceiver(store).ingest(FIXTURE.read_bytes(), "application/json", None, TENANT) + assert count == 6 + to_thread.assert_called_once() + + +@pytest.mark.asyncio +async def test_empty_export_writes_nothing(): + store = _fake_store() + assert await TraceReceiver(store).ingest(b"", "application/x-protobuf", None, TENANT) == 0 + store.insert_spans.assert_awaited_once_with(()) + + +@pytest.mark.asyncio +async def test_reads_delegate_to_store(): + store = _fake_store() + tracing = TraceReceiver(store) + scope: TraceScope = {"team_ids": ("team-research",), "api_key_hash": ""} + assert await tracing.get_trace("t1", scope) is None + store.get_trace.assert_awaited_once_with("t1", scope, "") diff --git a/tests/test_litellm/tracing/test_store.py b/tests/test_litellm/tracing/test_store.py new file mode 100644 index 00000000000..39ce3162073 --- /dev/null +++ b/tests/test_litellm/tracing/test_store.py @@ -0,0 +1,311 @@ +""" +Tests for the pure read-side helpers in litellm/tracing/store.py (no ClickHouse needed). +""" + +from typing import Any +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from litellm.tracing.store import ( + ClickHouseTraceStore, + agent_nodes, + decode_cursor, + encode_cursor, + span_from_row, + trace_from_rows, + trace_summary_from_row, +) +from litellm.tracing.types import TraceScope + +T0 = 1_790_742_989_000_000_000 # ns +MS = 1_000_000 + + +def _row( + span_id: str, + parent: str, + name: str, + type_: str, + agent: str, + start_ms: float = 0, + duration_ms: float = 10, + status: str = "STATUS_CODE_OK", + **extra: Any, +) -> dict[str, Any]: + return { + "span_id": span_id, + "parent_span_id": parent, + "name": name, + "type": type_, + "agent": agent, + "status": status, + "start_ns": T0 + int(start_ms * MS), + "duration_ns": int(duration_ms * MS), + "service": "agent-demo", + "input_preview": f"input of {name}", + "model": "", + "input_tokens": 0, + "output_tokens": 0, + "litellm_request_id": "", + **extra, + } + + +def _llm_row(span_id: str, parent: str, agent: str, request_id: str, start_ms: float = 1) -> dict: + return _row( + span_id, + parent, + "ChatOpenAI", + "llm", + agent, + start_ms=start_ms, + duration_ms=100, + model="claude-sonnet-4-5", + input_tokens=100, + output_tokens=20, + litellm_request_id=request_id, + ) + + +def _deep_agent_rows(researcher_invocations: int = 1) -> list[dict[str, Any]]: + """root agent -> llm, task tool -> researcher subagent (N times) -> llm + search_docs tool.""" + rows = [ + _row("root", "", "deep_research_agent", "agent", "deep_research_agent", duration_ms=1000), + _llm_row("llm-root", "root", "deep_research_agent", "chatcmpl-root"), + _row("task", "root", "task", "tool", "deep_research_agent", start_ms=200, duration_ms=700), + ] + for i in range(researcher_invocations): + rows += [ + _row(f"res-{i}", "task", "researcher", "agent", "researcher", start_ms=201, duration_ms=5), + _llm_row(f"res-llm-{i}", f"res-{i}", "researcher", f"chatcmpl-res-{i}", start_ms=202), + _row(f"res-tool-{i}", f"res-{i}", "search_docs", "tool", "researcher", start_ms=203, duration_ms=1), + _row(f"res-mw-{i}", f"res-{i}", "FilesystemMiddleware.wrap_model_call", "framework", "researcher"), + ] + return rows + + +# ---------------------------------------------------------------- trace_from_rows + + +def test_empty_rows_is_none(): + assert trace_from_rows("abc", []) is None + + +def test_llm_response_id_is_preserved_without_spend_enrichment(): + trace = trace_from_rows("t1", _deep_agent_rows()) + assert trace is not None + spans = {span["span_id"]: span for span in trace["spans"]} + assert spans["llm-root"]["litellm_request_id"] == "chatcmpl-root" + assert spans["task"]["litellm_request_id"] is None + assert "spend" not in trace["summary"] + + +def test_summary_totals(): + trace = trace_from_rows("t1", _deep_agent_rows()) + assert trace is not None + summary = trace["summary"] + assert summary["trace_id"] == "t1" + assert summary["name"] == "deep_research_agent" + assert summary["service"] == "agent-demo" + assert summary["input_preview"] == "input of deep_research_agent" + assert summary["status"] == "ok" + assert summary["span_count"] == 7 + assert summary["agent_count"] == 2 + assert summary["llm_calls"] == 2 + assert summary["tool_calls"] == 2 + assert summary["error_count"] == 0 + assert (summary["input_tokens"], summary["output_tokens"]) == (200, 40) + assert summary["models"] == ("claude-sonnet-4-5",) + assert summary["duration_ms"] == 1000 + assert summary["start_time"].startswith("2026-09-30T") + + +def test_error_count_counts_error_spans(): + rows = _deep_agent_rows() + rows[2]["status"] = "STATUS_CODE_ERROR" + trace = trace_from_rows("t1", rows) + assert trace is not None + assert trace["summary"]["error_count"] == 1 + assert trace["summary"]["status"] == "ok" # root span status; the UI uses error_count for "failed" + assert trace["spans"][2]["status"] == "error" + + +def test_offsets_are_relative_to_trace_start_in_ms(): + trace = trace_from_rows("t1", _deep_agent_rows()) + assert trace is not None + spans = {s["span_id"]: s for s in trace["spans"]} + assert spans["root"]["start_offset_ms"] == 0 + assert spans["task"]["start_offset_ms"] == 200 + assert spans["task"]["duration_ms"] == 700 + assert spans["root"]["parent_span_id"] is None + assert spans["task"]["parent_span_id"] == "root" + + +def test_span_from_row_optional_fields(): + span = span_from_row(_row("s", "", "x", "chain", "a", status="STATUS_CODE_UNSET"), T0) + assert (span["model"], span["parent_span_id"], span["status"], span["litellm_request_id"]) == ( + None, + None, + "unset", + None, + ) + + +def test_agent_nodes_parent_and_per_agent_counts(): + trace = trace_from_rows("t1", _deep_agent_rows()) + assert trace is not None + assert trace["agents"] == ( + { + "name": "deep_research_agent", + "parent_agent": None, + "invocations": 1, + "llm_calls": 1, + "tool_calls": 1, + "duration_ms": 1000, + }, + { + "name": "researcher", + "parent_agent": "deep_research_agent", + "invocations": 1, + "llm_calls": 1, + "tool_calls": 1, + "duration_ms": 5, + }, + ) + + +def test_200_subagent_invocations_aggregate_into_one_node(): + trace = trace_from_rows("t1", _deep_agent_rows(researcher_invocations=200)) + assert trace is not None + assert [a["name"] for a in trace["agents"]] == ["deep_research_agent", "researcher"] + researcher = trace["agents"][1] + assert researcher["parent_agent"] == "deep_research_agent" + assert researcher["invocations"] == 200 + assert researcher["llm_calls"] == 200 + assert researcher["tool_calls"] == 200 + assert researcher["duration_ms"] == pytest.approx(1000) + assert trace["summary"]["agent_count"] == 2 + assert trace["summary"]["span_count"] == 3 + 4 * 200 + + +def test_parent_agent_skips_same_name_ancestors(): + """A recursive agent (researcher -> researcher) still reports the nearest *different* agent.""" + rows = [ + _row("root", "", "lead", "agent", "lead"), + _row("r1", "root", "researcher", "agent", "researcher"), + _row("r2", "r1", "researcher", "agent", "researcher"), + ] + spans = [span_from_row(r, T0) for r in rows] + nodes = {n["name"]: n for n in agent_nodes(spans)} + assert nodes["researcher"]["parent_agent"] == "lead" + assert nodes["researcher"]["invocations"] == 2 + + +def test_parent_agent_stops_at_cyclic_parents(): + rows = [ + _row("self", "self", "researcher", "agent", "researcher"), + _row("first", "second", "researcher", "agent", "researcher"), + _row("second", "first", "researcher", "agent", "researcher"), + ] + spans = [span_from_row(row, T0) for row in rows] + assert agent_nodes(spans)[0]["parent_agent"] is None + + +def test_agent_nodes_ignores_spans_of_unknown_agents(): + spans = [span_from_row(_row("t", "", "tool", "tool", "ghost"), T0)] + assert agent_nodes(spans) == () + + +# ---------------------------------------------------------------- list helpers + + +def test_cursor_round_trip(): + cursor = encode_cursor(1790742989377, "4bad42b84e9de3ba46fc870185f8f023") + assert decode_cursor(cursor) == (1790742989377, "4bad42b84e9de3ba46fc870185f8f023") + assert decode_cursor(None) == (0, "") + assert decode_cursor("") == (0, "") + + +@pytest.mark.parametrize("cursor", ["abc", "bm90LWpzb24=", "WzEsIDJd", "WzAsICJ0Il0="]) +def test_invalid_cursor_is_rejected(cursor): + with pytest.raises(ValueError, match="Invalid trace cursor"): + decode_cursor(cursor) + + +def test_trace_summary_from_row(): + summary = trace_summary_from_row( + { + "trace_id": "t1", + "name": "deep_research_agent", + "service": "agent-demo", + "input_preview": "hi", + "start_ms": 1790742989377, + "duration_ms": 51385, + "status": "STATUS_CODE_OK", + "span_count": "126", + "agent_count": "2", + "llm_calls": "7", + "tool_calls": "26", + "error_count": "1", + "input_tokens": "30175", + "output_tokens": "2620", + "models": ["claude-sonnet-4-5"], + } + ) + assert summary["status"] == "ok" + assert (summary["span_count"], summary["error_count"]) == (126, 1) + assert summary["start_time"] == "2026-09-30T04:36:29.377000+00:00" + + +@pytest.mark.asyncio +async def test_list_traces_sets_next_cursor_on_full_page(): + client = MagicMock() + row = { + "trace_id": "t2", + "trace_ref": "ref2", + "name": "a", + "service": "s", + "input_preview": "", + "start_ms": 1000, + "duration_ms": 1, + "status": "STATUS_CODE_OK", + "span_count": 1, + "agent_count": 1, + "llm_calls": 0, + "tool_calls": 0, + "error_count": 0, + "input_tokens": 0, + "output_tokens": 0, + "models": [], + } + client.query = AsyncMock(return_value=[row, {**row, "trace_id": "t1", "trace_ref": "ref1", "start_ms": 900}]) + store = ClickHouseTraceStore(client) + scope: TraceScope = {"team_ids": ("team-a",), "api_key_hash": ""} + + page = await store.list_traces(scope, 0, 2000, limit=2) + assert [t["trace_id"] for t in page["data"]] == ["t2", "t1"] + assert page["next_cursor"] is not None + assert decode_cursor(page["next_cursor"]) == (900, "ref1") + params = client.query.call_args.args[1] + assert params["team_ids"] == ("team-a",) and params["limit"] == 2 and params["cursor_ms"] == 0 + + page = await store.list_traces(scope, 0, 2000, cursor=page["next_cursor"], limit=3) + assert page["next_cursor"] is None + assert client.query.call_args.args[1]["cursor_trace_id"] == "ref1" + + +@pytest.mark.asyncio +async def test_get_span_not_found_and_found(): + client = MagicMock() + client.query = AsyncMock(return_value=[]) + store = ClickHouseTraceStore(client) + scope: TraceScope = {"team_ids": (), "api_key_hash": ""} + assert await store.get_span("t", "s", scope) is None + client.query = AsyncMock(return_value=[{"span_id": "s", "input": "i", "output": "o", "attributes": {"k": "v"}}]) + assert await store.get_span("t", "s", scope) == { + "span_id": "s", + "input": "i", + "output": "o", + "attributes": {"k": "v"}, + } diff --git a/tests/test_litellm_rust/test_traces.py b/tests/test_litellm_rust/test_traces.py index 21f61ad24bc..447ca2ce4bb 100644 --- a/tests/test_litellm_rust/test_traces.py +++ b/tests/test_litellm_rust/test_traces.py @@ -1,11 +1,12 @@ import base64 +import gzip import json from typing import Final from urllib.parse import parse_qs, urlsplit import pytest -from litellm.rust_bridge.traces import encode_rows, ensure_schema, query +from litellm.rust_bridge._native import NativeTraceStorage from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec pytestmark = pytest.mark.requires_rust_extension @@ -14,14 +15,9 @@ pytestmark = pytest.mark.requires_rust_extension @pytest.mark.asyncio async def test_trace_reader_projects_connection_and_parameters(recording_server: RecordingServer) -> None: recording_server.enqueue(ResponseSpec(body={"data": [{"trace_id": "trace-1"}]})) - rows: Final = await query( - recording_server.base_url + "?database=wrong&user=wrong&password=wrong", - "trace_test", - "reader", - "p@ss/word%", - "SELECT {trace_id:String} AS trace_id", - {"trace_id": "trace-1"}, - ) + reader_url: Final = recording_server.base_url.replace("http://", "http://reader:p%40ss%2Fword%25@") + storage: Final = NativeTraceStorage("trace_test", recording_server.base_url, reader_url + "?database=wrong") + rows: Final = json.loads(await storage.query("SELECT {trace_id:String} AS trace_id", {"trace_id": "trace-1"})) request: Final = recording_server.requests[0] parameters: Final = parse_qs(urlsplit(request.path).query) assert rows == [{"trace_id": "trace-1"}] @@ -37,15 +33,22 @@ async def test_trace_reader_projects_connection_and_parameters(recording_server: @pytest.mark.asyncio async def test_trace_reader_rejects_success_status_with_embedded_error(recording_server: RecordingServer) -> None: recording_server.enqueue(ResponseSpec(body={"data": [], "exception": "query failed"})) + storage: Final = NativeTraceStorage("trace_test", recording_server.base_url, recording_server.base_url) with pytest.raises(RuntimeError, match="invalid or failed JSON"): - await query(recording_server.base_url, "trace_test", "reader", "password", "SELECT 1", {}) + await storage.query("SELECT 1", {}) @pytest.mark.asyncio -@pytest.mark.parametrize("database,retention", [("db; DROP DATABASE default", 7), ("traces", 0)]) -async def test_schema_binding_preserves_configuration_validation(database: str, retention: int) -> None: +async def test_schema_binding_rejects_invalid_database() -> None: with pytest.raises(ValueError, match=r"database.*retention"): - await ensure_schema("http://localhost:8123", database, "writer", "password", retention, 14) + NativeTraceStorage("db; DROP DATABASE default", "http://localhost:8123") + + +@pytest.mark.asyncio +async def test_schema_binding_rejects_non_positive_retention() -> None: + storage: Final = NativeTraceStorage("traces", "http://localhost:8123") + with pytest.raises(ValueError, match=r"database.*retention"): + await storage.ensure_schema(0, 14) @pytest.mark.asyncio @@ -53,15 +56,10 @@ async def test_schema_setup_uses_writer_credentials_and_rejects_failed_statement recording_server.expected_requests = 2 recording_server.enqueue(ResponseSpec(body="")) recording_server.enqueue(ResponseSpec(status=403, body="denied")) + writer_url: Final = recording_server.base_url.replace("http://", "http://writer:p%40ss%2Fword%25@") + storage: Final = NativeTraceStorage("trace_test", writer_url + "?database=wrong&readonly=1") with pytest.raises(RuntimeError, match="schema setup failed with HTTP status 403"): - await ensure_schema( - recording_server.base_url + "?database=wrong&readonly=1", - "trace_test", - "writer", - "p@ss/word%", - 7, - 14, - ) + await storage.ensure_schema(7, 14) assert len(recording_server.requests) == 2 assert recording_server.requests[0].raw_body.startswith(b"CREATE DATABASE IF NOT EXISTS") assert recording_server.requests[1].raw_body.startswith(b"CREATE TABLE IF NOT EXISTS") @@ -71,8 +69,15 @@ async def test_schema_setup_uses_writer_credentials_and_rejects_failed_statement ).decode() -def test_insert_encoding_preserves_nanoseconds_through_bridge() -> None: - assert json.loads(encode_rows([{"Timestamp": 1_234_567_890, "Input": "hello"}])) == { +@pytest.mark.asyncio +async def test_insert_encodes_and_sends_rows(recording_server: RecordingServer) -> None: + recording_server.enqueue(ResponseSpec(body="")) + storage: Final = NativeTraceStorage("trace_test", recording_server.base_url) + await storage.insert_rows("otel_traces", [{"Timestamp": 1_234_567_890, "Input": "hello"}]) + request: Final = recording_server.requests[0] + assert json.loads(gzip.decompress(request.raw_body)) == { "Input": "hello", "Timestamp": "1970-01-01T00:00:01.23456789Z", } + assert parse_qs(urlsplit(request.path).query)["query"] == ["INSERT INTO `trace_test`.otel_traces FORMAT JSONEachRow"] + assert request.headers["content-encoding"] == "gzip" diff --git a/ui/litellm-dashboard/package-lock.json b/ui/litellm-dashboard/package-lock.json index 03e29c02021..1cb1951d96a 100644 --- a/ui/litellm-dashboard/package-lock.json +++ b/ui/litellm-dashboard/package-lock.json @@ -25,7 +25,7 @@ "jwt-decode": "4.0.0", "lucide-react": "0.513.0", "moment": "2.31.0", - "next": "16.3.3", + "next": "16.3.6", "next-themes": "^0.4.6", "nuqs": "^2.9.4", "openai": "4.104.0", @@ -2061,9 +2061,9 @@ } }, "node_modules/@next/env": { - "version": "16.3.3", - "resolved": "https://registry.npmjs.org/@next/env/-/env-16.3.3.tgz", - "integrity": "sha512-U2eYQRwXj+dsqxV79zFqExDdatnNY/ZWc2nsJU1p/OgT7fd3dXwlF6OjYaFQCfMoeTA19PWq+wVmYgimVA+V+g==", + "version": "16.3.6", + "resolved": "https://registry.npmjs.org/@next/env/-/env-16.3.6.tgz", + "integrity": "sha512-x9Vblze1EbtltQYnNH38xCPWU3TVfBd1eXqA3+w9+BTpedkkdNpAaltXlGQ/nsc1+E0mVTNrtcbX3GoO09zeLQ==", "license": "MIT" }, "node_modules/@next/eslint-plugin-next": { @@ -2078,9 +2078,9 @@ } }, "node_modules/@next/swc-darwin-arm64": { - "version": "16.3.3", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-16.3.3.tgz", - "integrity": "sha512-8Hiv32QJPwdV6KYJ8meR9SBA061tQqnIKTJDocvOXlEQqib0xMFpzArosuffFUUc0sslbh7QQ8a3Yey1QV8EIw==", + "version": "16.3.6", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-16.3.6.tgz", + "integrity": "sha512-E/7GEqaUkt8mk/T8v9lAnrhzR06kdq1ZBkC12F8tAMkdIadwNp3H1KqHynDHrpcTlGCUdq/qu6vUL2aYVyYBdw==", "cpu": [ "arm64" ], @@ -2094,9 +2094,9 @@ } }, "node_modules/@next/swc-darwin-x64": { - "version": "16.3.3", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-16.3.3.tgz", - "integrity": "sha512-A1lgKgwVchRYmSe467zdwhxT9040dd8lH+o65sL5Jet8fjB4kegw/rDyPIpYVRb6jAqwXFOJpjIXJLxQKLiE3A==", + "version": "16.3.6", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-16.3.6.tgz", + "integrity": "sha512-yBE893/nDWTlaiBD1p+qgt7NUen4U5R6FXyH0s67Npq1S3E0cVSef1WIXC2xBRgQvwAvJq6DnS6Y6PrY0cy4Ew==", "cpu": [ "x64" ], @@ -2110,9 +2110,9 @@ } }, "node_modules/@next/swc-linux-arm64-gnu": { - "version": "16.3.3", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-16.3.3.tgz", - "integrity": "sha512-bf0FIssMFueU2dm7vQEWWxk0c8UjKTdW0yzuh0sQsD8pf1+KCLDdaqhYZNMYGmXwEOiHAUzgBKudovIlcvvBjg==", + "version": "16.3.6", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-16.3.6.tgz", + "integrity": "sha512-KJDpjBqBPYlvkivmyrp+Qys6k/7ksbqGQvRVc6ZEGfR+cjQxx+nUkJaWmNZJsmoOrqYNbaXByF8wa0lBwDhB3Q==", "cpu": [ "arm64" ], @@ -2129,9 +2129,9 @@ } }, "node_modules/@next/swc-linux-arm64-musl": { - "version": "16.3.3", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-16.3.3.tgz", - "integrity": "sha512-W7viwCk9JY/cAkdz/A273rd5bb3RgT/IHwR7Upv90tunjBWNtAAhGhoecHh+teRNRSinuAFmE+l7fwZ4YKkrXg==", + "version": "16.3.6", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-16.3.6.tgz", + "integrity": "sha512-mqNg2K+hvWskSRb/QM+Ix412DvBsuSF0XV+frTSw5vmoucNnIlynFwKYew8D01bfATErMOM7Bujrf0BA5DRKFA==", "cpu": [ "arm64" ], @@ -2148,9 +2148,9 @@ } }, "node_modules/@next/swc-linux-x64-gnu": { - "version": "16.3.3", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-16.3.3.tgz", - "integrity": "sha512-0W46zw1N3ODpI6n0GeivHvvob1pooozgZVqy65k0mh4/7vr+FbY9+WpHzNVXjHipJf/A3FDheBG19H1s5A25rA==", + "version": "16.3.6", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-16.3.6.tgz", + "integrity": "sha512-nFncBNGAYouRHjRVaITs9beZRfhX4ssVwpnvPIAbkZVH6LtGoAVlH4bJ8Cnf9SOo9bsXgPFer/GdHtEE3JNOkw==", "cpu": [ "x64" ], @@ -2167,9 +2167,9 @@ } }, "node_modules/@next/swc-linux-x64-musl": { - "version": "16.3.3", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-16.3.3.tgz", - "integrity": "sha512-H4mBso8ZTMBPtdT0PN0pBx2ayTvQuTuvS6qT13d77yVFJXAPCxkyIhLTmdMaGTJs0krQYI/qpzdHijCeihXhbg==", + "version": "16.3.6", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-16.3.6.tgz", + "integrity": "sha512-5Mf3cHDGR/Iz0ng2Bj3zUR3p5QS9YK3Hn2QiAfavFmyF48zwThAjpFoiTKNIcOHLYS4zEk+gzyJ/9deQ2ZB8yQ==", "cpu": [ "x64" ], @@ -2186,9 +2186,9 @@ } }, "node_modules/@next/swc-win32-arm64-msvc": { - "version": "16.3.3", - "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-16.3.3.tgz", - "integrity": "sha512-cTMUJpcEGmeywofCUfhR+rSsoE33+rVPnPEYNTNdLNlsOeEg/vktOsKUSTb28vUGqD2jkm4Zaskcwn7OCI6FQg==", + "version": "16.3.6", + "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-16.3.6.tgz", + "integrity": "sha512-0jkJy0C2kbrJWTk4YLa3xk80pVBpx8FCHJym7CnUfDAXe/FWv5qT7SQJbR0KuemyxaEDlEx5WT4VQJoTW+/9Qw==", "cpu": [ "arm64" ], @@ -2202,9 +2202,9 @@ } }, "node_modules/@next/swc-win32-x64-msvc": { - "version": "16.3.3", - "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-16.3.3.tgz", - "integrity": "sha512-2VR4cTBzHXaBjnGsuH6GyJjENzQOmHeAh11uY1iUhjm3j5dEUrVJuUj+VL78jaGi/Dik8xS76zEj18BsFhlVZQ==", + "version": "16.3.6", + "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-16.3.6.tgz", + "integrity": "sha512-/YXjI1e5OXcZ7YpxRwgP/1jAV/SBKTzeVKqN2mk7mLpcICsyn3Gl5+dIfDTJp70M0ccMhyMMRso4v6mPDCGepg==", "cpu": [ "x64" ], @@ -9712,12 +9712,12 @@ "license": "MIT" }, "node_modules/next": { - "version": "16.3.3", - "resolved": "https://registry.npmjs.org/next/-/next-16.3.3.tgz", - "integrity": "sha512-tuRTx1nQ/yVw83cwJBo9F+njGUgMn3UHQycreWHB8XsStvvAh1AthbI8/4IpKnFaF58F+iSiHejYOlMQ/eq83g==", + "version": "16.3.6", + "resolved": "https://registry.npmjs.org/next/-/next-16.3.6.tgz", + "integrity": "sha512-L+otWM/aQbYTx98aZhgEoMb4bZAXx1YVW4UMA/vuCyCoWG5HJyZUili8QAkqzrcC+5///tsz3s0M+SlyB5bLMw==", "license": "MIT", "dependencies": { - "@next/env": "16.3.3", + "@next/env": "16.3.6", "@swc/helpers": "0.5.23", "baseline-browser-mapping": "^2.9.19", "caniuse-lite": "^1.0.30001579", @@ -9731,15 +9731,15 @@ "node": ">=20.9.0" }, "optionalDependencies": { - "@next/swc-darwin-arm64": "16.3.3", - "@next/swc-darwin-x64": "16.3.3", - "@next/swc-linux-arm64-gnu": "16.3.3", - "@next/swc-linux-arm64-musl": "16.3.3", - "@next/swc-linux-x64-gnu": "16.3.3", - "@next/swc-linux-x64-musl": "16.3.3", - "@next/swc-win32-arm64-msvc": "16.3.3", - "@next/swc-win32-x64-msvc": "16.3.3", - "sharp": "^0.35.3" + "@next/swc-darwin-arm64": "16.3.6", + "@next/swc-darwin-x64": "16.3.6", + "@next/swc-linux-arm64-gnu": "16.3.6", + "@next/swc-linux-arm64-musl": "16.3.6", + "@next/swc-linux-x64-gnu": "16.3.6", + "@next/swc-linux-x64-musl": "16.3.6", + "@next/swc-win32-arm64-msvc": "16.3.6", + "@next/swc-win32-x64-msvc": "16.3.6", + "sharp": "^0.35.4" }, "peerDependencies": { "@opentelemetry/api": "^1.1.0", diff --git a/ui/litellm-dashboard/package.json b/ui/litellm-dashboard/package.json index 3bf32d37faf..0830e233bbe 100644 --- a/ui/litellm-dashboard/package.json +++ b/ui/litellm-dashboard/package.json @@ -41,7 +41,7 @@ "jwt-decode": "4.0.0", "lucide-react": "0.513.0", "moment": "2.31.0", - "next": "16.3.3", + "next": "16.3.6", "next-themes": "^0.4.6", "nuqs": "^2.9.4", "openai": "4.104.0", diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index d0a5349364f..7ffe0e0ec85 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -2121,12 +2121,21 @@ export const agentTraceListCall = async ({ return apiClient.get(`/v1/traces`, { accessToken, query }); }; -export const agentTraceCall = async (accessToken: string, traceId: string): Promise => - apiClient.get(`/v1/traces/${encodeURIComponent(traceId)}`, { accessToken }); +export const agentTraceCall = async (accessToken: string, traceId: string, traceRef?: string): Promise => + apiClient.get(`/v1/traces/${encodeURIComponent(traceId)}`, { + accessToken, + query: { trace_ref: traceRef || undefined }, + }); -export const agentTraceSpanCall = async (accessToken: string, traceId: string, spanId: string): Promise => +export const agentTraceSpanCall = async ( + accessToken: string, + traceId: string, + spanId: string, + traceRef?: string, +): Promise => apiClient.get(`/v1/traces/${encodeURIComponent(traceId)}/spans/${encodeURIComponent(spanId)}`, { accessToken, + query: { trace_ref: traceRef || undefined }, }); export const adminSpendLogsCall = async (accessToken: string) => { diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.tsx index 2bfa3c320a7..61e4c737b24 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.tsx @@ -67,7 +67,7 @@ export function AgentTracesSection({ timeControls, onRunOpenChange, }: AgentTracesSectionProps) { - const [openTraceId, setOpenTraceId] = useState(null); + const [openTrace, setOpenTrace] = useState(null); const [query, setQuery] = useState(""); const [service, setService] = useState(ALL_SERVICES); const [status, setStatus] = useState("all"); @@ -96,9 +96,9 @@ export function AgentTracesSection({ apply(hours); }; - const openRun = (traceId: string | null) => { - setOpenTraceId(traceId); - onRunOpenChange?.(traceId !== null); + const openRun = (trace: TraceSummary | null) => { + setOpenTrace(trace); + onRunOpenChange?.(trace !== null); }; if (traces.notEnabledDetail !== null) return ; @@ -120,8 +120,15 @@ export function AgentTracesSection({ ); } - if (openTraceId !== null) { - return openRun(null)} />; + if (openTrace !== null) { + return ( + openRun(null)} + /> + ); } return ( diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesTable.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesTable.tsx index e4faff1621a..d3912b7be5c 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesTable.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesTable.tsx @@ -14,7 +14,7 @@ interface AgentTracesTableProps { error: Error | null; hasMore: boolean; onLoadMore: () => void; - onOpenTrace: (traceId: string) => void; + onOpenTrace: (trace: TraceSummary) => void; } /** Spend is only on summaries once the spend-enrichment PR lands; show Cost when it's there. */ @@ -82,9 +82,9 @@ export function AgentTracesTable({ {traces.map((run) => ( onOpenTrace(run.trace_id)} + onClick={() => onOpenTrace(run)} className="h-9 cursor-pointer border-b border-border/60 text-[12px] hover:bg-accent/50" > const errorReason = (headline: string): string => /^([A-Za-z_][\w.]*)\(/.exec(headline)?.[1] ?? "error"; /** Shared lazy fetch of one span's full input / output / attributes. */ -export function useSpanDetail(accessToken: string, traceId: string, spanId: string | null) { +export function useSpanDetail(accessToken: string, traceId: string, spanId: string | null, traceRef?: string) { const queryOptions: UseQueryOptions = { - queryKey: ["agentTraceSpan", traceId, spanId, accessToken], - queryFn: () => agentTraceSpanCall(accessToken, traceId, spanId as string), + queryKey: ["agentTraceSpan", traceId, traceRef, spanId, accessToken], + queryFn: () => agentTraceSpanCall(accessToken, traceId, spanId as string, traceRef), enabled: spanId !== null, staleTime: Infinity, }; @@ -121,12 +121,13 @@ function Payload({ label, value, mono }: { label: string; value: string; mono: b interface DetailContentProps { accessToken: string; traceId: string; + traceRef?: string; span: Span; } /** Content tab: the error first (if any), then what went in and what came out. */ -export function DetailContent({ accessToken, traceId, span }: DetailContentProps) { - const detailQuery = useSpanDetail(accessToken, traceId, span.span_id); +export function DetailContent({ accessToken, traceId, traceRef, span }: DetailContentProps) { + const detailQuery = useSpanDetail(accessToken, traceId, span.span_id, traceRef); const detail = detailQuery.data; const isTool = span.type === "tool"; const empty = detail && !detail.input && !detail.output; diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/DetailPane.test.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/DetailPane.test.tsx index a15584e6d41..9a541d26966 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/DetailPane.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/DetailPane.test.tsx @@ -125,7 +125,7 @@ describe("DetailPane", () => { expect(screen.getByRole("tab", { name: "Attributes" })).toBeInTheDocument(); expect(await screen.findByText("You are a LiteLLM support agent.")).toBeInTheDocument(); expect(screen.getByText("get_customer_plan")).toBeInTheDocument(); - expect(vi.mocked(agentTraceSpanCall)).toHaveBeenCalledWith("sk-test", "t1", "llm1"); + expect(vi.mocked(agentTraceSpanCall)).toHaveBeenCalledWith("sk-test", "t1", "llm1", undefined); }); it("shows a tool failure as 'Tool · ' with the exception line and no traceback", async () => { diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/DetailPane.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/DetailPane.tsx index 0a4dd3c5029..4424b95cd52 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/DetailPane.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/DetailPane.tsx @@ -67,7 +67,12 @@ function SpanPane({ }) { const [tab, setTab] = useState("content"); const traceId = trace.summary.trace_id; - const detailQuery = useSpanDetail(accessToken, traceId, tab === "attributes" ? span.span_id : null); + const detailQuery = useSpanDetail( + accessToken, + traceId, + tab === "attributes" ? span.span_id : null, + trace.summary.trace_ref, + ); const tokens = span.input_tokens + span.output_tokens; return (