From e8745e9eb37462ee48235bf8aeee0d88a756fe32 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 2 Sep 2026 23:39:34 +0000 Subject: [PATCH 1/5] feat(cli): add lite debug claude session report and /debug-lite slash command Co-Authored-By: Ishaan Jaffer <155045088+ishaan-berri@users.noreply.github.com> --- litellm/proxy/client/cli/commands/debug.py | 341 ++++++++++++++++++ litellm/proxy/client/cli/main.py | 3 + .../proxy/client/cli/test_debug_commands.py | 174 +++++++++ 3 files changed, 518 insertions(+) create mode 100644 litellm/proxy/client/cli/commands/debug.py create mode 100644 tests/test_litellm/proxy/client/cli/test_debug_commands.py diff --git a/litellm/proxy/client/cli/commands/debug.py b/litellm/proxy/client/cli/commands/debug.py new file mode 100644 index 00000000000..1163dcf44f5 --- /dev/null +++ b/litellm/proxy/client/cli/commands/debug.py @@ -0,0 +1,341 @@ +"""`lite debug claude`: one-shot debug report for a Claude Code session routed through the proxy. + +Claude Code puts its session id in `metadata.user_id`, which the proxy lifts into +`LiteLLM_SpendLogs.session_id`. This command pulls every turn of that session, plus +the request / response bodies for failures and the most recent turns, and renders a +single markdown report that can be pasted into a bug report or handed to another agent. +""" + +import json +import os +from collections.abc import Mapping, Sequence +from datetime import datetime, timezone +from pathlib import Path +from typing import Final + +import click +from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter, ValidationError, field_validator + +from ...http_client import HTTPClient +from ._cli_context import cli_context_values + +CLAUDE_DIR: Final = Path.home() / ".claude" +REPORT_DIR: Final = Path.home() / ".litellm" / "debug" +SESSION_ID_ENV: Final = "CLAUDE_SESSION_ID" +SLASH_COMMAND_NAME: Final = "debug-lite" +SLASH_COMMAND_BODY: Final = """--- +description: Pull the LiteLLM debug report (spend, request, response, error) for this Claude Code session +allowed-tools: Bash(lite debug claude:*) +--- +Below is the LiteLLM debug report for this Claude Code session. Summarize the failing +request(s) in a few sentences (model, error, request id) and tell me the path the full +report was saved to so I can hand it off. If nothing failed, say so. + +!`lite debug claude $ARGUMENTS` +""" + + +class DebugError(Exception): + """Raised for any user-actionable failure while building the report.""" + + +class ErrorInformation(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + error_code: str | None = None + error_class: str | None = None + error_message: str | None = None + llm_provider: str | None = None + + +class SpendLogMetadata(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + status: str | None = None + error_information: ErrorInformation | None = None + + +class SpendLogRow(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore", populate_by_name=True) + + request_id: str + start_time: str | None = Field(default=None, alias="startTime") + end_time: str | None = Field(default=None, alias="endTime") + model: str | None = None + model_group: str | None = None + custom_llm_provider: str | None = None + api_base: str | None = None + call_type: str | None = None + status: str | None = None + spend: float = 0.0 + prompt_tokens: int = 0 + completion_tokens: int = 0 + total_tokens: int = 0 + metadata: SpendLogMetadata = SpendLogMetadata() + + @field_validator("metadata", mode="before") + @classmethod + def _parse_metadata(cls, value: object) -> object: + if value is None: + return SpendLogMetadata() + if isinstance(value, str): + return json.loads(value) if value else SpendLogMetadata() + return value + + @property + def failed(self) -> bool: + return (self.status or self.metadata.status) == "failure" + + @property + def error(self) -> ErrorInformation | None: + return self.metadata.error_information + + +class SessionLogsPage(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + data: tuple[SpendLogRow, ...] + total: int + total_pages: int + + +class RequestResponsePayload(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + proxy_server_request: JsonValue = None + response: JsonValue = None + messages: JsonValue = None + + +_SESSION_PAGE: Final = TypeAdapter(SessionLogsPage) +_PAYLOAD: Final[TypeAdapter[RequestResponsePayload | None]] = TypeAdapter(RequestResponsePayload | None) +_JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) + +_SESSION_PAGE_SIZE: Final = 100 + + +def detect_claude_session_id(env: Mapping[str, str], claude_dir: Path) -> str | None: + """Explicit env var first, else the transcript Claude Code touched most recently.""" + explicit: Final = env.get(SESSION_ID_ENV) + if explicit: + return explicit + transcripts: Final = tuple(claude_dir.glob("projects/*/*.jsonl")) + if not transcripts: + return None + newest: Final = max(transcripts, key=lambda p: p.stat().st_mtime) + return newest.stem + + +class SpendLogsFetcher: + """Thin typed wrapper over the two spend-log endpoints the report needs.""" + + def __init__(self, http: HTTPClient) -> None: + self._http = http + + def session_rows(self, session_id: str) -> tuple[SpendLogRow, ...]: + first: Final = self._page(session_id, 1) + rest: Final = tuple( + row for page in range(2, first.total_pages + 1) for row in self._page(session_id, page).data + ) + rows: Final = first.data + rest + return tuple(sorted(rows, key=lambda r: r.start_time or "")) + + def _get(self, uri: str, params: Mapping[str, str | int] | None = None) -> JsonValue: + return _JSON.validate_python(self._http.request("GET", uri, params=params)) # pyright: ignore[reportUnknownMemberType] # HTTPClient.request is untyped + + def _page(self, session_id: str, page: int) -> SessionLogsPage: + raw: Final = self._get( + "/spend/logs/session/ui", + {"session_id": session_id, "page": page, "page_size": _SESSION_PAGE_SIZE}, + ) + try: + return _SESSION_PAGE.validate_python(raw) + except ValidationError as e: + raise DebugError(f"Unexpected /spend/logs/session/ui response: {e}") from e + + def payload(self, request_id: str) -> RequestResponsePayload | None: + raw: Final = self._get(f"/spend/logs/ui/{request_id}") + try: + return _PAYLOAD.validate_python(raw) + except ValidationError as e: + raise DebugError(f"Unexpected /spend/logs/ui/{request_id} response: {e}") from e + + +def _fmt_json(value: JsonValue, max_chars: int) -> str: + text: Final = value if isinstance(value, str) else json.dumps(value, indent=2, default=str) + if len(text) <= max_chars: + return text + return f"{text[:max_chars]}\n... (truncated, {len(text) - max_chars} more chars)" + + +def _row_section(row: SpendLogRow, index: int, payload: RequestResponsePayload | None, max_chars: int) -> str: + err: Final = row.error + error_lines: Final = ( + ( + f"- error: `{err.error_code or '?'}` {err.error_class or ''}".rstrip(), + f"\n```\n{err.error_message or ''}\n```", + ) + if err is not None and row.failed + else () + ) + body_lines: Final = ( + ( + "", + "
request body", + "", + "```json", + _fmt_json(payload.proxy_server_request, max_chars), + "```", + "
", + "", + "
response", + "", + "```json", + _fmt_json(payload.response, max_chars), + "```", + "
", + ) + if payload is not None + else () + ) + header: Final = f"### {index}. {'FAILED' if row.failed else 'ok'} {row.model or row.model_group or '?'}" + facts: Final = ( + f"- request_id: `{row.request_id}`", + f"- time: {row.start_time} -> {row.end_time}", + f"- provider: {row.custom_llm_provider or '?'} ({row.api_base or 'n/a'}), call_type: {row.call_type or '?'}", + f"- spend: ${row.spend:.6f}, tokens: {row.prompt_tokens} in / {row.completion_tokens} out", + ) + return "\n".join((header, *facts, *error_lines, *body_lines)) + + +def render_report( + *, + session_id: str, + base_url: str, + rows: Sequence[SpendLogRow], + payloads: Mapping[str, RequestResponsePayload | None], + max_chars: int, +) -> str: + failures: Final = tuple(r for r in rows if r.failed) + summary: Final = ( + f"# LiteLLM debug report: Claude Code session `{session_id}`", + "", + f"- proxy: {base_url}", + f"- generated: {datetime.now(timezone.utc).isoformat(timespec='seconds')}", + f"- turns: {len(rows)}, failed: {len(failures)}", + f"- total spend: ${sum(r.spend for r in rows):.6f}", + f"- models: {', '.join(sorted({r.model or r.model_group or '?' for r in rows})) or 'n/a'}", + "", + "Bodies are included for failed turns and the most recent turns. " + "Bodies are empty unless the proxy runs with `general_settings.store_prompts_in_spend_logs: true`.", + "", + "## Turns", + "", + ) + sections: Final = tuple( + _row_section(row, i, payloads.get(row.request_id), max_chars) for i, row in enumerate(rows, start=1) + ) + return "\n".join(summary) + "\n\n".join(sections) + "\n" + + +def build_report( + *, + fetcher: SpendLogsFetcher, + session_id: str, + base_url: str, + recent_bodies: int, + max_chars: int, +) -> str: + rows: Final = fetcher.session_rows(session_id) + if not rows: + raise DebugError( + f"No spend logs found for session {session_id!r} on {base_url}. " + "Is Claude Code routed through this proxy (`lite up`), and does your key have log access?" + ) + wanted: Final = frozenset(r.request_id for r in rows if r.failed) | frozenset( + r.request_id for r in rows[-recent_bodies:] if recent_bodies > 0 + ) + payloads: Final = {rid: fetcher.payload(rid) for rid in wanted} + return render_report(session_id=session_id, base_url=base_url, rows=rows, payloads=payloads, max_chars=max_chars) + + +def write_report(report: str, session_id: str, report_dir: Path) -> Path: + report_dir.mkdir(parents=True, exist_ok=True) + path: Final = report_dir / f"claude-{session_id}.md" + path.write_text(report, encoding="utf-8") + path.chmod(0o600) + return path + + +def install_slash_command(claude_dir: Path) -> Path: + commands_dir: Final = claude_dir / "commands" + commands_dir.mkdir(parents=True, exist_ok=True) + path: Final = commands_dir / f"{SLASH_COMMAND_NAME}.md" + path.write_text(SLASH_COMMAND_BODY, encoding="utf-8") + return path + + +@click.group() +def debug() -> None: + """Pull debug reports (spend, request, response, error) for coding-agent sessions""" + + +@debug.command("claude") +@click.option( + "--session-id", + default=None, + help=f"Claude Code session id. Defaults to ${SESSION_ID_ENV}, else the most recently used transcript in ~/.claude", +) +@click.option( + "--recent-bodies", + default=3, + show_default=True, + type=click.IntRange(min=0), + help="Also include request/response bodies for the N most recent turns (failed turns always get bodies)", +) +@click.option( + "--max-body-chars", + default=20_000, + show_default=True, + type=click.IntRange(min=100), + help="Truncate each request/response body to this many characters", +) +@click.option("--no-save", is_flag=True, help="Print only, do not write the report under ~/.litellm/debug") +@click.pass_context +def debug_claude( + ctx: click.Context, session_id: str | None, recent_bodies: int, max_body_chars: int, no_save: bool +) -> None: + """Render a markdown debug report for one Claude Code session routed through the proxy + + Examples: + lite debug claude + lite debug claude --session-id e96634a3-fa28-4083-b354-55542e2dca01 + """ + resolved: Final = session_id or detect_claude_session_id(os.environ, CLAUDE_DIR) + if resolved is None: + raise click.ClickException(f"Could not find a Claude Code session. Pass --session-id or set ${SESSION_ID_ENV}.") + values: Final = cli_context_values(ctx) + base_url: Final = values["base_url"] + fetcher: Final = SpendLogsFetcher(HTTPClient(base_url, values["api_key"])) + try: + report: Final = build_report( + fetcher=fetcher, + session_id=resolved, + base_url=base_url, + recent_bodies=recent_bodies, + max_chars=max_body_chars, + ) + except DebugError as e: + raise click.ClickException(str(e)) from e + click.echo(report) + if not no_save: + path: Final = write_report(report, resolved, REPORT_DIR) + click.echo(f"Saved to {path}", err=True) + + +@debug.command("install-claude-command") +def debug_install_claude_command() -> None: + """Install the /debug-lite slash command into ~/.claude/commands so Claude Code can run `lite debug claude`""" + path: Final = install_slash_command(CLAUDE_DIR) + click.echo(f"Installed /{SLASH_COMMAND_NAME}: {path}") + click.echo("Restart Claude Code (or start a new session), then type /debug-lite.") diff --git a/litellm/proxy/client/cli/main.py b/litellm/proxy/client/cli/main.py index 2674bf49ff0..b78d542085a 100644 --- a/litellm/proxy/client/cli/main.py +++ b/litellm/proxy/client/cli/main.py @@ -14,6 +14,7 @@ from .commands.autoroute.commands import autoroute_group from .commands.chat import chat from .commands.config import config_commands, get_config_value, hidden_command_names from .commands.credentials import credentials +from .commands.debug import debug from .commands.encryption import encryption from .commands.http import http from .commands.keys import keys @@ -143,6 +144,8 @@ cli.add_command(encryption) cli.add_command(chat) # Add the http command group cli.add_command(http) +# Add the debug command group (session debug reports for coding agents) +cli.add_command(debug) # Add the keys command group cli.add_command(keys) # Add the teams command group diff --git a/tests/test_litellm/proxy/client/cli/test_debug_commands.py b/tests/test_litellm/proxy/client/cli/test_debug_commands.py new file mode 100644 index 00000000000..6249f105019 --- /dev/null +++ b/tests/test_litellm/proxy/client/cli/test_debug_commands.py @@ -0,0 +1,174 @@ +import json +import os +import time +from unittest.mock import patch + +import pytest +from click.testing import CliRunner + +from litellm.proxy.client.cli import cli +from litellm.proxy.client.cli.commands import debug as debug_module +from litellm.proxy.client.cli.commands.debug import ( + SLASH_COMMAND_NAME, + detect_claude_session_id, + install_slash_command, +) + +SESSION = "e96634a3-fa28-4083-b354-55542e2dca01" + +OK_ROW = { + "request_id": "req-ok", + "startTime": "2026-09-02T10:00:00", + "endTime": "2026-09-02T10:00:02", + "model": "claude-opus-4-1", + "custom_llm_provider": "anthropic", + "status": "success", + "spend": 0.0125, + "prompt_tokens": 100, + "completion_tokens": 20, + "metadata": {"status": "success"}, +} +FAILED_ROW = { + "request_id": "req-failed", + "startTime": "2026-09-02T10:01:00", + "endTime": "2026-09-02T10:01:01", + "model": "claude-opus-4-1", + "custom_llm_provider": "anthropic", + "status": "failure", + "spend": 0.0, + "prompt_tokens": 0, + "completion_tokens": 0, + # query_raw hands metadata back as a JSON string on some paths + "metadata": json.dumps( + { + "status": "failure", + "error_information": { + "error_code": "400", + "error_class": "BadRequestError", + "error_message": "`prompt` is required when `stop` is not true.", + }, + } + ), +} + + +def _fake_http(rows, payloads): + calls = [] + + class FakeHTTP: + def __init__(self, *_args, **_kwargs): + pass + + def request(self, method, uri, **kwargs): + calls.append(uri) + if uri == "/spend/logs/session/ui": + assert kwargs["params"]["session_id"] == SESSION + return {"data": rows, "total": len(rows), "page": 1, "page_size": 100, "total_pages": 1} + request_id = uri.rsplit("/", 1)[1] + return payloads.get(request_id) + + return FakeHTTP, calls + + +@pytest.fixture(autouse=True) +def env(monkeypatch, tmp_path): + monkeypatch.setenv("LITELLM_PROXY_URL", "http://localhost:4000") + monkeypatch.setenv("LITELLM_PROXY_API_KEY", "sk-test") + monkeypatch.setattr(debug_module, "REPORT_DIR", tmp_path / "reports") + monkeypatch.setattr(debug_module, "CLAUDE_DIR", tmp_path / "claude") + + +def test_report_includes_spend_error_and_bodies_for_failed_turn(tmp_path): + payloads = { + "req-failed": { + "proxy_server_request": {"body": {"model": "claude-opus-4-1", "messages": [{"role": "user"}]}}, + "response": {"error": {"message": "`prompt` is required"}}, + }, + "req-ok": {"proxy_server_request": {"body": {"model": "claude-opus-4-1"}}, "response": {"id": "msg_1"}}, + } + FakeHTTP, calls = _fake_http([FAILED_ROW, OK_ROW], payloads) + with patch.object(debug_module, "HTTPClient", FakeHTTP): + result = CliRunner().invoke(cli, ["debug", "claude", "--session-id", SESSION, "--recent-bodies", "0"]) + + assert result.exit_code == 0, result.output + assert "turns: 2, failed: 1" in result.output + assert "total spend: $0.012500" in result.output + assert "### 1. ok claude-opus-4-1" in result.output + assert "### 2. FAILED claude-opus-4-1" in result.output + assert "`400` BadRequestError" in result.output + assert "`prompt` is required when `stop` is not true." in result.output + assert '"messages"' in result.output + assert "msg_1" not in result.output + assert calls == ["/spend/logs/session/ui", "/spend/logs/ui/req-failed"] + saved = tmp_path / "reports" / f"claude-{SESSION}.md" + assert result.stdout.startswith(saved.read_text()) + assert "### 2. FAILED" in saved.read_text() + + +def test_recent_bodies_fetches_latest_turns_even_when_successful(): + payloads = {"req-ok": {"proxy_server_request": {"body": {"x": 1}}, "response": {"id": "msg_1"}}} + FakeHTTP, calls = _fake_http([OK_ROW], payloads) + with patch.object(debug_module, "HTTPClient", FakeHTTP): + result = CliRunner().invoke(cli, ["debug", "claude", "--session-id", SESSION, "--no-save"]) + + assert result.exit_code == 0, result.output + assert "msg_1" in result.output + assert calls == ["/spend/logs/session/ui", "/spend/logs/ui/req-ok"] + + +def test_bodies_are_truncated_to_max_chars(): + payloads = {"req-ok": {"proxy_server_request": {"body": "a" * 5000}, "response": None}} + FakeHTTP, _ = _fake_http([OK_ROW], payloads) + with patch.object(debug_module, "HTTPClient", FakeHTTP): + result = CliRunner().invoke( + cli, ["debug", "claude", "--session-id", SESSION, "--no-save", "--max-body-chars", "200"] + ) + + assert result.exit_code == 0, result.output + assert "truncated" in result.output + assert "a" * 300 not in result.output + + +def test_no_rows_is_a_clear_error(): + FakeHTTP, _ = _fake_http([], {}) + with patch.object(debug_module, "HTTPClient", FakeHTTP): + result = CliRunner().invoke(cli, ["debug", "claude", "--session-id", SESSION]) + + assert result.exit_code != 0 + assert "No spend logs found for session" in result.output + + +def test_no_session_id_anywhere_is_a_clear_error(monkeypatch): + monkeypatch.delenv("CLAUDE_SESSION_ID", raising=False) + result = CliRunner().invoke(cli, ["debug", "claude"]) + assert result.exit_code != 0 + assert "Could not find a Claude Code session" in result.output + + +def test_detect_session_id_prefers_env_then_newest_transcript(tmp_path): + project = tmp_path / "projects" / "-Users-me-repo" + project.mkdir(parents=True) + old = project / "old-session.jsonl" + new = project / "new-session.jsonl" + old.write_text("{}") + new.write_text("{}") + now = time.time() + os.utime(old, (now - 100, now - 100)) + os.utime(new, (now, now)) + + assert detect_claude_session_id({}, tmp_path) == "new-session" + assert detect_claude_session_id({"CLAUDE_SESSION_ID": "from-env"}, tmp_path) == "from-env" + assert detect_claude_session_id({}, tmp_path / "missing") is None + + +def test_install_slash_command_writes_runnable_command_file(tmp_path): + path = install_slash_command(tmp_path) + assert path == tmp_path / "commands" / f"{SLASH_COMMAND_NAME}.md" + body = path.read_text() + assert body.startswith("---\n") + assert "allowed-tools: Bash(lite debug claude:*)" in body + assert "!`lite debug claude $ARGUMENTS`" in body + + result = CliRunner().invoke(cli, ["debug", "install-claude-command"]) + assert result.exit_code == 0, result.output + assert "/debug-lite" in result.output From bc2640ee0ef49b416812f089244c75e8f5201a7a Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 2 Sep 2026 23:44:20 +0000 Subject: [PATCH 2/5] fix(cli): freeze collections in debug report builder to satisfy LIT002 Co-Authored-By: Ishaan Jaffer <155045088+ishaan-berri@users.noreply.github.com> --- litellm/proxy/client/cli/commands/debug.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/client/cli/commands/debug.py b/litellm/proxy/client/cli/commands/debug.py index 1163dcf44f5..b5774822df7 100644 --- a/litellm/proxy/client/cli/commands/debug.py +++ b/litellm/proxy/client/cli/commands/debug.py @@ -11,6 +11,7 @@ import os from collections.abc import Mapping, Sequence from datetime import datetime, timezone from pathlib import Path +from types import MappingProxyType from typing import Final import click @@ -146,7 +147,7 @@ class SpendLogsFetcher: def _page(self, session_id: str, page: int) -> SessionLogsPage: raw: Final = self._get( "/spend/logs/session/ui", - {"session_id": session_id, "page": page, "page_size": _SESSION_PAGE_SIZE}, + MappingProxyType({"session_id": session_id, "page": page, "page_size": _SESSION_PAGE_SIZE}), ) try: return _SESSION_PAGE.validate_python(raw) @@ -224,7 +225,7 @@ def render_report( f"- generated: {datetime.now(timezone.utc).isoformat(timespec='seconds')}", f"- turns: {len(rows)}, failed: {len(failures)}", f"- total spend: ${sum(r.spend for r in rows):.6f}", - f"- models: {', '.join(sorted({r.model or r.model_group or '?' for r in rows})) or 'n/a'}", + f"- models: {', '.join(sorted(frozenset(r.model or r.model_group or '?' for r in rows))) or 'n/a'}", "", "Bodies are included for failed turns and the most recent turns. " "Bodies are empty unless the proxy runs with `general_settings.store_prompts_in_spend_logs: true`.", @@ -255,7 +256,7 @@ def build_report( wanted: Final = frozenset(r.request_id for r in rows if r.failed) | frozenset( r.request_id for r in rows[-recent_bodies:] if recent_bodies > 0 ) - payloads: Final = {rid: fetcher.payload(rid) for rid in wanted} + payloads: Final = MappingProxyType({rid: fetcher.payload(rid) for rid in wanted}) return render_report(session_id=session_id, base_url=base_url, rows=rows, payloads=payloads, max_chars=max_chars) From 306427f5e4879bf77720b167bcd9ff7e623c3f9a Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 2 Sep 2026 23:50:11 +0000 Subject: [PATCH 3/5] test(cli): mock the HTTP boundary with responses instead of patching HTTPClient Co-Authored-By: Ishaan Jaffer <155045088+ishaan-berri@users.noreply.github.com> --- .../proxy/client/cli/test_debug_commands.py | 63 +++++++++---------- 1 file changed, 30 insertions(+), 33 deletions(-) diff --git a/tests/test_litellm/proxy/client/cli/test_debug_commands.py b/tests/test_litellm/proxy/client/cli/test_debug_commands.py index 6249f105019..42ce3aaf4d2 100644 --- a/tests/test_litellm/proxy/client/cli/test_debug_commands.py +++ b/tests/test_litellm/proxy/client/cli/test_debug_commands.py @@ -1,9 +1,9 @@ import json import os import time -from unittest.mock import patch import pytest +import responses from click.testing import CliRunner from litellm.proxy.client.cli import cli @@ -52,32 +52,32 @@ FAILED_ROW = { } -def _fake_http(rows, payloads): - calls = [] +PROXY = "http://localhost:4000" - class FakeHTTP: - def __init__(self, *_args, **_kwargs): - pass - def request(self, method, uri, **kwargs): - calls.append(uri) - if uri == "/spend/logs/session/ui": - assert kwargs["params"]["session_id"] == SESSION - return {"data": rows, "total": len(rows), "page": 1, "page_size": 100, "total_pages": 1} - request_id = uri.rsplit("/", 1)[1] - return payloads.get(request_id) +def _mock_proxy(rows, payloads): + responses.get( + f"{PROXY}/spend/logs/session/ui", + json={"data": rows, "total": len(rows), "page": 1, "page_size": 100, "total_pages": 1}, + match=[responses.matchers.query_param_matcher({"session_id": SESSION}, strict_match=False)], + ) + for request_id, payload in payloads.items(): + responses.get(f"{PROXY}/spend/logs/ui/{request_id}", json=payload) - return FakeHTTP, calls + +def _called_paths(): + return [c.request.path_url.split("?")[0] for c in responses.calls] @pytest.fixture(autouse=True) def env(monkeypatch, tmp_path): - monkeypatch.setenv("LITELLM_PROXY_URL", "http://localhost:4000") + monkeypatch.setenv("LITELLM_PROXY_URL", PROXY) monkeypatch.setenv("LITELLM_PROXY_API_KEY", "sk-test") monkeypatch.setattr(debug_module, "REPORT_DIR", tmp_path / "reports") monkeypatch.setattr(debug_module, "CLAUDE_DIR", tmp_path / "claude") +@responses.activate def test_report_includes_spend_error_and_bodies_for_failed_turn(tmp_path): payloads = { "req-failed": { @@ -86,9 +86,8 @@ def test_report_includes_spend_error_and_bodies_for_failed_turn(tmp_path): }, "req-ok": {"proxy_server_request": {"body": {"model": "claude-opus-4-1"}}, "response": {"id": "msg_1"}}, } - FakeHTTP, calls = _fake_http([FAILED_ROW, OK_ROW], payloads) - with patch.object(debug_module, "HTTPClient", FakeHTTP): - result = CliRunner().invoke(cli, ["debug", "claude", "--session-id", SESSION, "--recent-bodies", "0"]) + _mock_proxy([FAILED_ROW, OK_ROW], payloads) + result = CliRunner().invoke(cli, ["debug", "claude", "--session-id", SESSION, "--recent-bodies", "0"]) assert result.exit_code == 0, result.output assert "turns: 2, failed: 1" in result.output @@ -99,40 +98,38 @@ def test_report_includes_spend_error_and_bodies_for_failed_turn(tmp_path): assert "`prompt` is required when `stop` is not true." in result.output assert '"messages"' in result.output assert "msg_1" not in result.output - assert calls == ["/spend/logs/session/ui", "/spend/logs/ui/req-failed"] + assert _called_paths() == ["/spend/logs/session/ui", "/spend/logs/ui/req-failed"] saved = tmp_path / "reports" / f"claude-{SESSION}.md" assert result.stdout.startswith(saved.read_text()) assert "### 2. FAILED" in saved.read_text() +@responses.activate def test_recent_bodies_fetches_latest_turns_even_when_successful(): - payloads = {"req-ok": {"proxy_server_request": {"body": {"x": 1}}, "response": {"id": "msg_1"}}} - FakeHTTP, calls = _fake_http([OK_ROW], payloads) - with patch.object(debug_module, "HTTPClient", FakeHTTP): - result = CliRunner().invoke(cli, ["debug", "claude", "--session-id", SESSION, "--no-save"]) + _mock_proxy([OK_ROW], {"req-ok": {"proxy_server_request": {"body": {"x": 1}}, "response": {"id": "msg_1"}}}) + result = CliRunner().invoke(cli, ["debug", "claude", "--session-id", SESSION, "--no-save"]) assert result.exit_code == 0, result.output assert "msg_1" in result.output - assert calls == ["/spend/logs/session/ui", "/spend/logs/ui/req-ok"] + assert _called_paths() == ["/spend/logs/session/ui", "/spend/logs/ui/req-ok"] +@responses.activate def test_bodies_are_truncated_to_max_chars(): - payloads = {"req-ok": {"proxy_server_request": {"body": "a" * 5000}, "response": None}} - FakeHTTP, _ = _fake_http([OK_ROW], payloads) - with patch.object(debug_module, "HTTPClient", FakeHTTP): - result = CliRunner().invoke( - cli, ["debug", "claude", "--session-id", SESSION, "--no-save", "--max-body-chars", "200"] - ) + _mock_proxy([OK_ROW], {"req-ok": {"proxy_server_request": {"body": "a" * 5000}, "response": None}}) + result = CliRunner().invoke( + cli, ["debug", "claude", "--session-id", SESSION, "--no-save", "--max-body-chars", "200"] + ) assert result.exit_code == 0, result.output assert "truncated" in result.output assert "a" * 300 not in result.output +@responses.activate def test_no_rows_is_a_clear_error(): - FakeHTTP, _ = _fake_http([], {}) - with patch.object(debug_module, "HTTPClient", FakeHTTP): - result = CliRunner().invoke(cli, ["debug", "claude", "--session-id", SESSION]) + _mock_proxy([], {}) + result = CliRunner().invoke(cli, ["debug", "claude", "--session-id", SESSION]) assert result.exit_code != 0 assert "No spend logs found for session" in result.output From 02cb3daf2656be211c8c4ee3665c94f0686d2ea7 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 4 Sep 2026 19:15:21 -0700 Subject: [PATCH 4/5] fix(cli): return debug failures as values, survive transport errors, size report fences to content --- litellm/proxy/client/cli/commands/debug.py | 109 +++++++++++------- litellm/proxy/client/cli/main.py | 1 - .../proxy/client/cli/test_debug_commands.py | 54 ++++++++- 3 files changed, 121 insertions(+), 43 deletions(-) diff --git a/litellm/proxy/client/cli/commands/debug.py b/litellm/proxy/client/cli/commands/debug.py index b5774822df7..c5f147eb37a 100644 --- a/litellm/proxy/client/cli/commands/debug.py +++ b/litellm/proxy/client/cli/commands/debug.py @@ -8,13 +8,16 @@ single markdown report that can be pasted into a bug report or handed to another import json import os +import re from collections.abc import Mapping, Sequence +from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path from types import MappingProxyType from typing import Final import click +import requests from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter, ValidationError, field_validator from ...http_client import HTTPClient @@ -36,8 +39,9 @@ report was saved to so I can hand it off. If nothing failed, say so. """ -class DebugError(Exception): - """Raised for any user-actionable failure while building the report.""" +@dataclass(frozen=True, slots=True) +class DebugFailure: + message: str class ErrorInformation(BaseModel): @@ -113,10 +117,10 @@ _PAYLOAD: Final[TypeAdapter[RequestResponsePayload | None]] = TypeAdapter(Reques _JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) _SESSION_PAGE_SIZE: Final = 100 +_TRANSPORT_BODY_CHARS: Final = 500 def detect_claude_session_id(env: Mapping[str, str], claude_dir: Path) -> str | None: - """Explicit env var first, else the transcript Claude Code touched most recently.""" explicit: Final = env.get(SESSION_ID_ENV) if explicit: return explicit @@ -127,39 +131,54 @@ def detect_claude_session_id(env: Mapping[str, str], claude_dir: Path) -> str | return newest.stem -class SpendLogsFetcher: - """Thin typed wrapper over the two spend-log endpoints the report needs.""" +def _transport_failure(uri: str, error: requests.exceptions.RequestException) -> DebugFailure: + body: Final = error.response.text[:_TRANSPORT_BODY_CHARS] if error.response is not None else "" + detail: Final = f"\n{body}" if body else "" + return DebugFailure(f"GET {uri} failed: {error}{detail}") + +class SpendLogsFetcher: def __init__(self, http: HTTPClient) -> None: self._http = http - def session_rows(self, session_id: str) -> tuple[SpendLogRow, ...]: + def session_rows(self, session_id: str) -> tuple[SpendLogRow, ...] | DebugFailure: first: Final = self._page(session_id, 1) - rest: Final = tuple( - row for page in range(2, first.total_pages + 1) for row in self._page(session_id, page).data - ) - rows: Final = first.data + rest + if isinstance(first, DebugFailure): + return first + rest: Final = tuple(self._page(session_id, page) for page in range(2, first.total_pages + 1)) + failed_page: Final = next((page for page in rest if isinstance(page, DebugFailure)), None) + if failed_page is not None: + return failed_page + rows: Final = first.data + tuple(row for page in rest if isinstance(page, SessionLogsPage) for row in page.data) return tuple(sorted(rows, key=lambda r: r.start_time or "")) - def _get(self, uri: str, params: Mapping[str, str | int] | None = None) -> JsonValue: - return _JSON.validate_python(self._http.request("GET", uri, params=params)) # pyright: ignore[reportUnknownMemberType] # HTTPClient.request is untyped + def _get(self, uri: str, params: Mapping[str, str | int] | None = None) -> JsonValue | DebugFailure: + try: + return _JSON.validate_python(self._http.request("GET", uri, params=params)) # pyright: ignore[reportUnknownMemberType] # HTTPClient.request is untyped + except requests.exceptions.RequestException as e: + return _transport_failure(uri, e) - def _page(self, session_id: str, page: int) -> SessionLogsPage: + def _page(self, session_id: str, page: int) -> SessionLogsPage | DebugFailure: + uri: Final = "/spend/logs/session/ui" raw: Final = self._get( - "/spend/logs/session/ui", - MappingProxyType({"session_id": session_id, "page": page, "page_size": _SESSION_PAGE_SIZE}), + uri, MappingProxyType({"session_id": session_id, "page": page, "page_size": _SESSION_PAGE_SIZE}) ) + if isinstance(raw, DebugFailure): + return raw try: return _SESSION_PAGE.validate_python(raw) except ValidationError as e: - raise DebugError(f"Unexpected /spend/logs/session/ui response: {e}") from e + return DebugFailure(f"Unexpected {uri} response: {e}") - def payload(self, request_id: str) -> RequestResponsePayload | None: - raw: Final = self._get(f"/spend/logs/ui/{request_id}") + def payload(self, request_id: str) -> RequestResponsePayload | None | DebugFailure: + uri: Final = f"/spend/logs/ui/{request_id}" + raw: Final = self._get(uri) + if isinstance(raw, DebugFailure): + return raw try: return _PAYLOAD.validate_python(raw) except ValidationError as e: - raise DebugError(f"Unexpected /spend/logs/ui/{request_id} response: {e}") from e + return DebugFailure(f"Unexpected {uri} response: {e}") def _fmt_json(value: JsonValue, max_chars: int) -> str: @@ -169,12 +188,19 @@ def _fmt_json(value: JsonValue, max_chars: int) -> str: return f"{text[:max_chars]}\n... (truncated, {len(text) - max_chars} more chars)" +def _fenced(text: str, info: str = "") -> tuple[str, str, str]: + longest_run: Final = max((len(run) for run in re.findall(r"`+", text)), default=0) + fence: Final = "`" * max(3, longest_run + 1) + return (f"{fence}{info}", text, fence) + + def _row_section(row: SpendLogRow, index: int, payload: RequestResponsePayload | None, max_chars: int) -> str: err: Final = row.error error_lines: Final = ( ( f"- error: `{err.error_code or '?'}` {err.error_class or ''}".rstrip(), - f"\n```\n{err.error_message or ''}\n```", + "", + *_fenced(err.error_message or ""), ) if err is not None and row.failed else () @@ -184,16 +210,12 @@ def _row_section(row: SpendLogRow, index: int, payload: RequestResponsePayload | "", "
request body", "", - "```json", - _fmt_json(payload.proxy_server_request, max_chars), - "```", + *_fenced(_fmt_json(payload.proxy_server_request, max_chars), "json"), "
", "", "
response", "", - "```json", - _fmt_json(payload.response, max_chars), - "```", + *_fenced(_fmt_json(payload.response, max_chars), "json"), "
", ) if payload is not None @@ -246,17 +268,23 @@ def build_report( base_url: str, recent_bodies: int, max_chars: int, -) -> str: +) -> str | DebugFailure: rows: Final = fetcher.session_rows(session_id) + if isinstance(rows, DebugFailure): + return rows if not rows: - raise DebugError( + return DebugFailure( f"No spend logs found for session {session_id!r} on {base_url}. " "Is Claude Code routed through this proxy (`lite up`), and does your key have log access?" ) wanted: Final = frozenset(r.request_id for r in rows if r.failed) | frozenset( r.request_id for r in rows[-recent_bodies:] if recent_bodies > 0 ) - payloads: Final = MappingProxyType({rid: fetcher.payload(rid) for rid in wanted}) + fetched: Final = MappingProxyType({rid: fetcher.payload(rid) for rid in sorted(wanted)}) + failed_payload: Final = next((p for p in fetched.values() if isinstance(p, DebugFailure)), None) + if failed_payload is not None: + return failed_payload + payloads: Final = MappingProxyType({rid: p for rid, p in fetched.items() if not isinstance(p, DebugFailure)}) return render_report(session_id=session_id, base_url=base_url, rows=rows, payloads=payloads, max_chars=max_chars) @@ -318,19 +346,18 @@ def debug_claude( values: Final = cli_context_values(ctx) base_url: Final = values["base_url"] fetcher: Final = SpendLogsFetcher(HTTPClient(base_url, values["api_key"])) - try: - report: Final = build_report( - fetcher=fetcher, - session_id=resolved, - base_url=base_url, - recent_bodies=recent_bodies, - max_chars=max_body_chars, - ) - except DebugError as e: - raise click.ClickException(str(e)) from e - click.echo(report) + outcome: Final = build_report( + fetcher=fetcher, + session_id=resolved, + base_url=base_url, + recent_bodies=recent_bodies, + max_chars=max_body_chars, + ) + if isinstance(outcome, DebugFailure): + raise click.ClickException(outcome.message) + click.echo(outcome) if not no_save: - path: Final = write_report(report, resolved, REPORT_DIR) + path: Final = write_report(outcome, resolved, REPORT_DIR) click.echo(f"Saved to {path}", err=True) diff --git a/litellm/proxy/client/cli/main.py b/litellm/proxy/client/cli/main.py index b78d542085a..eae1b0f5bc9 100644 --- a/litellm/proxy/client/cli/main.py +++ b/litellm/proxy/client/cli/main.py @@ -144,7 +144,6 @@ cli.add_command(encryption) cli.add_command(chat) # Add the http command group cli.add_command(http) -# Add the debug command group (session debug reports for coding agents) cli.add_command(debug) # Add the keys command group cli.add_command(keys) diff --git a/tests/test_litellm/proxy/client/cli/test_debug_commands.py b/tests/test_litellm/proxy/client/cli/test_debug_commands.py index 42ce3aaf4d2..1853d0c3468 100644 --- a/tests/test_litellm/proxy/client/cli/test_debug_commands.py +++ b/tests/test_litellm/proxy/client/cli/test_debug_commands.py @@ -3,6 +3,7 @@ import os import time import pytest +import requests import responses from click.testing import CliRunner @@ -38,7 +39,6 @@ FAILED_ROW = { "spend": 0.0, "prompt_tokens": 0, "completion_tokens": 0, - # query_raw hands metadata back as a JSON string on some paths "metadata": json.dumps( { "status": "failure", @@ -169,3 +169,55 @@ def test_install_slash_command_writes_runnable_command_file(tmp_path): result = CliRunner().invoke(cli, ["debug", "install-claude-command"]) assert result.exit_code == 0, result.output assert "/debug-lite" in result.output + + +@responses.activate +def test_rejected_key_is_a_clear_error_not_a_traceback(): + responses.get( + f"{PROXY}/spend/logs/session/ui", + status=401, + json={"error": {"message": "Authentication Error, Invalid proxy server token passed", "code": "401"}}, + ) + result = CliRunner().invoke(cli, ["debug", "claude", "--session-id", SESSION]) + + assert isinstance(result.exception, SystemExit), result.exception + assert result.exit_code == 1 + assert "401" in result.output + assert "Invalid proxy server token passed" in result.output + + +@responses.activate +def test_unreachable_proxy_is_a_clear_error_not_a_traceback(): + responses.get(f"{PROXY}/spend/logs/session/ui", body=requests.ConnectionError("Connection refused")) + result = CliRunner().invoke(cli, ["debug", "claude", "--session-id", SESSION]) + + assert isinstance(result.exception, SystemExit), result.exception + assert result.exit_code == 1 + assert "Connection refused" in result.output + + +@responses.activate +def test_non_json_proxy_response_is_a_clear_error_not_a_traceback(): + responses.get(f"{PROXY}/spend/logs/session/ui", body="502 Bad Gateway") + result = CliRunner().invoke(cli, ["debug", "claude", "--session-id", SESSION]) + + assert isinstance(result.exception, SystemExit), result.exception + assert result.exit_code == 1 + assert "/spend/logs/session/ui failed" in result.output + + +@responses.activate +def test_logged_content_with_code_fences_stays_inside_its_fence(): + fenced_error_row = { + **FAILED_ROW, + "metadata": { + "status": "failure", + "error_information": {"error_code": "400", "error_message": "bad\n```\nrequest"}, + }, + } + _mock_proxy([fenced_error_row], {"req-failed": {"proxy_server_request": None, "response": "x\n````\ny"}}) + result = CliRunner().invoke(cli, ["debug", "claude", "--session-id", SESSION, "--no-save"]) + + assert result.exit_code == 0, result.output + assert "````\nbad\n```\nrequest\n````\n" in result.output + assert "`````json\nx\n````\ny\n`````\n" in result.output From 296cd8c1f5bb9945f7afa30ba38ac2f39bc4a530 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 4 Sep 2026 19:45:42 -0700 Subject: [PATCH 5/5] fix(cli): read CLAUDE_CODE_SESSION_ID and skip subagent transcripts when detecting the Claude Code session --- litellm/proxy/client/cli/commands/debug.py | 7 ++++-- .../proxy/client/cli/test_debug_commands.py | 22 +++++++++++++------ 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/litellm/proxy/client/cli/commands/debug.py b/litellm/proxy/client/cli/commands/debug.py index c5f147eb37a..4e3914143d8 100644 --- a/litellm/proxy/client/cli/commands/debug.py +++ b/litellm/proxy/client/cli/commands/debug.py @@ -25,7 +25,7 @@ from ._cli_context import cli_context_values CLAUDE_DIR: Final = Path.home() / ".claude" REPORT_DIR: Final = Path.home() / ".litellm" / "debug" -SESSION_ID_ENV: Final = "CLAUDE_SESSION_ID" +SESSION_ID_ENV: Final = "CLAUDE_CODE_SESSION_ID" SLASH_COMMAND_NAME: Final = "debug-lite" SLASH_COMMAND_BODY: Final = """--- description: Pull the LiteLLM debug report (spend, request, response, error) for this Claude Code session @@ -118,13 +118,16 @@ _JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) _SESSION_PAGE_SIZE: Final = 100 _TRANSPORT_BODY_CHARS: Final = 500 +_SESSION_TRANSCRIPT_STEM: Final = re.compile(r"[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}") def detect_claude_session_id(env: Mapping[str, str], claude_dir: Path) -> str | None: explicit: Final = env.get(SESSION_ID_ENV) if explicit: return explicit - transcripts: Final = tuple(claude_dir.glob("projects/*/*.jsonl")) + transcripts: Final = tuple( + path for path in claude_dir.glob("projects/*/*.jsonl") if _SESSION_TRANSCRIPT_STEM.fullmatch(path.stem) + ) if not transcripts: return None newest: Final = max(transcripts, key=lambda p: p.stat().st_mtime) diff --git a/tests/test_litellm/proxy/client/cli/test_debug_commands.py b/tests/test_litellm/proxy/client/cli/test_debug_commands.py index 1853d0c3468..419fd821cfb 100644 --- a/tests/test_litellm/proxy/client/cli/test_debug_commands.py +++ b/tests/test_litellm/proxy/client/cli/test_debug_commands.py @@ -136,25 +136,33 @@ def test_no_rows_is_a_clear_error(): def test_no_session_id_anywhere_is_a_clear_error(monkeypatch): - monkeypatch.delenv("CLAUDE_SESSION_ID", raising=False) + monkeypatch.delenv("CLAUDE_CODE_SESSION_ID", raising=False) result = CliRunner().invoke(cli, ["debug", "claude"]) assert result.exit_code != 0 assert "Could not find a Claude Code session" in result.output -def test_detect_session_id_prefers_env_then_newest_transcript(tmp_path): +OLD_SESSION = "0f3c2b1a-1111-4222-8333-444455556666" +NEW_SESSION = "2d79c54d-4644-4708-b03e-95395ef9ecbd" + + +def test_detect_session_id_prefers_env_then_newest_session_transcript(tmp_path): project = tmp_path / "projects" / "-Users-me-repo" project.mkdir(parents=True) - old = project / "old-session.jsonl" - new = project / "new-session.jsonl" + old = project / f"{OLD_SESSION}.jsonl" + new = project / f"{NEW_SESSION}.jsonl" + subagent = project / "agent-a1b2c3d4.jsonl" old.write_text("{}") new.write_text("{}") + subagent.write_text("{}") now = time.time() os.utime(old, (now - 100, now - 100)) - os.utime(new, (now, now)) + os.utime(new, (now - 50, now - 50)) + os.utime(subagent, (now, now)) - assert detect_claude_session_id({}, tmp_path) == "new-session" - assert detect_claude_session_id({"CLAUDE_SESSION_ID": "from-env"}, tmp_path) == "from-env" + assert detect_claude_session_id({}, tmp_path) == NEW_SESSION + assert detect_claude_session_id({"CLAUDE_CODE_SESSION_ID": "from-env"}, tmp_path) == "from-env" + assert detect_claude_session_id({"CLAUDE_SESSION_ID": "stale-name"}, tmp_path) == NEW_SESSION assert detect_claude_session_id({}, tmp_path / "missing") is None