From da2dbbb3a92903828732bfbf64ad051ce58361bd Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Thu, 7 May 2026 11:09:12 -0700 Subject: [PATCH] test(v2 managed agents): add runnable QA happy-path script (Flow 1/2/3) --- .../managed_agents/qa_happy_path.sh | 191 ++++++++++++++++++ 1 file changed, 191 insertions(+) create mode 100755 tests/test_litellm/managed_agents/qa_happy_path.sh diff --git a/tests/test_litellm/managed_agents/qa_happy_path.sh b/tests/test_litellm/managed_agents/qa_happy_path.sh new file mode 100755 index 00000000000..411d98e3db3 --- /dev/null +++ b/tests/test_litellm/managed_agents/qa_happy_path.sh @@ -0,0 +1,191 @@ +#!/usr/bin/env bash +# QA script for v2 managed agents PR #27400. +# +# Walks all three happy-path flows from .claude/v2_api_contract.md against a +# real LiteLLM proxy + a real opencode server. No Krrish dependency — we +# hand-INSERT a sessions row pointing at local opencode. +# +# Usage: +# bash tests/test_litellm/managed_agents/qa_happy_path.sh +# +# Required on PATH: opencode, jq, psql, curl +# Optional env: +# LITELLM_PROXY_URL (default: http://localhost:4000) +# LITELLM_API_KEY (default: sk-1234) +# OC_PORT (default: 1234) +# LITELLM_DB (default: litellm) +set -euo pipefail + +PROXY_URL="${LITELLM_PROXY_URL:-http://localhost:4000}" +PROXY_KEY="${LITELLM_API_KEY:-sk-1234}" +OC_PORT="${OC_PORT:-1234}" +DB_NAME="${LITELLM_DB:-litellm}" +RUN_ID="qa_$$_$(date +%s)" +SES_ID="ses_$RUN_ID" + +log() { printf "\033[1;34m▸\033[0m %s\n" "$*" >&2; } +ok() { printf " \033[1;32m✓\033[0m %s\n" "$*" >&2; } +warn() { printf " \033[1;33m!\033[0m %s\n" "$*" >&2; } +fail() { printf " \033[1;31m✗\033[0m %s\n" "$*" >&2; exit 1; } + +# ------------------------------------------------------------------ +# 0. Preflight +# ------------------------------------------------------------------ +log "preflight" +command -v opencode >/dev/null || fail "opencode not on PATH — install via 'npm i -g opencode-ai'" +command -v jq >/dev/null || fail "jq not on PATH" +command -v psql >/dev/null || fail "psql not on PATH" +curl -sf "$PROXY_URL/health/readiness" -o /dev/null \ + || fail "proxy not reachable at $PROXY_URL — start it with: pip install -e --no-deps && litellm --config proxy_server_config.yaml --port 4000" +psql -d "$DB_NAME" -c "SELECT 1" -tA >/dev/null 2>&1 \ + || fail "can't connect to db '$DB_NAME' — set LITELLM_DB or check your local postgres" +ok "opencode, jq, psql, proxy ($PROXY_URL), db ($DB_NAME)" + +# ------------------------------------------------------------------ +# 1. Bring up local opencode (or reuse existing) +# ------------------------------------------------------------------ +STARTED_OC=0 +if curl -sf "http://127.0.0.1:$OC_PORT/global/health" >/dev/null 2>&1; then + log "opencode already running on $OC_PORT — reusing" +else + log "starting opencode on port $OC_PORT (logs: /tmp/opencode-$RUN_ID.log)" + opencode serve --port "$OC_PORT" >/tmp/opencode-$RUN_ID.log 2>&1 & + OC_PID=$! + STARTED_OC=1 + for i in {1..30}; do + sleep 1 + if curl -sf "http://127.0.0.1:$OC_PORT/global/health" >/dev/null 2>&1; then + ok "opencode up (pid $OC_PID)"; break + fi + done + curl -sf "http://127.0.0.1:$OC_PORT/global/health" >/dev/null \ + || fail "opencode never became healthy — check /tmp/opencode-$RUN_ID.log" +fi + +cleanup() { + log "cleanup" + psql -d "$DB_NAME" -c "DELETE FROM \"LiteLLM_ManagedAgentSession\" WHERE id='$SES_ID'" >/dev/null 2>&1 || true + psql -d "$DB_NAME" -c "DELETE FROM \"LiteLLM_ManagedAgent\" WHERE id='${AGENT_ID:-x}'" >/dev/null 2>&1 || true + if [[ "$STARTED_OC" == "1" && -n "${OC_PID:-}" ]]; then + kill "$OC_PID" 2>/dev/null || true + ok "stopped opencode (pid $OC_PID)" + fi +} +trap cleanup EXIT + +# ------------------------------------------------------------------ +# Flow 1 — agent → session → message → response +# ------------------------------------------------------------------ +log "Flow 1.1: POST /v2/agents" +AGENT_RESP=$(curl -sS -X POST "$PROXY_URL/v2/agents" \ + -H "Authorization: Bearer $PROXY_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "name": "'"qa-agent-$RUN_ID"'", + "config": { + "model": "anthropic/claude-3-haiku", + "system_prompt": "You are a brief assistant. Reply in one sentence.", + "tools": [], + "litellm_api_key": "'"$PROXY_KEY"'", + "litellm_base_url": "'"$PROXY_URL"'" + } + }') +echo "$AGENT_RESP" | jq . +AGENT_ID=$(echo "$AGENT_RESP" | jq -r .id) +[[ "$AGENT_ID" =~ ^agt_ ]] || fail "expected agt_* id, got: $AGENT_ID" +[[ $(echo "$AGENT_RESP" | jq -r .config.litellm_api_key) == *"****"* ]] || fail "litellm_api_key not masked" +ok "agent: $AGENT_ID (api key masked)" + +log "Flow 1.2: stand up session manually (replaces Krrish's POST /v2/sessions)" +OC_SID=$(curl -sS -X POST "http://127.0.0.1:$OC_PORT/session" -d '{}' | jq -r .id) +[[ -n "$OC_SID" && "$OC_SID" != "null" ]] || fail "opencode /session returned no id" +ok "opencode session: $OC_SID" + +psql -d "$DB_NAME" >/dev/null </dev/null | head -50 || true +ok "stream observed (truncated at 10s)" + +# ------------------------------------------------------------------ +# Flow 2 — followup on same session +# ------------------------------------------------------------------ +log "Flow 2.1: POST followup message" +curl -sS -X POST "$PROXY_URL/v2/sessions/$SES_ID/messages" \ + -H "Authorization: Bearer $PROXY_KEY" \ + -H "Content-Type: application/json" \ + -d '{"content":"What is 2+2?"}' | jq '{id, role, status}' +ok "followup accepted" + +log "Flow 2.3: GET /v2/sessions/$SES_ID/messages (verify history)" +sleep 3 +LIST_RESP=$(curl -sS "$PROXY_URL/v2/sessions/$SES_ID/messages?limit=10" \ + -H "Authorization: Bearer $PROXY_KEY") +echo "$LIST_RESP" | jq '{count: (.data | length), roles: [.data[].role], statuses: [.data[].status]}' +COUNT=$(echo "$LIST_RESP" | jq '.data | length') +[[ $COUNT -ge 2 ]] || warn "expected >= 2 messages, got $COUNT (LLM may still be processing)" + +# ------------------------------------------------------------------ +# Flow 3 — resume + sandbox-death failure mode +# ------------------------------------------------------------------ +log "Flow 3.2: GET /v2/sessions/$SES_ID (verify still ready, no internal leaks)" +GET_RESP=$(curl -sS "$PROXY_URL/v2/sessions/$SES_ID" -H "Authorization: Bearer $PROXY_KEY") +echo "$GET_RESP" | jq . +[[ $(echo "$GET_RESP" | jq -r .status) == "ready" ]] || fail "expected status=ready" +[[ $(echo "$GET_RESP" | jq 'has("sandbox_url")') == "false" ]] || fail "sandbox_url leaked in response" +[[ $(echo "$GET_RESP" | jq 'has("sandbox_metadata")') == "false" ]] || fail "sandbox_metadata leaked" +ok "internal fields stripped from response" + +log "Flow 3.4 (fail-closed): point row at unreachable URL, expect 504" +psql -d "$DB_NAME" -c "UPDATE \"LiteLLM_ManagedAgentSession\" SET sandbox_url='http://127.0.0.1:1' WHERE id='$SES_ID'" >/dev/null +DEATH=$(curl -sS -w "\n%{http_code}" -X POST "$PROXY_URL/v2/sessions/$SES_ID/messages" \ + -H "Authorization: Bearer $PROXY_KEY" \ + -H "Content-Type: application/json" \ + -d '{"content":"will fail"}') +DEATH_CODE=$(printf '%s\n' "$DEATH" | tail -n 1) +DEATH_BODY=$(printf '%s\n' "$DEATH" | sed '$d') +echo "$DEATH_BODY" | jq . +[[ "$DEATH_CODE" == "504" ]] || fail "expected 504, got $DEATH_CODE" +ok "sandbox-unreachable correctly returns 504" + +# Validator smoke: prove the SandboxSpec validator catches bad inputs (this is +# a Pydantic validation, surfaces as 422 if Krrish's POST /v2/sessions ever +# tries to validate manually — but our POST /v2/agents does NOT take sandbox, +# so we exercise it via Python rather than HTTP). +log "smoke: SandboxSpec validator rejects idle > timeout" +python3 -c " +from litellm.managed_agents.types import SandboxSpec +try: + SandboxSpec(type='opencode', timeout_minutes=10, idle_timeout_minutes=120) + raise SystemExit('FAIL: validator did not catch idle > timeout') +except Exception as e: + print(f' caught: {type(e).__name__}') +" 2>&1 | tail -5 +ok "validator works" + +printf "\n\033[1;32mALL CHECKS PASSED\033[0m (run id: %s)\n" "$RUN_ID"