From 3545584a00c951a81d0f82c90fa360a899ccf4a7 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 25 Feb 2026 23:20:03 -0800 Subject: [PATCH 1/2] Development environment setup (#22160) * feat: add pretty view for realtime API logs in dashboard - Create RealtimePrettyView component that renders structured session config, conversation turns with transcripts, and token breakdowns - Update PrettyMessagesView to detect realtime responses (via isRealtimeResponse helper) and delegate to the new component - Session card shows model, voice, modalities, temperature, instructions in a collapsible panel - Conversation turns show status, per-turn token usage, and audio/text transcripts with appropriate icons - Add 24 tests for RealtimePrettyView and 3 tests for PrettyMessagesView - All 75 LogDetailsDrawer tests pass Co-authored-by: Ishaan Jaff * chore: remove dev_config.yaml from tracked files Co-authored-by: Ishaan Jaff * feat: show turn count in realtime pretty view session header and output header - Add purple 'N turns' tag to Session card header for at-a-glance turn count - Add 'Turns: N' to the Output section header next to tokens/cost - Extend SectionHeader to accept optional turnCount prop - Add 3 new tests for turn count display (singular, plural, output header) Co-authored-by: Ishaan Jaff * fix: address Greptile review feedback - Remove response.audio.done and conversation.item.created from isRealtimeResponse() detection since the view doesn't render them; prevents misleading fallback for responses with only those events - Remove dead code: index >= 0 is always true in .map() callback Co-authored-by: Ishaan Jaff --------- Co-authored-by: Cursor Agent Co-authored-by: Ishaan Jaff --- ui/litellm-dashboard/package-lock.json | 97 +-- .../PrettyMessagesView.test.tsx | 81 +++ .../LogDetailsDrawer/PrettyMessagesView.tsx | 8 +- .../RealtimePrettyView.test.tsx | 392 ++++++++++++ .../LogDetailsDrawer/RealtimePrettyView.tsx | 570 ++++++++++++++++++ .../LogDetailsDrawer/SectionHeader.tsx | 10 +- 6 files changed, 1064 insertions(+), 94 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/PrettyMessagesView.test.tsx create mode 100644 ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RealtimePrettyView.test.tsx create mode 100644 ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RealtimePrettyView.tsx diff --git a/ui/litellm-dashboard/package-lock.json b/ui/litellm-dashboard/package-lock.json index cc04e674003..503ed4a62a8 100644 --- a/ui/litellm-dashboard/package-lock.json +++ b/ui/litellm-dashboard/package-lock.json @@ -90,7 +90,6 @@ "version": "5.2.0", "resolved": "https://registry.npmjs.org/@alloc/quick-lru/-/quick-lru-5.2.0.tgz", "integrity": "sha512-UrcABB+4bUrFABwbluTIBErXwvbsU/V7TZWfmbgJfbkwiBuziS9gxdODUyuiecfdGQ85jglMW6juS3+z5TsKLw==", - "dev": true, "license": "MIT", "engines": { "node": ">=10" @@ -1772,7 +1771,6 @@ "version": "0.3.13", "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", - "dev": true, "license": "MIT", "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.0", @@ -1783,7 +1781,6 @@ "version": "3.1.2", "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", - "dev": true, "license": "MIT", "engines": { "node": ">=6.0.0" @@ -1793,14 +1790,12 @@ "version": "1.5.5", "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", - "dev": true, "license": "MIT" }, "node_modules/@jridgewell/trace-mapping": { "version": "0.3.31", "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", - "dev": true, "license": "MIT", "dependencies": { "@jridgewell/resolve-uri": "^3.1.0", @@ -1978,7 +1973,6 @@ "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", - "dev": true, "license": "MIT", "dependencies": { "@nodelib/fs.stat": "2.0.5", @@ -1992,7 +1986,6 @@ "version": "2.0.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", - "dev": true, "license": "MIT", "engines": { "node": ">= 8" @@ -2002,7 +1995,6 @@ "version": "1.2.8", "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", - "dev": true, "license": "MIT", "dependencies": { "@nodelib/fs.scandir": "2.1.5", @@ -2326,7 +2318,7 @@ "version": "1.58.1", "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.58.1.tgz", "integrity": "sha512-6LdVIUERWxQMmUSSQi0I53GgCBYgM2RpGngCPY7hSeju+VrKjq3lvs7HpJoPbDiY5QM5EYRtRX5fvrinnMAz3w==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "dependencies": { "playwright": "1.58.1" @@ -3431,14 +3423,12 @@ "version": "15.7.15", "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.15.tgz", "integrity": "sha512-F6bEyamV9jKGAFBEmlQnesRPGOQqS2+Uwi0Em15xenOxHaf2hv6L8YCVn3rPdPJOiJfPiCnLIRyvwVaqMY3MIw==", - "dev": true, "license": "MIT" }, "node_modules/@types/react": { "version": "18.2.48", "resolved": "https://registry.npmjs.org/@types/react/-/react-18.2.48.tgz", "integrity": "sha512-qboRCl6Ie70DQQG9hhNREz81jqC1cs9EVNcjQ1AU+jH6NFfSAhVVbrrY/+nSF+Bsk4AOwm9Qa61InvMCyV+H3w==", - "dev": true, "license": "MIT", "dependencies": { "@types/prop-types": "*", @@ -3480,7 +3470,6 @@ "version": "0.26.0", "resolved": "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.26.0.tgz", "integrity": "sha512-WFHp9YUJQ6CKshqoC37iOlHnQSmxNc795UhB26CyBBttrN9svdIrUjl/NjnNmfcwtncN0h/0PPAFWv9ovP8mLA==", - "dev": true, "license": "MIT" }, "node_modules/@types/unist": { @@ -4341,14 +4330,12 @@ "version": "1.3.0", "resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz", "integrity": "sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A==", - "dev": true, "license": "MIT" }, "node_modules/anymatch": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", - "dev": true, "license": "ISC", "dependencies": { "normalize-path": "^3.0.0", @@ -4362,7 +4349,6 @@ "version": "2.3.1", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", - "dev": true, "license": "MIT", "engines": { "node": ">=8.6" @@ -4375,7 +4361,6 @@ "version": "5.0.2", "resolved": "https://registry.npmjs.org/arg/-/arg-5.0.2.tgz", "integrity": "sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==", - "dev": true, "license": "MIT" }, "node_modules/argparse": { @@ -4747,7 +4732,6 @@ "version": "2.3.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", "integrity": "sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==", - "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -4773,7 +4757,6 @@ "version": "3.0.3", "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", - "dev": true, "license": "MIT", "dependencies": { "fill-range": "^7.1.1" @@ -4889,7 +4872,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/camelcase-css/-/camelcase-css-2.0.1.tgz", "integrity": "sha512-QOSvevhslijgYwRx6Rv7zKdMF8lbRmx+uQGx2+vDc+KI/eBnsy9kit5aj23AgGu3pa4t9AgwbnXWqS+iOY+2aA==", - "dev": true, "license": "MIT", "engines": { "node": ">= 6" @@ -5013,7 +4995,6 @@ "version": "3.6.0", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz", "integrity": "sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==", - "dev": true, "license": "MIT", "dependencies": { "anymatch": "~3.1.2", @@ -5038,7 +5019,6 @@ "version": "5.1.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", - "dev": true, "license": "ISC", "dependencies": { "is-glob": "^4.0.1" @@ -5114,7 +5094,6 @@ "version": "4.1.1", "resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz", "integrity": "sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==", - "dev": true, "license": "MIT", "engines": { "node": ">= 6" @@ -5175,7 +5154,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/cssesc/-/cssesc-3.0.0.tgz", "integrity": "sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==", - "dev": true, "license": "MIT", "bin": { "cssesc": "bin/cssesc" @@ -5589,14 +5567,12 @@ "version": "1.2.2", "resolved": "https://registry.npmjs.org/didyoumean/-/didyoumean-1.2.2.tgz", "integrity": "sha512-gxtyfqMg7GKyhQmb056K7M3xszy/myH8w+B4RT+QXBQsvAOdc3XymqDDPHx1BgPgsdAA5SIifona89YtRATDzw==", - "dev": true, "license": "Apache-2.0" }, "node_modules/dlv": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/dlv/-/dlv-1.1.3.tgz", "integrity": "sha512-+HlytyjlPKnIG8XuRG8WvmBP8xs8P71y+SKKS6ZXWoEgLuePxtDoUEiH7WkdePWrQ5JBpE6aoVqfZfJUQkjXwA==", - "dev": true, "license": "MIT" }, "node_modules/doctrine": { @@ -6510,7 +6486,6 @@ "version": "1.20.1", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.1.tgz", "integrity": "sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw==", - "dev": true, "license": "ISC", "dependencies": { "reusify": "^1.0.4" @@ -6543,7 +6518,6 @@ "version": "6.5.0", "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", - "dev": true, "license": "MIT", "engines": { "node": ">=12.0.0" @@ -6581,7 +6555,6 @@ "version": "7.1.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", - "dev": true, "license": "MIT", "dependencies": { "to-regex-range": "^5.0.1" @@ -6742,7 +6715,6 @@ "version": "2.3.2", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", - "dev": true, "hasInstallScript": true, "license": "MIT", "optional": true, @@ -6893,7 +6865,6 @@ "version": "6.0.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", - "dev": true, "license": "ISC", "dependencies": { "is-glob": "^4.0.3" @@ -7391,7 +7362,6 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", - "dev": true, "license": "MIT", "dependencies": { "binary-extensions": "^2.0.0" @@ -7444,7 +7414,6 @@ "version": "2.16.1", "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.16.1.tgz", "integrity": "sha512-UfoeMA6fIJ8wTYFEUjelnaGI67v6+N7qXJEvQuIGa99l4xsCruSYOVSQ0uPANn4dAzm8lkYPaKLrrijLq7x23w==", - "dev": true, "license": "MIT", "dependencies": { "hasown": "^2.0.2" @@ -7505,7 +7474,6 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", - "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -7551,7 +7519,6 @@ "version": "4.0.3", "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", - "dev": true, "license": "MIT", "dependencies": { "is-extglob": "^2.1.1" @@ -7600,7 +7567,6 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", - "dev": true, "license": "MIT", "engines": { "node": ">=0.12.0" @@ -7877,7 +7843,6 @@ "version": "1.21.7", "resolved": "https://registry.npmjs.org/jiti/-/jiti-1.21.7.tgz", "integrity": "sha512-/imKNG4EbWNrVjoNC/1H5/9GFy+tqjGBHCaSsN+P2RnPqjsLmv6UD3Ej+Kj8nBWaRAwyk7kK5ZUc+OEatnTR3A==", - "dev": true, "license": "MIT", "bin": { "jiti": "bin/jiti.js" @@ -8163,7 +8128,6 @@ "version": "3.1.3", "resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-3.1.3.tgz", "integrity": "sha512-/vlFKAoH5Cgt3Ie+JLhRbwOsCQePABiU3tJ1egGvyQ+33R/vcwM2Zl2QR/LzjsBeItPt3oSVXapn+m4nQDvpzw==", - "dev": true, "license": "MIT", "engines": { "node": ">=14" @@ -8176,7 +8140,6 @@ "version": "1.2.4", "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", - "dev": true, "license": "MIT" }, "node_modules/locate-path": { @@ -8491,7 +8454,6 @@ "version": "1.4.1", "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", - "dev": true, "license": "MIT", "engines": { "node": ">= 8" @@ -8943,7 +8905,6 @@ "version": "4.0.8", "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", - "dev": true, "license": "MIT", "dependencies": { "braces": "^3.0.3", @@ -8957,7 +8918,6 @@ "version": "2.3.1", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", - "dev": true, "license": "MIT", "engines": { "node": ">=8.6" @@ -9072,7 +9032,6 @@ "version": "2.7.0", "resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz", "integrity": "sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==", - "dev": true, "license": "MIT", "dependencies": { "any-promise": "^1.0.0", @@ -9284,7 +9243,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", - "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -9303,7 +9261,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/object-hash/-/object-hash-3.0.0.tgz", "integrity": "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==", - "dev": true, "license": "MIT", "engines": { "node": ">= 6" @@ -9648,7 +9605,6 @@ "version": "1.0.7", "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", - "dev": true, "license": "MIT" }, "node_modules/path-scurry": { @@ -9695,7 +9651,6 @@ "version": "4.0.3", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", - "dev": true, "license": "MIT", "engines": { "node": ">=12" @@ -9708,7 +9663,6 @@ "version": "2.3.0", "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", "integrity": "sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==", - "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -9718,7 +9672,6 @@ "version": "4.0.7", "resolved": "https://registry.npmjs.org/pirates/-/pirates-4.0.7.tgz", "integrity": "sha512-TfySrs/5nm8fQJDcBDuUng3VOUKsd7S+zqvbOTiGXHfxX4wK31ard+hoNuvkicM/2YFzlpDgABOevKSsB4G/FA==", - "dev": true, "license": "MIT", "engines": { "node": ">= 6" @@ -9728,7 +9681,7 @@ "version": "1.58.1", "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.58.1.tgz", "integrity": "sha512-+2uTZHxSCcxjvGc5C891LrS1/NlxglGxzrC4seZiVjcYVQfUa87wBL6rTDqzGjuoWNjnBzRqKmF6zRYGMvQUaQ==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "dependencies": { "playwright-core": "1.58.1" @@ -9747,7 +9700,7 @@ "version": "1.58.1", "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.58.1.tgz", "integrity": "sha512-bcWzOaTxcW+VOOGBCQgnaKToLJ65d6AqfLVKEWvexyS3AS6rbXl+xdpYRMGSRBClPvyj44njOWoxjNdL/H9UNg==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "bin": { "playwright-core": "cli.js" @@ -9770,7 +9723,6 @@ "version": "8.5.6", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.6.tgz", "integrity": "sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg==", - "dev": true, "funding": [ { "type": "opencollective", @@ -9799,7 +9751,6 @@ "version": "15.1.0", "resolved": "https://registry.npmjs.org/postcss-import/-/postcss-import-15.1.0.tgz", "integrity": "sha512-hpr+J05B2FVYUAXHeK1YyI267J/dDDhMU6B6civm8hSY1jYJnBXxzKDKDswzJmtLHryrjhnDjqqp/49t8FALew==", - "dev": true, "license": "MIT", "dependencies": { "postcss-value-parser": "^4.0.0", @@ -9817,7 +9768,6 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/postcss-js/-/postcss-js-4.1.0.tgz", "integrity": "sha512-oIAOTqgIo7q2EOwbhb8UalYePMvYoIeRY2YKntdpFQXNosSu3vLrniGgmH9OKs/qAkfoj5oB3le/7mINW1LCfw==", - "dev": true, "funding": [ { "type": "opencollective", @@ -9843,7 +9793,6 @@ "version": "6.0.1", "resolved": "https://registry.npmjs.org/postcss-load-config/-/postcss-load-config-6.0.1.tgz", "integrity": "sha512-oPtTM4oerL+UXmx+93ytZVN82RrlY/wPUV8IeDxFrzIjXOLF1pN+EmKPLbubvKHT2HC20xXsCAH2Z+CKV6Oz/g==", - "dev": true, "funding": [ { "type": "opencollective", @@ -9886,7 +9835,6 @@ "version": "6.2.0", "resolved": "https://registry.npmjs.org/postcss-nested/-/postcss-nested-6.2.0.tgz", "integrity": "sha512-HQbt28KulC5AJzG+cZtj9kvKB93CFCdLvog1WFLf1D+xmMvPGlBstkpTEZfK5+AN9hfJocyBFCNiqyS48bpgzQ==", - "dev": true, "funding": [ { "type": "opencollective", @@ -9912,7 +9860,6 @@ "version": "6.1.2", "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.1.2.tgz", "integrity": "sha512-Q8qQfPiZ+THO/3ZrOrO0cJJKfpYCagtMUkXbnEfmgUjwXg6z/WBeOyS9APBBPCTSiDV+s4SwQGu8yFsiMRIudg==", - "dev": true, "license": "MIT", "dependencies": { "cssesc": "^3.0.0", @@ -9926,7 +9873,6 @@ "version": "4.2.0", "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-4.2.0.tgz", "integrity": "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==", - "dev": true, "license": "MIT" }, "node_modules/prelude-ls": { @@ -10040,7 +9986,6 @@ "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", - "dev": true, "funding": [ { "type": "github", @@ -10829,7 +10774,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/read-cache/-/read-cache-1.0.0.tgz", "integrity": "sha512-Owdv/Ft7IjOgm/i0xvNDZ1LrRANRfew4b2prF3OWMQLxLfu3bS8FVhCsrSCMK4lR56Y9ya+AThoTpDCTxCmpRA==", - "dev": true, "license": "MIT", "dependencies": { "pify": "^2.3.0" @@ -10839,7 +10783,6 @@ "version": "3.6.0", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", - "dev": true, "license": "MIT", "dependencies": { "picomatch": "^2.2.1" @@ -10852,7 +10795,6 @@ "version": "2.3.1", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", - "dev": true, "license": "MIT", "engines": { "node": ">=8.6" @@ -11117,7 +11059,6 @@ "version": "1.22.11", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.11.tgz", "integrity": "sha512-RfqAvLnMl313r7c9oclB1HhUEAezcpLjz95wFH4LVuhk9JF/r22qmVP9AMmOU4vMX7Q8pN8jwNg/CSpdFnMjTQ==", - "dev": true, "license": "MIT", "dependencies": { "is-core-module": "^2.16.1", @@ -11158,7 +11099,6 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz", "integrity": "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==", - "dev": true, "license": "MIT", "engines": { "iojs": ">=1.0.0", @@ -11214,7 +11154,6 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", - "dev": true, "funding": [ { "type": "github", @@ -11855,7 +11794,6 @@ "version": "3.35.1", "resolved": "https://registry.npmjs.org/sucrase/-/sucrase-3.35.1.tgz", "integrity": "sha512-DhuTmvZWux4H1UOnWMB3sk0sbaCVOoQZjv8u1rDoTV0HTdGem9hkAZtl4JZy8P2z4Bg0nT+YMeOFyVr4zcG5Tw==", - "dev": true, "license": "MIT", "dependencies": { "@jridgewell/gen-mapping": "^0.3.2", @@ -11891,7 +11829,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", - "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -11927,7 +11864,6 @@ "version": "3.4.19", "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-3.4.19.tgz", "integrity": "sha512-3ofp+LL8E+pK/JuPLPggVAIaEuhvIz4qNcf3nA1Xn2o/7fb7s/TYpHhwGDv1ZU3PkBluUVaF8PyCHcm48cKLWQ==", - "dev": true, "license": "MIT", "dependencies": { "@alloc/quick-lru": "^5.2.0", @@ -11965,7 +11901,6 @@ "version": "3.3.3", "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz", "integrity": "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==", - "dev": true, "license": "MIT", "dependencies": { "@nodelib/fs.stat": "^2.0.2", @@ -11982,7 +11917,6 @@ "version": "5.1.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", - "dev": true, "license": "ISC", "dependencies": { "is-glob": "^4.0.1" @@ -12010,7 +11944,6 @@ "version": "3.3.1", "resolved": "https://registry.npmjs.org/thenify/-/thenify-3.3.1.tgz", "integrity": "sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw==", - "dev": true, "license": "MIT", "dependencies": { "any-promise": "^1.0.0" @@ -12020,7 +11953,6 @@ "version": "1.6.0", "resolved": "https://registry.npmjs.org/thenify-all/-/thenify-all-1.6.0.tgz", "integrity": "sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA==", - "dev": true, "license": "MIT", "dependencies": { "thenify": ">= 3.1.0 < 4" @@ -12062,7 +11994,6 @@ "version": "0.2.15", "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.15.tgz", "integrity": "sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==", - "dev": true, "license": "MIT", "dependencies": { "fdir": "^6.5.0", @@ -12129,7 +12060,6 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", - "dev": true, "license": "MIT", "dependencies": { "is-number": "^7.0.0" @@ -12217,7 +12147,6 @@ "version": "0.1.13", "resolved": "https://registry.npmjs.org/ts-interface-checker/-/ts-interface-checker-0.1.13.tgz", "integrity": "sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA==", - "dev": true, "license": "Apache-2.0" }, "node_modules/tsconfig-paths": { @@ -12334,7 +12263,7 @@ "version": "5.3.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.3.3.tgz", "integrity": "sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", @@ -12536,7 +12465,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", - "dev": true, "license": "MIT" }, "node_modules/uuid": { @@ -12990,7 +12918,7 @@ "version": "8.19.0", "resolved": "https://registry.npmjs.org/ws/-/ws-8.19.0.tgz", "integrity": "sha512-blAT2mjOEIi0ZzruJfIhb3nps74PRWTCz1IjglWEEpQl5XS/UNama6u2/rjFkDDouqr4L67ry+1aGIALViWjDg==", - "dev": true, + "devOptional": true, "license": "MIT", "engines": { "node": ">=10.0.0" @@ -13056,21 +12984,6 @@ "type": "github", "url": "https://github.com/sponsors/wooorm" } - }, - "node_modules/@next/swc-win32-ia32-msvc": { - "version": "14.2.33", - "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.2.33.tgz", - "integrity": "sha512-pc9LpGNKhJ0dXQhZ5QMmYxtARwwmWLpeocFmVG5Z0DzWq5Uf0izcI8tLc+qOpqxO1PWqZ5A7J1blrUIKrIFc7Q==", - "cpu": [ - "ia32" - ], - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 10" - } } } } diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/PrettyMessagesView.test.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/PrettyMessagesView.test.tsx new file mode 100644 index 00000000000..b73dcafcdc3 --- /dev/null +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/PrettyMessagesView.test.tsx @@ -0,0 +1,81 @@ +import React from "react"; +import { render, screen } from "@testing-library/react"; +import { describe, it, expect, vi } from "vitest"; +import { PrettyMessagesView } from "./PrettyMessagesView"; + +vi.mock("antd", async () => { + const actual = await vi.importActual("antd"); + return { + ...actual, + message: { + success: vi.fn(), + }, + }; +}); + +describe("PrettyMessagesView", () => { + it("should render the component for standard chat completions", () => { + const request = { + messages: [{ role: "user", content: "Hello" }], + }; + const response = { + choices: [{ message: { role: "assistant", content: "Hi there!" } }], + }; + + render(); + expect(screen.getByText("Hello")).toBeInTheDocument(); + expect(screen.getByText("Hi there!")).toBeInTheDocument(); + }); + + it("should render the realtime pretty view for realtime API responses", () => { + const request = {}; + const response = { + results: [ + { + type: "session.created", + session: { + id: "sess_123", + model: "gpt-4o-mini-realtime-preview", + voice: "alloy", + modalities: ["audio", "text"], + }, + }, + { + type: "response.done", + response: { + id: "resp_1", + status: "completed", + output: [ + { + id: "item_1", + role: "assistant", + type: "message", + content: [{ type: "audio", transcript: "Hello from realtime!" }], + }, + ], + }, + }, + ], + }; + + render(); + expect(screen.getByText("Session")).toBeInTheDocument(); + expect(screen.getByText("Hello from realtime!")).toBeInTheDocument(); + const modelElements = screen.getAllByText("gpt-4o-mini-realtime-preview"); + expect(modelElements.length).toBeGreaterThanOrEqual(1); + }); + + it("should render standard view when response has results but no realtime events", () => { + const request = { + messages: [{ role: "user", content: "Test" }], + }; + const response = { + results: [{ type: "some.other.type" }], + choices: [{ message: { role: "assistant", content: "Reply" } }], + }; + + render(); + expect(screen.getByText("Test")).toBeInTheDocument(); + expect(screen.getByText("Reply")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/PrettyMessagesView.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/PrettyMessagesView.tsx index 2d4a14d6f5d..06a5bbb5f9b 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/PrettyMessagesView.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/PrettyMessagesView.tsx @@ -1,11 +1,13 @@ /** * PrettyMessagesView - Datadog-style view with Input/Output cards - * Two main cards showing request and response with token counts and costs + * Two main cards showing request and response with token counts and costs. + * Detects realtime API responses and renders a specialized view. */ import { parseMessages } from './prettyMessagesUtils'; import { InputCard } from './InputCard'; import { OutputCard } from './OutputCard'; +import { isRealtimeResponse, RealtimePrettyView } from './RealtimePrettyView'; interface PrettyMessagesViewProps { request: any; @@ -19,6 +21,10 @@ interface PrettyMessagesViewProps { } export function PrettyMessagesView({ request, response, metrics }: PrettyMessagesViewProps) { + if (isRealtimeResponse(response)) { + return ; + } + const { requestMessages, responseMessage } = parseMessages(request, response); return ( diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RealtimePrettyView.test.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RealtimePrettyView.test.tsx new file mode 100644 index 00000000000..4a0adecc967 --- /dev/null +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RealtimePrettyView.test.tsx @@ -0,0 +1,392 @@ +import React from "react"; +import { render, screen, fireEvent, act, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { RealtimePrettyView, isRealtimeResponse } from "./RealtimePrettyView"; + +vi.mock("antd", async () => { + const actual = await vi.importActual("antd"); + return { + ...actual, + message: { + success: vi.fn(), + }, + }; +}); + +const sampleRealtimeResponse = { + usage: { + total_tokens: 587, + prompt_tokens: 294, + completion_tokens: 293, + }, + results: [ + { + type: "session.created", + session: { + id: "sess_DDNQlPKHjLsokSJPAOWY0", + model: "gpt-4o-mini-realtime-preview", + tools: [], + voice: "alloy", + modalities: ["audio", "text"], + temperature: 0.8, + tool_choice: "auto", + instructions: "You are a helpful assistant.", + turn_detection: { + type: "server_vad", + threshold: 0.5, + }, + input_audio_format: "pcm16", + output_audio_format: "pcm16", + max_response_output_tokens: "inf", + }, + event_id: "event_DDNQlB4VNUlpqTVIjBbm3", + }, + { + type: "response.done", + event_id: "event_DDNQnagYJCZyZATdJCn0L", + response: { + id: "resp_DDNQnlXGHZJB46D5JhJ95", + usage: { + input_tokens: 116, + total_tokens: 162, + output_tokens: 46, + input_token_details: { + text_tokens: 116, + audio_tokens: 0, + }, + output_token_details: { + text_tokens: 16, + audio_tokens: 30, + }, + }, + voice: "alloy", + object: "realtime.response", + output: [ + { + id: "item_DDNQnz5uN1b8NEvPOPZOM", + role: "assistant", + type: "message", + status: "completed", + content: [ + { + type: "audio", + transcript: "Hello! How's your day going?", + }, + ], + }, + ], + status: "completed", + conversation_id: "conv_DDNQlpNllPYhCCfXCtT8X", + max_output_tokens: "inf", + }, + }, + { + type: "response.done", + event_id: "event_DDNR0VmrRTVU69RxGC29U", + response: { + id: "resp_DDNQy6S4PBZxW4qsKq6Ah", + usage: { + input_tokens: 178, + total_tokens: 425, + output_tokens: 247, + }, + voice: "alloy", + object: "realtime.response", + output: [ + { + id: "item_DDNQywctWVnYmujg4FSTZ", + role: "assistant", + type: "message", + status: "completed", + content: [ + { + type: "audio", + transcript: + "I'm here to help with information and general questions.", + }, + ], + }, + ], + status: "completed", + conversation_id: "conv_DDNQlpNllPYhCCfXCtT8X", + max_output_tokens: "inf", + }, + }, + ], +}; + +describe("isRealtimeResponse", () => { + it("should return true for a valid realtime response with session.created", () => { + expect(isRealtimeResponse(sampleRealtimeResponse)).toBe(true); + }); + + it("should return true for response with only response.done events", () => { + const resp = { + results: [{ type: "response.done", response: { id: "r1" } }], + }; + expect(isRealtimeResponse(resp)).toBe(true); + }); + + it("should return false for a standard chat completion response", () => { + const chatResponse = { + choices: [{ message: { role: "assistant", content: "Hello" } }], + }; + expect(isRealtimeResponse(chatResponse)).toBe(false); + }); + + it("should return false for null/undefined", () => { + expect(isRealtimeResponse(null)).toBe(false); + expect(isRealtimeResponse(undefined)).toBe(false); + }); + + it("should return false for empty results array", () => { + expect(isRealtimeResponse({ results: [] })).toBe(false); + }); + + it("should return false for results with unrecognized event types", () => { + const resp = { + results: [{ type: "some.unknown.event" }], + }; + expect(isRealtimeResponse(resp)).toBe(false); + }); +}); + +describe("RealtimePrettyView", () => { + const mockWriteText = vi.fn().mockResolvedValue(undefined); + + beforeEach(() => { + vi.clearAllMocks(); + Object.defineProperty(navigator, "clipboard", { + value: { writeText: mockWriteText }, + writable: true, + configurable: true, + }); + }); + + it("should render the component successfully", () => { + render(); + expect(screen.getByText("Session")).toBeInTheDocument(); + }); + + it("should display the session model name", () => { + render(); + const modelElements = screen.getAllByText("gpt-4o-mini-realtime-preview"); + expect(modelElements.length).toBeGreaterThanOrEqual(1); + }); + + it("should display the session voice tag", () => { + render(); + const voiceElements = screen.getAllByText("alloy"); + expect(voiceElements.length).toBeGreaterThanOrEqual(1); + }); + + it("should display modality tags", () => { + render(); + expect(screen.getByText("audio")).toBeInTheDocument(); + expect(screen.getByText("text")).toBeInTheDocument(); + }); + + it("should display the turn count in session header", () => { + render(); + expect(screen.getByText("2 turns")).toBeInTheDocument(); + }); + + it("should display singular 'turn' for a single response event", () => { + const singleTurnResponse = { + results: [ + { + type: "session.created", + session: { + id: "sess_1", + model: "gpt-4o-mini-realtime-preview", + voice: "alloy", + modalities: ["audio"], + }, + }, + { + type: "response.done", + response: { + id: "r1", + status: "completed", + output: [ + { + id: "item1", + role: "assistant", + type: "message", + content: [{ type: "audio", transcript: "Hi!" }], + }, + ], + }, + }, + ], + }; + render(); + expect(screen.getByText("1 turn")).toBeInTheDocument(); + }); + + it("should display the turn count in the output section header", () => { + render(); + expect(screen.getByText("Turns: 2")).toBeInTheDocument(); + }); + + it("should display the Output section header", () => { + render(); + expect(screen.getByText("Output")).toBeInTheDocument(); + }); + + it("should display transcript text from response turns", () => { + render(); + expect( + screen.getByText("Hello! How's your day going?") + ).toBeInTheDocument(); + expect( + screen.getByText( + "I'm here to help with information and general questions." + ) + ).toBeInTheDocument(); + }); + + it("should display completed status tags for response turns", () => { + render(); + const completedTags = screen.getAllByText("completed"); + expect(completedTags.length).toBe(2); + }); + + it("should display token usage per turn", () => { + render(); + expect(screen.getByText("116 in / 46 out tokens")).toBeInTheDocument(); + expect(screen.getByText("178 in / 247 out tokens")).toBeInTheDocument(); + }); + + it("should expand session details when session header is clicked", async () => { + const user = userEvent.setup(); + render(); + + await user.click(screen.getByText("Session")); + + await waitFor(() => { + expect(screen.getByText("Temperature")).toBeInTheDocument(); + }); + }); + + it("should display session instructions when expanded", async () => { + const user = userEvent.setup(); + render(); + + await user.click(screen.getByText("Session")); + + await waitFor(() => { + expect(screen.getByText("Instructions")).toBeInTheDocument(); + expect( + screen.getByText("You are a helpful assistant.") + ).toBeInTheDocument(); + }); + }); + + it("should display session audio format when expanded", async () => { + const user = userEvent.setup(); + render(); + + await user.click(screen.getByText("Session")); + + await waitFor(() => { + expect(screen.getByText("Input Audio Format")).toBeInTheDocument(); + expect(screen.getAllByText("pcm16").length).toBeGreaterThanOrEqual(1); + }); + }); + + it("should display ASSISTANT label for output messages", () => { + render(); + const assistantLabels = screen.getAllByText("ASSISTANT"); + expect(assistantLabels.length).toBe(2); + }); + + it("should display fallback message when no recognized events exist", () => { + const emptyResponse = { + results: [{ type: "unknown.event" }], + }; + render(); + expect( + screen.getByText("No recognized realtime events found") + ).toBeInTheDocument(); + }); + + it("should handle response with no output items gracefully", () => { + const noOutputResponse = { + results: [ + { + type: "response.done", + response: { + id: "r1", + status: "completed", + output: [], + }, + }, + ], + }; + render(); + expect(screen.getByText("completed")).toBeInTheDocument(); + }); + + it("should display metrics tokens when provided", () => { + render( + + ); + expect(screen.getByText(/Tokens: 500/)).toBeInTheDocument(); + expect(screen.getByText(/Cost: \$0\.005000/)).toBeInTheDocument(); + }); + + it("should toggle output section collapse when header is clicked", async () => { + const user = userEvent.setup(); + render(); + + const transcript = screen.getByText("Hello! How's your day going?"); + expect(transcript).toBeVisible(); + + const outputHeader = screen.getByText("Output").closest("div"); + if (outputHeader) { + await user.click(outputHeader); + await waitFor(() => { + expect(transcript).not.toBeVisible(); + }); + } + }); + + it("should display token breakdown tags when input_token_details are present", async () => { + render(); + expect(screen.getByText(/Text Tokens: 116/)).toBeInTheDocument(); + }); + + it("should handle text content type in addition to audio", () => { + const textResponse = { + results: [ + { + type: "response.done", + response: { + id: "r1", + status: "completed", + output: [ + { + id: "item1", + role: "assistant", + type: "message", + content: [ + { + type: "text", + text: "This is a text response", + }, + ], + }, + ], + }, + }, + ], + }; + render(); + expect(screen.getByText("This is a text response")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RealtimePrettyView.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RealtimePrettyView.tsx new file mode 100644 index 00000000000..4029fbdb835 --- /dev/null +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RealtimePrettyView.tsx @@ -0,0 +1,570 @@ +/** + * RealtimePrettyView - Structured pretty view for OpenAI Realtime API logs + * Displays session config, conversation turns, and token usage + * in a readable format instead of raw JSON. + */ + +import { useState } from 'react'; +import { Typography, Tag, Tooltip } from 'antd'; +import { + SoundOutlined, + MessageOutlined, + SettingOutlined, + AudioOutlined, + DownOutlined, + UpOutlined, +} from '@ant-design/icons'; +import { SectionHeader } from './SectionHeader'; + +const { Text } = Typography; + +interface RealtimeEvent { + type: string; + event_id?: string; + session?: RealtimeSession; + response?: RealtimeResponse; +} + +interface RealtimeSession { + id: string; + model: string; + voice?: string; + modalities?: string[]; + temperature?: number; + tools?: any[]; + instructions?: string; + turn_detection?: Record; + input_audio_format?: string; + output_audio_format?: string; + max_response_output_tokens?: string | number; + [key: string]: any; +} + +interface RealtimeResponse { + id: string; + status: string; + usage?: { + input_tokens?: number; + output_tokens?: number; + total_tokens?: number; + input_token_details?: Record; + output_token_details?: Record; + }; + output?: RealtimeOutputItem[]; + modalities?: string[]; + voice?: string; + conversation_id?: string; + [key: string]: any; +} + +interface RealtimeOutputItem { + id: string; + role: string; + type: string; + status?: string; + content?: Array<{ + type: string; + transcript?: string; + text?: string; + }>; +} + +interface RealtimePrettyViewProps { + response: any; + metrics?: { + prompt_tokens?: number; + completion_tokens?: number; + input_cost?: number; + output_cost?: number; + }; +} + +export function isRealtimeResponse(response: any): boolean { + if (!response || !response.results || !Array.isArray(response.results) || response.results.length === 0) { + return false; + } + + return response.results.some( + (r: any) => + r.type === 'session.created' || + r.type === 'session.updated' || + r.type === 'response.done' + ); +} + +export function RealtimePrettyView({ response, metrics }: RealtimePrettyViewProps) { + const events: RealtimeEvent[] = response?.results || []; + const usage = response?.usage; + + const sessionEvent = events.find( + (e) => e.type === 'session.created' || e.type === 'session.updated' + ); + const responseEvents = events.filter((e) => e.type === 'response.done'); + + return ( +
+ {/* Session Configuration Card */} + {sessionEvent?.session && ( + + )} + + {/* Conversation Turns */} + {responseEvents.length > 0 && ( + e.response!).filter(Boolean)} + totalUsage={usage} + metrics={metrics} + /> + )} + + {/* Fallback if no recognized events */} + {!sessionEvent && responseEvents.length === 0 && ( +
+ No recognized realtime events found +
+ )} +
+ ); +} + +function SessionCard({ session, turnCount }: { session: RealtimeSession; turnCount: number }) { + const [isCollapsed, setIsCollapsed] = useState(true); + + return ( +
+
setIsCollapsed(!isCollapsed)} + style={{ + display: 'flex', + alignItems: 'center', + justifyContent: 'space-between', + padding: '10px 16px', + borderBottom: isCollapsed ? 'none' : '1px solid #f0f0f0', + background: '#fafafa', + cursor: 'pointer', + transition: 'background 0.15s ease', + }} + onMouseEnter={(e) => { + e.currentTarget.style.background = '#f5f5f5'; + }} + onMouseLeave={(e) => { + e.currentTarget.style.background = '#fafafa'; + }} + > +
+
+ {isCollapsed ? ( + + ) : ( + + )} +
+
+ + Session +
+ + {session.model} + + {turnCount > 0 && ( + + {turnCount} {turnCount === 1 ? 'turn' : 'turns'} + + )} + {session.voice && ( + + {session.voice} + + )} + {session.modalities && ( +
+ {session.modalities.map((m) => ( + + {m === 'audio' ? : } {m} + + ))} +
+ )} +
+
+ +
+
+
+ + + + + + + {session.turn_detection && ( + + )} + {session.tools && session.tools.length > 0 && ( + + )} +
+ + {session.instructions && ( +
+ + Instructions + +
+ {session.instructions} +
+
+ )} +
+
+
+ ); +} + +function ConversationCard({ + responses, + totalUsage, + metrics, +}: { + responses: RealtimeResponse[]; + totalUsage?: any; + metrics?: RealtimePrettyViewProps['metrics']; +}) { + const [isCollapsed, setIsCollapsed] = useState(false); + + const totalTokens = totalUsage?.total_tokens; + const turnCount = responses.length; + const handleCopy = () => { + const transcripts = responses + .flatMap((r) => + (r.output || []).flatMap((o) => + (o.content || []).map( + (c) => `${o.role}: ${c.transcript || c.text || ''}` + ) + ) + ) + .join('\n'); + navigator.clipboard.writeText(transcripts); + }; + + return ( +
+ setIsCollapsed(!isCollapsed)} + turnCount={turnCount} + /> + +
+
+ {responses.map((resp, idx) => ( + + ))} +
+
+
+ ); +} + +function ResponseTurn({ + response, + index, +}: { + response: RealtimeResponse; + index: number; +}) { + const outputs = response.output || []; + const usage = response.usage; + + return ( +
+ {/* Turn header */} +
+ + {response.status || 'unknown'} + + {usage && ( + + {usage.input_tokens ?? 0} in / {usage.output_tokens ?? 0} out tokens + + )} + {response.conversation_id && ( + + + conv: {response.conversation_id.slice(0, 12)}... + + + )} +
+ + {/* Output messages / transcripts */} + {outputs.map((output, oIdx) => ( + + ))} + + {/* Token breakdown if available */} + {usage?.input_token_details && ( + + )} + {usage?.output_token_details && ( + + )} +
+ ); +} + +function OutputMessage({ output }: { output: RealtimeOutputItem }) { + const contents = output.content || []; + const hasTranscripts = contents.some((c) => c.transcript || c.text); + + if (!hasTranscripts) return null; + + return ( +
+ + {output.role?.toUpperCase() || 'ASSISTANT'} + + {contents.map((c, cIdx) => { + const text = c.transcript || c.text; + if (!text) return null; + return ( +
+ {c.type === 'audio' && ( + + )} + {c.type === 'text' && ( + + )} +
+ {text} +
+
+ ); + })} +
+ ); +} + +function TokenBreakdown({ + label, + details, +}: { + label: string; + details: Record; +}) { + const entries = Object.entries(details).filter( + ([, v]) => + typeof v === 'number' || + (typeof v === 'object' && v !== null) + ); + + if (entries.length === 0) return null; + + return ( +
+ + {label} Token Breakdown + +
+ {entries.map(([key, value]) => { + if (typeof value === 'number') { + return ( + + {formatTokenLabel(key)}: {value.toLocaleString()} + + ); + } + return null; + })} +
+
+ ); +} + +function ConfigRow({ + label, + value, +}: { + label: string; + value: any; +}) { + if (value === undefined || value === null) return null; + return ( +
+ + {label} + +
+ {String(value)} +
+
+ ); +} + +function formatTokenLabel(key: string): string { + return key + .replace(/_/g, ' ') + .replace(/\b\w/g, (c) => c.toUpperCase()); +} diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/SectionHeader.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/SectionHeader.tsx index e667e125149..632ea961853 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/SectionHeader.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/SectionHeader.tsx @@ -19,9 +19,10 @@ interface SectionHeaderProps { onCopy: () => void; isCollapsed?: boolean; onToggleCollapse?: () => void; + turnCount?: number; } -export function SectionHeader({ type, tokens, cost, onCopy, isCollapsed, onToggleCollapse }: SectionHeaderProps) { +export function SectionHeader({ type, tokens, cost, onCopy, isCollapsed, onToggleCollapse, turnCount }: SectionHeaderProps) { return (
)} + + {/* Turn count */} + {turnCount !== undefined && turnCount > 0 && ( + + Turns: {turnCount} + + )}
{/* Copy Button */} From 819581f6bfbe6a49c65e12be62b0fbd41e82b603 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 25 Feb 2026 23:43:13 -0800 Subject: [PATCH 2/2] fix(realtime): guardrails with pre_call/post_call mode now work on realtime WebSocket (#22161) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(realtime): guardrails with pre_call/post_call mode now work on realtime WebSocket; return error directly to consumer * fix(realtime guardrails): address code review feedback - Restore session.update injection for audio/VAD path, but only when realtime_input_transcription guardrails are configured (not pre_call). Forward session.created to the client first so no error arrives before the client sees the session. - Change _swallow_next_response_create bool to int counter so consecutive blocked items are handled correctly. - Extract _build_litellm_metadata() helper to eliminate duplicated metadata-building logic across OpenAI/Azure/XAI provider branches. - Plumb litellm_metadata and user_api_key_dict to Azure and XAI handlers so guardrails work for those providers too. - Add tests for session.update injection, no-inject for pre_call-only, and consecutive-block counter. * simplify: remove response.create swallowing after guardrail block When an item is blocked, the error event is already sent to the client. The subsequent response.create from the client is fine to forward through — the LLM may respond to previous context which is acceptable behavior. Removing the swallow counter eliminates unnecessary state tracking. --- .../litellm_core_utils/realtime_streaming.py | 144 ++++++----- litellm/llms/azure/realtime/handler.py | 10 +- litellm/llms/openai/realtime/handler.py | 2 + litellm/realtime_api/main.py | 14 + .../test_realtime_streaming.py | 240 +++++++++++++++--- 5 files changed, 321 insertions(+), 89 deletions(-) diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index 5d7a5bfe318..231f3a975dc 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -43,6 +43,7 @@ class RealTimeStreaming: provider_config: Optional[BaseRealtimeConfig] = None, model: str = "", user_api_key_dict: Optional[Any] = None, + request_data: Optional[Dict] = None, ): self.websocket = websocket self.backend_ws = backend_ws @@ -68,6 +69,7 @@ class RealTimeStreaming: self.current_delta_type: Optional[ALL_DELTA_TYPES] = None self.session_configuration_request: Optional[str] = None self.user_api_key_dict = user_api_key_dict + self.request_data: Dict = request_data or {} def _should_store_message( self, @@ -231,14 +233,40 @@ class RealTimeStreaming: await self.backend_ws.send(message) def _has_realtime_guardrails(self) -> bool: - """Return True if any callback is registered for realtime_input_transcription.""" + """Return True if any callback is registered for realtime guardrail event types.""" + from litellm.integrations.custom_guardrail import CustomGuardrail + from litellm.types.guardrails import GuardrailEventHooks + + _realtime_event_types = [ + GuardrailEventHooks.realtime_input_transcription, + GuardrailEventHooks.pre_call, + GuardrailEventHooks.post_call, + ] + return any( + isinstance(cb, CustomGuardrail) + and any( + cb.should_run_guardrail( + data=self.request_data, + event_type=et, + ) + for et in _realtime_event_types + ) + for cb in litellm.callbacks + ) + + def _has_audio_transcription_guardrails(self) -> bool: + """Return True if any callback needs to run on audio transcriptions (VAD path). + + When this returns True, we inject a session.update to disable the LLM's + auto-response so the guardrail can gate it first. + """ from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.types.guardrails import GuardrailEventHooks return any( isinstance(cb, CustomGuardrail) and cb.should_run_guardrail( - data={}, + data=self.request_data, event_type=GuardrailEventHooks.realtime_input_transcription, ) for cb in litellm.callbacks @@ -258,17 +286,25 @@ class RealTimeStreaming: from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.types.guardrails import GuardrailEventHooks + _realtime_event_types = [ + GuardrailEventHooks.realtime_input_transcription, + GuardrailEventHooks.pre_call, + GuardrailEventHooks.post_call, + ] + _check_data = {**self.request_data, "transcript": transcript} + _already_run: set = set() + for callback in litellm.callbacks: if not isinstance(callback, CustomGuardrail): continue - if ( - callback.should_run_guardrail( - data={"transcript": transcript}, - event_type=GuardrailEventHooks.realtime_input_transcription, - ) - is not True + if id(callback) in _already_run: + continue + if not any( + callback.should_run_guardrail(data=_check_data, event_type=et) + for et in _realtime_event_types ): continue + _already_run.add(id(callback)) try: await callback.apply_guardrail( inputs={"texts": [transcript], "images": []}, @@ -293,20 +329,15 @@ class RealTimeStreaming: safe_msg = str(detail) else: safe_msg = str(e) or "I'm sorry, that request was blocked by the content filter." - # Cancel any in-flight response before speaking the warning. - # This handles the race where create_response fired before we could intercept. - await self._send_to_backend(json.dumps({"type": "response.cancel"})) - # Ask the model to speak the warning — TTS audio plays naturally in the client - await self._send_to_backend( + # Return the error directly to the WebSocket consumer. + await self.websocket.send_text( json.dumps( { - "type": "response.create", - "response": { - "modalities": ["text", "audio"], - "instructions": ( - f"Say exactly and only: \"{safe_msg}\". " - "Do not add anything else." - ), + "type": "error", + "error": { + "type": "guardrail_violation", + "message": safe_msg, + "code": "content_policy_violation", }, } ) @@ -348,25 +379,25 @@ class RealTimeStreaming: if isinstance(transformed_response, list) else [transformed_response] ) - for event in events: - ## GUARDRAIL: inject create_response=false on session.created - if isinstance(event, dict) and event.get("type") == "session.created": - if self._has_realtime_guardrails(): - await self._send_to_backend( - json.dumps( - { - "type": "session.update", - "session": { - "turn_detection": { - "type": "server_vad", - "create_response": False, - } - }, - } - ) - ) for event in events: event_str = json.dumps(event) + ## For audio/VAD guardrail path: forward session.created first, then inject. + if ( + isinstance(event, dict) + and event.get("type") == "session.created" + and self._has_audio_transcription_guardrails() + ): + self.store_message(event_str) + await self.websocket.send_text(event_str) + await self._send_to_backend( + json.dumps( + { + "type": "session.update", + "session": {"turn_detection": {"create_response": False}}, + } + ) + ) + continue ## GUARDRAIL: run on transcription events in provider_config path too if ( isinstance(event, dict) @@ -397,27 +428,26 @@ class RealTimeStreaming: try: event_obj = json.loads(raw_response) - if event_obj.get("type") == "session.created": - # If any realtime guardrails are registered, proactively - # set create_response=false so the LLM never auto-responds - # before our guardrail has a chance to run. - if self._has_realtime_guardrails(): - await self._send_to_backend( - json.dumps( - { - "type": "session.update", - "session": { - "turn_detection": { - "type": "server_vad", - "create_response": False, - } - }, - } - ) - ) - verbose_logger.debug( - "[realtime guardrail] injected create_response=false into session" + # For audio/VAD guardrail path: once the session is ready, tell the backend + # not to auto-respond after VAD detects end-of-speech. We send the + # session.created to the client FIRST so the client is always in sync, then + # inject the session.update so a potential error from the backend doesn't + # arrive before the client sees session.created. + if ( + event_obj.get("type") == "session.created" + and self._has_audio_transcription_guardrails() + ): + self.store_message(raw_response) + await self.websocket.send_text(raw_response) + await self._send_to_backend( + json.dumps( + { + "type": "session.update", + "session": {"turn_detection": {"create_response": False}}, + } ) + ) + return True if ( event_obj.get("type") diff --git a/litellm/llms/azure/realtime/handler.py b/litellm/llms/azure/realtime/handler.py index e533978e07a..8f4291ec271 100644 --- a/litellm/llms/azure/realtime/handler.py +++ b/litellm/llms/azure/realtime/handler.py @@ -6,13 +6,13 @@ This requires websockets, and is currently only supported on LiteLLM Proxy. from typing import Any, Optional, cast +from litellm._logging import verbose_proxy_logger from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES from ....litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from ....litellm_core_utils.realtime_streaming import RealTimeStreaming from ....llms.custom_httpx.http_handler import get_shared_realtime_ssl_context from ..azure import AzureChatCompletion -from litellm._logging import verbose_proxy_logger # BACKEND_WS_URL = "ws://localhost:8080/v1/realtime?model=gpt-4o-realtime-preview-2024-10-01" @@ -77,6 +77,8 @@ class AzureOpenAIRealtime(AzureChatCompletion): client: Optional[Any] = None, timeout: Optional[float] = None, realtime_protocol: Optional[str] = None, + user_api_key_dict: Optional[Any] = None, + litellm_metadata: Optional[dict] = None, ): import websockets from websockets.asyncio.client import ClientConnection @@ -101,7 +103,11 @@ class AzureOpenAIRealtime(AzureChatCompletion): ssl=ssl_context, ) as backend_ws: realtime_streaming = RealTimeStreaming( - websocket, cast(ClientConnection, backend_ws), logging_obj + websocket, + cast(ClientConnection, backend_ws), + logging_obj, + user_api_key_dict=user_api_key_dict, + request_data={"litellm_metadata": litellm_metadata or {}}, ) await realtime_streaming.bidirectional_forward() diff --git a/litellm/llms/openai/realtime/handler.py b/litellm/llms/openai/realtime/handler.py index c2fccfc7289..05915e36a69 100644 --- a/litellm/llms/openai/realtime/handler.py +++ b/litellm/llms/openai/realtime/handler.py @@ -99,6 +99,7 @@ class OpenAIRealtime(OpenAIChatCompletion): timeout: Optional[float] = None, query_params: Optional[RealtimeQueryParams] = None, user_api_key_dict: Optional[Any] = None, + litellm_metadata: Optional[dict] = None, **kwargs: Any, ): import websockets @@ -142,6 +143,7 @@ class OpenAIRealtime(OpenAIChatCompletion): cast(ClientConnection, backend_ws), logging_obj, user_api_key_dict=user_api_key_dict, + request_data={"litellm_metadata": litellm_metadata or {}}, ) await realtime_streaming.bidirectional_forward() diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index e4c8f648190..df49d4c54b2 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -32,6 +32,15 @@ vertex_llm_base = VertexBase() base_llm_http_handler = BaseLLMHTTPHandler() +def _build_litellm_metadata(kwargs: dict) -> dict: + """Build the litellm_metadata dict for guardrail checking (internal only, not forwarded to provider).""" + metadata: dict = {**(kwargs.get("litellm_metadata") or {})} + guardrails = (kwargs.get("metadata") or {}).get("guardrails") or kwargs.get("guardrails") or [] + if guardrails: + metadata["guardrails"] = guardrails + return metadata + + @wrapper_client async def _arealtime( model: str, @@ -134,6 +143,8 @@ async def _arealtime( timeout=timeout, logging_obj=litellm_logging_obj, realtime_protocol=realtime_protocol, + user_api_key_dict=kwargs.get("user_api_key_dict"), + litellm_metadata=_build_litellm_metadata(kwargs), ) elif _custom_llm_provider == "openai": api_base = ( @@ -160,6 +171,7 @@ async def _arealtime( timeout=timeout, query_params=query_params, user_api_key_dict=kwargs.get("user_api_key_dict"), + litellm_metadata=_build_litellm_metadata(kwargs), ) elif _custom_llm_provider == "bedrock": # Extract AWS parameters from kwargs @@ -217,6 +229,8 @@ async def _arealtime( client=None, timeout=timeout, query_params=query_params, + user_api_key_dict=kwargs.get("user_api_key_dict"), + litellm_metadata=_build_litellm_metadata(kwargs), ) elif _custom_llm_provider == "vertex_ai": vertex_credentials = ( diff --git a/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py b/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py index aaaab95ce6f..8db626a4d33 100644 --- a/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py +++ b/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py @@ -416,32 +416,32 @@ async def test_realtime_guardrail_blocks_prompt_injection(): streaming = RealTimeStreaming(client_ws, backend_ws, logging_obj) await streaming.backend_to_client_send_messages() - # ASSERT 1: no bare response.create was sent to backend (injection blocked). - # The only response.create allowed is the warning one (has "instructions" field). + # ASSERT 1: no response.create was sent to backend (injection blocked). sent_to_backend = [ json.loads(c.args[0]) for c in backend_ws.send.call_args_list if c.args ] - bare_response_creates = [ + response_creates = [ e for e in sent_to_backend if e.get("type") == "response.create" - and "instructions" not in e.get("response", {}) ] - assert len(bare_response_creates) == 0, ( - f"Guardrail should prevent bare response.create for injected content, " - f"but got: {bare_response_creates}" + assert len(response_creates) == 0, ( + f"Guardrail should prevent response.create for injected content, " + f"but got: {response_creates}" ) - # ASSERT 2: warning response.create was sent to backend (to speak the block message) - warning_creates = [ - e for e in sent_to_backend - if e.get("type") == "response.create" - and "instructions" in e.get("response", {}) + # ASSERT 2: error event was sent directly to the client WebSocket + sent_to_client = [ + json.loads(c.args[0]) for c in client_ws.send_text.call_args_list + if c.args ] - assert len(warning_creates) > 0, ( - f"Backend should receive a response.create with warning instructions, " - f"but got: {sent_to_backend}" + error_events = [e for e in sent_to_client if e.get("type") == "error"] + assert len(error_events) == 1, ( + f"Expected one error event sent to client, got: {sent_to_client}" + ) + assert error_events[0]["error"]["type"] == "guardrail_violation", ( + f"Expected guardrail_violation error type, got: {error_events[0]}" ) litellm.callbacks = [] # cleanup @@ -514,11 +514,91 @@ async def test_realtime_guardrail_allows_clean_transcript(): @pytest.mark.asyncio -async def test_realtime_session_created_injects_create_response_false(): +async def test_realtime_text_input_guardrail_blocks_and_returns_error(): """ - Test that when session.created arrives from the backend and realtime guardrails - are registered, the proxy injects a session.update with create_response=False - so the LLM never auto-responds before the guardrail runs. + Test that when conversation.item.create arrives with text that triggers a guardrail, + the proxy blocks it (doesn't forward to backend) and returns an error event directly + to the client WebSocket. + """ + from fastapi import HTTPException + + import litellm + from litellm.integrations.custom_guardrail import CustomGuardrail + from litellm.types.guardrails import GuardrailEventHooks + + class BlockingGuardrail(CustomGuardrail): + async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): + texts = inputs.get("texts", []) + for text in texts: + if "@" in text: + raise HTTPException( + status_code=403, + detail={"error": "email address detected"}, + ) + return inputs + + guardrail = BlockingGuardrail( + guardrail_name="email-blocker", + event_hook=GuardrailEventHooks.pre_call, + default_on=True, + ) + litellm.callbacks = [guardrail] + + client_ws = MagicMock() + client_ws.send_text = AsyncMock() + + backend_ws = MagicMock() + backend_ws.send = AsyncMock() + backend_ws.recv = AsyncMock(side_effect=ConnectionClosed(None, None)) + + logging_obj = MagicMock() + logging_obj.pre_call = MagicMock() + + streaming = RealTimeStreaming(client_ws, backend_ws, logging_obj) + + item_create_msg = json.dumps({ + "type": "conversation.item.create", + "item": { + "role": "user", + "content": [{"type": "input_text", "text": "My email is test@example.com"}], + }, + }) + + # Simulate the client sending a conversation.item.create with an email + client_ws.receive_text = AsyncMock( + side_effect=[ + item_create_msg, + Exception("connection closed"), # stop the loop + ] + ) + + await streaming.client_ack_messages() + + # ASSERT: error event was sent to client + assert client_ws.send_text.called, "Expected error to be sent to client websocket" + sent_texts = [json.loads(c.args[0]) for c in client_ws.send_text.call_args_list] + error_events = [e for e in sent_texts if e.get("type") == "error"] + assert len(error_events) == 1, f"Expected one error event, got: {sent_texts}" + assert error_events[0]["error"]["type"] == "guardrail_violation" + + # ASSERT: blocked item was NOT forwarded to the backend + sent_to_backend = [c.args[0] for c in backend_ws.send.call_args_list if c.args] + forwarded_items = [ + json.loads(m) for m in sent_to_backend + if isinstance(m, str) and json.loads(m).get("type") == "conversation.item.create" + ] + assert len(forwarded_items) == 0, ( + f"Blocked item should not be forwarded to backend, got: {forwarded_items}" + ) + + litellm.callbacks = [] # cleanup + + +@pytest.mark.asyncio +async def test_realtime_text_input_guardrail_uses_pre_call_mode(): + """ + Test that _has_realtime_guardrails returns True for a guardrail configured with + pre_call mode (not just realtime_input_transcription). """ import litellm from litellm.integrations.custom_guardrail import CustomGuardrail @@ -529,7 +609,46 @@ async def test_realtime_session_created_injects_create_response_false(): return inputs guardrail = DummyGuardrail( - guardrail_name="dummy", + guardrail_name="pre-call-guardrail", + event_hook=GuardrailEventHooks.pre_call, + default_on=True, + ) + litellm.callbacks = [guardrail] + + client_ws = MagicMock() + backend_ws = MagicMock() + logging_obj = MagicMock() + streaming = RealTimeStreaming(client_ws, backend_ws, logging_obj) + + assert streaming._has_realtime_guardrails() is True, ( + "pre_call guardrail should be recognized as a realtime guardrail" + ) + # pre_call guardrail should NOT trigger the audio/VAD session.update injection + assert streaming._has_audio_transcription_guardrails() is False, ( + "pre_call guardrail should not trigger audio transcription guardrail path" + ) + + litellm.callbacks = [] # cleanup + + +@pytest.mark.asyncio +async def test_realtime_session_created_injects_session_update_for_audio_guardrail(): + """ + Test that when an audio transcription guardrail is configured, a session.created + event from the backend triggers a session.update injection (create_response: false) + AFTER forwarding session.created to the client. This prevents the LLM from + auto-responding before the guardrail can run on the transcript. + """ + import litellm + from litellm.integrations.custom_guardrail import CustomGuardrail + from litellm.types.guardrails import GuardrailEventHooks + + class AudioGuardrail(CustomGuardrail): + async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): + return inputs + + guardrail = AudioGuardrail( + guardrail_name="audio-guardrail", event_hook=GuardrailEventHooks.realtime_input_transcription, default_on=True, ) @@ -538,34 +657,95 @@ async def test_realtime_session_created_injects_create_response_false(): client_ws = MagicMock() client_ws.send_text = AsyncMock() - session_created_event = json.dumps({"type": "session.created"}).encode() + session_created_event = json.dumps( + {"type": "session.created", "session": {"id": "sess_abc"}} + ).encode() backend_ws = MagicMock() backend_ws.recv = AsyncMock( - side_effect=[ - session_created_event, - ConnectionClosed(None, None), - ] + side_effect=[session_created_event, ConnectionClosed(None, None)] ) backend_ws.send = AsyncMock() logging_obj = MagicMock() logging_obj.async_success_handler = AsyncMock() logging_obj.success_handler = MagicMock() + streaming = RealTimeStreaming(client_ws, backend_ws, logging_obj) await streaming.backend_to_client_send_messages() - # ASSERT: proxy injected session.update with create_response=False to backend + # session.created must be forwarded to the client + sent_to_client = [ + json.loads(c.args[0]) for c in client_ws.send_text.call_args_list if c.args + ] + session_created_events = [e for e in sent_to_client if e.get("type") == "session.created"] + assert len(session_created_events) == 1, ( + f"session.created should be forwarded to client, got: {sent_to_client}" + ) + + # session.update must be sent to the backend AFTER session.created was forwarded sent_to_backend = [ json.loads(c.args[0]) for c in backend_ws.send.call_args_list if c.args ] session_updates = [e for e in sent_to_backend if e.get("type") == "session.update"] assert len(session_updates) == 1, ( - f"Expected proxy to inject session.update, got: {sent_to_backend}" + f"Expected one session.update injected to backend, got: {sent_to_backend}" ) - td = session_updates[0]["session"]["turn_detection"] - assert td["create_response"] is False, ( - f"Expected create_response=False, got: {td}" + assert session_updates[0]["session"]["turn_detection"]["create_response"] is False + + litellm.callbacks = [] # cleanup + + +@pytest.mark.asyncio +async def test_realtime_session_created_no_injection_for_pre_call_only(): + """ + Test that when only a pre_call guardrail is configured (no audio transcription), + session.created does NOT trigger the session.update injection. + """ + import litellm + from litellm.integrations.custom_guardrail import CustomGuardrail + from litellm.types.guardrails import GuardrailEventHooks + + class PreCallGuardrail(CustomGuardrail): + async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): + return inputs + + guardrail = PreCallGuardrail( + guardrail_name="pre-call-only", + event_hook=GuardrailEventHooks.pre_call, + default_on=True, + ) + litellm.callbacks = [guardrail] + + client_ws = MagicMock() + client_ws.send_text = AsyncMock() + + session_created_event = json.dumps( + {"type": "session.created", "session": {"id": "sess_xyz"}} + ).encode() + + backend_ws = MagicMock() + backend_ws.recv = AsyncMock( + side_effect=[session_created_event, ConnectionClosed(None, None)] + ) + backend_ws.send = AsyncMock() + + logging_obj = MagicMock() + logging_obj.async_success_handler = AsyncMock() + logging_obj.success_handler = MagicMock() + + streaming = RealTimeStreaming(client_ws, backend_ws, logging_obj) + await streaming.backend_to_client_send_messages() + + # No session.update should be injected + sent_to_backend = [ + json.loads(c.args[0]) for c in backend_ws.send.call_args_list if c.args + ] + session_updates = [e for e in sent_to_backend if e.get("type") == "session.update"] + assert len(session_updates) == 0, ( + f"pre_call guardrail should NOT inject session.update, got: {sent_to_backend}" ) litellm.callbacks = [] # cleanup + +