From b96842f62c4a6cd0a79a85616d05885966781778 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 20:35:30 -0700 Subject: [PATCH 01/15] test(e2e-ui): check the MCP Tools tab against the upstream's own tools/list (#42397) * test(e2e-ui): check the MCP Tools tab against the upstream's own tools/list DeepWiki renamed ask_question to ask_wiki_question, and the spec hardcoded the old name, so e2e_ui_testing went red on main for something that is not a litellm regression. The spec now asks the upstream server for its tool list with the official MCP TypeScript SDK and expects the tab to show exactly those cards, so a vendor rename cannot turn the job red again. * test(e2e-ui): cite the pinned DeepWiki tool name and drop the helper docstring --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- tests/e2e/ui/helpers/mcp.ts | 13 + tests/e2e/ui/package-lock.json | 1282 +++++++++++++++++++++++ tests/e2e/ui/package.json | 1 + tests/e2e/ui/tests/mcp/mcpTools.spec.ts | 20 +- 4 files changed, 1309 insertions(+), 7 deletions(-) diff --git a/tests/e2e/ui/helpers/mcp.ts b/tests/e2e/ui/helpers/mcp.ts index 554177e11bc..399f746f3a8 100644 --- a/tests/e2e/ui/helpers/mcp.ts +++ b/tests/e2e/ui/helpers/mcp.ts @@ -1,8 +1,21 @@ import { expect, Page as PwPage } from "@playwright/test"; +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js"; import { navigateToPage } from "./navigation"; import { Page } from "../fixtures/pages"; import { masterKey } from "./traffic"; +export async function listUpstreamToolNames(url: string): Promise { + const client = new Client({ name: "litellm-ui-e2e", version: "0.0.0" }); + await client.connect(new StreamableHTTPClientTransport(new URL(url))); + try { + const { tools } = await client.listTools(); + return tools.map((tool) => tool.name); + } finally { + await client.close(); + } +} + /** Creates an MCP server through the UI's discovery to custom-form flow and returns its name. */ export async function createMcpServer(page: PwPage, url: string): Promise { await navigateToPage(page, Page.McpServers); diff --git a/tests/e2e/ui/package-lock.json b/tests/e2e/ui/package-lock.json index b22673a3535..f56e00506e9 100644 --- a/tests/e2e/ui/package-lock.json +++ b/tests/e2e/ui/package-lock.json @@ -8,11 +8,66 @@ "name": "litellm-ui-e2e", "version": "0.0.0", "devDependencies": { + "@modelcontextprotocol/sdk": "1.30.0", "@playwright/test": "1.58.1", "@types/node": "20.19.37", "typescript": "5.9.3" } }, + "node_modules/@hono/node-server": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-2.1.1.tgz", + "integrity": "sha512-ELuehkj5VCBdgEw9zs+ivkKwyzzUCSQuE96YmiPvn1ECBoZCczbFXJLeEGMTYjphP6gydh4pHMqEYPVMYUVgQg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20" + }, + "peerDependencies": { + "hono": "^4" + } + }, + "node_modules/@modelcontextprotocol/sdk": { + "version": "1.30.0", + "resolved": "https://registry.npmjs.org/@modelcontextprotocol/sdk/-/sdk-1.30.0.tgz", + "integrity": "sha512-xKd8OIzlqNzcqcNumGAa6g+PW2kjD5vrpcKOnfldAUPP3j7lnqMPwlTXQm8gF+UwH72z0lqaRbjr9hqGz0eITA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@hono/node-server": "^1.19.9 || ^2.0.5", + "ajv": "^8.17.1", + "ajv-formats": "^3.0.1", + "content-type": "^1.0.5", + "cors": "^2.8.5", + "cross-spawn": "^7.0.5", + "eventsource": "^3.0.2", + "eventsource-parser": "^3.0.0", + "express": "^5.2.1", + "express-rate-limit": "^8.2.1", + "hono": "^4.11.4", + "jose": "^6.1.3", + "json-schema-typed": "^8.0.2", + "pkce-challenge": "^5.0.0", + "raw-body": "^3.0.0", + "zod": "^3.25 || ^4.0", + "zod-to-json-schema": "^3.25.1" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@cfworker/json-schema": "^4.1.1", + "zod": "^3.25 || ^4.0" + }, + "peerDependenciesMeta": { + "@cfworker/json-schema": { + "optional": true + }, + "zod": { + "optional": false + } + } + }, "node_modules/@playwright/test": { "version": "1.58.1", "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.58.1.tgz", @@ -39,6 +94,475 @@ "undici-types": "~6.21.0" } }, + "node_modules/accepts": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", + "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", + "dev": true, + "license": "MIT", + "dependencies": { + "mime-types": "^3.0.0", + "negotiator": "^1.0.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/ajv": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/body-parser": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.3.0.tgz", + "integrity": "sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw==", + "dev": true, + "license": "MIT", + "dependencies": { + "bytes": "^3.1.2", + "content-type": "^2.0.0", + "debug": "^4.4.3", + "http-errors": "^2.0.1", + "iconv-lite": "^0.7.2", + "on-finished": "^2.4.1", + "qs": "^6.15.2", + "raw-body": "^3.0.2", + "type-is": "^2.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/body-parser/node_modules/content-type": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", + "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/content-disposition": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.1.0.tgz", + "integrity": "sha512-5jRCH9Z/+DRP7rkvY83B+yGIGX96OYdJmzngqnw2SBSxqCFPd0w2km3s5iawpGX8krnwSGmF0FW5Nhr0Hfai3g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", + "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.6.0" + } + }, + "node_modules/cors": { + "version": "2.8.6", + "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.6.tgz", + "integrity": "sha512-tJtZBBHA6vjIAaF6EnIaq6laBBP9aq/Y3ouVJjEfoHbRBcHBAHYcMh/w8LDrk2PvIMMq8gmopa5D4V8RmbrxGw==", + "dev": true, + "license": "MIT", + "dependencies": { + "object-assign": "^4", + "vary": "^1" + }, + "engines": { + "node": ">= 0.10" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "dev": true, + "license": "MIT" + }, + "node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.2.tgz", + "integrity": "sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "dev": true, + "license": "MIT" + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/eventsource": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-3.0.7.tgz", + "integrity": "sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA==", + "dev": true, + "license": "MIT", + "dependencies": { + "eventsource-parser": "^3.0.1" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/eventsource-parser": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.1.1.tgz", + "integrity": "sha512-EKN1vKAMcZ8MlYMpaNuxN6R9yakzH6uajHcHVTqWJzvu5pWw9DyhbP35HH8MVBQ+dZjAfDxk+A8NiR9KWaXiyQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/express": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", + "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", + "dev": true, + "license": "MIT", + "dependencies": { + "accepts": "^2.0.0", + "body-parser": "^2.2.1", + "content-disposition": "^1.0.0", + "content-type": "^1.0.5", + "cookie": "^0.7.1", + "cookie-signature": "^1.2.1", + "debug": "^4.4.0", + "depd": "^2.0.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "finalhandler": "^2.1.0", + "fresh": "^2.0.0", + "http-errors": "^2.0.0", + "merge-descriptors": "^2.0.0", + "mime-types": "^3.0.0", + "on-finished": "^2.4.1", + "once": "^1.4.0", + "parseurl": "^1.3.3", + "proxy-addr": "^2.0.7", + "qs": "^6.14.0", + "range-parser": "^1.2.1", + "router": "^2.2.0", + "send": "^1.1.0", + "serve-static": "^2.2.0", + "statuses": "^2.0.1", + "type-is": "^2.0.1", + "vary": "^1.1.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/express-rate-limit": { + "version": "8.7.0", + "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-8.7.0.tgz", + "integrity": "sha512-hOwV7WOxXfjRpAM1DSJWZDXx3GhplwD8IfwuwvogD8i1Qnkgosw/H45s4ZnFAUHDAhPjlY9hLBvJhKmGMyY26g==", + "dev": true, + "license": "MIT", + "dependencies": { + "debug": "^4.4.3", + "ip-address": "^10.2.0" + }, + "engines": { + "node": ">= 16" + }, + "funding": { + "url": "https://github.com/sponsors/express-rate-limit" + }, + "peerDependencies": { + "express": ">= 4.11" + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-uri": { + "version": "3.1.8", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.8.tgz", + "integrity": "sha512-GZMtZUTNRpOVIECoXwLNZS5xUGE+mVNbTB8h/7Rwh2TFWcBQiPzTgyZi05BF9UMZKkLJv8XBRJTlU7zg8+ZfMg==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/finalhandler": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", + "integrity": "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==", + "dev": true, + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "on-finished": "^2.4.1", + "parseurl": "^1.3.3", + "statuses": "^2.0.1" + }, + "engines": { + "node": ">= 18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", + "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/fsevents": { "version": "2.3.2", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", @@ -54,6 +578,396 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "dev": true, + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", + "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", + "dev": true, + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/hono": { + "version": "4.13.8", + "resolved": "https://registry.npmjs.org/hono/-/hono-4.13.8.tgz", + "integrity": "sha512-/Gng7NfoykZl2pjukW5Z6+8Yxm3BPRf86GTbQnt0SbySkvax4fyL4H3HhY1cCpBGmiW9XDRFzRV+CXK2W8QudQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=16.9.0" + } + }, + "node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/iconv-lite": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz", + "integrity": "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/ip-address": { + "version": "10.7.2", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.7.2.tgz", + "integrity": "sha512-7H/2gFSIitxc0hG3nOI1glS8QLo/EHBFFLk8vEUjXY/xu0AdL8jZ9U1IzO2PUm0d2D/ofQcAifb0g6OBkt8U7w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/is-promise": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", + "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true, + "license": "ISC" + }, + "node_modules/jose": { + "version": "6.2.12", + "resolved": "https://registry.npmjs.org/jose/-/jose-6.2.12.tgz", + "integrity": "sha512-9NiFmJEex0sy2Dk58j2UGBSHgUs2ypF9eZSu4L6vjOX3Dp96Sw1F3uL+H+D1sx02jZZdzUT0HgvCy59CuvXcWw==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "dev": true, + "license": "MIT" + }, + "node_modules/json-schema-typed": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/json-schema-typed/-/json-schema-typed-8.0.2.tgz", + "integrity": "sha512-fQhoXdcvc3V28x7C7BMs4P5+kNlgUURe2jmUT1T//oBRMDrqy1QPelJimwZGo7Hg9VPV3EQV5Bnq4hbFy2vetA==", + "dev": true, + "license": "BSD-2-Clause" + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/media-typer": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.1.tgz", + "integrity": "sha512-yz3xRaG20c6/BOzvYoDaGtPmGscs7YivItZEEqe6GbwNfHuxu9YNmvnEkMzKldAGY4/80pRcQRZSEnhquk9XuQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/merge-descriptors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", + "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/negotiator": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.1.0.tgz", + "integrity": "sha512-NMPBRMJgiQHjbd8phG3Vebdx4kZ1H121rbl5IkMqeOsahptB9BKo/d7oJ3zTXqTgagn2bWlNSXkh0QUGM31RYg==", + "dev": true, + "license": "MIT", + "dependencies": { + "content-type": "^2.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/negotiator/node_modules/content-type": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", + "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-inspect": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "dev": true, + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dev": true, + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-to-regexp": { + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.2.tgz", + "integrity": "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA==", + "dev": true, + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/pkce-challenge": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/pkce-challenge/-/pkce-challenge-5.0.1.tgz", + "integrity": "sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=16.20.0" + } + }, "node_modules/playwright": { "version": "1.58.1", "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.58.1.tgz", @@ -86,6 +1000,311 @@ "node": ">=18" } }, + "node_modules/proxy-addr": { + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.8.tgz", + "integrity": "sha512-5nnx0yGyVUcY6t9RnWcARWtwT9F1D8O9rt08htPvnd49W1IgZtmLkhu9WfMzQj1cFxjHIO6connUNVW5k7AVyQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/qs": { + "version": "6.16.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.16.0.tgz", + "integrity": "sha512-h6fhOIaRrID2CbEY2fqs+7t+UXZo+MLAnU5gRIq85uFtdiUPCdsApMlHhXogKVM4HM2DVbIjGNTTYH2OcmP1vA==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "es-define-property": "^1.0.1", + "side-channel": "^1.1.1" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/range-parser": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.3.0.tgz", + "integrity": "sha512-hek2mFQpPuI4E1BBKrSto+BU3e3x4xuarsbiwr3+lf7p44juvFMV0XFWQAP3xUyqXA4RrXLIoaSUGbSt056ZMw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/raw-body": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz", + "integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==", + "dev": true, + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.7.0", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/router": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz", + "integrity": "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "depd": "^2.0.0", + "is-promise": "^4.0.0", + "parseurl": "^1.3.3", + "path-to-regexp": "^8.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "dev": true, + "license": "MIT" + }, + "node_modules/send": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz", + "integrity": "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "debug": "^4.4.3", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "fresh": "^2.0.0", + "http-errors": "^2.0.1", + "mime-types": "^3.0.2", + "ms": "^2.1.3", + "on-finished": "^2.4.1", + "range-parser": "^1.2.1", + "statuses": "^2.0.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/serve-static": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz", + "integrity": "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==", + "dev": true, + "license": "MIT", + "dependencies": { + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "parseurl": "^1.3.3", + "send": "^1.2.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "dev": true, + "license": "ISC" + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/side-channel": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.1.tgz", + "integrity": "sha512-6x6dK6zJdpTzF4sQeNYxwtvBzf6Eg4GtlesS94HOvTudUeyK2WXAaIfmDgsyslYrRBeFIlsi54AYsFGUuhmvrQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4", + "side-channel-list": "^1.0.1", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz", + "integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/type-is": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.1.0.tgz", + "integrity": "sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==", + "dev": true, + "license": "MIT", + "dependencies": { + "content-type": "^2.0.0", + "media-typer": "^1.1.0", + "mime-types": "^3.0.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/type-is/node_modules/content-type": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", + "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/typescript": { "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", @@ -106,6 +1325,69 @@ "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", "dev": true, "license": "MIT" + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/zod": { + "version": "4.6.5", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.6.5.tgz", + "integrity": "sha512-v5l/aFXZQeai4awLbOpSoHecE9UiMrnfx75tEXLjNonXVARxQ5mOeipTjROUchszUNCqnE+hqAMujRsRHsut2Q==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/zod-to-json-schema": { + "version": "3.25.2", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.25.2.tgz", + "integrity": "sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA==", + "dev": true, + "license": "ISC", + "peerDependencies": { + "zod": "^3.25.28 || ^4" + } } } } diff --git a/tests/e2e/ui/package.json b/tests/e2e/ui/package.json index ede759d97cb..78130412be9 100644 --- a/tests/e2e/ui/package.json +++ b/tests/e2e/ui/package.json @@ -9,6 +9,7 @@ "e2e:migration:root": "playwright test --config migration.serverRootPath.config.ts" }, "devDependencies": { + "@modelcontextprotocol/sdk": "1.30.0", "@playwright/test": "1.58.1", "@types/node": "20.19.37", "typescript": "5.9.3" diff --git a/tests/e2e/ui/tests/mcp/mcpTools.spec.ts b/tests/e2e/ui/tests/mcp/mcpTools.spec.ts index 225ca8b9449..2390a78755e 100644 --- a/tests/e2e/ui/tests/mcp/mcpTools.spec.ts +++ b/tests/e2e/ui/tests/mcp/mcpTools.spec.ts @@ -1,17 +1,20 @@ import { test, expect, Locator } from "@playwright/test"; import { ADMIN_STORAGE_PATH } from "../../constants"; -import { createMcpServer, deleteMcpServerByName, openMcpToolsTab } from "../../helpers/mcp"; +import { createMcpServer, deleteMcpServerByName, listUpstreamToolNames, openMcpToolsTab } from "../../helpers/mcp"; // Listing and calling MCP tools, which needs a server that really answers; the create-only spec // points at an unreachable URL on purpose. // -// This spec makes a read-only network call to DeepWiki's public MCP server, from the proxy rather -// than the browser. It needs no credentials, so there is no secret to leak from a public repo. +// This spec makes read-only network calls to DeepWiki's public MCP server: from the proxy, and from +// the test runner to learn which tools the upstream advertises today, so the tool list is never +// pinned here. It needs no credentials, so there is no secret to leak from a public repo. // // A DeepWiki outage turns this red for something that is not a litellm regression. That is left // visible rather than auto-skipped: skipping on connection trouble also skips when the proxy's own // MCP client breaks, which is the regression this exists to catch. E2E_SKIP_EXTERNAL_MCP=1 opts out. const MCP_SERVER_URL = "https://mcp.deepwiki.com/mcp"; +// Read from DeepWiki's tools/list on 2026-09-22. One name has to be pinned so the call-tool test can +// fill a known input (repoName); the listing test checks it is still advertised before the UI checks. const TOOL_NAME = "read_wiki_structure"; const TOOL_ARG_REPO = "BerriAI/litellm"; @@ -36,14 +39,17 @@ test.describe("MCP Tools", () => { }); test("MCP Tools tab lists the tools the upstream server advertises", async ({ page }) => { + const upstreamTools = await listUpstreamToolNames(MCP_SERVER_URL); + expect(upstreamTools).toContain(TOOL_NAME); + // Fetched through the proxy on mount, so allow for a cold upstream connection. const toolList = page.locator(".mcp-tools-scrollable"); await expect(toolList).toBeVisible({ timeout: 30_000 }); - // Non-empty would still pass if the proxy returned some other server's tools. - await expect(toolCard(toolList, TOOL_NAME)).toBeVisible(); - await expect(toolCard(toolList, "ask_question")).toBeVisible(); - await expect(toolCard(toolList, "read_wiki_contents")).toBeVisible(); + for (const name of upstreamTools) { + await expect(toolCard(toolList, name)).toBeVisible(); + } + await expect(toolList.locator("h4.font-mono")).toHaveCount(upstreamTools.length); // No other tool's name or description contains this string, so exactly one card survives. await page.getByPlaceholder("Search tools...").fill(TOOL_NAME); From ad263b01f4add19936d43786908005da55216ec7 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 20:36:53 -0700 Subject: [PATCH 02/15] test(integration): chain a proxy-issued previous_response_id in the cost suite (#42396) The gpt-5.6-responses_previous_response_id case sent a literal id the proxy never issued, which the Responses id security hook refuses with a 403 at production defaults. The case now primes a response through the proxy and chains the id it hands back, so the harness drops allow_unmanaged_response_ids and the security hook stays exercised Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../cost_calculation/cost_tracking_case.py | 26 +++++++++ .../cost_calculation/cost_tracking_cases.json | 2 +- .../cost_calculation/test_cost_tracking.py | 54 +++++++++++++------ tests/integration/proxy_config.yaml | 1 - 4 files changed, 64 insertions(+), 19 deletions(-) diff --git a/tests/integration/cost_calculation/cost_tracking_case.py b/tests/integration/cost_calculation/cost_tracking_case.py index ea8bf230d05..ac3fcd33d2e 100644 --- a/tests/integration/cost_calculation/cost_tracking_case.py +++ b/tests/integration/cost_calculation/cost_tracking_case.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json from collections.abc import Mapping from pathlib import Path from types import MappingProxyType @@ -8,6 +9,7 @@ from typing import Annotated, Final, Literal, TypeAlias from pydantic import BaseModel, ConfigDict, Field, JsonValue, field_validator, model_validator CASES_PATH: Final = Path(__file__).resolve().parent / "cost_tracking_cases.json" +PRIOR_RESPONSE_ID_MARKER: Final = "$PRIOR_RESPONSE_ID" class SearchContextCostPerQuery(BaseModel): @@ -312,6 +314,21 @@ class CostTrackingTestCase(BaseModel): usage: Final = self.response.body.get("usage") return isinstance(usage, dict) and isinstance(usage.get("cost"), (int, float)) + @property + def chains_prior_response(self) -> bool: + return self.request.get("previous_response_id") == PRIOR_RESPONSE_ID_MARKER + + @property + def can_chain_prior_response(self) -> bool: + return ( + self.chains_prior_response + and self.endpoint == "/v1/responses" + and isinstance(self.response, JsonResponse) + and isinstance(self.response.body.get("id"), str) + and not isinstance(self.expected, FailureExpected) + and not (isinstance(self.expected, ExactExpected) and self.expected.rollups) + ) + class BatchOutputLine(BaseModel): model_config = ConfigDict(frozen=True, extra="forbid") @@ -681,6 +698,11 @@ def data_errors() -> tuple[str, ...]: for marker in ('"id": "call_$REQUEST_ID"', '"id": "toolu_$REQUEST_ID"') ) ) + invalid_prior_response_chains: Final = sorted( + case.name + for case in CASES + if PRIOR_RESPONSE_ID_MARKER in json.dumps(case.request) and not case.can_chain_prior_response + ) return tuple( message for message in ( @@ -700,6 +722,10 @@ def data_errors() -> tuple[str, ...]: f"pinned tool IDs contain $REQUEST_ID: {invalid_pinned_tool_ids}" if invalid_pinned_tool_ids else None, + f"{PRIOR_RESPONSE_ID_MARKER} needs a non-rollup, non-failure /v1/responses JSON response with a string id" + f" as previous_response_id: {invalid_prior_response_chains}" + if invalid_prior_response_chains + else None, ) if message is not None ) diff --git a/tests/integration/cost_calculation/cost_tracking_cases.json b/tests/integration/cost_calculation/cost_tracking_cases.json index 17ebc793fae..09dfa66012f 100644 --- a/tests/integration/cost_calculation/cost_tracking_cases.json +++ b/tests/integration/cost_calculation/cost_tracking_cases.json @@ -27733,7 +27733,7 @@ "request": { "model": "$MODEL", "input": "continue this text", - "previous_response_id": "resp_scripted_prior" + "previous_response_id": "$PRIOR_RESPONSE_ID" }, "response": { "content_type": "application/json", diff --git a/tests/integration/cost_calculation/test_cost_tracking.py b/tests/integration/cost_calculation/test_cost_tracking.py index efab17acba4..82878634677 100644 --- a/tests/integration/cost_calculation/test_cost_tracking.py +++ b/tests/integration/cost_calculation/test_cost_tracking.py @@ -9,13 +9,14 @@ import time import uuid import wave import zlib +from collections.abc import Mapping from hashlib import sha256 from itertools import islice from typing import Final, cast import httpx import pytest -from integration._support.client import JSON_OBJECT, Gateway +from integration._support.client import JSON_OBJECT, Gateway, string_value from integration._support.upstream import delete_scenario, register_scenario from integration.cost_calculation.assertions import assert_exact, assert_recount from integration.cost_calculation.conftest import ( @@ -112,6 +113,19 @@ def _replace_model(value: JsonValue, model_name: str) -> JsonValue: return value +def _prime_prior_response( + gateway: Gateway, request_path: str, request_values: Mapping[str, JsonValue], key: str +) -> str: + primed: Final = gateway.request( + "POST", + request_path, + {field: value for field, value in request_values.items() if field != "previous_response_id"}, + key=key, + ) + assert primed.is_success, f"priming response failed: {primed.status_code}: {primed.text[:400]}" + return string_value(JSON_OBJECT.validate_json(primed.content)["id"]) + + @pytest.mark.parametrize("case", _CASES) def test_case_bills_expected_cost(gateway: Gateway, case: CostTrackingTestCase) -> None: marker: Final = sha256(case.name.encode()).hexdigest()[:12] @@ -175,21 +189,6 @@ def test_case_bills_expected_cost(gateway: Gateway, case: CostTrackingTestCase) if isinstance(expected, ExactExpected) and expected.rollups else None ) - request_body: Final = JSON_OBJECT.validate_python( - { - **base_request_values, - **( - {"model": fallback_deployment.model_name, "fallbacks": [model_name]} - if fallback_deployment is not None - else {} - ), - **( - {"user": end_user_id, "cache": {"no-cache": True}} - if end_user_id is not None - else {} - ), - } - ) request_headers: Final = ( { "x-pass-x-scripted-scenario": scenario_id, @@ -207,6 +206,27 @@ def test_case_bills_expected_cost(gateway: Gateway, case: CostTrackingTestCase) if passthrough_provider is not None else case.endpoint ) + prior_response_id: Final = ( + _prime_prior_response(gateway, request_path, base_request_values, key) + if case.chains_prior_response + else None + ) + request_body: Final = JSON_OBJECT.validate_python( + { + **base_request_values, + **( + {"model": fallback_deployment.model_name, "fallbacks": [model_name]} + if fallback_deployment is not None + else {} + ), + **( + {"user": end_user_id, "cache": {"no-cache": True}} + if end_user_id is not None + else {} + ), + **({"previous_response_id": prior_response_id} if prior_response_id is not None else {}), + } + ) if case.disconnect_after_frames is not None: with gateway.client.stream( "POST", @@ -250,7 +270,7 @@ def test_case_bills_expected_cost(gateway: Gateway, case: CostTrackingTestCase) assert response.is_success, f"{case.name}: proxy returned {response.status_code}: {response.text[:400]}" if case.response.content_type == "text/event-stream": _assert_stream_has_no_error(response.text) - rows: Final = poll_rows(key, len(responses)) + rows: Final = poll_rows(key, len(responses) + (prior_response_id is not None)) if isinstance(expected, RecountExpected): row: Final = rows[0] assert_recount(case.name, expected, row) diff --git a/tests/integration/proxy_config.yaml b/tests/integration/proxy_config.yaml index 34e48228dd7..a3b07f76d2f 100644 --- a/tests/integration/proxy_config.yaml +++ b/tests/integration/proxy_config.yaml @@ -5,7 +5,6 @@ general_settings: store_model_in_db: true disable_spend_logs: false proxy_batch_write_at: 1 - allow_unmanaged_response_ids: true litellm_settings: enable_redis_auth_cache: true cache: true From 3106d9c573c6f1898280581e499aade3db8ac13c Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 03:41:04 +0000 Subject: [PATCH 03/15] feat(rust-bridge): add cache and secret migration foundations (#42328) * docs(rust): plan Python interop foundation * fix(rust): preserve Python settings coercion at the native boundary * chore(rust): drop interop planning note Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(rust): resolve OCR provider secrets through an async SecretSource before transformation Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(rust): project the Python secret manager into the bridge and resolve OCR secrets through it Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(rust): drop premium_user from the secret manager snapshot Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(rust-bridge): read the private key management globals once in the settings snapshot Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(rust): bound the bridge secret manager state cache to the active snapshot Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(rust): inline coercion unit tests * fix(rust): preserve Python secret manager bindings * refactor(rust-bridge): let settings projectors own their contract specs Each settings group now declares its SettingSpec rows next to the projector that reads them, and the manifest test derives python_settings.json from those tables instead of a hand-copied duplicate. Field carries (group, name) instead of a dotted path, and coercion gains the dict-item reader plus the Redis Boolean, certificate-requirement, non-empty string, and numeric adapters that the cache configuration projection adopts next. Co-Authored-By: Claude Fable 5.1 * refactor(rust-bridge): capture the secret manager binding in one settings read The secret_manager accessor now carries the live client and settings objects, so the bridge classifies the binding from a single snapshot instead of re-reading litellm globals. The unreachable native arm and the service alias go away, the binding-to-state mapping moves next to the snapshot, and the Python callback precomputes its key_manager name. Co-Authored-By: Claude Fable 5.1 * refactor(rust-bridge): execute typed settings field declarations * refactor(rust-bridge): compare cache backends by identity behind one exact trait cache-response gains an object-safe ExactResponseCache so every exact-match backend sits behind one pointer; WriteBuffer flushes through it. The bridge's NativeResponseCache shrinks from nine variants and fifteen per-backend accessors to an exact service plus the three semantic backends, and facade mismatch detection compares BackendIdentity values instead of matching on each backend type. Request projections move next to NativeRequest. Co-Authored-By: Claude Fable 5.1 * refactor(rust-bridge): drive both Python-embedded semantic caches through one execution Redis-semantic and Valkey-semantic operations now share one SemanticExecution body: await the Python embedder, seed the task-local vector, run the native backend, repeat per batch entry. Valkey drops its with_embedder path in favor of the same seeded embedder, and each backend keeps its own embedding-failure policy. PythonEmbedder exposes one call shape. Redis-semantic thresholds are compared at the backend's f32 width, which un-breaks the redis-stack parity tests that a 0.8 facade threshold failed before this branch. Co-Authored-By: Claude Fable 5.1 * wip * feat(rust-bridge): complete response cache runtime surface * fix(rust-bridge): preserve secret manager callback exceptions * refactor(rust-bridge): unify route cache and secret rollout catalog --------- Co-authored-by: Yujong Lee Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Claude Fable 5.1 --- litellm-rust/Cargo.lock | 7 + litellm-rust/crates/auth-aws/src/aws.rs | 20 + litellm-rust/crates/auth-aws/src/constants.rs | 13 + litellm-rust/crates/auth-azure/src/lib.rs | 2 +- litellm-rust/crates/auth-azure/src/resolve.rs | 32 +- litellm-rust/crates/auth-gcp/src/lib.rs | 32 + litellm-rust/crates/auth-types/src/secret.rs | 7 + litellm-rust/crates/cache-response/README.md | 2 + .../crates/cache-response/src/buffer.rs | 8 +- .../crates/cache-response/src/exact.rs | 148 +++ litellm-rust/crates/cache-response/src/lib.rs | 2 + .../crates/core-utils/src/serde_compat.rs | 5 + litellm-rust/crates/core/Cargo.toml | 1 + litellm-rust/crates/core/src/ocr/handler.rs | 7 +- litellm-rust/crates/core/src/ocr/prepare.rs | 20 +- .../crates/core/src/ocr/provider_config.rs | 27 + litellm-rust/crates/core/tests/ocr.rs | 85 +- litellm-rust/crates/llms/Cargo.toml | 1 + .../ocr/analyze_transformation.rs | 4 + .../src/aws_textract/ocr/transformation.rs | 4 + .../ocr/cohere_parse_transformation.rs | 4 + .../document_intelligence/transformation.rs | 13 +- .../llms/src/azure_ai/ocr/transformation.rs | 12 + .../crates/llms/src/base_llm/inference/mod.rs | 1 + .../llms/src/base_llm/inference/secrets.rs | 19 + litellm-rust/crates/llms/src/base_llm/mod.rs | 1 + .../crates/llms/src/base_llm/ocr/error.rs | 2 + .../crates/llms/src/base_llm/ocr/handler.rs | 15 +- .../crates/llms/src/base_llm/ocr/settings.rs | 4 +- .../llms/src/base_llm/ocr/transformation.rs | 13 +- .../llms/src/cohere/ocr/transformation.rs | 4 + .../llms/src/mistral/ocr/transformation.rs | 8 + .../llms/src/reducto/ocr/transformation.rs | 8 + .../vertex_ai/ocr/deepseek_transformation.rs | 4 + .../llms/src/vertex_ai/ocr/transformation.rs | 4 + litellm-rust/crates/python-bridge/Cargo.toml | 6 + litellm-rust/crates/python-bridge/README.md | 5 + .../crates/python-bridge/python_settings.json | 154 --- .../crates/python-bridge/src/cache/binding.rs | 31 +- .../crates/python-bridge/src/cache/config.rs | 209 +--- .../python-bridge/src/cache/embedder.rs | 86 +- .../crates/python-bridge/src/cache/facade.rs | 47 +- .../crates/python-bridge/src/cache/handle.rs | 2 +- .../python-bridge/src/cache/identity.rs | 511 +++++++++ .../crates/python-bridge/src/cache/mod.rs | 2 +- .../crates/python-bridge/src/cache/native.rs | 992 +++++------------- .../crates/python-bridge/src/cache/request.rs | 184 +++- .../python-bridge/src/cache/semantic.rs | 212 ++-- .../python-bridge/src/cache/semantic_step.rs | 249 ----- .../crates/python-bridge/src/coercion.rs | 485 +++++++-- .../python-bridge/src/coercion/tests.rs | 372 ------- litellm-rust/crates/python-bridge/src/http.rs | 187 +++- litellm-rust/crates/python-bridge/src/lib.rs | 7 +- .../python-bridge/src/python_settings.rs | 318 ++---- .../python-bridge/src/routes/ocr/host.rs | 5 + .../python-bridge/src/routes/ocr/mod.rs | 94 +- .../python-bridge/src/secrets/callback.rs | 349 ++++++ .../python-bridge/src/secrets/config.rs | 338 ++++++ .../crates/python-bridge/src/secrets/mod.rs | 3 + .../python-bridge/src/secrets/resolved.rs | 252 +++++ .../crates/secrets-types/src/config.rs | 6 +- litellm-rust/crates/secrets/src/error.rs | 2 + litellm-rust/crates/secrets/src/handler.rs | 20 + litellm-rust/crates/secrets/src/lib.rs | 2 +- litellm-rust/crates/secrets/src/resolver.rs | 1 + litellm/chat_completions/dispatch.py | 6 +- .../bedrock/audio_transcription/__init__.py | 6 +- litellm/llms/custom_httpx/llm_http_handler.py | 4 +- litellm/messages/dispatch.py | 6 +- litellm/ocr/dispatch.py | 6 +- litellm/responses/dispatch.py | 6 +- litellm/rust_bridge/_native.pyi | 6 +- litellm/rust_bridge/catalog.py | 80 +- litellm/rust_bridge/configuration.py | 2 +- litellm/rust_bridge/dispatch.py | 6 +- litellm/rust_bridge/response_cache.py | 180 ++++ litellm/rust_bridge/runtime.py | 12 +- litellm/rust_bridge/settings.py | 59 ++ tests/test_litellm/responses/test_dispatch.py | 15 +- .../rust_bridge/ocr/test_secrets.py | 112 ++ .../test_litellm/rust_bridge/test_catalog.py | 147 ++- .../test_litellm/rust_bridge/test_dispatch.py | 48 +- .../test_litellm/rust_bridge/test_runtime.py | 26 +- .../test_litellm/rust_bridge/test_settings.py | 101 +- tests/test_litellm_rust/ocr/test_requests.py | 6 +- tests/test_litellm_rust/test_cache.py | 115 +- tests/unit/chat_completions/test_dispatch.py | 8 +- tests/unit/messages/test_dispatch.py | 4 +- tests/unit/ocr/test_dispatch.py | 10 +- 89 files changed, 4138 insertions(+), 2503 deletions(-) create mode 100644 litellm-rust/crates/cache-response/src/exact.rs create mode 100644 litellm-rust/crates/llms/src/base_llm/inference/mod.rs create mode 100644 litellm-rust/crates/llms/src/base_llm/inference/secrets.rs create mode 100644 litellm-rust/crates/python-bridge/README.md delete mode 100644 litellm-rust/crates/python-bridge/python_settings.json create mode 100644 litellm-rust/crates/python-bridge/src/cache/identity.rs delete mode 100644 litellm-rust/crates/python-bridge/src/cache/semantic_step.rs delete mode 100644 litellm-rust/crates/python-bridge/src/coercion/tests.rs create mode 100644 litellm-rust/crates/python-bridge/src/secrets/callback.rs create mode 100644 litellm-rust/crates/python-bridge/src/secrets/config.rs create mode 100644 litellm-rust/crates/python-bridge/src/secrets/mod.rs create mode 100644 litellm-rust/crates/python-bridge/src/secrets/resolved.rs create mode 100644 litellm/rust_bridge/response_cache.py create mode 100644 tests/test_litellm/rust_bridge/ocr/test_secrets.py diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 464edb3b104..03e0dabbc17 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -2894,6 +2894,7 @@ dependencies = [ "litellm-host", "litellm-http", "litellm-llms", + "litellm-secrets", "litellm-types", "mime_guess", "moka", @@ -3006,6 +3007,7 @@ dependencies = [ "litellm-framing", "litellm-host", "litellm-http", + "litellm-secrets", "litellm-types", "reqwest 0.12.28", "rstest", @@ -3024,6 +3026,7 @@ dependencies = [ name = "litellm-python-bridge" version = "0.1.0" dependencies = [ + "aws-sdk-secretsmanager", "bytes", "criterion", "futures-util", @@ -3047,6 +3050,9 @@ dependencies = [ "litellm-host-python", "litellm-http", "litellm-llms", + "litellm-secrets", + "litellm-secrets-aws", + "litellm-secrets-types", "litellm-token-counter", "litellm-types", "pyo3", @@ -3062,6 +3068,7 @@ dependencies = [ "tokio", "tokio-tungstenite", "url", + "wiremock", ] [[package]] diff --git a/litellm-rust/crates/auth-aws/src/aws.rs b/litellm-rust/crates/auth-aws/src/aws.rs index bbcb0f016c8..cb9195ffeb6 100644 --- a/litellm-rust/crates/auth-aws/src/aws.rs +++ b/litellm-rust/crates/auth-aws/src/aws.rs @@ -621,6 +621,26 @@ mod tests { None } + #[test] + fn secret_names_cover_environment_reads() { + let seen = std::sync::Arc::new(std::sync::Mutex::new( + std::collections::BTreeSet::::new(), + )); + let recorded = seen.clone(); + let env = |name: &str| { + recorded.lock().unwrap().insert(name.to_string()); + None + }; + resolve_aws_region(None, &Map::new(), &env); + aws_auth_config(&Map::new(), &env); + assert!( + seen.lock() + .unwrap() + .iter() + .all(|name| crate::constants::SECRET_NAMES.contains(&name.as_str())) + ); + } + #[test] fn a_region_comes_from_the_call_then_the_model_then_the_environment() { let params = Map::from_iter([("aws_region_name".to_string(), Value::from("eu-west-1"))]); diff --git a/litellm-rust/crates/auth-aws/src/constants.rs b/litellm-rust/crates/auth-aws/src/constants.rs index 9e7c6bfab43..26df4f2a350 100644 --- a/litellm-rust/crates/auth-aws/src/constants.rs +++ b/litellm-rust/crates/auth-aws/src/constants.rs @@ -14,6 +14,19 @@ pub const AWS_WEB_IDENTITY_TOKEN_FILE: &str = "AWS_WEB_IDENTITY_TOKEN_FILE"; pub const AWS_STS_ENDPOINT: &str = "AWS_STS_ENDPOINT"; pub const AWS_EXTERNAL_ID: &str = "AWS_EXTERNAL_ID"; pub const AWS_BEARER_TOKEN_BEDROCK: &str = "AWS_BEARER_TOKEN_BEDROCK"; +pub const SECRET_NAMES: &[&str] = &[ + AWS_ACCESS_KEY_ID, + AWS_SECRET_ACCESS_KEY, + AWS_SESSION_TOKEN, + AWS_REGION_NAME, + AWS_REGION, + AWS_SESSION_NAME, + AWS_PROFILE_NAME, + AWS_ROLE_NAME, + AWS_WEB_IDENTITY_TOKEN, + AWS_STS_ENDPOINT, + AWS_EXTERNAL_ID, +]; /// Headers SigV4 covers, beyond the `x-amz-` / `x-amzn-` prefixes. Mirrors /// Python's `_filter_headers_for_aws_signature` allowlist. diff --git a/litellm-rust/crates/auth-azure/src/lib.rs b/litellm-rust/crates/auth-azure/src/lib.rs index 5c7c654b69d..9ed505e4160 100644 --- a/litellm-rust/crates/auth-azure/src/lib.rs +++ b/litellm-rust/crates/auth-azure/src/lib.rs @@ -3,5 +3,5 @@ mod native; mod resolve; mod types; -pub use resolve::AzureAuthService; +pub use resolve::{AzureAuthService, SECRET_NAMES}; pub use types::{AzureAuthInputs, ConfigValue}; diff --git a/litellm-rust/crates/auth-azure/src/resolve.rs b/litellm-rust/crates/auth-azure/src/resolve.rs index 4d564b6e68a..9a7afe645db 100644 --- a/litellm-rust/crates/auth-azure/src/resolve.rs +++ b/litellm-rust/crates/auth-azure/src/resolve.rs @@ -19,6 +19,17 @@ const AZURE_AUTHORITY_HOST_ENV: &str = "AZURE_AUTHORITY_HOST"; const AZURE_CREDENTIAL_ENV: &str = "AZURE_CREDENTIAL"; const AZURE_FEDERATED_TOKEN_FILE_ENV: &str = "AZURE_FEDERATED_TOKEN_FILE"; +pub const SECRET_NAMES: &[&str] = &[ + AZURE_AD_TOKEN_ENV, + AZURE_TENANT_ID_ENV, + AZURE_CLIENT_ID_ENV, + AZURE_CLIENT_SECRET_ENV, + AZURE_SCOPE_ENV, + AZURE_AUTHORITY_HOST_ENV, + AZURE_CREDENTIAL_ENV, + AZURE_FEDERATED_TOKEN_FILE_ENV, +]; + #[derive(Clone, Debug)] pub(crate) enum AzureCredentialPlan { Supplied(Sourced), @@ -440,13 +451,14 @@ fn non_empty_reference(value: &str, kind: &str) -> Result { #[cfg(test)] mod tests { + use std::collections::BTreeSet; use std::future::Future; use std::sync::{Arc, Mutex}; use serde_json::json; use super::{ - AzureAuthService, AzureCredentialPlan, AzureTokenAcquirer, oidc_reference, + AzureAuthService, AzureCredentialPlan, AzureTokenAcquirer, SECRET_NAMES, oidc_reference, resolve_reference, select_auth_plan, }; use crate::native::ValidatedAzureRequest; @@ -517,6 +529,24 @@ mod tests { assert!(matches!(plan, AzureCredentialPlan::Native(_))); } + #[test] + fn secret_names_cover_environment_reads() { + let seen = std::sync::Arc::new(std::sync::Mutex::new(BTreeSet::::new())); + let recorded = seen.clone(); + let inputs = AzureAuthInputs::default(); + select_auth_plan(&inputs, &|name| { + recorded.lock().unwrap().insert(name.to_string()); + None + }) + .unwrap(); + assert!( + seen.lock() + .unwrap() + .iter() + .all(|name| SECRET_NAMES.contains(&name.as_str())) + ); + } + #[test] fn supplied_token_does_not_require_refresh() { let params = json!({"azure_ad_token": "token"}); diff --git a/litellm-rust/crates/auth-gcp/src/lib.rs b/litellm-rust/crates/auth-gcp/src/lib.rs index 534d85acdb0..682f1af5fe1 100644 --- a/litellm-rust/crates/auth-gcp/src/lib.rs +++ b/litellm-rust/crates/auth-gcp/src/lib.rs @@ -23,6 +23,16 @@ const VERTEXAI_PROJECT_ENV: &str = "VERTEXAI_PROJECT"; const VERTEXAI_LOCATION_ENV: &str = "VERTEXAI_LOCATION"; const VERTEX_LOCATION_ENV: &str = "VERTEX_LOCATION"; +pub const SECRET_NAMES: &[&str] = &[ + VERTEX_AI_API_KEY_ENV, + VERTEXAI_API_KEY_ENV, + VERTEXAI_CREDENTIALS_ENV, + GOOGLE_APPLICATION_CREDENTIALS_ENV, + VERTEXAI_PROJECT_ENV, + VERTEXAI_LOCATION_ENV, + VERTEX_LOCATION_ENV, +]; + #[derive(Clone, Debug, Default)] pub struct VertexConfig { credentials: Option>, @@ -406,6 +416,7 @@ fn auth_acquisition_error(error: gcp_auth::Error) -> Error { #[cfg(test)] mod tests { + use std::collections::BTreeSet; use std::sync::atomic::{AtomicUsize, Ordering}; use serde_json::json; @@ -476,6 +487,27 @@ mod tests { ); } + #[tokio::test] + async fn secret_names_cover_environment_reads() { + let seen = Arc::new(std::sync::Mutex::new(BTreeSet::::new())); + let recorded = seen.clone(); + let env = |name: &str| { + recorded.lock().unwrap().insert(name.to_string()); + None + }; + let auth = auth(Arc::new(AtomicUsize::new(0)), Arc::new(AtomicUsize::new(0))); + auth.validate_environment(Vec::new(), None, &VertexConfig::default(), &env) + .await + .unwrap(); + get_vertex_ai_location(&VertexConfig::default(), &env); + assert!( + seen.lock() + .unwrap() + .iter() + .all(|name| SECRET_NAMES.contains(&name.as_str())) + ); + } + #[test] fn empty_primary_values_fall_back_to_python_aliases() { let config = config(json!({ diff --git a/litellm-rust/crates/auth-types/src/secret.rs b/litellm-rust/crates/auth-types/src/secret.rs index a07fe3eaad9..7e6789deef7 100644 --- a/litellm-rust/crates/auth-types/src/secret.rs +++ b/litellm-rust/crates/auth-types/src/secret.rs @@ -1,4 +1,5 @@ use serde::Deserialize; +use std::hash::{Hash, Hasher}; use veil::Redact; #[derive(Redact, Clone, Deserialize)] @@ -23,6 +24,12 @@ impl PartialEq for SecretValue { impl Eq for SecretValue {} +impl Hash for SecretValue { + fn hash(&self, state: &mut H) { + self.0.hash(state); + } +} + #[cfg(test)] mod tests { use super::SecretValue; diff --git a/litellm-rust/crates/cache-response/README.md b/litellm-rust/crates/cache-response/README.md index 46e561ddad1..d048afb69f8 100644 --- a/litellm-rust/crates/cache-response/README.md +++ b/litellm-rust/crates/cache-response/README.md @@ -36,6 +36,8 @@ Callers supply Unix time for response freshness. Backend TTL uses its own clock. The extension keeps a private test harness for memory and Redis single and batch response lookup and storage. Batch lookup returns ordered values plus missing indices for embedding partial-hit wiring. No bridge-only cache type is part of the public API +The bridge also exposes a production-shaped response cache runtime selected through the Rust catalog. Its shipped rule set is empty, so current SDK, Router, and proxy calls stay on Python and do not construct native cache resources. Tests can inject a rule and build the native memory runtime from an ordinary Python `Cache` configuration without changing the legacy cache classes + Object responses are written as they are, and every other response shape is written as a serialized string, which is the pair of shapes Python reads. A string on the wire is therefore always a serialized response, so string-valued responses round trip. Typed backends such as memory never pass through the codec The resolver reads the namespace's `cache` attribute each time it resolves. A captured binding retains the selected service for its operation, including background writes. `None` disables caching. Custom Python cache objects keep their original methods, arguments, returned awaitables, exceptions, and caller-task execution diff --git a/litellm-rust/crates/cache-response/src/buffer.rs b/litellm-rust/crates/cache-response/src/buffer.rs index 606c21410c7..68f2348e28b 100644 --- a/litellm-rust/crates/cache-response/src/buffer.rs +++ b/litellm-rust/crates/cache-response/src/buffer.rs @@ -1,9 +1,9 @@ use std::{sync::Mutex, time::Duration}; -use litellm_cache::{BaseCache, Error, ExactCacheContext}; +use litellm_cache::Error; use serde_json::Value; -use crate::{CacheEntry, ResponseCache, ResponseCacheRequest}; +use crate::{ExactResponseCache, ResponseCacheRequest}; pub struct WriteBuffer { flush_size: usize, @@ -18,9 +18,9 @@ impl WriteBuffer { } } - pub async fn async_store>( + pub async fn async_store( &self, - cache: &ResponseCache, + cache: &dyn ExactResponseCache, request: &ResponseCacheRequest, response: Value, now: Duration, diff --git a/litellm-rust/crates/cache-response/src/exact.rs b/litellm-rust/crates/cache-response/src/exact.rs new file mode 100644 index 00000000000..f5e86b2598c --- /dev/null +++ b/litellm-rust/crates/cache-response/src/exact.rs @@ -0,0 +1,148 @@ +use std::{future::Future, pin::Pin, time::Duration}; + +use litellm_cache::{ + BaseCache, BatchCache, CacheConnectionResult, Error, ExactCacheContext, FlushCache, +}; +use serde_json::Value; + +use crate::{CacheEntry, PartialHits, ResponseCache, ResponseCacheRequest}; + +type BoxFuture<'a, T> = Pin + Send + 'a>>; + +/// Object-safe view of a `ResponseCache` over an exact-match backend, so hosts can hold every +/// exact backend behind one pointer without erasing which backend it is elsewhere. +pub trait ExactResponseCache: Send + Sync { + fn default_ttl(&self) -> Option; + + fn lookup(&self, request: &ResponseCacheRequest, now: Duration) + -> Result, Error>; + + fn store( + &self, + request: &ResponseCacheRequest, + response: Value, + now: Duration, + ) -> Result<(), Error>; + + fn lookup_batch( + &self, + requests: &[ResponseCacheRequest], + now: Duration, + ) -> Result; + + fn async_lookup<'a>( + &'a self, + request: &'a ResponseCacheRequest, + now: Duration, + ) -> BoxFuture<'a, Result, Error>>; + + fn async_store<'a>( + &'a self, + request: &'a ResponseCacheRequest, + response: Value, + now: Duration, + ) -> BoxFuture<'a, Result<(), Error>>; + + fn async_lookup_batch<'a>( + &'a self, + requests: &'a [ResponseCacheRequest], + now: Duration, + ) -> BoxFuture<'a, Result>; + + fn async_store_batch<'a>( + &'a self, + entries: Vec<(ResponseCacheRequest, Value)>, + now: Duration, + ) -> BoxFuture<'a, Result<(), Error>>; + + fn async_store_entries<'a>( + &'a self, + entries: Vec<(ResponseCacheRequest, Value, Duration)>, + ) -> BoxFuture<'a, Result<(), Error>>; + + fn async_flush<'a>(&'a self) -> BoxFuture<'a, Result<(), Error>>; + + fn test_connection<'a>(&'a self) -> BoxFuture<'a, Result>; +} + +impl ExactResponseCache for ResponseCache +where + B: BaseCache + BatchCache + FlushCache, +{ + fn default_ttl(&self) -> Option { + ResponseCache::default_ttl(self) + } + + fn lookup( + &self, + request: &ResponseCacheRequest, + now: Duration, + ) -> Result, Error> { + ResponseCache::lookup(self, request, now) + } + + fn store( + &self, + request: &ResponseCacheRequest, + response: Value, + now: Duration, + ) -> Result<(), Error> { + ResponseCache::store(self, request, response, now) + } + + fn lookup_batch( + &self, + requests: &[ResponseCacheRequest], + now: Duration, + ) -> Result { + ResponseCache::lookup_batch(self, requests, now) + } + + fn async_lookup<'a>( + &'a self, + request: &'a ResponseCacheRequest, + now: Duration, + ) -> BoxFuture<'a, Result, Error>> { + Box::pin(ResponseCache::async_lookup(self, request, now)) + } + + fn async_store<'a>( + &'a self, + request: &'a ResponseCacheRequest, + response: Value, + now: Duration, + ) -> BoxFuture<'a, Result<(), Error>> { + Box::pin(ResponseCache::async_store(self, request, response, now)) + } + + fn async_lookup_batch<'a>( + &'a self, + requests: &'a [ResponseCacheRequest], + now: Duration, + ) -> BoxFuture<'a, Result> { + Box::pin(ResponseCache::async_lookup_batch(self, requests, now)) + } + + fn async_store_batch<'a>( + &'a self, + entries: Vec<(ResponseCacheRequest, Value)>, + now: Duration, + ) -> BoxFuture<'a, Result<(), Error>> { + Box::pin(ResponseCache::async_store_batch(self, entries, now)) + } + + fn async_store_entries<'a>( + &'a self, + entries: Vec<(ResponseCacheRequest, Value, Duration)>, + ) -> BoxFuture<'a, Result<(), Error>> { + Box::pin(ResponseCache::async_store_entries(self, entries)) + } + + fn async_flush<'a>(&'a self) -> BoxFuture<'a, Result<(), Error>> { + Box::pin(ResponseCache::async_flush(self)) + } + + fn test_connection<'a>(&'a self) -> BoxFuture<'a, Result> { + Box::pin(ResponseCache::test_connection(self)) + } +} diff --git a/litellm-rust/crates/cache-response/src/lib.rs b/litellm-rust/crates/cache-response/src/lib.rs index 91b36ebe24b..ab9867ac8db 100644 --- a/litellm-rust/crates/cache-response/src/lib.rs +++ b/litellm-rust/crates/cache-response/src/lib.rs @@ -2,6 +2,7 @@ mod buffer; mod caching; mod codec; mod embedding; +mod exact; mod response; pub use buffer::WriteBuffer; @@ -11,4 +12,5 @@ pub use caching::{ }; pub use codec::ResponseCacheCodec; pub use embedding::PartialHits; +pub use exact::ExactResponseCache; pub use response::{ResponseCache, ResponseCacheRequest}; diff --git a/litellm-rust/crates/core-utils/src/serde_compat.rs b/litellm-rust/crates/core-utils/src/serde_compat.rs index c767c709f50..fddab1d80e3 100644 --- a/litellm-rust/crates/core-utils/src/serde_compat.rs +++ b/litellm-rust/crates/core-utils/src/serde_compat.rs @@ -17,6 +17,11 @@ pub fn parse_str_bool(value: &str) -> Option { token.eq_ignore_ascii_case("false").then_some(false) } +/// `redis-py` string Booleans: only `1`, `true`, and `yes` (case-insensitive) are true. +pub fn parse_redis_bool(value: &str) -> bool { + value == "1" || value.eq_ignore_ascii_case("true") || value.eq_ignore_ascii_case("yes") +} + impl<'de> DeserializeAs<'de, i64> for LaxI64 { fn deserialize_as>(deserializer: D) -> Result { deserializer.deserialize_any(Self) diff --git a/litellm-rust/crates/core/Cargo.toml b/litellm-rust/crates/core/Cargo.toml index 69ae8004d46..3bfc5bae925 100644 --- a/litellm-rust/crates/core/Cargo.toml +++ b/litellm-rust/crates/core/Cargo.toml @@ -36,6 +36,7 @@ url.workspace = true veil.workspace = true [dev-dependencies] +litellm-secrets.workspace = true litellm-auth-gcp.workspace = true litellm-llms = { workspace = true, features = ["test-support"] } rstest.workspace = true diff --git a/litellm-rust/crates/core/src/ocr/handler.rs b/litellm-rust/crates/core/src/ocr/handler.rs index 19037e49033..c1265e1e91c 100644 --- a/litellm-rust/crates/core/src/ocr/handler.rs +++ b/litellm-rust/crates/core/src/ocr/handler.rs @@ -22,7 +22,12 @@ pub(crate) async fn perform_ocr_request( ) -> Result { request.response_format()?; let config = request.config; - let request = prepare_request(request, caller_document, client); + let secrets = client + .secret_source() + .resolve(&config.secret_names()) + .await + .map_err(|error| Error::Secret(std::sync::Arc::new(error)))?; + let request = prepare_request(request, caller_document, client, secrets); let hooks = OcrCallHooks::new(host.clone(), &request, config); config.ocr(client, &request, &hooks).await } diff --git a/litellm-rust/crates/core/src/ocr/prepare.rs b/litellm-rust/crates/core/src/ocr/prepare.rs index 715aedc69df..54960256faa 100644 --- a/litellm-rust/crates/core/src/ocr/prepare.rs +++ b/litellm-rust/crates/core/src/ocr/prepare.rs @@ -1,7 +1,10 @@ use litellm_auth::{InputSource, SecretValue, Sourced}; -use litellm_llms::base_llm::ocr::{ - handler::OcrClient, - transformation::{OcrConnection, OcrCredentialInputs, PreparedOcrRequest}, +use litellm_llms::base_llm::{ + inference::secrets::Secrets, + ocr::{ + handler::OcrClient, + transformation::{OcrConnection, OcrCredentialInputs, PreparedOcrRequest}, + }, }; use super::provider_config::OcrProvider; @@ -11,6 +14,7 @@ pub(crate) fn prepare_request( request: ResolvedOcrRequest, caller_document: bool, client: &OcrClient, + secrets: Secrets, ) -> PreparedOcrRequest { let credentials = request.credentials.clone(); let (preferred_api_key_env, api_base_env) = match request.config.provider() { @@ -24,7 +28,7 @@ pub(crate) fn prepare_request( | OcrProvider::Reducto | OcrProvider::VertexAi => (None, None), }; - let secret = |name: &str| client.secrets().truthy(name); + let secret = |name: &str| secrets.truthy(name); let dynamic_api_key = credentials.dynamic_api_key.or_else(|| { credentials.api_key.clone().or_else(|| { preferred_api_key_env @@ -60,12 +64,7 @@ pub(crate) fn prepare_request( PreparedOcrRequest { model, document, - connection: OcrConnection::new( - resolved, - transport, - client.settings().clone(), - client.secrets().clone(), - ), + connection: OcrConnection::new(resolved, transport, client.settings().clone(), secrets), caller_document, optional_params, input_sources, @@ -79,6 +78,7 @@ pub(crate) fn prepare_request_for_test(request: ResolvedOcrRequest) -> PreparedO request, true, &OcrClient::for_test(reqwest::Client::new(), reqwest::Client::new()), + std::sync::Arc::new(litellm_core_utils::settings::ProcessEnvironment), ) } diff --git a/litellm-rust/crates/core/src/ocr/provider_config.rs b/litellm-rust/crates/core/src/ocr/provider_config.rs index d38d87b92cc..0b09e9be354 100644 --- a/litellm-rust/crates/core/src/ocr/provider_config.rs +++ b/litellm-rust/crates/core/src/ocr/provider_config.rs @@ -114,6 +114,10 @@ impl OcrConfigKind { with_config!(self, config => config.get_api_key_env_var()) } + pub(crate) fn secret_names(self) -> Vec<&'static str> { + with_config!(self, config => config.secret_names()) + } + pub(crate) fn get_health_check_document(self) -> OcrDocument { with_config!(self, config => config.get_health_check_document()) } @@ -213,6 +217,8 @@ fn is_document_intelligence_model(model: &str) -> bool { #[cfg(test)] mod tests { + use std::collections::HashSet; + use litellm_auth::{InputSource, Sourced}; use litellm_llms::{ base_llm::ocr::document::InlineDocument, cohere::ocr::transformation::validate_document, @@ -221,6 +227,27 @@ mod tests { use super::*; + #[rstest] + #[case(OcrConfigKind::AwsTextract)] + #[case(OcrConfigKind::AwsTextractAnalyze)] + #[case(OcrConfigKind::Cohere)] + #[case(OcrConfigKind::Mistral)] + #[case(OcrConfigKind::AzureAi)] + #[case(OcrConfigKind::AzureCohere)] + #[case(OcrConfigKind::AzureDocumentIntelligence)] + #[case(OcrConfigKind::ReductoLegacy)] + #[case(OcrConfigKind::ReductoV3)] + #[case(OcrConfigKind::VertexAi)] + #[case(OcrConfigKind::VertexDeepSeek)] + fn secret_names_include_api_keys_without_duplicates(#[case] config: OcrConfigKind) { + let names = config.secret_names(); + let unique = names.iter().collect::>(); + assert_eq!(names.len(), unique.len()); + if let Some(api_key) = config.get_api_key_env_var() { + assert!(names.contains(&api_key)); + } + } + #[rstest] #[case("cohere")] #[case("mistral")] diff --git a/litellm-rust/crates/core/tests/ocr.rs b/litellm-rust/crates/core/tests/ocr.rs index 3aedc7b9023..d376f0df784 100644 --- a/litellm-rust/crates/core/tests/ocr.rs +++ b/litellm-rust/crates/core/tests/ocr.rs @@ -1,5 +1,6 @@ use std::sync::{Arc, Mutex}; +use futures_util::future::BoxFuture; use litellm_auth_gcp::VertexAuth; use litellm_host::{ event::{CallEvent, MachineEvent, WireRequest}, @@ -10,11 +11,14 @@ use litellm_http::{ HttpClientPool, HttpSettings, Resolution, media::{PublicDnsResolver, UrlPolicy}, }; +use litellm_llms::base_llm::inference::secrets::{SecretSource, Secrets}; use litellm_llms::base_llm::ocr::{ error::Error as OcrError, handler::OcrClient, settings::OcrSettings, - transformation::{LiteLLMOcrResponse, OCR_RESPONSE_MAX_BYTES, OcrTransportConfig}, + transformation::{ + BaseOcrConfig, LiteLLMOcrResponse, OCR_RESPONSE_MAX_BYTES, OcrTransportConfig, + }, }; use rstest::rstest; use serde_json::{Value, json}; @@ -27,6 +31,32 @@ use super::{ }; use crate::ocr::route::{LocalOcrHost, OcrOp, OcrOpResult, ocr_machine}; +struct RecordingSecretSource { + names: Arc>>, + values: &'static [(&'static str, &'static str)], + api_base: String, +} + +impl SecretSource for RecordingSecretSource { + fn resolve<'a>( + &'a self, + names: &'a [&'static str], + ) -> BoxFuture<'a, Result> { + *self.names.lock().unwrap() = names.to_vec(); + let values = self.values; + let api_base = self.api_base.clone(); + Box::pin(async move { + Ok(Arc::new(move |name: &str| match name { + "MISTRAL_AZURE_API_BASE" => Some(api_base.clone()), + _ => values + .iter() + .find(|(key, _)| *key == name) + .map(|(_, value)| value.to_string()), + }) as Secrets) + }) + } +} + #[rstest] #[case::mistral("mistral/model", json!({}))] #[case::vertex("vertex_ai/mistral-ocr-latest", json!({"vertex_project":"test-project", "vertex_location":"us-central1"}))] @@ -184,14 +214,11 @@ async fn mistral_env_fallbacks_follow_python_through_the_injected_secret_source( #[case] expected_key: &str, ) { let (base, seen, server) = mock_server(vec![MockResponse::json(json!({"pages":[]}))]).await; - let secret_base = base.clone(); - let client = ocr_client().with_secrets(Arc::new(move |name: &str| match name { - "MISTRAL_AZURE_API_BASE" => Some(secret_base.clone()), - "MISTRAL_API_BASE" => Some("http://127.0.0.1:9/never-read".into()), - _ => secrets - .iter() - .find(|(key, _)| *key == name) - .map(|(_, value)| value.to_string()), + let names = Arc::new(Mutex::new(Vec::new())); + let client = ocr_client().with_secrets(Arc::new(RecordingSecretSource { + names: names.clone(), + values: secrets, + api_base: base.clone(), })); let request = decode_request(OcrWireRequest { model: "mistral/model".into(), @@ -208,9 +235,47 @@ async fn mistral_env_fallbacks_follow_python_through_the_injected_secret_source( crate::ocr::client::perform(&client, request).await.unwrap(); server.await.unwrap(); + assert_eq!( + *names.lock().unwrap(), + litellm_llms::mistral::ocr::transformation::MistralOcrConfig.secret_names() + ); assert!(seen.lock().unwrap()[0].contains(&format!("authorization: Bearer {expected_key}"))); } +#[tokio::test] +async fn mistral_ocr_resolves_provider_secrets_before_transformation() { + let (base, seen, server) = mock_server(vec![MockResponse::json(json!({"pages":[]}))]).await; + let names = Arc::new(Mutex::new(Vec::new())); + let client = ocr_client().with_secrets(Arc::new(RecordingSecretSource { + names: names.clone(), + values: &[("MISTRAL_API_KEY", "source-key")], + api_base: base.clone(), + })); + let request = decode_request(OcrWireRequest { + model: "mistral/mistral-ocr-latest".into(), + document: json!({ + "type":"document_url", + "document_url":"data:application/pdf;base64,YWJj" + }), + api_key: None, + api_base: None, + custom_llm_provider: None, + extra_headers: None, + optional_params: Default::default(), + input_sources: Default::default(), + timeout_seconds: Some(2.0), + }) + .unwrap(); + + crate::ocr::client::perform(&client, request).await.unwrap(); + server.await.unwrap(); + assert_eq!( + *names.lock().unwrap(), + litellm_llms::mistral::ocr::transformation::MistralOcrConfig.secret_names() + ); + assert!(seen.lock().unwrap()[0].contains("authorization: Bearer source-key")); +} + #[tokio::test] async fn ocr_client_uses_the_injected_http_pool_configuration() { let (base, seen, server) = mock_server(vec![MockResponse::json(json!({"pages":[]}))]).await; @@ -224,7 +289,7 @@ async fn ocr_client_uses_the_injected_http_pool_configuration() { UrlPolicy::default(), VertexAuth::default(), OcrSettings::default(), - Arc::new(litellm_core_utils::settings::ProcessEnvironment), + Arc::new(litellm_llms::base_llm::inference::secrets::EnvironmentSecrets), ) .unwrap(); crate::ocr::client::perform(&client, wire_request("mistral/model", &base, json!({}))) diff --git a/litellm-rust/crates/llms/Cargo.toml b/litellm-rust/crates/llms/Cargo.toml index f04b78feee1..ed15d9f7cdb 100644 --- a/litellm-rust/crates/llms/Cargo.toml +++ b/litellm-rust/crates/llms/Cargo.toml @@ -18,6 +18,7 @@ litellm-auth-gcp.workspace = true litellm-host.workspace = true litellm-framing.workspace = true litellm-http.workspace = true +litellm-secrets.workspace = true base64.workspace = true bytes.workspace = true data-url = "0.3.2" diff --git a/litellm-rust/crates/llms/src/aws_textract/ocr/analyze_transformation.rs b/litellm-rust/crates/llms/src/aws_textract/ocr/analyze_transformation.rs index d476861e6e1..2ce1b0da51b 100644 --- a/litellm-rust/crates/llms/src/aws_textract/ocr/analyze_transformation.rs +++ b/litellm-rust/crates/llms/src/aws_textract/ocr/analyze_transformation.rs @@ -40,6 +40,10 @@ impl BaseOcrConfig for TextractAnalyzeDocumentConfig { type ProviderRequest = AnalyzeDocumentRequest; type Environment = TextractEnvironment; + fn secret_names(&self) -> Vec<&'static str> { + litellm_auth_aws::constants::SECRET_NAMES.to_vec() + } + fn get_supported_ocr_params(&self, _model: &str) -> &'static [&'static str] { &["feature_types"] } diff --git a/litellm-rust/crates/llms/src/aws_textract/ocr/transformation.rs b/litellm-rust/crates/llms/src/aws_textract/ocr/transformation.rs index ad630a1ca4c..6eb195defaa 100644 --- a/litellm-rust/crates/llms/src/aws_textract/ocr/transformation.rs +++ b/litellm-rust/crates/llms/src/aws_textract/ocr/transformation.rs @@ -29,6 +29,10 @@ impl BaseOcrConfig for TextractDetectTextConfig { type ProviderRequest = DetectDocumentTextRequest; type Environment = TextractEnvironment; + fn secret_names(&self) -> Vec<&'static str> { + litellm_auth_aws::constants::SECRET_NAMES.to_vec() + } + fn get_health_check_document(&self) -> OcrDocument { health_check_document() } diff --git a/litellm-rust/crates/llms/src/azure_ai/ocr/cohere_parse_transformation.rs b/litellm-rust/crates/llms/src/azure_ai/ocr/cohere_parse_transformation.rs index 045d8744bc9..09639481cdf 100644 --- a/litellm-rust/crates/llms/src/azure_ai/ocr/cohere_parse_transformation.rs +++ b/litellm-rust/crates/llms/src/azure_ai/ocr/cohere_parse_transformation.rs @@ -28,6 +28,10 @@ impl BaseOcrConfig for AzureAICohereParseConfig { super::transformation::AzureAiOcrConfig.get_api_key_env_var() } + fn secret_names(&self) -> Vec<&'static str> { + super::transformation::AzureAiOcrConfig.secret_names() + } + fn get_health_check_document(&self) -> OcrDocument { CohereParseConfig.get_health_check_document() } diff --git a/litellm-rust/crates/llms/src/azure_ai/ocr/document_intelligence/transformation.rs b/litellm-rust/crates/llms/src/azure_ai/ocr/document_intelligence/transformation.rs index 9b27fdbb568..bfe0d76aab1 100644 --- a/litellm-rust/crates/llms/src/azure_ai/ocr/document_intelligence/transformation.rs +++ b/litellm-rust/crates/llms/src/azure_ai/ocr/document_intelligence/transformation.rs @@ -2,7 +2,7 @@ use std::{collections::BTreeSet, time::Duration}; use base64::{Engine, engine::general_purpose::STANDARD}; use litellm_auth::{InputSource, Sourced}; -use litellm_auth_azure::AzureAuthInputs; +use litellm_auth_azure::{AzureAuthInputs, SECRET_NAMES as AZURE_AUTH_SECRET_NAMES}; use litellm_core_utils::{ call_arguments::CallArguments, serde_compat::{FiniteF64, LaxI64}, @@ -141,6 +141,17 @@ impl BaseOcrConfig for AzureDocumentIntelligenceOcrConfig { Some(AZURE_DI_API_KEY_ENV) } + fn secret_names(&self) -> Vec<&'static str> { + [ + [AZURE_DI_API_KEY_ENV, AZURE_DI_ENDPOINT_ENV].as_slice(), + AZURE_AUTH_SECRET_NAMES, + ] + .into_iter() + .flatten() + .copied() + .collect() + } + fn resolve_connection_params(&self, inputs: OcrCredentialInputs) -> ResolvedOcrCredentials { ResolvedOcrCredentials { api_key: inputs.api_key.and_then(|key| { diff --git a/litellm-rust/crates/llms/src/azure_ai/ocr/transformation.rs b/litellm-rust/crates/llms/src/azure_ai/ocr/transformation.rs index 6df83e57eab..1fad860f757 100644 --- a/litellm-rust/crates/llms/src/azure_ai/ocr/transformation.rs +++ b/litellm-rust/crates/llms/src/azure_ai/ocr/transformation.rs @@ -1,5 +1,6 @@ use litellm_auth::{InputSource, Sourced}; use litellm_auth_azure::AzureAuthInputs; +use litellm_auth_azure::SECRET_NAMES as AZURE_AUTH_SECRET_NAMES; use litellm_core_utils::{call_arguments::CallArguments, params::OpaqueParams, url_utils::ApiUrl}; use serde_json::Value; @@ -37,6 +38,17 @@ impl BaseOcrConfig for AzureAiOcrConfig { Some(AZURE_AI_API_KEY_ENV) } + fn secret_names(&self) -> Vec<&'static str> { + [ + [AZURE_AI_API_KEY_ENV, AZURE_AI_API_BASE_ENV].as_slice(), + AZURE_AUTH_SECRET_NAMES, + ] + .into_iter() + .flatten() + .copied() + .collect() + } + fn map_ocr_params( &self, non_default_params: &CallArguments, diff --git a/litellm-rust/crates/llms/src/base_llm/inference/mod.rs b/litellm-rust/crates/llms/src/base_llm/inference/mod.rs new file mode 100644 index 00000000000..10c0454f947 --- /dev/null +++ b/litellm-rust/crates/llms/src/base_llm/inference/mod.rs @@ -0,0 +1 @@ +pub mod secrets; diff --git a/litellm-rust/crates/llms/src/base_llm/inference/secrets.rs b/litellm-rust/crates/llms/src/base_llm/inference/secrets.rs new file mode 100644 index 00000000000..eb13fe95116 --- /dev/null +++ b/litellm-rust/crates/llms/src/base_llm/inference/secrets.rs @@ -0,0 +1,19 @@ +use std::sync::Arc; + +use futures_util::future::BoxFuture; +use litellm_core_utils::settings::{Lookup, ProcessEnvironment}; +use litellm_secrets::Error; + +pub type Secrets = Arc; + +pub trait SecretSource: Send + Sync { + fn resolve<'a>(&'a self, names: &'a [&'static str]) -> BoxFuture<'a, Result>; +} + +pub struct EnvironmentSecrets; + +impl SecretSource for EnvironmentSecrets { + fn resolve<'a>(&'a self, _names: &'a [&'static str]) -> BoxFuture<'a, Result> { + Box::pin(async { Ok(Arc::new(ProcessEnvironment) as Secrets) }) + } +} diff --git a/litellm-rust/crates/llms/src/base_llm/mod.rs b/litellm-rust/crates/llms/src/base_llm/mod.rs index 8ed37da4573..9cced64b687 100644 --- a/litellm-rust/crates/llms/src/base_llm/mod.rs +++ b/litellm-rust/crates/llms/src/base_llm/mod.rs @@ -2,5 +2,6 @@ pub mod anthropic_messages; pub mod audio_transcription; pub mod base_model_iterator; pub mod chat; +pub mod inference; pub mod ocr; pub mod responses; diff --git a/litellm-rust/crates/llms/src/base_llm/ocr/error.rs b/litellm-rust/crates/llms/src/base_llm/ocr/error.rs index e09842e2856..b3df8fc18c8 100644 --- a/litellm-rust/crates/llms/src/base_llm/ocr/error.rs +++ b/litellm-rust/crates/llms/src/base_llm/ocr/error.rs @@ -98,6 +98,8 @@ pub enum Error { "Missing REDUCTO_API_KEY - set it in the environment or pass api_key to litellm.ocr()/litellm.aocr()" )] MissingReductoApiKey, + #[error("secret resolution failed: {0}")] + Secret(#[source] std::sync::Arc), #[error(transparent)] Auth(#[from] litellm_auth::Error), #[error(transparent)] diff --git a/litellm-rust/crates/llms/src/base_llm/ocr/handler.rs b/litellm-rust/crates/llms/src/base_llm/ocr/handler.rs index 245261d9f92..3ec9de8197f 100644 --- a/litellm-rust/crates/llms/src/base_llm/ocr/handler.rs +++ b/litellm-rust/crates/llms/src/base_llm/ocr/handler.rs @@ -1,3 +1,5 @@ +use std::sync::Arc; + use bytes::{Bytes, BytesMut}; use futures_util::future::BoxFuture; use litellm_auth_gcp::VertexAuth; @@ -11,9 +13,10 @@ use litellm_http::{ use serde::{Serialize, de::DeserializeOwned}; use serde_json::Value; +use crate::base_llm::inference::secrets::SecretSource; use crate::base_llm::ocr::{ error::Error, - settings::{OcrSettings, Secrets}, + settings::OcrSettings, transformation::{ BaseOcrConfig, DecodedOcrResponse, LiteLLMOcrResponse, OcrDocument, OcrResponseContext, PreparedOcrRequest, decode_request_value, decode_response, @@ -35,7 +38,7 @@ pub struct OcrClient { document_fetcher: MediaFetcher, vertex_auth: VertexAuth, settings: OcrSettings, - secrets: Secrets, + secrets: Arc, } impl OcrClient { @@ -45,7 +48,7 @@ impl OcrClient { url_policy: UrlPolicy, vertex_auth: VertexAuth, settings: OcrSettings, - secrets: Secrets, + secrets: Arc, ) -> Result { Ok(Self { provider_http: pool.client(config, ClientVariant::Provider)?, @@ -77,7 +80,7 @@ impl OcrClient { &self.settings } - pub fn secrets(&self) -> &Secrets { + pub fn secret_source(&self) -> &Arc { &self.secrets } @@ -92,7 +95,7 @@ impl OcrClient { document_fetcher: MediaFetcher::for_test(document_http), vertex_auth: VertexAuth::default(), settings: OcrSettings::default(), - secrets: std::sync::Arc::new(litellm_core_utils::settings::ProcessEnvironment), + secrets: Arc::new(crate::base_llm::inference::secrets::EnvironmentSecrets), } } @@ -102,7 +105,7 @@ impl OcrClient { } #[cfg(any(test, feature = "test-support"))] - pub fn with_secrets(self, secrets: Secrets) -> Self { + pub fn with_secrets(self, secrets: Arc) -> Self { Self { secrets, ..self } } } diff --git a/litellm-rust/crates/llms/src/base_llm/ocr/settings.rs b/litellm-rust/crates/llms/src/base_llm/ocr/settings.rs index f5954599b43..87461cd36aa 100644 --- a/litellm-rust/crates/llms/src/base_llm/ocr/settings.rs +++ b/litellm-rust/crates/llms/src/base_llm/ocr/settings.rs @@ -1,9 +1,7 @@ -use std::{sync::Arc, time::Duration}; +use std::time::Duration; use litellm_core_utils::settings::Lookup; -pub type Secrets = Arc; - #[derive(Clone, Debug, PartialEq)] pub struct OcrSettings { pub request_timeout: Duration, diff --git a/litellm-rust/crates/llms/src/base_llm/ocr/transformation.rs b/litellm-rust/crates/llms/src/base_llm/ocr/transformation.rs index e02a4b7f266..0506ff3d6df 100644 --- a/litellm-rust/crates/llms/src/base_llm/ocr/transformation.rs +++ b/litellm-rust/crates/llms/src/base_llm/ocr/transformation.rs @@ -14,10 +14,13 @@ use serde::{ use serde_json::{Map, Value}; use serde_with::serde_as; -use crate::base_llm::ocr::{ - error::Error, - handler::{CallHooks, OcrClient, read_response_bytes, transform_request_body}, - settings::{OcrSettings, Secrets}, +use crate::base_llm::{ + inference::secrets::Secrets, + ocr::{ + error::Error, + handler::{CallHooks, OcrClient, read_response_bytes, transform_request_body}, + settings::OcrSettings, + }, }; pub const OCR_RESPONSE_MAX_BYTES: usize = 64 * 1024 * 1024; @@ -436,6 +439,8 @@ pub trait BaseOcrConfig: Send + Sync + Sized + 'static { None } + fn secret_names(&self) -> Vec<&'static str>; + fn resolve_connection_params(&self, inputs: OcrCredentialInputs) -> ResolvedOcrCredentials { ResolvedOcrCredentials { api_key: inputs diff --git a/litellm-rust/crates/llms/src/cohere/ocr/transformation.rs b/litellm-rust/crates/llms/src/cohere/ocr/transformation.rs index d141c68db38..c0bb4c60563 100644 --- a/litellm-rust/crates/llms/src/cohere/ocr/transformation.rs +++ b/litellm-rust/crates/llms/src/cohere/ocr/transformation.rs @@ -102,6 +102,10 @@ impl BaseOcrConfig for CohereParseConfig { Some(COHERE_API_KEY_ENV) } + fn secret_names(&self) -> Vec<&'static str> { + vec![COHERE_API_KEY_ENV] + } + fn get_health_check_document(&self) -> OcrDocument { OcrDocument::ImageUrl { image_url: COHERE_PARSE_HEALTH_CHECK_IMAGE_DATA_URI.into(), diff --git a/litellm-rust/crates/llms/src/mistral/ocr/transformation.rs b/litellm-rust/crates/llms/src/mistral/ocr/transformation.rs index 2b14372fbec..149e8056789 100644 --- a/litellm-rust/crates/llms/src/mistral/ocr/transformation.rs +++ b/litellm-rust/crates/llms/src/mistral/ocr/transformation.rs @@ -69,6 +69,14 @@ impl BaseOcrConfig for MistralOcrConfig { Some(MISTRAL_OCR_API_KEY_ENV_VAR) } + fn secret_names(&self) -> Vec<&'static str> { + vec![ + MISTRAL_OCR_API_KEY_ENV_VAR, + "MISTRAL_AZURE_API_KEY", + "MISTRAL_AZURE_API_BASE", + ] + } + fn map_ocr_params( &self, non_default_params: &CallArguments, diff --git a/litellm-rust/crates/llms/src/reducto/ocr/transformation.rs b/litellm-rust/crates/llms/src/reducto/ocr/transformation.rs index 5272be97c24..f00259984ba 100644 --- a/litellm-rust/crates/llms/src/reducto/ocr/transformation.rs +++ b/litellm-rust/crates/llms/src/reducto/ocr/transformation.rs @@ -92,6 +92,10 @@ impl BaseOcrConfig for ReductoParseV3Config { type ProviderRequest = ReductoV3Request; type Environment = Vec<(String, String)>; + fn secret_names(&self) -> Vec<&'static str> { + vec![REDUCTO_API_KEY_ENV] + } + fn get_supported_ocr_params(&self, _model: &str) -> &'static [&'static str] { &["formatting", "retrieval", "settings"] } @@ -180,6 +184,10 @@ impl BaseOcrConfig for ReductoParseLegacyConfig { type ProviderRequest = ReductoLegacyRequest; type Environment = Vec<(String, String)>; + fn secret_names(&self) -> Vec<&'static str> { + vec![REDUCTO_API_KEY_ENV] + } + fn get_supported_ocr_params(&self, _model: &str) -> &'static [&'static str] { &["enhance"] } diff --git a/litellm-rust/crates/llms/src/vertex_ai/ocr/deepseek_transformation.rs b/litellm-rust/crates/llms/src/vertex_ai/ocr/deepseek_transformation.rs index 9a23deefb89..86231d50f9c 100644 --- a/litellm-rust/crates/llms/src/vertex_ai/ocr/deepseek_transformation.rs +++ b/litellm-rust/crates/llms/src/vertex_ai/ocr/deepseek_transformation.rs @@ -105,6 +105,10 @@ impl BaseOcrConfig for VertexAIDeepSeekOCRConfig { VertexAiOcrConfig.get_api_key_env_var() } + fn secret_names(&self) -> Vec<&'static str> { + VertexAiOcrConfig.secret_names() + } + fn map_ocr_params( &self, _arguments: &CallArguments, diff --git a/litellm-rust/crates/llms/src/vertex_ai/ocr/transformation.rs b/litellm-rust/crates/llms/src/vertex_ai/ocr/transformation.rs index 2d505ba4342..c9342c87e9a 100644 --- a/litellm-rust/crates/llms/src/vertex_ai/ocr/transformation.rs +++ b/litellm-rust/crates/llms/src/vertex_ai/ocr/transformation.rs @@ -34,6 +34,10 @@ impl BaseOcrConfig for VertexAiOcrConfig { Some("VERTEX_AI_API_KEY") } + fn secret_names(&self) -> Vec<&'static str> { + litellm_auth_gcp::SECRET_NAMES.to_vec() + } + fn map_ocr_params( &self, non_default_params: &CallArguments, diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml index 6b4476d897c..4e6c510d104 100644 --- a/litellm-rust/crates/python-bridge/Cargo.toml +++ b/litellm-rust/crates/python-bridge/Cargo.toml @@ -20,6 +20,7 @@ tiktoken = ["litellm-token-counter/tiktoken"] [dependencies] bytes.workspace = true +futures-util.workspace = true litellm-cache.workspace = true litellm-cache-azure-blob.workspace = true litellm-cache-memory.workspace = true @@ -41,6 +42,8 @@ litellm-core-utils.workspace = true litellm-auth-gcp.workspace = true litellm-http.workspace = true litellm-llms.workspace = true +litellm-secrets = { workspace = true, features = ["aws"] } +litellm-secrets-types.workspace = true litellm-types.workspace = true litellm-host-python.workspace = true litellm-token-counter = { path = "../token-counter", default-features = false } @@ -53,6 +56,7 @@ url.workspace = true tokio = { workspace = true, features = ["rt", "sync"] } [dev-dependencies] +litellm-secrets-aws.workspace = true serde.workspace = true serde_with.workspace = true criterion.workspace = true @@ -60,6 +64,8 @@ futures-util.workspace = true rstest.workspace = true sha2.workspace = true tokio-tungstenite.workspace = true +wiremock = "0.6.5" +aws-sdk-secretsmanager = "1.117.0" [[bench]] name = "serialization" diff --git a/litellm-rust/crates/python-bridge/README.md b/litellm-rust/crates/python-bridge/README.md new file mode 100644 index 00000000000..faaca233f5a --- /dev/null +++ b/litellm-rust/crates/python-bridge/README.md @@ -0,0 +1,5 @@ +Native OCR uses `SecretSource` with `EnvironmentSecrets`, preserving process-environment reads. Readable Python secret managers still make OCR decline to the existing Python implementation. `ResolvedSecrets` and the separate `secret_manager_binding()` snapshot are inactive foundations for a later rollout + +Cache and secret-manager catalog entries remain Python-only, including when `LITELLM_RUST=1`. The new cache runtime is not connected to SDK or gateway caching + +OCR provider requests use the shared `litellm-http` pool. AWS and Google secret-manager SDK clients keep their SDK transports, which do not yet inherit the pool's proxy, TLS, certificate, timeout, or observability configuration. Preserve those SDK transports and configure them equivalently instead of forcing them through reqwest diff --git a/litellm-rust/crates/python-bridge/python_settings.json b/litellm-rust/crates/python-bridge/python_settings.json deleted file mode 100644 index ea53d1d2025..00000000000 --- a/litellm-rust/crates/python-bridge/python_settings.json +++ /dev/null @@ -1,154 +0,0 @@ -{ - "http_settings": { - "version": 1, - "fields": { - "ssl_verify": { - "adapter": "SslVerifyInput", - "required": true, - "precedence": "module_global", - "sensitive": false, - "shapes": [ - "none", - "bool", - "str" - ], - "unsupported_live": "configuration_error" - }, - "ssl_certificate": { - "adapter": "OptionalStrictString", - "required": true, - "precedence": "module_global", - "sensitive": false, - "shapes": [], - "unsupported_live": null - }, - "ssl_security_level": { - "adapter": "TuningString", - "required": true, - "precedence": "module_global", - "sensitive": false, - "shapes": [], - "unsupported_live": null - }, - "ssl_ecdh_curve": { - "adapter": "TuningString", - "required": true, - "precedence": "module_global", - "sensitive": false, - "shapes": [], - "unsupported_live": null - }, - "force_ipv4": { - "adapter": "Truthy", - "required": true, - "precedence": "module_global", - "sensitive": false, - "shapes": [], - "unsupported_live": null - }, - "http2": { - "adapter": "ExactTrue", - "required": true, - "precedence": "module_global", - "sensitive": false, - "shapes": [], - "unsupported_live": null - }, - "aiohttp_trust_env": { - "adapter": "Truthy", - "required": true, - "precedence": "module_global", - "sensitive": false, - "shapes": [], - "unsupported_live": null - }, - "disable_aiohttp_trust_env": { - "adapter": "Truthy", - "required": true, - "precedence": "module_global", - "sensitive": false, - "shapes": [], - "unsupported_live": null - }, - "disable_aiohttp_transport": { - "adapter": "ExactTrue", - "required": true, - "precedence": "module_global", - "sensitive": false, - "shapes": [], - "unsupported_live": null - }, - "user_agent": { - "adapter": "StrictString", - "required": true, - "precedence": "accessor", - "sensitive": false, - "shapes": [], - "unsupported_live": null - } - } - }, - "url_policy": { - "version": 1, - "fields": { - "user_url_validation": { - "adapter": "Truthy", - "required": true, - "precedence": "module_global", - "sensitive": false, - "shapes": [], - "unsupported_live": null - }, - "user_url_allowed_hosts": { - "adapter": "HostCollection", - "required": true, - "precedence": "module_global", - "sensitive": false, - "shapes": [], - "unsupported_live": null - } - } - }, - "provider_defaults": { - "version": 1, - "fields": { - "vertex_project": { - "adapter": "FalsyOptionalString", - "required": true, - "precedence": "module_global", - "sensitive": true, - "shapes": [], - "unsupported_live": null - }, - "vertex_location": { - "adapter": "FalsyOptionalString", - "required": true, - "precedence": "module_global", - "sensitive": true, - "shapes": [], - "unsupported_live": null - }, - "enable_azure_ad_token_refresh": { - "adapter": "ExactTrue", - "required": true, - "precedence": "module_global", - "sensitive": false, - "shapes": [], - "unsupported_live": null - } - } - }, - "secret_manager": { - "version": 1, - "fields": { - "readable": { - "adapter": "StrictBool", - "required": true, - "precedence": "accessor", - "sensitive": false, - "shapes": [], - "unsupported_live": null - } - } - } -} diff --git a/litellm-rust/crates/python-bridge/src/cache/binding.rs b/litellm-rust/crates/python-bridge/src/cache/binding.rs index 2ff73238202..273d3f9ca4e 100644 --- a/litellm-rust/crates/python-bridge/src/cache/binding.rs +++ b/litellm-rust/crates/python-bridge/src/cache/binding.rs @@ -11,10 +11,12 @@ use serde_json::Value; use super::{ cache_error, callback::PythonCallback, + config::{CacheBackendConfig, CacheConfigProjection, NativeCacheConfig}, future::{ready_none, ready_value}, native::NativeResponseCache, request::{now, request, requests}, }; +use crate::errors::RustBridgeDeclined; pub(super) enum CacheBinding { Disabled, @@ -22,7 +24,7 @@ pub(super) enum CacheBinding { PythonCallback(PythonCallback), } -#[pyclass(frozen, name = "_CacheTestBinding")] +#[pyclass(frozen, name = "_ResponseCacheRuntime")] pub(crate) struct ResolvedCache { binding: CacheBinding, pid: u32, @@ -66,6 +68,33 @@ impl ResolvedCache { #[pymethods] impl ResolvedCache { + #[staticmethod] + fn from_cache(cache: &Bound<'_, PyAny>) -> PyResult { + let config = match NativeCacheConfig::project(cache)? { + CacheConfigProjection::Native(config) => *config, + CacheConfigProjection::Unsupported(reason) => { + return Err(RustBridgeDeclined::new_err(reason.message())); + } + }; + let service = match config.backend { + CacheBackendConfig::Memory(memory) => NativeResponseCache::memory( + memory.capacity, + memory.default_ttl, + memory.max_entry_bytes, + ), + _ => { + return Err(RustBridgeDeclined::new_err( + "native response cache activation is not implemented for this backend", + )); + } + }; + Ok(Self::new(CacheBinding::Native( + service + .with_scope(config.policy.semantic_cache_scope) + .with_redis_flush_size(config.policy.redis_flush_size), + ))) + } + #[getter] fn kind(&self) -> &'static str { match self.binding { diff --git a/litellm-rust/crates/python-bridge/src/cache/config.rs b/litellm-rust/crates/python-bridge/src/cache/config.rs index 766d526cf5f..b6e08102e18 100644 --- a/litellm-rust/crates/python-bridge/src/cache/config.rs +++ b/litellm-rust/crates/python-bridge/src/cache/config.rs @@ -11,7 +11,7 @@ use pyo3::{ types::{PyAny, PyBool, PyDict, PyList, PyString}, }; -use super::{native::NativeResponseCache, request::duration}; +use super::{identity::BackendIdentity, native::NativeResponseCache, request::duration}; #[allow(dead_code, reason = "consumed by the cache activation follow-up")] pub(super) struct CachePolicy { @@ -293,161 +293,58 @@ impl NativeCacheConfig { } pub(super) fn service_mismatch(&self, service: &NativeResponseCache) -> Option<&'static str> { - let default_ttl = match &self.backend { - CacheBackendConfig::Memory(config) => Some(config.default_ttl), - CacheBackendConfig::Redis(config) => Some(config.default_ttl), - CacheBackendConfig::S3(_) => None, - CacheBackendConfig::ValkeySemantic(_) => Some(Duration::ZERO), - CacheBackendConfig::Disk(_) - | CacheBackendConfig::AzureBlob(_) - | CacheBackendConfig::Gcs(_) - | CacheBackendConfig::RedisSemantic(_) - | CacheBackendConfig::QdrantSemantic(_) => None, - }; - if !matches!(self.backend, CacheBackendConfig::ValkeySemantic(_)) - && service.default_ttl() != default_ttl - { - return Some("facade and native backend default TTLs must match"); - } - match &self.backend { - CacheBackendConfig::Memory(config) if service.kind() != "memory" => { - Some("facade and native backend types must match") - } - CacheBackendConfig::Memory(config) if service.capacity() != Some(config.capacity) => { - Some("facade and native backend capacities must match") - } - CacheBackendConfig::Memory(config) - if service.max_entry_bytes() != Some(config.max_entry_bytes) => - { - Some("facade and native backend item limits must match") - } - CacheBackendConfig::Memory(_) => None, - CacheBackendConfig::Redis(_) if service.kind() != "redis" => { - Some("facade and native backend types must match") - } - CacheBackendConfig::Redis(config) if service.topology() != Some(&config.topology) => { - Some("facade and native backend topologies must match") - } - CacheBackendConfig::Redis(config) => (service.namespace() - != config.namespace.as_deref()) - .then_some("facade and native backend namespaces must match"), - CacheBackendConfig::S3(_) if service.kind() != "s3" => { - Some("facade and native backend types must match") - } - CacheBackendConfig::S3(config) if service.bucket() != Some(config.bucket.as_str()) => { - Some("facade and native backend buckets must match") - } - CacheBackendConfig::S3(config) - if service.key_prefix() != Some(config.key_prefix.as_str()) => - { - Some("facade and native backend key prefixes must match") - } - CacheBackendConfig::S3(config) if service.region() != Some(config.region.as_str()) => { - Some("facade and native backend regions must match") - } - CacheBackendConfig::S3(config) - if service.endpoint() - != config - .endpoint - .as_ref() - .map(|endpoint| endpoint.url.as_str()) => - { - Some("facade and native backend endpoints must match") - } - CacheBackendConfig::S3(_) => None, - CacheBackendConfig::Gcs(_) if service.kind() != "gcs" => { - Some("facade and native backend types must match") - } - CacheBackendConfig::Gcs(config) - if service - .gcs_backend() - .is_none_or(|backend| backend.bucket_name() != config.bucket_name) => - { - Some("facade and native backend buckets must match") - } - CacheBackendConfig::Gcs(config) - if service - .gcs_backend() - .is_none_or(|backend| backend.key_prefix() != config.key_prefix) => - { - Some("facade and native backend key prefixes must match") - } - CacheBackendConfig::Gcs(config) - if service.gcs_backend().is_none_or(|backend| { - backend.path_service_account() != config.path_service_account.as_deref() - }) => - { - Some("facade and native backend credentials must match") - } - CacheBackendConfig::Gcs(_) => None, - CacheBackendConfig::ValkeySemantic(config) => { - if service.kind() != "valkey-semantic" { - return Some("facade and native backend types must match"); - } - let Some((threshold, index_name)) = service.semantic_config() else { - return Some("facade and native backend types must match"); - }; - (threshold != config.similarity_threshold || index_name != config.index_name) - .then_some("facade and native semantic settings must match") - } - CacheBackendConfig::Disk(_) if service.kind() != "disk" => { - Some("facade and native backend types must match") - } - CacheBackendConfig::Disk(config) => { - let Some(directory) = service.directory() else { - return Some("facade and native backend types must match"); - }; - let native = std::fs::canonicalize(directory).ok(); - let facade = std::fs::canonicalize(&config.directory).ok(); - (native != facade).then_some("facade and native backend directories must match") - } - CacheBackendConfig::RedisSemantic(_) if service.kind() != "redis_semantic" => { - Some("facade and native backend types must match") - } - CacheBackendConfig::RedisSemantic(config) - if service.index_name() != Some(config.index_name.as_str()) => - { - Some("facade and native backend index names must match") - } - CacheBackendConfig::RedisSemantic(config) - if service.similarity_threshold() - != Some(f64::from(config.similarity_threshold as f32)) => - { - Some("facade and native backend similarity thresholds must match") - } - CacheBackendConfig::RedisSemantic(_) => None, - CacheBackendConfig::QdrantSemantic(config) if service.kind() != "qdrant_semantic" => { - Some("facade and native backend types must match") - } - CacheBackendConfig::QdrantSemantic(config) - if service.collection_name() != Some(config.collection_name.as_str()) => - { - Some("facade and native backend collections must match") - } - CacheBackendConfig::QdrantSemantic(config) - if service.similarity_threshold() != Some(config.similarity_threshold) => - { - Some("facade and native backend similarity thresholds must match") - } - CacheBackendConfig::QdrantSemantic(config) - if service.vector_size() != Some(config.vector_size) => - { - Some("facade and native backend vector sizes must match") - } - CacheBackendConfig::QdrantSemantic(config) - if service.embedding_model() != Some(config.embedding.model.as_str()) => - { - Some("facade and native backend embedding models must match") - } - CacheBackendConfig::QdrantSemantic(_) => None, - CacheBackendConfig::AzureBlob(config) => match service.azure_blob_identity() { - None => Some("facade and native backend types must match"), - Some((account_url, container)) - if account_url != config.account_url || container != config.container => - { - Some("facade and native backend containers must match") - } - Some(_) => None, + self.backend.identity().mismatch(&service.identity()) + } +} + +impl CacheBackendConfig { + /// The identity a native backend must have for this facade configuration to describe it. + pub(super) fn identity(&self) -> BackendIdentity { + match self { + Self::Memory(config) => BackendIdentity::Memory { + capacity: config.capacity, + max_entry_bytes: Some(config.max_entry_bytes), + default_ttl: Some(config.default_ttl), + }, + Self::Redis(config) => BackendIdentity::Redis { + topology: config.topology.clone(), + namespace: config.namespace.clone(), + default_ttl: Some(config.default_ttl), + }, + Self::S3(config) => BackendIdentity::S3 { + bucket: config.bucket.clone(), + key_prefix: config.key_prefix.clone(), + region: config.region.clone(), + endpoint: config + .endpoint + .as_ref() + .map(|endpoint| endpoint.url.clone()), + }, + Self::Gcs(config) => BackendIdentity::Gcs { + bucket_name: config.bucket_name.clone(), + key_prefix: config.key_prefix.clone(), + path_service_account: config.path_service_account.clone(), + }, + Self::ValkeySemantic(config) => BackendIdentity::ValkeySemantic { + index_name: config.index_name.clone(), + similarity_threshold: config.similarity_threshold, + }, + Self::Disk(config) => BackendIdentity::Disk { + directory: config.directory.clone(), + }, + Self::AzureBlob(config) => BackendIdentity::AzureBlob { + account_url: config.account_url.clone(), + container: config.container.clone(), + }, + Self::RedisSemantic(config) => BackendIdentity::RedisSemantic { + index_name: config.index_name.clone(), + similarity_threshold: config.similarity_threshold as f32, + }, + Self::QdrantSemantic(config) => BackendIdentity::QdrantSemantic { + collection_name: config.collection_name.clone(), + similarity_threshold: config.similarity_threshold, + vector_size: config.vector_size, + embedding_model: config.embedding.model.clone(), }, } } diff --git a/litellm-rust/crates/python-bridge/src/cache/embedder.rs b/litellm-rust/crates/python-bridge/src/cache/embedder.rs index 9398e5a862b..ffd72e33e1b 100644 --- a/litellm-rust/crates/python-bridge/src/cache/embedder.rs +++ b/litellm-rust/crates/python-bridge/src/cache/embedder.rs @@ -9,6 +9,8 @@ tokio::task_local! { static PREPARED_EMBEDDING: Result, Error>; } +/// Runs `future` with the vector the Python embedder already produced, so the backend's +/// `async_embed` never has to call back into Python from the runtime. pub(super) fn with_prepared_embedding( vector: Result, Error>, future: F, @@ -16,6 +18,7 @@ pub(super) fn with_prepared_embedding( PREPARED_EMBEDDING.scope(vector, future) } +/// The Python object that owns embedding for a semantic backend. pub(super) struct PythonEmbedder(Py); impl Clone for PythonEmbedder { @@ -29,10 +32,6 @@ impl PythonEmbedder { Self(object) } - pub(super) fn from_backend(backend: &Bound<'_, PyAny>) -> PyResult { - Ok(Self(backend.clone().unbind())) - } - pub(super) fn object(&self) -> &Py { &self.0 } @@ -41,18 +40,6 @@ impl PythonEmbedder { visit.call(&self.0) } - pub(super) fn async_embed_awaitable<'py>( - &self, - py: Python<'py>, - prompt: &str, - metadata: &Option, - ) -> PyResult> { - let metadata = to_py(py, metadata)?; - self.0 - .bind(py) - .call_method1("_get_async_embedding", (prompt, metadata)) - } - fn metadata_kwargs<'py>( py: Python<'py>, metadata: Option<&Value>, @@ -62,7 +49,8 @@ impl PythonEmbedder { Ok(kwargs) } - pub(super) fn async_embedding_coroutine( + /// The awaitable of `_get_async_embedding(prompt, metadata=...)`, to run in the caller's loop. + pub(super) fn async_embedding( &self, py: Python<'_>, prompt: &str, @@ -82,35 +70,8 @@ impl PythonEmbedder { .map(|value| value as f32) .collect()) } -} -impl litellm_cache_valkey_semantic::Embedder for PythonEmbedder { - fn embed(&self, prompt: &str, metadata: Option<&Value>) -> Result, Error> { - let result = Python::attach(|py| -> PyResult> { - let metadata = to_py(py, &metadata)?; - self.0 - .bind(py) - .call_method1("_get_embedding", (prompt, metadata))? - .extract() - }) - .map_err(|_| Error::Unavailable)?; - Ok(result.into_iter().map(|value| value as f32).collect()) - } - - fn async_embed( - &self, - _prompt: &str, - _metadata: Option<&Value>, - ) -> impl Future, Error>> + Send { - let seeded = PREPARED_EMBEDDING - .try_with(Clone::clone) - .unwrap_or(Err(Error::Unavailable)); - std::future::ready(seeded) - } -} - -impl litellm_cache_redis_semantic::Embedder for PythonEmbedder { - fn embed(&self, prompt: &str, metadata: Option<&Value>) -> Result, Error> { + fn embed_sync(&self, prompt: &str, metadata: Option<&Value>) -> Result, Error> { Python::attach(|py| { let kwargs = Self::metadata_kwargs(py, metadata)?; Self::extract(self.0.bind(py).call_method( @@ -122,15 +83,38 @@ impl litellm_cache_redis_semantic::Embedder for PythonEmbedder { .map_err(|_| Error::Unavailable) } + fn seeded_embedding() -> Result, Error> { + PREPARED_EMBEDDING + .try_with(Clone::clone) + .unwrap_or(Err(Error::Unavailable)) + } +} + +impl litellm_cache_valkey_semantic::Embedder for PythonEmbedder { + fn embed(&self, prompt: &str, metadata: Option<&Value>) -> Result, Error> { + self.embed_sync(prompt, metadata) + } + fn async_embed( &self, _prompt: &str, _metadata: Option<&Value>, ) -> impl Future, Error>> + Send { - let seeded = PREPARED_EMBEDDING - .try_with(Clone::clone) - .unwrap_or(Err(Error::Unavailable)); - std::future::ready(seeded) + std::future::ready(Self::seeded_embedding()) + } +} + +impl litellm_cache_redis_semantic::Embedder for PythonEmbedder { + fn embed(&self, prompt: &str, metadata: Option<&Value>) -> Result, Error> { + self.embed_sync(prompt, metadata) + } + + fn async_embed( + &self, + _prompt: &str, + _metadata: Option<&Value>, + ) -> impl Future, Error>> + Send { + std::future::ready(Self::seeded_embedding()) } } @@ -152,5 +136,9 @@ mod tests { let unscoped = litellm_cache_redis_semantic::Embedder::async_embed(&embedder, "prompt", None).await; assert_eq!(unscoped, Err(Error::Unavailable)); + let valkey = with_prepared_embedding(Ok(vec![0.5]), async move { + litellm_cache_valkey_semantic::Embedder::async_embed(&embedder, "prompt", None).await + }); + assert_eq!(valkey.await, Ok(vec![0.5])); } } diff --git a/litellm-rust/crates/python-bridge/src/cache/facade.rs b/litellm-rust/crates/python-bridge/src/cache/facade.rs index 43b53584943..17fa278ae5e 100644 --- a/litellm-rust/crates/python-bridge/src/cache/facade.rs +++ b/litellm-rust/crates/python-bridge/src/cache/facade.rs @@ -11,6 +11,7 @@ use serde_json::Value; use super::{ config::{CacheConfigProjection, NativeCacheConfig}, handle::CacheTestHandle, + identity::BackendIdentity, native::NativeResponseCache, }; @@ -352,47 +353,41 @@ impl FacadeGuard { facade: &Bound<'_, PyAny>, service: &NativeResponseCache, ) -> PyResult { - let kind = service.kind(); + let identity = service.identity(); + let kind = identity.kind(); let cache_type = py.import("litellm.caching.caching")?.getattr("Cache")?; if !facade.get_type().is(&cache_type) { return Err(PyTypeError::new_err( "only exact built-in Cache facades can be registered", )); } - let cluster = matches!(service.topology(), Some(RedisTopology::Cluster { .. })); - let (module, name, cache_kind) = match (kind, cluster) { - ("memory", _) => ("litellm.caching.in_memory_cache", "InMemoryCache", "local"), - ("redis", false) => ("litellm.caching.redis_cache", "RedisCache", "redis"), - ("redis_semantic", _) => ( - "litellm.caching.redis_semantic_cache", - "RedisSemanticCache", - "redis-semantic", - ), + let cluster = matches!( + identity, + BackendIdentity::Redis { + topology: RedisTopology::Cluster { .. }, + .. + } + ); + let (module, name) = match (kind, cluster) { + ("memory", _) => ("litellm.caching.in_memory_cache", "InMemoryCache"), + ("redis", false) => ("litellm.caching.redis_cache", "RedisCache"), + ("redis", true) => ("litellm.caching.redis_cluster_cache", "RedisClusterCache"), + ("redis_semantic", _) => ("litellm.caching.redis_semantic_cache", "RedisSemanticCache"), ("qdrant_semantic", _) => ( "litellm.caching.qdrant_semantic_cache", "QdrantSemanticCache", - "qdrant-semantic", ), - ("redis", true) => ( - "litellm.caching.redis_cluster_cache", - "RedisClusterCache", - "redis", - ), - ("gcs", _) => ("litellm.caching.gcs_cache", "GCSCache", "gcs"), - ("valkey-semantic", false) => ( + ("gcs", _) => ("litellm.caching.gcs_cache", "GCSCache"), + ("valkey-semantic", _) => ( "litellm.caching.valkey_semantic_cache", "ValkeySemanticCache", - "valkey-semantic", ), - ("disk", _) => ("litellm.caching.disk_cache", "DiskCache", "disk"), - ("azure-blob", _) => ( - "litellm.caching.azure_blob_cache", - "AzureBlobCache", - "azure-blob", - ), - ("s3", _) => ("litellm.caching.s3_cache", "S3Cache", "s3"), + ("disk", _) => ("litellm.caching.disk_cache", "DiskCache"), + ("azure-blob", _) => ("litellm.caching.azure_blob_cache", "AzureBlobCache"), + ("s3", _) => ("litellm.caching.s3_cache", "S3Cache"), _ => unreachable!(), }; + let cache_kind = identity.cache_type(); let backend = facade.getattr("cache")?; if facade.getattr("type")?.extract::()? != cache_kind || !backend.get_type().is(&py.import(module)?.getattr(name)?) diff --git a/litellm-rust/crates/python-bridge/src/cache/handle.rs b/litellm-rust/crates/python-bridge/src/cache/handle.rs index 51e1b02c405..61993f42279 100644 --- a/litellm-rust/crates/python-bridge/src/cache/handle.rs +++ b/litellm-rust/crates/python-bridge/src/cache/handle.rs @@ -261,7 +261,7 @@ impl CacheTestHandle { index_name: String, embedder: &Bound<'_, PyAny>, ) -> PyResult { - let python_embedder = PythonEmbedder::from_backend(embedder)?; + let python_embedder = PythonEmbedder::new(embedder.clone().unbind()); let service = NativeResponseCache::valkey_semantic( &url, similarity_threshold, diff --git a/litellm-rust/crates/python-bridge/src/cache/identity.rs b/litellm-rust/crates/python-bridge/src/cache/identity.rs new file mode 100644 index 00000000000..835bafd3ff1 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/cache/identity.rs @@ -0,0 +1,511 @@ +use std::{path::PathBuf, time::Duration}; + +use litellm_cache_redis::RedisTopology; + +/// What makes a native backend the one a Python facade describes: the configuration a user can +/// observe on the Python object, captured once so facade projection and native construction +/// compare plain data instead of reaching into each backend type. +#[derive(Clone, Debug, PartialEq)] +pub(super) enum BackendIdentity { + Memory { + capacity: usize, + max_entry_bytes: Option, + default_ttl: Option, + }, + Redis { + topology: RedisTopology, + namespace: Option, + default_ttl: Option, + }, + S3 { + bucket: String, + key_prefix: String, + region: String, + endpoint: Option, + }, + Gcs { + bucket_name: String, + key_prefix: String, + path_service_account: Option, + }, + Disk { + directory: PathBuf, + }, + AzureBlob { + account_url: String, + container: String, + }, + RedisSemantic { + index_name: String, + /// The backend stores the threshold as `f32`; a facade's `f64` is compared at that width. + similarity_threshold: f32, + }, + ValkeySemantic { + index_name: String, + similarity_threshold: f64, + }, + QdrantSemantic { + collection_name: String, + similarity_threshold: f64, + vector_size: u64, + embedding_model: String, + }, +} + +const TYPES: &str = "facade and native backend types must match"; + +impl BackendIdentity { + /// The native backend name reported to Python through `_CacheTestHandle.backend`. + pub(super) fn kind(&self) -> &'static str { + match self { + Self::Memory { .. } => "memory", + Self::Redis { .. } => "redis", + Self::S3 { .. } => "s3", + Self::Gcs { .. } => "gcs", + Self::ValkeySemantic { .. } => "valkey-semantic", + Self::RedisSemantic { .. } => "redis_semantic", + Self::QdrantSemantic { .. } => "qdrant_semantic", + Self::Disk { .. } => "disk", + Self::AzureBlob { .. } => "azure-blob", + } + } + + /// The `LiteLLMCacheType` value a facade of this backend carries in `Cache.type`. + pub(super) fn cache_type(&self) -> &'static str { + match self { + Self::Memory { .. } => "local", + Self::Redis { .. } => "redis", + Self::S3 { .. } => "s3", + Self::Gcs { .. } => "gcs", + Self::ValkeySemantic { .. } => "valkey-semantic", + Self::RedisSemantic { .. } => "redis-semantic", + Self::QdrantSemantic { .. } => "qdrant-semantic", + Self::Disk { .. } => "disk", + Self::AzureBlob { .. } => "azure-blob", + } + } + + /// The first difference between the facade's configuration (`self`) and the native + /// backend (`native`), in the order Python users see the attributes. + pub(super) fn mismatch(&self, native: &Self) -> Option<&'static str> { + let mut differences: Vec<(bool, &'static str)> = Vec::new(); + let mut differs = |condition: bool, message: &'static str| { + differences.push((condition, message)); + }; + match (self, native) { + ( + Self::Memory { + capacity, + max_entry_bytes, + default_ttl, + }, + Self::Memory { + capacity: native_capacity, + max_entry_bytes: native_max_entry_bytes, + default_ttl: native_default_ttl, + }, + ) => { + differs( + default_ttl != native_default_ttl, + "facade and native backend default TTLs must match", + ); + differs( + capacity != native_capacity, + "facade and native backend capacities must match", + ); + differs( + max_entry_bytes != native_max_entry_bytes, + "facade and native backend item limits must match", + ); + } + ( + Self::Redis { + topology, + namespace, + default_ttl, + }, + Self::Redis { + topology: native_topology, + namespace: native_namespace, + default_ttl: native_default_ttl, + }, + ) => { + differs( + default_ttl != native_default_ttl, + "facade and native backend default TTLs must match", + ); + differs( + topology != native_topology, + "facade and native backend topologies must match", + ); + differs( + namespace != native_namespace, + "facade and native backend namespaces must match", + ); + } + ( + Self::S3 { + bucket, + key_prefix, + region, + endpoint, + }, + Self::S3 { + bucket: native_bucket, + key_prefix: native_key_prefix, + region: native_region, + endpoint: native_endpoint, + }, + ) => { + differs( + bucket != native_bucket, + "facade and native backend buckets must match", + ); + differs( + key_prefix != native_key_prefix, + "facade and native backend key prefixes must match", + ); + differs( + region != native_region, + "facade and native backend regions must match", + ); + differs( + endpoint != native_endpoint, + "facade and native backend endpoints must match", + ); + } + ( + Self::Gcs { + bucket_name, + key_prefix, + path_service_account, + }, + Self::Gcs { + bucket_name: native_bucket_name, + key_prefix: native_key_prefix, + path_service_account: native_path_service_account, + }, + ) => { + differs( + bucket_name != native_bucket_name, + "facade and native backend buckets must match", + ); + differs( + key_prefix != native_key_prefix, + "facade and native backend key prefixes must match", + ); + differs( + path_service_account != native_path_service_account, + "facade and native backend credentials must match", + ); + } + ( + Self::Disk { directory }, + Self::Disk { + directory: native_directory, + }, + ) => { + let canonical = |path: &PathBuf| std::fs::canonicalize(path).ok(); + differs( + canonical(directory) != canonical(native_directory), + "facade and native backend directories must match", + ); + } + ( + Self::AzureBlob { + account_url, + container, + }, + Self::AzureBlob { + account_url: native_account_url, + container: native_container, + }, + ) => { + differs( + account_url != native_account_url || container != native_container, + "facade and native backend containers must match", + ); + } + ( + Self::RedisSemantic { + index_name, + similarity_threshold, + }, + Self::RedisSemantic { + index_name: native_index_name, + similarity_threshold: native_similarity_threshold, + }, + ) => { + differs( + index_name != native_index_name, + "facade and native backend index names must match", + ); + differs( + similarity_threshold != native_similarity_threshold, + "facade and native backend similarity thresholds must match", + ); + } + ( + Self::ValkeySemantic { + index_name, + similarity_threshold, + }, + Self::ValkeySemantic { + index_name: native_index_name, + similarity_threshold: native_similarity_threshold, + }, + ) => { + differs( + index_name != native_index_name + || similarity_threshold != native_similarity_threshold, + "facade and native semantic settings must match", + ); + } + ( + Self::QdrantSemantic { + collection_name, + similarity_threshold, + vector_size, + embedding_model, + }, + Self::QdrantSemantic { + collection_name: native_collection_name, + similarity_threshold: native_similarity_threshold, + vector_size: native_vector_size, + embedding_model: native_embedding_model, + }, + ) => { + differs( + collection_name != native_collection_name, + "facade and native backend collections must match", + ); + differs( + similarity_threshold != native_similarity_threshold, + "facade and native backend similarity thresholds must match", + ); + differs( + vector_size != native_vector_size, + "facade and native backend vector sizes must match", + ); + differs( + embedding_model != native_embedding_model, + "facade and native backend embedding models must match", + ); + } + _ => return Some(TYPES), + } + differences + .into_iter() + .find_map(|(condition, message)| condition.then_some(message)) + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use litellm_cache_redis::{RedisNode, RedisTopology}; + + use super::BackendIdentity; + + fn memory() -> BackendIdentity { + BackendIdentity::Memory { + capacity: 200, + max_entry_bytes: Some(1024), + default_ttl: Some(Duration::from_secs(60)), + } + } + + fn redis() -> BackendIdentity { + BackendIdentity::Redis { + topology: RedisTopology::Standalone, + namespace: Some("team".into()), + default_ttl: Some(Duration::from_secs(60)), + } + } + + fn s3() -> BackendIdentity { + BackendIdentity::S3 { + bucket: "bucket".into(), + key_prefix: "cache/".into(), + region: "us-east-1".into(), + endpoint: None, + } + } + + fn gcs() -> BackendIdentity { + BackendIdentity::Gcs { + bucket_name: "bucket".into(), + key_prefix: "cache/".into(), + path_service_account: Some("credentials.json".into()), + } + } + + fn azure() -> BackendIdentity { + BackendIdentity::AzureBlob { + account_url: "https://account.blob.core.windows.net".into(), + container: "cache".into(), + } + } + + fn redis_semantic() -> BackendIdentity { + BackendIdentity::RedisSemantic { + index_name: "idx".into(), + similarity_threshold: 0.8, + } + } + + #[test] + fn redis_semantic_thresholds_compare_at_backend_precision() { + let facade = BackendIdentity::RedisSemantic { + index_name: "idx".into(), + similarity_threshold: 0.8_f64 as f32, + }; + assert_eq!(facade.mismatch(&redis_semantic()), None); + } + + fn valkey_semantic() -> BackendIdentity { + BackendIdentity::ValkeySemantic { + index_name: "idx".into(), + similarity_threshold: 0.8, + } + } + + fn qdrant() -> BackendIdentity { + BackendIdentity::QdrantSemantic { + collection_name: "collection".into(), + similarity_threshold: 0.8, + vector_size: 1536, + embedding_model: "text-embedding-3-small".into(), + } + } + + #[test] + fn identical_identities_have_no_mismatch() { + for identity in [ + memory(), + redis(), + s3(), + gcs(), + azure(), + redis_semantic(), + valkey_semantic(), + qdrant(), + BackendIdentity::Disk { + directory: std::env::temp_dir(), + }, + ] { + assert_eq!(identity.mismatch(&identity), None, "{identity:?}"); + } + } + + #[test] + fn different_kinds_report_a_type_mismatch() { + assert_eq!( + memory().mismatch(&redis()), + Some("facade and native backend types must match") + ); + assert_eq!( + redis_semantic().mismatch(&valkey_semantic()), + Some("facade and native backend types must match") + ); + } + + #[test] + fn the_first_differing_field_names_the_mismatch() { + let BackendIdentity::Memory { capacity, .. } = memory() else { + unreachable!() + }; + assert_eq!( + memory().mismatch(&BackendIdentity::Memory { + capacity: capacity + 1, + max_entry_bytes: Some(1), + default_ttl: Some(Duration::from_secs(60)), + }), + Some("facade and native backend capacities must match") + ); + assert_eq!( + memory().mismatch(&BackendIdentity::Memory { + capacity, + max_entry_bytes: Some(1), + default_ttl: Some(Duration::from_secs(61)), + }), + Some("facade and native backend default TTLs must match") + ); + assert_eq!( + redis().mismatch(&BackendIdentity::Redis { + topology: RedisTopology::Cluster { + startup_nodes: vec![RedisNode { + host: "node".into(), + port: 7000, + }], + }, + namespace: None, + default_ttl: Some(Duration::from_secs(60)), + }), + Some("facade and native backend topologies must match") + ); + assert_eq!( + s3().mismatch(&BackendIdentity::S3 { + bucket: "bucket".into(), + key_prefix: "cache/".into(), + region: "us-east-1".into(), + endpoint: Some("http://localhost:9000".into()), + }), + Some("facade and native backend endpoints must match") + ); + assert_eq!( + gcs().mismatch(&BackendIdentity::Gcs { + bucket_name: "bucket".into(), + key_prefix: "cache/".into(), + path_service_account: None, + }), + Some("facade and native backend credentials must match") + ); + assert_eq!( + azure().mismatch(&BackendIdentity::AzureBlob { + account_url: "https://account.blob.core.windows.net".into(), + container: "other".into(), + }), + Some("facade and native backend containers must match") + ); + assert_eq!( + valkey_semantic().mismatch(&BackendIdentity::ValkeySemantic { + index_name: "idx".into(), + similarity_threshold: 0.9, + }), + Some("facade and native semantic settings must match") + ); + assert_eq!( + qdrant().mismatch(&BackendIdentity::QdrantSemantic { + collection_name: "collection".into(), + similarity_threshold: 0.8, + vector_size: 1536, + embedding_model: "text-embedding-3-large".into(), + }), + Some("facade and native backend embedding models must match") + ); + } + + #[test] + fn disk_directories_compare_canonically() { + let directory = std::env::temp_dir(); + let mut indirect = directory.clone(); + indirect.push("."); + assert_eq!( + BackendIdentity::Disk { + directory: directory.clone() + } + .mismatch(&BackendIdentity::Disk { + directory: indirect + }), + None + ); + assert_eq!( + BackendIdentity::Disk { directory }.mismatch(&BackendIdentity::Disk { + directory: "/definitely/missing".into() + }), + Some("facade and native backend directories must match") + ); + } +} diff --git a/litellm-rust/crates/python-bridge/src/cache/mod.rs b/litellm-rust/crates/python-bridge/src/cache/mod.rs index fa028518559..28dd6c3e798 100644 --- a/litellm-rust/crates/python-bridge/src/cache/mod.rs +++ b/litellm-rust/crates/python-bridge/src/cache/mod.rs @@ -5,11 +5,11 @@ mod embedder; mod facade; mod future; mod handle; +mod identity; mod native; mod request; mod resolver; mod semantic; -mod semantic_step; use litellm_cache::Error; use pyo3::{ diff --git a/litellm-rust/crates/python-bridge/src/cache/native.rs b/litellm-rust/crates/python-bridge/src/cache/native.rs index 7fef6f55611..254b9cdea4d 100644 --- a/litellm-rust/crates/python-bridge/src/cache/native.rs +++ b/litellm-rust/crates/python-bridge/src/cache/native.rs @@ -1,8 +1,6 @@ -use std::{path::Path, sync::Arc, time::Duration}; +use std::{sync::Arc, time::Duration}; -use litellm_cache::{ - CacheCodec, CacheConnectionResult, Error, ExactCacheContext, SemanticCacheContext, -}; +use litellm_cache::{CacheCodec, CacheConnectionResult, Error}; use litellm_cache_azure_blob::AzureBlobCache; use litellm_cache_disk::DiskCache; use litellm_cache_gcs::{GcsCache, GcsConfig, StaticTokenSource}; @@ -11,8 +9,7 @@ use litellm_cache_qdrant_semantic::{Embedder, OpenAiEmbedder, QdrantSemanticCach use litellm_cache_redis::{RedisCache, RedisTopology}; use litellm_cache_redis_semantic::{RedisSemanticCache, RedisSemanticConfig}; use litellm_cache_response::{ - CacheEntry, CacheKeyField, PartialHits, ResponseCache, ResponseCacheCodec, - ResponseCacheRequest, WriteBuffer, + ExactResponseCache, PartialHits, ResponseCache, ResponseCacheCodec, WriteBuffer, }; use litellm_cache_s3::{S3Cache, S3CacheConfig}; use litellm_cache_valkey_semantic::{ValkeySemanticCache, ValkeySemanticConfig}; @@ -22,69 +19,37 @@ use serde_json::Value; use super::{ config::QdrantSemanticCacheConfig, embedder::PythonEmbedder, - request::NativeRequest, - semantic::{SemanticBody, SemanticOperation, drive}, - semantic_step::{SemanticEmbedExecution, drive_semantic}, + identity::BackendIdentity, + request::{NativeRequest, now}, + semantic::{EmbeddingFailure, SemanticExecution, SemanticOperation, drive}, }; -fn semantic_key(request: &NativeRequest, scope: &str) -> litellm_cache_response::CacheKeyInput { - let mut key = request.key.clone(); - if key.preset.is_some() { - return key; +/// What the Python embedder receives for one semantic request. +pub(super) struct EmbeddingInput { + pub(super) prompt: String, + pub(super) metadata: Option, +} + +/// An exact-match backend behind one pointer, with the identity its facade must reproduce. +pub(super) struct ExactService { + cache: Arc, + buffer: Option, + identity: BackendIdentity, +} + +impl ExactService { + fn new(cache: Arc, identity: BackendIdentity) -> Arc { + Arc::new(Self { + cache, + buffer: None, + identity, + }) } - key.fields - .retain(|field| !matches!(field.name.as_str(), "messages" | "prompt" | "input")); - const TENANT: [&str; 3] = [ - "user_api_key", - "user_api_key_team_id", - "user_api_key_org_id", - ]; - let end_user = (scope == "end_user").then_some("user_api_key_end_user_id"); - for name in TENANT.into_iter().chain(end_user) { - let sources = [ - request.metadata.as_ref(), - request.litellm_metadata.as_ref(), - request - .litellm_params - .as_ref() - .and_then(|params| params.get("metadata")), - request - .litellm_params - .as_ref() - .and_then(|params| params.get("litellm_metadata")), - ]; - let Some(value) = sources.into_iter().flatten().find_map(|source| { - source - .as_object() - .and_then(|values| values.get(name)) - .filter(|value| !value.is_null()) - }) else { - continue; - }; - let value = match value { - Value::Null => continue, - Value::String(text) => text.clone(), - other => other.to_string(), - }; - key.fields.push(CacheKeyField { - name: name.to_owned(), - value: Some(value), - api_parameter: true, - internal_parameter: false, - }); - } - key } #[derive(Clone)] pub(super) enum NativeResponseCache { - Memory(Arc>>), - Redis { - cache: Arc>>, - buffer: Option>, - }, - S3(Arc>>), - Gcs(Arc>>), + Exact(Arc), ValkeySemantic { cache: Arc>>, embedder: PythonEmbedder, @@ -95,23 +60,25 @@ pub(super) enum NativeResponseCache { embedder: PythonEmbedder, }, QdrantSemantic(Arc>>), - Disk(Arc>>), - AzureBlob(Arc>>), } impl NativeResponseCache { pub fn memory(capacity: usize, ttl: Duration, max_entry_bytes: usize) -> Self { - Self::Memory(Arc::new(ResponseCache::new(Arc::new( - InMemoryCache::with_clock_and_size_measurement( - Some(capacity), - Some(ttl), - Some(max_entry_bytes), - Some(Arc::new(|entry| { - ResponseCacheCodec.encode(entry).map(|bytes| bytes.len()) - })), - super::request::now, - ), - )))) + let backend = InMemoryCache::with_clock_and_size_measurement( + Some(capacity), + Some(ttl), + Some(max_entry_bytes), + Some(Arc::new(|entry| { + ResponseCacheCodec.encode(entry).map(|bytes| bytes.len()) + })), + now, + ); + let identity = BackendIdentity::Memory { + capacity: backend.max_size_in_memory(), + max_entry_bytes: backend.max_entry_bytes(), + default_ttl: None, + }; + Self::exact(ResponseCache::new(Arc::new(backend)), identity) } pub fn redis( @@ -122,19 +89,97 @@ impl NativeResponseCache { ) -> Result { let backend = RedisCache::connect(url, topology, ttl, ResponseCacheCodec)?.with_namespace(namespace); - Ok(Self::Redis { - cache: Arc::new(ResponseCache::new(Arc::new(backend))), - buffer: None, - }) + let identity = BackendIdentity::Redis { + topology: backend.topology().clone(), + namespace: backend.namespace().map(str::to_owned), + default_ttl: None, + }; + Ok(Self::exact(ResponseCache::new(Arc::new(backend)), identity)) } pub async fn s3(config: S3CacheConfig) -> Self { let runtime = tokio::runtime::Handle::current(); - Self::S3(Arc::new(ResponseCache::new(Arc::new(S3Cache::new( - config, + let backend = S3Cache::new(config, ResponseCacheCodec, runtime); + let identity = BackendIdentity::S3 { + bucket: backend.bucket().to_owned(), + key_prefix: backend.key_prefix().to_owned(), + region: backend.region().to_owned(), + endpoint: backend.endpoint().map(str::to_owned), + }; + Self::exact(ResponseCache::new(Arc::new(backend)), identity) + } + + pub fn disk(directory: &str) -> Result { + let backend = DiskCache::open(directory, ResponseCacheCodec)?; + let identity = BackendIdentity::Disk { + directory: backend.directory().to_path_buf(), + }; + Ok(Self::exact(ResponseCache::new(Arc::new(backend)), identity)) + } + + pub fn gcs(config: GcsConfig, token: Option) -> Result { + let backend = match token { + Some(token) => GcsCache::with_token_source( + config, + ResponseCacheCodec, + Arc::new(StaticTokenSource(token)), + )?, + None => GcsCache::new(config, ResponseCacheCodec)?, + }; + let identity = BackendIdentity::Gcs { + bucket_name: backend.bucket_name().to_owned(), + key_prefix: backend.key_prefix().to_owned(), + path_service_account: backend.path_service_account().map(str::to_owned), + }; + Ok(Self::exact(ResponseCache::new(Arc::new(backend)), identity)) + } + + pub async fn azure_blob(account_url: &str, container: &str) -> Result { + let backend = AzureBlobCache::connect( + account_url, + container, ResponseCacheCodec, - runtime, - ))))) + tokio::runtime::Handle::current(), + ) + .await?; + let identity = BackendIdentity::AzureBlob { + account_url: backend.account_url().to_owned(), + container: backend.container_name().to_owned(), + }; + Ok(Self::exact(ResponseCache::new(Arc::new(backend)), identity)) + } + + /// Wraps a built exact backend; the TTL a facade must match comes from the built cache. + fn exact(cache: ResponseCache, identity: BackendIdentity) -> Self + where + ResponseCache: ExactResponseCache + 'static, + B: litellm_cache::BaseCache, + B::Context: Default + PartialEq, + { + let cache: Arc = Arc::new(cache); + let default_ttl = cache.default_ttl(); + let identity = match identity { + BackendIdentity::Memory { + capacity, + max_entry_bytes, + .. + } => BackendIdentity::Memory { + capacity, + max_entry_bytes, + default_ttl, + }, + BackendIdentity::Redis { + topology, + namespace, + .. + } => BackendIdentity::Redis { + topology, + namespace, + default_ttl, + }, + other => other, + }; + Self::Exact(ExactService::new(cache, identity)) } pub fn valkey_semantic( @@ -196,103 +241,39 @@ impl NativeResponseCache { )))) } - pub fn disk(directory: &str) -> Result { - let cache = DiskCache::open(directory, ResponseCacheCodec)?; - Ok(Self::Disk(Arc::new(ResponseCache::new(Arc::new(cache))))) - } - - pub fn gcs(config: GcsConfig, token: Option) -> Result { - let backend = match token { - Some(token) => GcsCache::with_token_source( - config, - ResponseCacheCodec, - Arc::new(StaticTokenSource(token)), - )?, - None => GcsCache::new(config, ResponseCacheCodec)?, - }; - Ok(Self::Gcs(Arc::new(ResponseCache::new(Arc::new(backend))))) - } - - pub async fn azure_blob(account_url: &str, container: &str) -> Result { - let backend = AzureBlobCache::connect( - account_url, - container, - ResponseCacheCodec, - tokio::runtime::Handle::current(), - ) - .await?; - Ok(Self::AzureBlob(Arc::new(ResponseCache::new(Arc::new( - backend, - ))))) - } - - pub fn azure_blob_identity(&self) -> Option<(&str, &str)> { + pub fn identity(&self) -> BackendIdentity { match self { - Self::AzureBlob(cache) => Some(( - cache.backend().account_url(), - cache.backend().container_name(), - )), - Self::Memory(_) - | Self::Redis { .. } - | Self::S3(_) - | Self::ValkeySemantic { .. } - | Self::RedisSemantic { .. } - | Self::QdrantSemantic(_) - | Self::Disk(_) - | Self::Gcs(_) => None, - } - } - - fn exact(request: &NativeRequest) -> ResponseCacheRequest { - ResponseCacheRequest { - key: request.key.clone(), - controls: request.controls, - context: ExactCacheContext { ttl: request.ttl }, - max_age: request.max_age, - } - } - - pub(super) fn semantic_request( - request: &NativeRequest, - ) -> ResponseCacheRequest { - ResponseCacheRequest { - key: request.key.clone(), - controls: request.controls, - context: SemanticCacheContext { - input: request.input.clone(), - messages: request.messages.clone(), - metadata: request.metadata.clone(), - scope: request.scope.clone(), - ttl: request.ttl, + Self::Exact(service) => service.identity.clone(), + Self::ValkeySemantic { cache, .. } => BackendIdentity::ValkeySemantic { + index_name: cache.backend().index_name().to_owned(), + similarity_threshold: cache.backend().similarity_threshold(), + }, + Self::RedisSemantic { cache, .. } => BackendIdentity::RedisSemantic { + index_name: cache.backend().index_name().to_owned(), + similarity_threshold: cache.backend().similarity_threshold(), + }, + Self::QdrantSemantic(cache) => BackendIdentity::QdrantSemantic { + collection_name: cache.backend().collection_name().to_owned(), + similarity_threshold: cache.backend().similarity_threshold(), + vector_size: cache.backend().vector_size(), + embedding_model: cache.backend().embedder().model().to_owned(), }, - max_age: request.max_age, } } - fn semantic( - request: &NativeRequest, - scope: &str, - ) -> ResponseCacheRequest { - ResponseCacheRequest { - key: semantic_key(request, scope), - controls: request.controls, - context: SemanticCacheContext { - input: request.input.clone(), - messages: request.messages.clone(), - metadata: request.metadata.clone(), - scope: Some(scope.to_owned()), - ttl: request.ttl, - }, - max_age: request.max_age, - } + pub fn kind(&self) -> &'static str { + self.identity().kind() } pub fn with_redis_flush_size(self, flush_size: Option) -> Self { match self { - Self::Redis { cache, .. } => Self::Redis { - cache, - buffer: flush_size.map(|size| Arc::new(WriteBuffer::new(size))), - }, + Self::Exact(service) if matches!(service.identity, BackendIdentity::Redis { .. }) => { + Self::Exact(Arc::new(ExactService { + cache: Arc::clone(&service.cache), + buffer: flush_size.map(WriteBuffer::new), + identity: service.identity.clone(), + })) + } value => value, } } @@ -310,191 +291,6 @@ impl NativeResponseCache { } } - pub fn kind(&self) -> &'static str { - match self { - Self::Memory(_) => "memory", - Self::Redis { .. } => "redis", - Self::S3(_) => "s3", - Self::Gcs(_) => "gcs", - Self::ValkeySemantic { .. } => "valkey-semantic", - Self::RedisSemantic { .. } => "redis_semantic", - Self::QdrantSemantic(_) => "qdrant_semantic", - Self::Disk(_) => "disk", - Self::AzureBlob(_) => "azure-blob", - } - } - - pub fn default_ttl(&self) -> Option { - match self { - Self::Memory(cache) => cache.default_ttl(), - Self::Redis { cache, .. } => cache.default_ttl(), - Self::S3(cache) => cache.default_ttl(), - Self::Gcs(cache) => cache.default_ttl(), - Self::ValkeySemantic { cache, .. } => cache.default_ttl(), - Self::RedisSemantic { cache, .. } => cache.default_ttl(), - Self::QdrantSemantic(_) => None, - Self::Disk(cache) => cache.default_ttl(), - Self::AzureBlob(cache) => cache.default_ttl(), - } - } - - pub fn bucket(&self) -> Option<&str> { - match self { - Self::S3(cache) => Some(cache.backend().bucket()), - _ => None, - } - } - - pub fn key_prefix(&self) -> Option<&str> { - match self { - Self::S3(cache) => Some(cache.backend().key_prefix()), - _ => None, - } - } - - pub fn region(&self) -> Option<&str> { - match self { - Self::S3(cache) => Some(cache.backend().region()), - _ => None, - } - } - - pub fn endpoint(&self) -> Option<&str> { - match self { - Self::S3(cache) => cache.backend().endpoint(), - _ => None, - } - } - - pub fn namespace(&self) -> Option<&str> { - match self { - Self::Memory(_) - | Self::S3(_) - | Self::ValkeySemantic { .. } - | Self::RedisSemantic { .. } - | Self::QdrantSemantic(_) - | Self::Disk(_) - | Self::AzureBlob(_) - | Self::Gcs(_) => None, - Self::Redis { cache, .. } => cache.backend().namespace(), - } - } - - pub fn topology(&self) -> Option<&RedisTopology> { - match self { - Self::Memory(_) - | Self::S3(_) - | Self::ValkeySemantic { .. } - | Self::RedisSemantic { .. } - | Self::QdrantSemantic(_) - | Self::Disk(_) - | Self::AzureBlob(_) - | Self::Gcs(_) => None, - Self::Redis { cache, .. } => Some(cache.backend().topology()), - } - } - - pub fn capacity(&self) -> Option { - match self { - Self::Memory(cache) => Some(cache.backend().max_size_in_memory()), - Self::Redis { .. } - | Self::S3(_) - | Self::ValkeySemantic { .. } - | Self::RedisSemantic { .. } - | Self::QdrantSemantic(_) - | Self::Disk(_) - | Self::AzureBlob(_) - | Self::Gcs(_) => None, - } - } - - pub fn max_entry_bytes(&self) -> Option { - match self { - Self::Memory(cache) => cache.backend().max_entry_bytes(), - Self::Redis { .. } - | Self::S3(_) - | Self::ValkeySemantic { .. } - | Self::RedisSemantic { .. } - | Self::QdrantSemantic(_) - | Self::Disk(_) - | Self::AzureBlob(_) - | Self::Gcs(_) => None, - } - } - - pub fn directory(&self) -> Option<&Path> { - match self { - Self::Disk(cache) => Some(cache.backend().directory()), - Self::Memory(_) - | Self::Redis { .. } - | Self::S3(_) - | Self::ValkeySemantic { .. } - | Self::RedisSemantic { .. } - | Self::QdrantSemantic(_) - | Self::AzureBlob(_) - | Self::Gcs(_) => None, - } - } - - pub fn semantic_config(&self) -> Option<(f64, &str)> { - match self { - Self::ValkeySemantic { cache, .. } => Some(( - cache.backend().similarity_threshold(), - cache.backend().index_name(), - )), - Self::RedisSemantic { cache, .. } => Some(( - f64::from(cache.backend().similarity_threshold()), - cache.backend().index_name(), - )), - _ => None, - } - } - - pub fn index_name(&self) -> Option<&str> { - match self { - Self::RedisSemantic { cache, .. } => Some(cache.backend().index_name()), - _ => None, - } - } - - pub fn similarity_threshold(&self) -> Option { - match self { - Self::RedisSemantic { cache, .. } => { - Some(f64::from(cache.backend().similarity_threshold())) - } - Self::QdrantSemantic(cache) => Some(cache.backend().similarity_threshold()), - _ => None, - } - } - - pub fn collection_name(&self) -> Option<&str> { - match self { - Self::QdrantSemantic(cache) => Some(cache.backend().collection_name()), - _ => None, - } - } - - pub fn vector_size(&self) -> Option { - match self { - Self::QdrantSemantic(cache) => Some(cache.backend().vector_size()), - _ => None, - } - } - - pub fn embedding_model(&self) -> Option<&str> { - match self { - Self::QdrantSemantic(cache) => Some(cache.backend().embedder().model()), - _ => None, - } - } - - pub fn semantic_embedder(&self) -> Option<&PythonEmbedder> { - match self { - Self::RedisSemantic { embedder, .. } => Some(embedder), - _ => None, - } - } - pub fn embedder_object(&self) -> Option<&Py> { match self { Self::RedisSemantic { embedder, .. } => Some(embedder.object()), @@ -502,21 +298,49 @@ impl NativeResponseCache { } } + /// The prompt and metadata this backend would embed for `request`, if it has a prompt. + pub(super) fn embedding_input(&self, request: &NativeRequest) -> Option { + let context = match self { + Self::ValkeySemantic { scope, .. } => request.scoped_semantic(scope).context, + Self::RedisSemantic { .. } => request.semantic().context, + Self::Exact(_) | Self::QdrantSemantic(_) => return None, + }; + let prompt = litellm_cache_redis_semantic::prompt_from_context(&context)?; + Some(EmbeddingInput { + prompt, + metadata: context.metadata, + }) + } + + /// Drives a semantic operation whose embedding comes from Python. + fn python_semantic<'py>( + &self, + py: Python<'py>, + operation: SemanticOperation, + ) -> PyResult> { + let (embedder, failure) = match self { + Self::ValkeySemantic { embedder, .. } => (embedder, EmbeddingFailure::Propagate), + Self::RedisSemantic { embedder, .. } => (embedder, EmbeddingFailure::Unavailable), + Self::Exact(_) | Self::QdrantSemantic(_) => { + return Err(pyo3::exceptions::PyRuntimeError::new_err( + "semantic execution requires a Python-embedded backend", + )); + } + }; + drive( + py, + SemanticExecution::new(self.clone(), embedder.clone(), failure, operation), + ) + } + pub fn lookup(&self, request: &NativeRequest, now: Duration) -> Result, Error> { match self { - Self::Memory(cache) => cache.lookup(&Self::exact(request), now), - Self::Redis { cache, .. } => cache.lookup(&Self::exact(request), now), - Self::S3(cache) => cache.lookup(&Self::exact(request), now), + Self::Exact(service) => service.cache.lookup(&request.exact(), now), Self::ValkeySemantic { cache, scope, .. } => { - cache.lookup(&Self::semantic(request, scope), now) + cache.lookup(&request.scoped_semantic(scope), now) } - Self::RedisSemantic { cache, .. } => { - cache.lookup(&Self::semantic_request(request), now) - } - Self::Gcs(cache) => cache.lookup(&Self::exact(request), now), - Self::QdrantSemantic(cache) => cache.lookup(&Self::semantic_request(request), now), - Self::Disk(cache) => cache.lookup(&Self::exact(request), now), - Self::AzureBlob(cache) => cache.lookup(&Self::exact(request), now), + Self::RedisSemantic { cache, .. } => cache.lookup(&request.semantic(), now), + Self::QdrantSemantic(cache) => cache.lookup(&request.semantic(), now), } } @@ -527,21 +351,12 @@ impl NativeResponseCache { now: Duration, ) -> Result<(), Error> { match self { - Self::Memory(cache) => cache.store(&Self::exact(request), response, now), - Self::Redis { cache, .. } => cache.store(&Self::exact(request), response, now), - Self::S3(cache) => cache.store(&Self::exact(request), response, now), + Self::Exact(service) => service.cache.store(&request.exact(), response, now), Self::ValkeySemantic { cache, scope, .. } => { - cache.store(&Self::semantic(request, scope), response, now) + cache.store(&request.scoped_semantic(scope), response, now) } - Self::RedisSemantic { cache, .. } => { - cache.store(&Self::semantic_request(request), response, now) - } - Self::Gcs(cache) => cache.store(&Self::exact(request), response, now), - Self::QdrantSemantic(cache) => { - cache.store(&Self::semantic_request(request), response, now) - } - Self::Disk(cache) => cache.store(&Self::exact(request), response, now), - Self::AzureBlob(cache) => cache.store(&Self::exact(request), response, now), + Self::RedisSemantic { cache, .. } => cache.store(&request.semantic(), response, now), + Self::QdrantSemantic(cache) => cache.store(&request.semantic(), response, now), } } @@ -551,29 +366,10 @@ impl NativeResponseCache { now: Duration, ) -> Result { match self { - Self::Memory(cache) => { - let requests = requests.iter().map(Self::exact).collect::>(); - cache.lookup_batch(&requests, now) - } - Self::Redis { cache, .. } => { - let requests = requests.iter().map(Self::exact).collect::>(); - cache.lookup_batch(&requests, now) - } - Self::S3(cache) => { - cache.lookup_batch(&requests.iter().map(Self::exact).collect::>(), now) - } + Self::Exact(service) => service.cache.lookup_batch(&exact_requests(requests), now), Self::ValkeySemantic { .. } | Self::RedisSemantic { .. } | Self::QdrantSemantic(_) => { Err(Error::UnsupportedOperation) } - Self::Gcs(cache) => { - cache.lookup_batch(&requests.iter().map(Self::exact).collect::>(), now) - } - Self::Disk(cache) => { - cache.lookup_batch(&requests.iter().map(Self::exact).collect::>(), now) - } - Self::AzureBlob(cache) => { - cache.lookup_batch(&requests.iter().map(Self::exact).collect::>(), now) - } } } @@ -583,27 +379,14 @@ impl NativeResponseCache { now: Duration, ) -> Result, Error> { match self { - Self::Memory(cache) => cache.async_lookup(&Self::exact(request), now).await, - Self::Redis { cache, .. } => cache.async_lookup(&Self::exact(request), now).await, - Self::S3(cache) => cache.async_lookup(&Self::exact(request), now).await, + Self::Exact(service) => service.cache.async_lookup(&request.exact(), now).await, Self::ValkeySemantic { cache, scope, .. } => { cache - .async_lookup(&Self::semantic(request, scope), now) + .async_lookup(&request.scoped_semantic(scope), now) .await } - Self::RedisSemantic { cache, .. } => { - cache - .async_lookup(&Self::semantic_request(request), now) - .await - } - Self::QdrantSemantic(cache) => { - cache - .async_lookup(&Self::semantic_request(request), now) - .await - } - Self::Gcs(cache) => cache.async_lookup(&Self::exact(request), now).await, - Self::Disk(cache) => cache.async_lookup(&Self::exact(request), now).await, - Self::AzureBlob(cache) => cache.async_lookup(&Self::exact(request), now).await, + Self::RedisSemantic { cache, .. } => cache.async_lookup(&request.semantic(), now).await, + Self::QdrantSemantic(cache) => cache.async_lookup(&request.semantic(), now).await, } } @@ -613,42 +396,16 @@ impl NativeResponseCache { request: NativeRequest, ) -> PyResult> { match self { - Self::Memory(_) - | Self::Redis { .. } - | Self::S3(_) - | Self::Disk(_) - | Self::AzureBlob(_) - | Self::Gcs(_) => { + Self::Exact(_) | Self::QdrantSemantic(_) => { let service = self.clone(); litellm_host_python::run_async( py, - async move { service.async_lookup(&request, super::request::now()).await }, + async move { service.async_lookup(&request, now()).await }, super::cache_error, ) } - Self::ValkeySemantic { - cache, - embedder, - scope, - } => drive_semantic( - py, - SemanticEmbedExecution::lookup( - Arc::clone(cache.backend_arc()), - embedder.clone(), - Self::semantic(&request, scope), - ), - ), - Self::RedisSemantic { .. } => drive( - py, - SemanticBody::new(self.clone(), SemanticOperation::Lookup(request)), - ), - Self::QdrantSemantic(_) => { - let service = self.clone(); - litellm_host_python::run_async( - py, - async move { service.async_lookup(&request, super::request::now()).await }, - super::cache_error, - ) + Self::ValkeySemantic { .. } | Self::RedisSemantic { .. } => { + self.python_semantic(py, SemanticOperation::Lookup(request)) } } } @@ -660,61 +417,29 @@ impl NativeResponseCache { now: Duration, ) -> Result<(), Error> { match self { - Self::Memory(cache) => { - cache - .async_store(&Self::exact(request), response, now) - .await - } - Self::Redis { - cache, - buffer: None, - } => { - cache - .async_store(&Self::exact(request), response, now) - .await - } - Self::Redis { - cache, - buffer: Some(buffer), - } => { - buffer - .async_store(cache, &Self::exact(request), response, now) - .await - } - Self::S3(cache) => { - cache - .async_store(&Self::exact(request), response, now) - .await - } + Self::Exact(service) => match &service.buffer { + None => { + service + .cache + .async_store(&request.exact(), response, now) + .await + } + Some(buffer) => { + buffer + .async_store(service.cache.as_ref(), &request.exact(), response, now) + .await + } + }, Self::ValkeySemantic { cache, scope, .. } => { cache - .async_store(&Self::semantic(request, scope), response, now) + .async_store(&request.scoped_semantic(scope), response, now) .await } Self::RedisSemantic { cache, .. } => { - cache - .async_store(&Self::semantic_request(request), response, now) - .await + cache.async_store(&request.semantic(), response, now).await } Self::QdrantSemantic(cache) => { - cache - .async_store(&Self::semantic_request(request), response, now) - .await - } - Self::Gcs(cache) => { - cache - .async_store(&Self::exact(request), response, now) - .await - } - Self::Disk(cache) => { - cache - .async_store(&Self::exact(request), response, now) - .await - } - Self::AzureBlob(cache) => { - cache - .async_store(&Self::exact(request), response, now) - .await + cache.async_store(&request.semantic(), response, now).await } } } @@ -726,51 +451,16 @@ impl NativeResponseCache { response: Value, ) -> PyResult> { match self { - Self::Memory(_) - | Self::Redis { .. } - | Self::S3(_) - | Self::Disk(_) - | Self::AzureBlob(_) - | Self::Gcs(_) => { + Self::Exact(_) | Self::QdrantSemantic(_) => { let service = self.clone(); litellm_host_python::run_async( py, - async move { - service - .async_store(&request, response, super::request::now()) - .await - }, + async move { service.async_store(&request, response, now()).await }, super::cache_error, ) } - Self::ValkeySemantic { - cache, - embedder, - scope, - } => drive_semantic( - py, - SemanticEmbedExecution::store( - Arc::clone(cache.backend_arc()), - embedder.clone(), - Self::semantic(&request, scope), - response, - ), - ), - Self::RedisSemantic { .. } => drive( - py, - SemanticBody::new(self.clone(), SemanticOperation::Store(request, response)), - ), - Self::QdrantSemantic(_) => { - let service = self.clone(); - litellm_host_python::run_async( - py, - async move { - service - .async_store(&request, response, super::request::now()) - .await - }, - super::cache_error, - ) + Self::ValkeySemantic { .. } | Self::RedisSemantic { .. } => { + self.python_semantic(py, SemanticOperation::Store(request, response)) } } } @@ -781,37 +471,15 @@ impl NativeResponseCache { now: Duration, ) -> Result { match self { - Self::Memory(cache) => { - let requests = requests.iter().map(Self::exact).collect::>(); - cache.async_lookup_batch(&requests, now).await - } - Self::Redis { cache, .. } => { - let requests = requests.iter().map(Self::exact).collect::>(); - cache.async_lookup_batch(&requests, now).await - } - Self::S3(cache) => { - cache - .async_lookup_batch(&requests.iter().map(Self::exact).collect::>(), now) + Self::Exact(service) => { + service + .cache + .async_lookup_batch(&exact_requests(requests), now) .await } Self::ValkeySemantic { .. } | Self::RedisSemantic { .. } | Self::QdrantSemantic(_) => { Err(Error::UnsupportedOperation) } - Self::Gcs(cache) => { - cache - .async_lookup_batch(&requests.iter().map(Self::exact).collect::>(), now) - .await - } - Self::Disk(cache) => { - cache - .async_lookup_batch(&requests.iter().map(Self::exact).collect::>(), now) - .await - } - Self::AzureBlob(cache) => { - cache - .async_lookup_batch(&requests.iter().map(Self::exact).collect::>(), now) - .await - } } } @@ -821,31 +489,17 @@ impl NativeResponseCache { now: Duration, ) -> Result<(), Error> { match self { - Self::Memory(cache) => { + Self::Exact(service) => { let entries = entries .into_iter() - .map(|(request, value)| (Self::exact(&request), value)) + .map(|(request, value)| (request.exact(), value)) .collect(); - cache.async_store_batch(entries, now).await - } - Self::Redis { cache, .. } => { - let entries = entries - .into_iter() - .map(|(request, value)| (Self::exact(&request), value)) - .collect(); - cache.async_store_batch(entries, now).await - } - Self::S3(cache) => { - let entries = entries - .into_iter() - .map(|(request, value)| (Self::exact(&request), value)) - .collect(); - cache.async_store_batch(entries, now).await + service.cache.async_store_batch(entries, now).await } Self::ValkeySemantic { cache, scope, .. } => { let entries = entries .into_iter() - .map(|(request, value)| (Self::semantic(&request, scope), value)) + .map(|(request, value)| (request.scoped_semantic(scope), value)) .collect(); cache.async_store_batch(entries, now).await } @@ -853,28 +507,7 @@ impl NativeResponseCache { Self::QdrantSemantic(cache) => { let entries = entries .into_iter() - .map(|(request, value)| (Self::semantic_request(&request), value)) - .collect(); - cache.async_store_batch(entries, now).await - } - Self::Gcs(cache) => { - let entries = entries - .into_iter() - .map(|(request, value)| (Self::exact(&request), value)) - .collect(); - cache.async_store_batch(entries, now).await - } - Self::Disk(cache) => { - let entries = entries - .into_iter() - .map(|(request, value)| (Self::exact(&request), value)) - .collect(); - cache.async_store_batch(entries, now).await - } - Self::AzureBlob(cache) => { - let entries = entries - .into_iter() - .map(|(request, value)| (Self::exact(&request), value)) + .map(|(request, value)| (request.semantic(), value)) .collect(); cache.async_store_batch(entries, now).await } @@ -887,185 +520,54 @@ impl NativeResponseCache { entries: Vec<(NativeRequest, Value)>, ) -> PyResult> { match self { - Self::Memory(_) - | Self::Redis { .. } - | Self::S3(_) - | Self::Disk(_) - | Self::AzureBlob(_) - | Self::Gcs(_) => { + Self::Exact(_) | Self::QdrantSemantic(_) => { let service = self.clone(); litellm_host_python::run_async( py, - async move { - service - .async_store_batch(entries, super::request::now()) - .await - }, + async move { service.async_store_batch(entries, now()).await }, super::cache_error, ) } - Self::ValkeySemantic { - cache, - embedder, - scope, - } => { - let (requests, responses): (Vec<_>, Vec<_>) = entries - .into_iter() - .map(|(request, response)| (Self::semantic(&request, scope), response)) - .unzip(); - drive_semantic( - py, - SemanticEmbedExecution::store_batch( - Arc::clone(cache.backend_arc()), - embedder.clone(), - requests, - responses, - ), - ) - } - Self::RedisSemantic { .. } => drive( - py, - SemanticBody::new(self.clone(), SemanticOperation::StoreBatch(entries.into())), - ), - Self::QdrantSemantic(_) => { - let service = self.clone(); - litellm_host_python::run_async( - py, - async move { - service - .async_store_batch(entries, super::request::now()) - .await - }, - super::cache_error, - ) + Self::ValkeySemantic { .. } | Self::RedisSemantic { .. } => { + self.python_semantic(py, SemanticOperation::StoreBatch(entries.into())) } } } pub async fn async_flush(&self) -> Result<(), Error> { match self { - Self::Memory(cache) => cache.async_flush().await, - Self::Redis { cache, buffer } => { - if let Some(buffer) = buffer { + Self::Exact(service) => { + if let Some(buffer) = &service.buffer { buffer.clear()?; } - cache.async_flush().await + service.cache.async_flush().await } - Self::S3(cache) => cache.async_flush().await, Self::ValkeySemantic { .. } | Self::RedisSemantic { .. } | Self::QdrantSemantic(_) => { Err(Error::UnsupportedOperation) } - Self::Gcs(cache) => cache.async_flush().await, - Self::Disk(cache) => cache.async_flush().await, - Self::AzureBlob(cache) => cache.async_flush().await, } } pub async fn test_connection(&self) -> Result { match self { - Self::Memory(cache) => cache.test_connection().await, - Self::Redis { cache, .. } => cache.test_connection().await, - Self::S3(cache) => cache.test_connection().await, + Self::Exact(service) => service.cache.test_connection().await, Self::ValkeySemantic { cache, .. } => cache.test_connection().await, Self::RedisSemantic { .. } | Self::QdrantSemantic(_) => { Err(Error::UnsupportedOperation) } - Self::Gcs(cache) => cache.test_connection().await, - Self::Disk(cache) => cache.test_connection().await, - Self::AzureBlob(cache) => cache.test_connection().await, } } pub(super) fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { match self { - Self::ValkeySemantic { embedder, .. } => embedder.traverse(visit)?, - Self::RedisSemantic { embedder, .. } => embedder.traverse(visit)?, - _ => {} - } - Ok(()) - } - - pub fn gcs_backend(&self) -> Option<&GcsCache> { - match self { - Self::Gcs(cache) => Some(cache.backend()), - _ => None, + Self::ValkeySemantic { embedder, .. } | Self::RedisSemantic { embedder, .. } => { + embedder.traverse(visit) + } + Self::Exact(_) | Self::QdrantSemantic(_) => Ok(()), } } } -#[cfg(test)] -mod tests { - use litellm_cache_response::{CacheControls, CacheKeyInput, cache_key}; - use serde_json::json; - use sha2::{Digest, Sha256}; - - use super::*; - - fn native_request(key: CacheKeyInput, metadata: Value) -> NativeRequest { - NativeRequest { - key, - controls: CacheControls::default(), - ttl: None, - max_age: None, - messages: Some(json!([{"role": "user", "content": "prompt"}])), - input: None, - metadata: Some(metadata), - litellm_metadata: None, - litellm_params: None, - scope: None, - } - } - - #[test] - fn semantic_key_matches_python_scope_material() { - let key = CacheKeyInput { - fields: vec![ - CacheKeyField { - name: "model".to_owned(), - value: Some("gpt-4.1".to_owned()), - api_parameter: true, - internal_parameter: false, - }, - CacheKeyField { - name: "messages".to_owned(), - value: Some("prompt".to_owned()), - api_parameter: true, - internal_parameter: false, - }, - ], - ..Default::default() - }; - let request = native_request( - key, - json!({"user_api_key": "k1", "user_api_key_team_id": null}), - ); - let expected = format!("{:x}", Sha256::digest(b"model: gpt-4.1user_api_key: k1")); - assert_eq!(cache_key(&semantic_key(&request, "key")), expected); - - let end_user_request = native_request( - request.key.clone(), - json!({"user_api_key": "k1", "user_api_key_end_user_id": "u1"}), - ); - let expected = format!( - "{:x}", - Sha256::digest(b"model: gpt-4.1user_api_key: k1user_api_key_end_user_id: u1") - ); - assert_eq!( - cache_key(&semantic_key(&end_user_request, "end_user")), - expected - ); - - let preset_request = native_request( - CacheKeyInput { - preset: Some("preset-key".to_owned()), - ..Default::default() - }, - json!({"user_api_key": "k1"}), - ); - assert_eq!( - semantic_key(&preset_request, "end_user").preset.as_deref(), - Some("preset-key") - ); - assert!(semantic_key(&preset_request, "end_user").fields.is_empty()); - } +fn exact_requests(requests: &[NativeRequest]) -> Vec { + requests.iter().map(NativeRequest::exact).collect() } diff --git a/litellm-rust/crates/python-bridge/src/cache/request.rs b/litellm-rust/crates/python-bridge/src/cache/request.rs index 3b4b910c1f0..627bf9f1840 100644 --- a/litellm-rust/crates/python-bridge/src/cache/request.rs +++ b/litellm-rust/crates/python-bridge/src/cache/request.rs @@ -1,7 +1,7 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH}; -use litellm_cache::ExactCacheContext; -use litellm_cache_response::{CacheControls, CacheKeyInput, ResponseCacheRequest}; +use litellm_cache::{ExactCacheContext, SemanticCacheContext}; +use litellm_cache_response::{CacheControls, CacheKeyField, CacheKeyInput, ResponseCacheRequest}; use litellm_host_python::from_py; use pyo3::{exceptions::PyValueError, prelude::*}; use serde::Deserialize; @@ -36,6 +36,99 @@ pub(super) struct NativeRequest { pub(super) scope: Option, } +impl NativeRequest { + pub(super) fn exact(&self) -> ResponseCacheRequest { + ResponseCacheRequest { + key: self.key.clone(), + controls: self.controls, + context: ExactCacheContext { ttl: self.ttl }, + max_age: self.max_age, + } + } + + /// The request as a semantic backend that keys on the caller's scope sees it. + pub(super) fn semantic(&self) -> ResponseCacheRequest { + self.semantic_with(self.key.clone(), self.scope.clone()) + } + + /// The request keyed the way Python's Valkey semantic cache keys it: prompt fields drop out + /// and the tenant identifiers for `scope` join the key. + pub(super) fn scoped_semantic( + &self, + scope: &str, + ) -> ResponseCacheRequest { + self.semantic_with(semantic_key(self, scope), Some(scope.to_owned())) + } + + fn semantic_with( + &self, + key: CacheKeyInput, + scope: Option, + ) -> ResponseCacheRequest { + ResponseCacheRequest { + key, + controls: self.controls, + context: SemanticCacheContext { + input: self.input.clone(), + messages: self.messages.clone(), + metadata: self.metadata.clone(), + scope, + ttl: self.ttl, + }, + max_age: self.max_age, + } + } +} + +fn semantic_key(request: &NativeRequest, scope: &str) -> CacheKeyInput { + let mut key = request.key.clone(); + if key.preset.is_some() { + return key; + } + key.fields + .retain(|field| !matches!(field.name.as_str(), "messages" | "prompt" | "input")); + const TENANT: [&str; 3] = [ + "user_api_key", + "user_api_key_team_id", + "user_api_key_org_id", + ]; + let end_user = (scope == "end_user").then_some("user_api_key_end_user_id"); + for name in TENANT.into_iter().chain(end_user) { + let sources = [ + request.metadata.as_ref(), + request.litellm_metadata.as_ref(), + request + .litellm_params + .as_ref() + .and_then(|params| params.get("metadata")), + request + .litellm_params + .as_ref() + .and_then(|params| params.get("litellm_metadata")), + ]; + let Some(value) = sources.into_iter().flatten().find_map(|source| { + source + .as_object() + .and_then(|values| values.get(name)) + .filter(|value| !value.is_null()) + }) else { + continue; + }; + let value = match value { + Value::Null => continue, + Value::String(text) => text.clone(), + other => other.to_string(), + }; + key.fields.push(CacheKeyField { + name: name.to_owned(), + value: Some(value), + api_parameter: true, + internal_parameter: false, + }); + } + key +} + pub(super) fn request(value: &Bound<'_, PyAny>) -> PyResult { let input: RequestInput = from_py(value)?; request_input(input) @@ -76,3 +169,90 @@ pub(super) fn now() -> Duration { .duration_since(UNIX_EPOCH) .unwrap_or_default() } + +#[cfg(test)] +mod tests { + use litellm_cache_response::{CacheControls, CacheKeyInput, cache_key}; + use serde_json::json; + use sha2::{Digest, Sha256}; + + use super::*; + + fn native_request(key: CacheKeyInput, metadata: Value) -> NativeRequest { + NativeRequest { + key, + controls: CacheControls::default(), + ttl: None, + max_age: None, + messages: Some(json!([{"role": "user", "content": "prompt"}])), + input: None, + metadata: Some(metadata), + litellm_metadata: None, + litellm_params: None, + scope: None, + } + } + + #[test] + fn semantic_key_matches_python_scope_material() { + let key = CacheKeyInput { + fields: vec![ + CacheKeyField { + name: "model".to_owned(), + value: Some("gpt-4.1".to_owned()), + api_parameter: true, + internal_parameter: false, + }, + CacheKeyField { + name: "messages".to_owned(), + value: Some("prompt".to_owned()), + api_parameter: true, + internal_parameter: false, + }, + ], + ..Default::default() + }; + let request = native_request( + key, + json!({"user_api_key": "k1", "user_api_key_team_id": null}), + ); + let expected = format!("{:x}", Sha256::digest(b"model: gpt-4.1user_api_key: k1")); + assert_eq!(cache_key(&semantic_key(&request, "key")), expected); + assert_eq!(cache_key(&request.scoped_semantic("key").key), expected); + + let end_user_request = native_request( + request.key.clone(), + json!({"user_api_key": "k1", "user_api_key_end_user_id": "u1"}), + ); + let expected = format!( + "{:x}", + Sha256::digest(b"model: gpt-4.1user_api_key: k1user_api_key_end_user_id: u1") + ); + assert_eq!( + cache_key(&semantic_key(&end_user_request, "end_user")), + expected + ); + + let preset_request = native_request( + CacheKeyInput { + preset: Some("preset-key".to_owned()), + ..Default::default() + }, + json!({"user_api_key": "k1"}), + ); + assert_eq!( + semantic_key(&preset_request, "end_user").preset.as_deref(), + Some("preset-key") + ); + assert!(semantic_key(&preset_request, "end_user").fields.is_empty()); + assert_eq!(preset_request.semantic().context.scope, None); + assert_eq!( + preset_request + .scoped_semantic("end_user") + .context + .scope + .as_deref(), + Some("end_user") + ); + } +} diff --git a/litellm-rust/crates/python-bridge/src/cache/semantic.rs b/litellm-rust/crates/python-bridge/src/cache/semantic.rs index 934de01e721..9f4d18d45cd 100644 --- a/litellm-rust/crates/python-bridge/src/cache/semantic.rs +++ b/litellm-rust/crates/python-bridge/src/cache/semantic.rs @@ -1,7 +1,6 @@ -use std::collections::VecDeque; +use std::{collections::VecDeque, time::Duration}; use litellm_cache::Error; -use litellm_cache_redis_semantic::prompt_from_context; use litellm_host_python::{Execution, ExecutionBody, ExecutionStep, run_async}; use pyo3::{ PyTraverseError, PyVisit, @@ -23,29 +22,104 @@ pub(super) enum SemanticOperation { StoreBatch(VecDeque<(NativeRequest, Value)>), } +/// What an exception from the Python embedder means for the operation. +#[derive(Clone, Copy)] +pub(super) enum EmbeddingFailure { + /// Raise the Python exception unchanged. + Propagate, + /// Treat the embedding as unavailable and let the backend report that. + Unavailable, +} + enum Phase { Start, AwaitingEmbedding, AwaitingBackend, } -pub(super) struct SemanticBody { +/// Runs a semantic cache operation whose embedding comes from Python: await the Python +/// embedder in the caller's event loop, seed the native backend with the vector, await the +/// backend, and repeat for each entry of a batch. +pub(super) struct SemanticExecution { service: NativeResponseCache, + embedder: PythonEmbedder, + failure: EmbeddingFailure, operation: SemanticOperation, pending: Option<(NativeRequest, Option)>, phase: Phase, + now: Duration, } -impl SemanticBody { - pub(super) fn new(service: NativeResponseCache, operation: SemanticOperation) -> Self { +impl SemanticExecution { + pub(super) fn new( + service: NativeResponseCache, + embedder: PythonEmbedder, + failure: EmbeddingFailure, + operation: SemanticOperation, + ) -> Self { Self { service, + embedder, + failure, operation, pending: None, phase: Phase::Start, + now: now(), } } + /// Takes the next entry of the operation; `None` once a batch is exhausted. + fn next_pending(&mut self) -> Option<(NativeRequest, Option)> { + match &mut self.operation { + SemanticOperation::Lookup(request) => Some((request.clone(), None)), + SemanticOperation::Store(request, response) => { + Some((request.clone(), Some(std::mem::take(response)))) + } + SemanticOperation::StoreBatch(queue) => queue + .pop_front() + .map(|(request, response)| (request, Some(response))), + } + } + + fn start(&mut self, py: Python<'_>) -> PyResult { + let Some(pending) = self.next_pending() else { + return Ok(ExecutionStep::Return(py.None())); + }; + let (request, response) = &pending; + let enabled = match response { + None => request.controls.reads(), + Some(_) => request.controls.writes(), + }; + let input = enabled + .then(|| self.service.embedding_input(request)) + .flatten(); + self.pending = Some(pending); + let Some(input) = input else { + return self.backend_step(py, Err(Error::Unavailable)); + }; + let awaitable = + self.embedder + .async_embedding(py, &input.prompt, input.metadata.as_ref())?; + self.phase = Phase::AwaitingEmbedding; + Ok(ExecutionStep::Await(awaitable)) + } + + fn embedded(&mut self, py: Python<'_>, result: PyResult>) -> PyResult { + let seed = match result { + Ok(vector) => { + PythonEmbedder::extract(vector.into_bound(py)).map_err(|_| Error::Unavailable) + } + Err(error) => match self.failure { + EmbeddingFailure::Propagate => return Err(error), + EmbeddingFailure::Unavailable if error.is_instance_of::(py) => { + Err(Error::Unavailable) + } + EmbeddingFailure::Unavailable => return Err(error), + }, + }; + self.backend_step(py, seed) + } + fn backend_step( &mut self, py: Python<'_>, @@ -56,11 +130,12 @@ impl SemanticBody { PyRuntimeError::new_err("semantic execution resumed without a pending operation") })?; let service = self.service.clone(); + let now = self.now; let future = async move { match response { - None => service.async_lookup(&request, now()).await, + None => service.async_lookup(&request, now).await, Some(response) => service - .async_store(&request, response, now()) + .async_store(&request, response, now) .await .map(|_| None), } @@ -68,106 +143,45 @@ impl SemanticBody { let awaitable = run_async(py, with_prepared_embedding(seed, future), cache_error)?; Ok(ExecutionStep::Await(awaitable.unbind())) } + + fn resume_py( + &mut self, + py: Python<'_>, + result: Option>>, + ) -> PyResult { + match (&self.phase, result) { + (Phase::Start, None) => self.start(py), + (Phase::AwaitingEmbedding, Some(result)) => self.embedded(py, result), + (Phase::AwaitingBackend, Some(Err(error))) => Err(error), + (Phase::AwaitingBackend, Some(Ok(value))) => { + let more = matches!( + &self.operation, + SemanticOperation::StoreBatch(queue) if !queue.is_empty() + ); + if more { + self.phase = Phase::Start; + return self.start(py); + } + Ok(ExecutionStep::Return(value)) + } + _ => Err(PyRuntimeError::new_err( + "invalid semantic cache execution state", + )), + } + } } -impl ExecutionBody for SemanticBody { - fn resume(&mut self, mut result: Option>>) -> PyResult { - Python::attach(|py| { - loop { - match self.phase { - Phase::Start => { - if result.is_some() { - return Err(PyRuntimeError::new_err( - "semantic execution received a result before starting", - )); - } - if self.pending.is_none() { - match &mut self.operation { - SemanticOperation::Lookup(request) => { - self.pending = Some((request.clone(), None)); - } - SemanticOperation::Store(request, response) => { - let response = std::mem::replace(response, Value::Null); - self.pending = Some((request.clone(), Some(response))); - } - SemanticOperation::StoreBatch(queue) => { - let Some((request, response)) = queue.pop_front() else { - return Ok(ExecutionStep::Return(py.None())); - }; - self.pending = Some((request, Some(response))); - } - } - } - let (request, _) = self.pending.as_ref().ok_or_else(|| { - PyRuntimeError::new_err("semantic execution has no pending operation") - })?; - let semantic = NativeResponseCache::semantic_request(request); - let Some(prompt) = prompt_from_context(&semantic.context) else { - return self.backend_step(py, Err(Error::Unavailable)); - }; - let embedder = self.service.semantic_embedder().ok_or_else(|| { - PyRuntimeError::new_err( - "semantic execution requires a redis-semantic backend", - ) - })?; - let coroutine = embedder.async_embedding_coroutine( - py, - &prompt, - semantic.context.metadata.as_ref(), - )?; - self.phase = Phase::AwaitingEmbedding; - return Ok(ExecutionStep::Await(coroutine)); - } - Phase::AwaitingEmbedding => { - let result = result.take().ok_or_else(|| { - PyRuntimeError::new_err( - "semantic execution expected an embedding result", - ) - })?; - let seed = match result { - Ok(value) => PythonEmbedder::extract(value.into_bound(py)) - .map_err(|_| Error::Unavailable), - Err(error) => { - if !error.is_instance_of::(py) { - return Err(error); - } - Err(Error::Unavailable) - } - }; - return self.backend_step(py, seed); - } - Phase::AwaitingBackend => { - let result = result.take().ok_or_else(|| { - PyRuntimeError::new_err("semantic execution expected a backend result") - })?; - let value = match result { - Ok(value) => value, - Err(error) => return Err(error), - }; - let more = matches!( - &self.operation, - SemanticOperation::StoreBatch(queue) if !queue.is_empty() - ); - if more { - self.phase = Phase::Start; - continue; - } - return Ok(ExecutionStep::Return(value)); - } - } - } - }) +impl ExecutionBody for SemanticExecution { + fn resume(&mut self, result: Option>>) -> PyResult { + Python::attach(|py| self.resume_py(py, result)) } fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { - if let Some(embedder) = self.service.semantic_embedder() { - embedder.traverse(visit)?; - } - Ok(()) + self.embedder.traverse(visit) } } -pub(super) fn drive(py: Python<'_>, body: SemanticBody) -> PyResult> { +pub(super) fn drive(py: Python<'_>, body: SemanticExecution) -> PyResult> { let execution = Py::new(py, Execution::new(body))?; py.import("litellm.rust_bridge.lifecycle")? .getattr("drive")? diff --git a/litellm-rust/crates/python-bridge/src/cache/semantic_step.rs b/litellm-rust/crates/python-bridge/src/cache/semantic_step.rs deleted file mode 100644 index 24caf3374d6..00000000000 --- a/litellm-rust/crates/python-bridge/src/cache/semantic_step.rs +++ /dev/null @@ -1,249 +0,0 @@ -use std::{sync::Arc, time::Duration}; - -use litellm_cache::SemanticCacheContext; -use litellm_cache_response::{ResponseCache, ResponseCacheCodec, ResponseCacheRequest}; -use litellm_cache_valkey_semantic::{PreparedEmbedding, ValkeySemanticCache, prompt_from_context}; -use litellm_host_python::{Execution, ExecutionBody, ExecutionStep, run_async}; -use pyo3::{PyTraverseError, PyVisit, exceptions::PyRuntimeError, prelude::*}; -use serde_json::Value; - -use super::{cache_error, embedder::PythonEmbedder}; - -pub(super) enum Op { - Lookup, - Store(Value), - StoreBatch(Vec), -} - -#[derive(Clone, Copy)] -enum State { - Start, - AwaitingEmbedding, - AwaitingStorage, - Done, -} - -pub(super) struct SemanticEmbedExecution { - backend: Arc>, - embedder: PythonEmbedder, - requests: Vec>, - op: Op, - now: Option, - prepared: Vec>>, - index: usize, - state: State, -} - -impl SemanticEmbedExecution { - pub(super) fn lookup( - backend: Arc>, - embedder: PythonEmbedder, - request: ResponseCacheRequest, - ) -> Self { - Self { - backend, - embedder, - requests: vec![request], - op: Op::Lookup, - now: None, - prepared: vec![None], - index: 0, - state: State::Start, - } - } - - pub(super) fn store( - backend: Arc>, - embedder: PythonEmbedder, - request: ResponseCacheRequest, - response: Value, - ) -> Self { - Self { - backend, - embedder, - requests: vec![request], - op: Op::Store(response), - now: None, - prepared: vec![None], - index: 0, - state: State::Start, - } - } - - pub(super) fn store_batch( - backend: Arc>, - embedder: PythonEmbedder, - requests: Vec>, - responses: Vec, - ) -> Self { - Self { - backend, - embedder, - prepared: vec![None; requests.len()], - requests, - op: Op::StoreBatch(responses), - now: None, - index: 0, - state: State::Start, - } - } - - fn start(&mut self, py: Python<'_>) -> PyResult { - if self.now.is_none() { - self.now = Some(super::request::now()); - } - while self.index < self.requests.len() { - let request = &self.requests[self.index]; - let enabled = match &self.op { - Op::Lookup => request.controls.reads(), - Op::Store(_) | Op::StoreBatch(_) => request.controls.writes(), - }; - if !enabled { - self.index += 1; - continue; - } - let Some(prompt) = prompt_from_context(&request.context) else { - self.index += 1; - continue; - }; - let metadata = request.context.metadata.clone(); - let awaitable = self - .embedder - .async_embed_awaitable(py, &prompt, &metadata)?; - self.state = State::AwaitingEmbedding; - return Ok(ExecutionStep::Await(awaitable.unbind())); - } - self.state = State::AwaitingStorage; - self.storage_step(py) - } - - fn storage_step(&self, py: Python<'_>) -> PyResult { - let requests = self.requests.clone(); - let prepared = self.prepared.clone(); - let backend = Arc::clone(&self.backend); - let now = self - .now - .ok_or_else(|| PyRuntimeError::new_err("semantic cache timestamp is unavailable"))?; - let awaitable = match &self.op { - Op::Lookup => { - let Some(request) = requests.into_iter().next() else { - return Err(PyRuntimeError::new_err( - "semantic lookup requires one request", - )); - }; - match prepared.into_iter().next().flatten() { - Some(values) => { - let backend = backend.with_embedder(PreparedEmbedding(values)); - let cache = Arc::new(ResponseCache::new(Arc::new(backend))); - run_async( - py, - async move { cache.async_lookup(&request, now).await }, - cache_error, - )? - } - None => { - let cache = Arc::new(ResponseCache::new(backend)); - run_async( - py, - async move { cache.async_lookup(&request, now).await }, - cache_error, - )? - } - } - } - Op::Store(response) => { - let Some(request) = requests.into_iter().next() else { - return Err(PyRuntimeError::new_err( - "semantic store requires one request", - )); - }; - let response = response.clone(); - match prepared.into_iter().next().flatten() { - Some(values) => { - let backend = backend.with_embedder(PreparedEmbedding(values)); - let cache = Arc::new(ResponseCache::new(Arc::new(backend))); - run_async( - py, - async move { cache.async_store(&request, response, now).await }, - cache_error, - )? - } - None => { - let cache = Arc::new(ResponseCache::new(backend)); - run_async( - py, - async move { cache.async_store(&request, response, now).await }, - cache_error, - )? - } - } - } - Op::StoreBatch(responses) => { - let responses = responses.clone(); - run_async( - py, - async move { - for ((request, response), prepared) in - requests.into_iter().zip(responses).zip(prepared) - { - let Some(values) = prepared else { - continue; - }; - let backend = backend.with_embedder(PreparedEmbedding(values)); - let cache = ResponseCache::new(Arc::new(backend)); - cache.async_store(&request, response, now).await?; - } - Ok(()) - }, - cache_error, - )? - } - }; - Ok(ExecutionStep::Await(awaitable.unbind())) - } - - fn resume_py( - &mut self, - py: Python<'_>, - result: Option>>, - ) -> PyResult { - match (self.state, result) { - (State::Start, None) => self.start(py), - (State::AwaitingEmbedding, Some(Ok(value))) => { - let values = value.bind(py).extract::>()?; - self.prepared[self.index] = - Some(values.into_iter().map(|value| value as f32).collect()); - self.index += 1; - self.start(py) - } - (State::AwaitingStorage, Some(Ok(value))) => { - self.state = State::Done; - Ok(ExecutionStep::Return(value)) - } - (_, Some(Err(error))) => Err(error), - _ => Err(PyRuntimeError::new_err( - "invalid semantic cache execution state", - )), - } - } -} - -impl ExecutionBody for SemanticEmbedExecution { - fn resume(&mut self, result: Option>>) -> PyResult { - Python::attach(|py| self.resume_py(py, result)) - } - - fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { - self.embedder.traverse(visit) - } -} - -pub(super) fn drive_semantic<'py>( - py: Python<'py>, - body: SemanticEmbedExecution, -) -> PyResult> { - let execution = Py::new(py, Execution::new(body))?; - py.import("litellm.rust_bridge.lifecycle")? - .getattr("drive")? - .call1((execution,)) -} diff --git a/litellm-rust/crates/python-bridge/src/coercion.rs b/litellm-rust/crates/python-bridge/src/coercion.rs index bb5b8b2d454..1b0b073b3d1 100644 --- a/litellm-rust/crates/python-bridge/src/coercion.rs +++ b/litellm-rust/crates/python-bridge/src/coercion.rs @@ -1,7 +1,4 @@ -use std::collections::BTreeSet; - use litellm_core_utils::serde_compat::parse_str_bool; -use litellm_http::SslVerify; use pyo3::{ exceptions::{PyAttributeError, PyRuntimeError, PyValueError}, prelude::*, @@ -33,36 +30,52 @@ impl From for PyErr { } } -pub(crate) struct Truthy(pub bool); -pub(crate) struct ExactTrue(pub bool); -pub(crate) struct StrBool(pub Option); -pub(crate) struct OptionalStrictString(pub Option); -pub(crate) struct FalsyOptionalString(pub Option); -pub(crate) struct TuningString(pub Option); -pub(crate) struct StringCollection(pub Vec); -pub(crate) struct SslVerifyInput(pub Option); +pub(crate) struct FieldSpec { + name: &'static str, + decode: fn(&Field<'_>) -> Result, +} + +impl FieldSpec { + pub(crate) const fn new( + name: &'static str, + decode: fn(&Field<'_>) -> Result, + ) -> Self { + Self { name, decode } + } + + pub(crate) fn read( + &self, + snapshot: &Bound<'_, PyAny>, + group: &'static str, + ) -> Result { + (self.decode)(&Field::read(snapshot, group, self.name)?) + } +} pub(crate) struct Field<'py> { - path: &'static str, + group: &'static str, + name: &'static str, value: Bound<'py, PyAny>, } impl<'py> Field<'py> { - pub(crate) fn new(path: &'static str, value: Bound<'py, PyAny>) -> Self { - Self { path, value } + pub(crate) fn new(group: &'static str, name: &'static str, value: Bound<'py, PyAny>) -> Self { + Self { group, name, value } } + /// Reads `snapshot.`, distinguishing a field the accessor never declared from a + /// descriptor that raised `AttributeError`. pub(crate) fn read( snapshot: &Bound<'py, PyAny>, - path: &'static str, + group: &'static str, + name: &'static str, ) -> Result { - let name = path.rsplit('.').next().unwrap_or(path); match snapshot.getattr(name) { - Ok(value) => Ok(Self::new(path, value)), + Ok(value) => Ok(Self::new(group, name, value)), Err(error) if error.is_instance_of::(snapshot.py()) => { match Self::missing_field(snapshot, name) { Ok(true) => Err(ProjectionError::InternalSchemaFailure(format!( - "{path}: missing snapshot field" + "{group}.{name}: missing snapshot field" ))), _ => Err(error.into()), } @@ -84,27 +97,49 @@ impl<'py> Field<'py> { && getter.is(object.getattr("__getattribute__")?)) } - fn expected(&self, expected: &'static str) -> Result { + pub(crate) fn path(&self) -> String { + format!("{}.{}", self.group, self.name) + } + + /// A member of this field's collection, reported under the same path. + pub(crate) fn member(&self, value: Bound<'py, PyAny>) -> Self { + Self::new(self.group, self.name, value) + } + + pub(crate) fn expected(&self, expected: &str) -> Result { Ok(format!( "{}: expected {expected}, got {}", - self.path, + self.path(), self.value.get_type().name()? )) } - fn invalid(&self, expected: &'static str) -> ProjectionError { + pub(crate) fn invalid(&self, expected: &str) -> ProjectionError { match self.expected(expected) { Ok(message) => ProjectionError::InvalidConfiguration(message), Err(error) => error, } } - pub(crate) fn truthy(&self) -> Result { - Ok(Truthy(self.value.is_truthy()?)) + pub(crate) fn value(&self) -> &Bound<'py, PyAny> { + &self.value } - pub(crate) fn exact_true(&self) -> ExactTrue { - ExactTrue(self.value.is(PyBool::new(self.value.py(), true))) + pub(crate) fn truthy(&self) -> Result { + Ok(self.value.is_truthy()?) + } + + pub(crate) fn exact_true(&self) -> bool { + self.value.is(PyBool::new(self.value.py(), true)) + } + + pub(crate) fn schema_bool(&self) -> Result { + if !self.value.is_instance_of::() { + return Err(ProjectionError::InternalSchemaFailure( + self.expected("a Boolean")?, + )); + } + Ok(self.exact_true()) } pub(crate) fn strict_string(&self) -> Result { @@ -124,108 +159,366 @@ impl<'py> Field<'py> { self.strict_string() } - pub(crate) fn schema_bool(&self) -> Result { - if !self.value.is_instance_of::() { - return Err(ProjectionError::InternalSchemaFailure( - self.expected("a Boolean")?, - )); - } - Ok(self.exact_true().0) - } - - pub(crate) fn str_bool(&self) -> Result { + pub(crate) fn str_bool(&self) -> Result, ProjectionError> { if self.value.is_none() { - return Ok(StrBool(None)); + return Ok(None); } - Ok(StrBool(parse_str_bool(&self.strict_string()?))) + Ok(parse_str_bool(&self.strict_string()?)) } - pub(crate) fn optional_strict_string(&self) -> Result { + pub(crate) fn optional_strict_string(&self) -> Result, ProjectionError> { if self.value.is_none() { - return Ok(OptionalStrictString(None)); + return Ok(None); } - self.strict_string().map(Some).map(OptionalStrictString) + self.strict_string().map(Some) } - pub(crate) fn falsy_optional_string(&self) -> Result { - if !self.truthy()?.0 { - return Ok(FalsyOptionalString(None)); + pub(crate) fn falsy_optional_string(&self) -> Result, ProjectionError> { + if !self.truthy()? { + return Ok(None); } - self.strict_string().map(Some).map(FalsyOptionalString) + self.strict_string().map(Some) } - pub(crate) fn tuning_string(&self) -> Result { - if !self.truthy()?.0 || !self.value.is_instance_of::() { - return Ok(TuningString(None)); + pub(crate) fn tuning_string(&self) -> Result, ProjectionError> { + if !self.truthy()? || !self.value.is_instance_of::() { + return Ok(None); } - self.strict_string().map(Some).map(TuningString) + self.strict_string().map(Some) } - pub(crate) fn string_collection(&self) -> Result { - if !self.truthy()?.0 { - return Ok(StringCollection(Vec::new())); + pub(crate) fn string_collection(&self) -> Result, ProjectionError> { + if !self.truthy()? { + return Ok(Vec::new()); } if self.value.is_instance_of::() { - return self - .strict_string() - .map(|value| StringCollection(vec![value])); + return self.strict_string().map(|value| vec![value]); } - let values = self - .value + self.value .try_iter()? .filter_map(|item| { let member = match item { - Ok(value) => Self::new(self.path, value), + Ok(value) => self.member(value), Err(error) => return Some(Err(error.into())), }; match member.truthy() { - Ok(Truthy(false)) => None, - Ok(Truthy(true)) => Some(member.strict_string()), + Ok(false) => None, + Ok(true) => Some(member.strict_string()), Err(error) => Some(Err(error)), } }) - .collect::, ProjectionError>>()?; - Ok(StringCollection(values)) + .collect() } - pub(crate) fn host_collection(&self) -> Result { - let values = self - .string_collection()? - .0 - .into_iter() - .map(|host| litellm_http::media::normalize_host(&host)) - .collect::>(); - Ok(StringCollection(values.into_iter().collect())) - } - - pub(crate) fn ssl_verify(&self) -> Result { + pub(crate) fn optional_string_collection( + &self, + ) -> Result>, ProjectionError> { if self.value.is_none() { - return Ok(SslVerifyInput(None)); + return Ok(None); } - if self.value.is_instance_of::() { - return Ok(SslVerifyInput(Some(if self.exact_true().0 { - SslVerify::Enabled - } else { - SslVerify::Disabled - }))); - } - if self.value.is_instance_of::() { - let parsed = match self.str_bool()?.0 { - Some(true) => SslVerify::Enabled, - Some(false) => SslVerify::Disabled, - None => SslVerify::CaBundle(self.strict_string()?.into()), - }; - return Ok(SslVerifyInput(Some(parsed))); - } - let context = self.value.py().import("ssl")?.getattr("SSLContext")?; - if self.value.is_instance(&context)? { - return Err(ProjectionError::UnsupportedLiveObject(self.expected( - "a Boolean, Boolean string, CA path, or None; live SSLContext is unsupported", - )?)); - } - Err(self.invalid("a Boolean, Boolean string, CA path, or None")) + self.string_collection().map(Some) + } + + pub(crate) fn python_binding(&self) -> Option> { + (!self.value.is_none()).then(|| self.value.clone().unbind()) } } #[cfg(test)] -mod tests; +mod tests { + use std::ffi::CString; + + use pyo3::{ + exceptions::{PyLookupError, PyRuntimeError, PyValueError}, + types::PyDict, + }; + use rstest::rstest; + + use super::*; + + fn evaluate<'py>(py: Python<'py>, source: &str) -> Bound<'py, PyAny> { + py.eval(&CString::new(source).unwrap(), None, None).unwrap() + } + + #[rstest] + #[case("None", false, false)] + #[case("False", false, false)] + #[case("True", true, true)] + #[case("0", false, false)] + #[case("1", true, false)] + #[case("''", false, false)] + #[case("'false'", true, false)] + #[case("[]", false, false)] + #[case("[0]", true, false)] + #[case("{}", false, false)] + #[case("object()", true, false)] + fn boolean_operations_have_distinct_python_semantics( + #[case] source: &str, + #[case] truth: bool, + #[case] exact: bool, + ) { + Python::initialize(); + Python::attach(|py| { + let value = evaluate(py, source); + let field = Field::new("test", "flag", value.clone()); + assert_eq!(field.truthy().unwrap(), truth); + assert_eq!(field.exact_true(), exact); + assert_eq!( + field.truthy().unwrap(), + py.import("builtins") + .unwrap() + .getattr("bool") + .unwrap() + .call1((value,)) + .unwrap() + .extract::() + .unwrap() + ); + }); + } + + #[rstest] + #[case("None", Ok(None), Ok(None), Ok(None))] + #[case("''", Ok(Some("")), Ok(None), Ok(None))] + #[case( + "' value '", + Ok(Some(" value ")), + Ok(Some(" value ")), + Ok(Some(" value ")) + )] + #[case("[]", Err(()), Ok(None), Ok(None))] + #[case("0", Err(()), Ok(None), Ok(None))] + #[case("1", Err(()), Err(()), Ok(None))] + #[case("object()", Err(()), Err(()), Ok(None))] + fn string_operations_do_not_conflate_absence_and_type_checks( + #[case] source: &str, + #[case] strict: Result, ()>, + #[case] fallback: Result, ()>, + #[case] tuning: Result, ()>, + ) { + Python::initialize(); + Python::attach(|py| { + let field = Field::new("test", "string", evaluate(py, source)); + let owned = + |expected: Result, ()>| expected.map(|value| value.map(str::to_owned)); + assert_eq!( + field.optional_strict_string().map_err(|_| ()), + owned(strict) + ); + assert_eq!( + field.falsy_optional_string().map_err(|_| ()), + owned(fallback) + ); + assert_eq!(field.tuning_string().map_err(|_| ()), owned(tuning)); + }); + } + + #[rstest] + #[case("None", None)] + #[case("' True '", Some(true))] + #[case("' fAlSe '", Some(false))] + #[case("'yes'", None)] + #[case("'1'", None)] + #[case("'unknown'", None)] + fn string_boolean_tokens_remain_separate_from_truthiness( + #[case] source: &str, + #[case] expected: Option, + ) { + Python::initialize(); + Python::attach(|py| { + assert_eq!( + Field::new("test", "flag", evaluate(py, source)) + .str_bool() + .unwrap(), + expected + ); + }); + } + + #[test] + fn protocol_errors_preserve_exception_identity_traceback_cause_and_context() { + Python::initialize(); + Python::attach(|py| { + let locals = PyDict::new(py); + py.run( + c" +failure = LookupError('protocol failed') +cause = ValueError('cause') +context = RuntimeError('context') +def fail(): + try: + raise context + except RuntimeError: + raise failure from cause +class Bool: + def __bool__(self): return fail() +class Length: + def __len__(self): return fail() +class Iter: + def __iter__(self): return fail() +class Next: + def __iter__(self): return self + def __next__(self): return fail() +class Descriptor: + @property + def flag(self): return fail() +values = (Bool(), Length(), Iter(), Next(), [Bool()]) +descriptor = Descriptor() +", + Some(&locals), + Some(&locals), + ) + .unwrap(); + let values = locals.get_item("values").unwrap().unwrap(); + for value in values.try_iter().unwrap() { + let error = Field::new("test", "flag", value.unwrap()) + .string_collection() + .err() + .unwrap(); + let error = PyErr::from(error); + assert!( + error + .value(py) + .is(locals.get_item("failure").unwrap().unwrap()) + ); + assert!(error.is_instance_of::(py)); + assert!(error.traceback(py).is_some()); + assert!( + error + .value(py) + .getattr("__cause__") + .unwrap() + .is(locals.get_item("cause").unwrap().unwrap()) + ); + assert!( + error + .value(py) + .getattr("__context__") + .unwrap() + .is(locals.get_item("context").unwrap().unwrap()) + ); + } + let error = Field::read( + &locals.get_item("descriptor").unwrap().unwrap(), + "test", + "flag", + ) + .err() + .unwrap(); + assert!( + PyErr::from(error) + .value(py) + .is(locals.get_item("failure").unwrap().unwrap()) + ); + }); + } + + #[test] + fn identity_and_string_contents_do_not_invoke_unrelated_protocols() { + Python::initialize(); + Python::attach(|py| { + let locals = PyDict::new(py); + py.run( + c" +class Hostile: + def __bool__(self): raise AssertionError('bool called') + def __eq__(self, other): raise AssertionError('eq called') + def __str__(self): raise AssertionError('str called') +class Text(str): + def __str__(self): raise AssertionError('str called') + def strip(self): raise AssertionError('strip called') + def lower(self): raise AssertionError('lower called') +hostile = Hostile() +text = Text(' False ') +", + Some(&locals), + Some(&locals), + ) + .unwrap(); + let hostile = Field::new("test", "flag", locals.get_item("hostile").unwrap().unwrap()); + assert!(!hostile.exact_true()); + assert!(matches!( + hostile.strict_string(), + Err(ProjectionError::InvalidConfiguration(_)) + )); + let text = Field::new("test", "flag", locals.get_item("text").unwrap().unwrap()); + assert_eq!(text.strict_string().unwrap(), " False "); + assert_eq!(text.str_bool().unwrap(), Some(false)); + }); + } + + #[test] + fn missing_snapshot_fields_and_descriptor_attribute_errors_are_distinct() { + Python::initialize(); + Python::attach(|py| { + let locals = PyDict::new(py); + py.run( + c" +failure = AttributeError('descriptor failed') +class Snapshot: + @property + def flag(self): raise failure +snapshot = Snapshot() +class Dynamic: + def __getattr__(self, name): raise failure +class Intercepted: + def __getattribute__(self, name): raise failure +dynamic = Dynamic() +intercepted = Intercepted() +", + Some(&locals), + Some(&locals), + ) + .unwrap(); + let snapshot = locals.get_item("snapshot").unwrap().unwrap(); + let descriptor = PyErr::from(Field::read(&snapshot, "test", "flag").err().unwrap()); + assert!( + descriptor + .value(py) + .is(locals.get_item("failure").unwrap().unwrap()) + ); + for name in ["dynamic", "intercepted"] { + let value = locals.get_item(name).unwrap().unwrap(); + let error = PyErr::from(Field::read(&value, "test", "flag").err().unwrap()); + assert!( + error + .value(py) + .is(locals.get_item("failure").unwrap().unwrap()) + ); + } + let missing = PyErr::from(Field::read(&snapshot, "test", "missing").err().unwrap()); + assert!(missing.is_instance_of::(py)); + assert!(missing.to_string().contains("test.missing")); + }); + } + + #[test] + fn configuration_errors_name_fields_without_exposing_values() { + Python::initialize(); + Python::attach(|py| { + for source in [ + "{'secret': 'do-not-print'}", + "['host.test', {'secret': 'do-not-print'}]", + ] { + let field = Field::new("test", "setting", evaluate(py, source)); + let error = PyErr::from(field.falsy_optional_string().err().unwrap()); + assert!(error.is_instance_of::(py)); + assert!(error.to_string().contains("test.setting")); + assert!(!error.to_string().contains("do-not-print")); + } + let hosts = Field::new( + "url_policy", + "user_url_allowed_hosts", + evaluate(py, "['host.test', 1]"), + ); + assert!(matches!( + hosts.string_collection(), + Err(ProjectionError::InvalidConfiguration(_)) + )); + assert!(matches!( + Field::new("test", "flag", evaluate(py, "1")).str_bool(), + Err(ProjectionError::InvalidConfiguration(_)) + )); + }); + } +} diff --git a/litellm-rust/crates/python-bridge/src/coercion/tests.rs b/litellm-rust/crates/python-bridge/src/coercion/tests.rs deleted file mode 100644 index 5ed237c3c64..00000000000 --- a/litellm-rust/crates/python-bridge/src/coercion/tests.rs +++ /dev/null @@ -1,372 +0,0 @@ -use std::ffi::CString; - -use pyo3::{ - exceptions::{PyLookupError, PyRuntimeError, PyValueError}, - types::PyDict, -}; -use rstest::rstest; - -use super::*; - -fn evaluate<'py>(py: Python<'py>, source: &str) -> Bound<'py, PyAny> { - py.eval(&CString::new(source).unwrap(), None, None).unwrap() -} - -#[rstest] -#[case("None", false, false)] -#[case("False", false, false)] -#[case("True", true, true)] -#[case("0", false, false)] -#[case("1", true, false)] -#[case("''", false, false)] -#[case("'false'", true, false)] -#[case("[]", false, false)] -#[case("[0]", true, false)] -#[case("{}", false, false)] -#[case("object()", true, false)] -fn boolean_operations_have_distinct_python_semantics( - #[case] source: &str, - #[case] truth: bool, - #[case] exact: bool, -) { - Python::initialize(); - Python::attach(|py| { - let value = evaluate(py, source); - let field = Field::new("test.flag", value.clone()); - assert_eq!(field.truthy().unwrap().0, truth); - assert_eq!(field.exact_true().0, exact); - assert_eq!( - field.truthy().unwrap().0, - py.import("builtins") - .unwrap() - .getattr("bool") - .unwrap() - .call1((value,)) - .unwrap() - .extract::() - .unwrap() - ); - }); -} - -#[rstest] -#[case("None", Ok(None), Ok(None), Ok(None))] -#[case("''", Ok(Some("")), Ok(None), Ok(None))] -#[case( - "' value '", - Ok(Some(" value ")), - Ok(Some(" value ")), - Ok(Some(" value ")) -)] -#[case("[]", Err(()), Ok(None), Ok(None))] -#[case("0", Err(()), Ok(None), Ok(None))] -#[case("1", Err(()), Err(()), Ok(None))] -#[case("object()", Err(()), Err(()), Ok(None))] -fn string_operations_do_not_conflate_absence_and_type_checks( - #[case] source: &str, - #[case] strict: Result, ()>, - #[case] fallback: Result, ()>, - #[case] tuning: Result, ()>, -) { - Python::initialize(); - Python::attach(|py| { - let field = Field::new("test.string", evaluate(py, source)); - let owned = - |expected: Result, ()>| expected.map(|value| value.map(str::to_owned)); - assert_eq!( - field - .optional_strict_string() - .map(|value| value.0) - .map_err(|_| ()), - owned(strict) - ); - assert_eq!( - field - .falsy_optional_string() - .map(|value| value.0) - .map_err(|_| ()), - owned(fallback) - ); - assert_eq!( - field.tuning_string().map(|value| value.0).map_err(|_| ()), - owned(tuning) - ); - }); -} - -#[rstest] -#[case("None", None)] -#[case("' True '", Some(true))] -#[case("' fAlSe '", Some(false))] -#[case("'yes'", None)] -#[case("'1'", None)] -#[case("'unknown'", None)] -fn string_boolean_tokens_remain_separate_from_truthiness( - #[case] source: &str, - #[case] expected: Option, -) { - Python::initialize(); - Python::attach(|py| { - assert_eq!( - Field::new("test.flag", evaluate(py, source)) - .str_bool() - .unwrap() - .0, - expected - ); - }); -} - -#[rstest] -#[case("'EXAMPLE.TEST.'", vec!["example.test"])] -#[case("['B.test', '', None, 0, [], 'A.test.', 'b.test']", vec!["a.test", "b.test"])] -#[case("('B.test', 'a.test')", vec!["a.test", "b.test"])] -#[case("{'B.test', 'a.test'}", vec!["a.test", "b.test"])] -#[case("(host for host in ['B.test', 'a.test'])", vec!["a.test", "b.test"])] -#[case("None", vec![])] -#[case("False", vec![])] -fn host_collection_is_owned_normalized_and_deterministic( - #[case] source: &str, - #[case] expected: Vec<&str>, -) { - Python::initialize(); - Python::attach(|py| { - assert_eq!( - Field::new("url_policy.user_url_allowed_hosts", evaluate(py, source)) - .host_collection() - .unwrap() - .0, - expected - ); - }); -} - -#[test] -fn protocol_errors_preserve_exception_identity_traceback_cause_and_context() { - Python::initialize(); - Python::attach(|py| { - let locals = PyDict::new(py); - py.run( - c" -failure = LookupError('protocol failed') -cause = ValueError('cause') -context = RuntimeError('context') -def fail(): - try: - raise context - except RuntimeError: - raise failure from cause -class Bool: - def __bool__(self): return fail() -class Length: - def __len__(self): return fail() -class Iter: - def __iter__(self): return fail() -class Next: - def __iter__(self): return self - def __next__(self): return fail() -class Descriptor: - @property - def flag(self): return fail() -values = (Bool(), Length(), Iter(), Next(), [Bool()]) -descriptor = Descriptor() -", - Some(&locals), - Some(&locals), - ) - .unwrap(); - let values = locals.get_item("values").unwrap().unwrap(); - for value in values.try_iter().unwrap() { - let error = Field::new("test.flag", value.unwrap()) - .host_collection() - .err() - .unwrap(); - let error = PyErr::from(error); - assert!( - error - .value(py) - .is(locals.get_item("failure").unwrap().unwrap()) - ); - assert!(error.is_instance_of::(py)); - assert!(error.traceback(py).is_some()); - assert!( - error - .value(py) - .getattr("__cause__") - .unwrap() - .is(locals.get_item("cause").unwrap().unwrap()) - ); - assert!( - error - .value(py) - .getattr("__context__") - .unwrap() - .is(locals.get_item("context").unwrap().unwrap()) - ); - } - let error = Field::read( - &locals.get_item("descriptor").unwrap().unwrap(), - "test.flag", - ) - .err() - .unwrap(); - assert!( - PyErr::from(error) - .value(py) - .is(locals.get_item("failure").unwrap().unwrap()) - ); - }); -} - -#[test] -fn identity_and_string_contents_do_not_invoke_unrelated_protocols() { - Python::initialize(); - Python::attach(|py| { - let locals = PyDict::new(py); - py.run( - c" -class Hostile: - def __bool__(self): raise AssertionError('bool called') - def __eq__(self, other): raise AssertionError('eq called') - def __str__(self): raise AssertionError('str called') -class Text(str): - def __str__(self): raise AssertionError('str called') - def strip(self): raise AssertionError('strip called') - def lower(self): raise AssertionError('lower called') -hostile = Hostile() -text = Text(' False ') -", - Some(&locals), - Some(&locals), - ) - .unwrap(); - let hostile = Field::new("test.flag", locals.get_item("hostile").unwrap().unwrap()); - assert!(!hostile.exact_true().0); - assert!(matches!( - hostile.strict_string(), - Err(ProjectionError::InvalidConfiguration(_)) - )); - let text = Field::new("test.flag", locals.get_item("text").unwrap().unwrap()); - assert_eq!(text.strict_string().unwrap(), " False "); - assert_eq!(text.str_bool().unwrap().0, Some(false)); - }); -} - -#[test] -fn missing_snapshot_fields_and_descriptor_attribute_errors_are_distinct() { - Python::initialize(); - Python::attach(|py| { - let locals = PyDict::new(py); - py.run( - c" -failure = AttributeError('descriptor failed') -class Snapshot: - @property - def flag(self): raise failure -snapshot = Snapshot() -class Dynamic: - def __getattr__(self, name): raise failure -class Intercepted: - def __getattribute__(self, name): raise failure -dynamic = Dynamic() -intercepted = Intercepted() -", - Some(&locals), - Some(&locals), - ) - .unwrap(); - let snapshot = locals.get_item("snapshot").unwrap().unwrap(); - let descriptor = PyErr::from(Field::read(&snapshot, "test.flag").err().unwrap()); - assert!( - descriptor - .value(py) - .is(locals.get_item("failure").unwrap().unwrap()) - ); - for name in ["dynamic", "intercepted"] { - let value = locals.get_item(name).unwrap().unwrap(); - let error = PyErr::from(Field::read(&value, "test.flag").err().unwrap()); - assert!( - error - .value(py) - .is(locals.get_item("failure").unwrap().unwrap()) - ); - } - let missing = PyErr::from(Field::read(&snapshot, "test.missing").err().unwrap()); - assert!(missing.is_instance_of::(py)); - assert!(missing.to_string().contains("test.missing")); - }); -} - -#[test] -fn configuration_errors_name_fields_without_exposing_values() { - Python::initialize(); - Python::attach(|py| { - for source in [ - "{'secret': 'do-not-print'}", - "['host.test', {'secret': 'do-not-print'}]", - ] { - let field = Field::new("test.setting", evaluate(py, source)); - let error = PyErr::from(field.falsy_optional_string().err().unwrap()); - assert!(error.is_instance_of::(py)); - assert!(error.to_string().contains("test.setting")); - assert!(!error.to_string().contains("do-not-print")); - } - let hosts = Field::new( - "url_policy.user_url_allowed_hosts", - evaluate(py, "['host.test', 1]"), - ); - assert!(matches!( - hosts.host_collection(), - Err(ProjectionError::InvalidConfiguration(_)) - )); - assert!(matches!( - Field::new("test.flag", evaluate(py, "1")).str_bool(), - Err(ProjectionError::InvalidConfiguration(_)) - )); - }); -} - -#[test] -fn projection_releases_the_source_collection() { - Python::initialize(); - Python::attach(|py| { - let source = evaluate(py, "['A.test']"); - let projected = Field::new("test.hosts", source.clone()) - .host_collection() - .unwrap() - .0; - source.call_method1("append", ("b.test",)).unwrap(); - assert_eq!(projected, ["a.test"]); - assert_eq!( - Field::new("test.hosts", source) - .host_collection() - .unwrap() - .0, - ["a.test", "b.test"] - ); - }); -} - -#[rstest] -#[case("True", Some(true))] -#[case("False", Some(false))] -#[case("1", None)] -#[case("None", None)] -#[case("[]", None)] -fn accessor_booleans_are_strict_schema_values( - #[case] source: &str, - #[case] expected: Option, -) { - Python::initialize(); - Python::attach(|py| { - let result = Field::new("secret_manager.readable", evaluate(py, source)).schema_bool(); - match expected { - Some(expected) => assert_eq!(result.unwrap(), expected), - None => { - let error = PyErr::from(result.unwrap_err()); - assert!(error.is_instance_of::(py)); - assert!(error.to_string().contains("secret_manager.readable")); - } - } - }); -} diff --git a/litellm-rust/crates/python-bridge/src/http.rs b/litellm-rust/crates/python-bridge/src/http.rs index 596a89a73d7..2515b409c54 100644 --- a/litellm-rust/crates/python-bridge/src/http.rs +++ b/litellm-rust/crates/python-bridge/src/http.rs @@ -1,5 +1,5 @@ use std::{ - collections::HashSet, + collections::{BTreeSet, HashSet}, path::{Path, PathBuf}, sync::{Arc, LazyLock, Mutex, PoisonError}, }; @@ -10,9 +10,75 @@ use litellm_http::{ TlsSource, Unsupported, media::{PublicDnsResolver, UrlPolicy}, }; -use pyo3::{exceptions::PyValueError, prelude::*, types::PyDict}; +use pyo3::{ + exceptions::PyValueError, + prelude::*, + types::{PyBool, PyDict, PyString}, +}; -use crate::{coercion::Field, python_settings::PythonSettings}; +use crate::{ + coercion::{Field, FieldSpec, ProjectionError}, + python_settings::{PythonSettings, Snapshot}, +}; + +const SSL_VERIFY: FieldSpec> = FieldSpec::new("ssl_verify", decode_ssl_verify); +const SSL_CERTIFICATE: FieldSpec> = + FieldSpec::new("ssl_certificate", |field| field.optional_strict_string()); +const SSL_SECURITY_LEVEL: FieldSpec> = + FieldSpec::new("ssl_security_level", |field| field.tuning_string()); +const SSL_ECDH_CURVE: FieldSpec> = + FieldSpec::new("ssl_ecdh_curve", |field| field.tuning_string()); +const FORCE_IPV4: FieldSpec = FieldSpec::new("force_ipv4", |field| field.truthy()); +const HTTP2: FieldSpec = FieldSpec::new("http2", |field| Ok(field.exact_true())); +const AIOHTTP_TRUST_ENV: FieldSpec = + FieldSpec::new("aiohttp_trust_env", |field| field.truthy()); +const DISABLE_AIOHTTP_TRUST_ENV: FieldSpec = + FieldSpec::new("disable_aiohttp_trust_env", |field| field.truthy()); +const DISABLE_AIOHTTP_TRANSPORT: FieldSpec = + FieldSpec::new("disable_aiohttp_transport", |field| Ok(field.exact_true())); +const USER_AGENT: FieldSpec = FieldSpec::new("user_agent", |field| field.schema_string()); +const USER_URL_VALIDATION: FieldSpec = + FieldSpec::new("user_url_validation", |field| field.truthy()); +const USER_URL_ALLOWED_HOSTS: FieldSpec> = + FieldSpec::new("user_url_allowed_hosts", decode_hosts); + +fn decode_hosts(field: &Field<'_>) -> Result, ProjectionError> { + Ok(field + .string_collection()? + .into_iter() + .map(|host| litellm_http::media::normalize_host(&host)) + .collect::>() + .into_iter() + .collect()) +} + +fn decode_ssl_verify(field: &Field<'_>) -> Result, ProjectionError> { + let value = field.value(); + if value.is_none() { + return Ok(None); + } + if value.is_instance_of::() { + return Ok(Some(if field.exact_true() { + SslVerify::Enabled + } else { + SslVerify::Disabled + })); + } + if value.is_instance_of::() { + return Ok(Some(match field.str_bool()? { + Some(true) => SslVerify::Enabled, + Some(false) => SslVerify::Disabled, + None => SslVerify::CaBundle(field.strict_string()?.into()), + })); + } + let context = value.py().import("ssl")?.getattr("SSLContext")?; + if value.is_instance(&context)? { + return Err(ProjectionError::UnsupportedLiveObject(field.expected( + "a Boolean, Boolean string, CA path, or None; live SSLContext is unsupported", + )?)); + } + Err(field.invalid("a Boolean, Boolean string, CA path, or None")) +} static POOL: LazyLock = LazyLock::new(|| HttpClientPool::new(Arc::new(PublicDnsResolver))); @@ -80,20 +146,20 @@ pub(crate) fn url_policy(py: Python<'_>) -> PyResult { project_url_policy(&PythonSettings::UrlPolicy.read(py)?) } -fn project_url_policy(value: &Bound<'_, PyAny>) -> PyResult { +fn project_url_policy(snapshot: &Snapshot<'_>) -> PyResult { Ok(UrlPolicy { - validate: Field::read(value, "url_policy.user_url_validation")? - .truthy()? - .0, - allowed_hosts: Field::read(value, "url_policy.user_url_allowed_hosts")? - .host_collection()? - .0, + validate: snapshot.read(&USER_URL_VALIDATION)?, + allowed_hosts: snapshot.read(&USER_URL_ALLOWED_HOSTS)?, }) } fn call_ssl_verify(kwargs: &Bound<'_, PyDict>) -> PyResult> { match kwargs.get_item("ssl_verify")? { - Some(value) => Ok(Field::new("request.ssl_verify", value).ssl_verify()?.0), + Some(value) => Ok(decode_ssl_verify(&Field::new( + "request", + "ssl_verify", + value, + ))?), None => Ok(None), } } @@ -106,39 +172,18 @@ fn for_call(call_ssl_verify: Option, asynchronous: bool) -> HttpSetti } } -fn configured(value: &Bound<'_, PyAny>) -> PyResult { +fn configured(snapshot: &Snapshot<'_>) -> PyResult { Ok(HttpSettingsLayer { - ssl_verify: Field::read(value, "http_settings.ssl_verify")? - .ssl_verify()? - .0, - ssl_certificate: Field::read(value, "http_settings.ssl_certificate")? - .optional_strict_string()? - .0 - .map(PathBuf::from), - ssl_security_level: Field::read(value, "http_settings.ssl_security_level")? - .tuning_string()? - .0, - ssl_ecdh_curve: Field::read(value, "http_settings.ssl_ecdh_curve")? - .tuning_string()? - .0, - force_ipv4: Some(Field::read(value, "http_settings.force_ipv4")?.truthy()?.0), - http2: Some(Field::read(value, "http_settings.http2")?.exact_true().0), - aiohttp_trust_env: Some( - Field::read(value, "http_settings.aiohttp_trust_env")? - .truthy()? - .0, - ), - disable_aiohttp_trust_env: Some( - Field::read(value, "http_settings.disable_aiohttp_trust_env")? - .truthy()? - .0, - ), - disable_aiohttp_transport: Some( - Field::read(value, "http_settings.disable_aiohttp_transport")? - .exact_true() - .0, - ), - user_agent: Some(Field::read(value, "http_settings.user_agent")?.schema_string()?), + ssl_verify: snapshot.read(&SSL_VERIFY)?, + ssl_certificate: snapshot.read(&SSL_CERTIFICATE)?.map(PathBuf::from), + ssl_security_level: snapshot.read(&SSL_SECURITY_LEVEL)?, + ssl_ecdh_curve: snapshot.read(&SSL_ECDH_CURVE)?, + force_ipv4: Some(snapshot.read(&FORCE_IPV4)?), + http2: Some(snapshot.read(&HTTP2)?), + aiohttp_trust_env: Some(snapshot.read(&AIOHTTP_TRUST_ENV)?), + disable_aiohttp_trust_env: Some(snapshot.read(&DISABLE_AIOHTTP_TRUST_ENV)?), + disable_aiohttp_transport: Some(snapshot.read(&DISABLE_AIOHTTP_TRANSPORT)?), + user_agent: Some(snapshot.read(&USER_AGENT)?), ..HttpSettingsLayer::default() }) } @@ -150,12 +195,15 @@ mod tests { use rstest::rstest; use super::*; - use crate::python_settings::CONTRACT; - fn python_settings<'py>(py: Python<'py>, overrides: &str) -> Bound<'py, PyAny> { + fn evaluate<'py>(py: Python<'py>, source: &str) -> Bound<'py, PyAny> { + py.eval(&std::ffi::CString::new(source).unwrap(), None, None) + .unwrap() + } + + fn python_settings<'py>(py: Python<'py>, overrides: &str) -> Snapshot<'py> { let source = format!( " -import json import types defaults = dict( ssl_verify=True, @@ -170,14 +218,13 @@ defaults = dict( user_agent='litellm/test', ) defaults.update(dict({overrides})) -settings = types.SimpleNamespace(**{{name: defaults[name] for name in json.loads(contract)['http_settings']['fields']}}) +settings = types.SimpleNamespace(**defaults) " ); let locals = PyDict::new(py); - locals.set_item("contract", CONTRACT).unwrap(); let source = std::ffi::CString::new(source).unwrap(); py.run(&source, Some(&locals), Some(&locals)).unwrap(); - locals.get_item("settings").unwrap().unwrap() + PythonSettings::Http.snapshot(locals.get_item("settings").unwrap().unwrap()) } #[test] @@ -395,7 +442,7 @@ user_agent='litellm/9.9.9', Python::attach(|py| { let value = py.eval(c"__import__('types').SimpleNamespace(user_url_validation=[], user_url_allowed_hosts=['B.test', 'a.test.', 'b.test'])", None, None).unwrap(); assert_eq!( - project_url_policy(&value).unwrap(), + project_url_policy(&PythonSettings::UrlPolicy.snapshot(value)).unwrap(), UrlPolicy { validate: false, allowed_hosts: vec!["a.test".into(), "b.test".into()], @@ -419,4 +466,44 @@ user_agent='litellm/9.9.9', let settings = HttpSettings::from_layers([for_call(None, asynchronous), opted_out]); assert_eq!(settings.trust_proxy_env, expected); } + #[rstest] + #[case("'EXAMPLE.TEST.'", vec!["example.test"])] + #[case("['B.test', '', None, 0, [], 'A.test.', 'b.test']", vec!["a.test", "b.test"])] + #[case("('B.test', 'a.test')", vec!["a.test", "b.test"])] + #[case("{'B.test', 'a.test'}", vec!["a.test", "b.test"])] + #[case("(host for host in ['B.test', 'a.test'])", vec!["a.test", "b.test"])] + #[case("None", vec![])] + #[case("False", vec![])] + fn host_collection_is_owned_normalized_and_deterministic( + #[case] source: &str, + #[case] expected: Vec<&str>, + ) { + Python::initialize(); + Python::attach(|py| { + assert_eq!( + decode_hosts(&Field::new( + "url_policy", + "user_url_allowed_hosts", + evaluate(py, source) + )) + .unwrap(), + expected + ); + }); + } + + #[test] + fn projection_releases_the_source_collection() { + Python::initialize(); + Python::attach(|py| { + let source = evaluate(py, "['A.test']"); + let projected = decode_hosts(&Field::new("test", "hosts", source.clone())).unwrap(); + source.call_method1("append", ("b.test",)).unwrap(); + assert_eq!(projected, ["a.test"]); + assert_eq!( + decode_hosts(&Field::new("test", "hosts", source)).unwrap(), + ["a.test", "b.test"] + ); + }); + } } diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index f13a3ad433f..ed9bc90f650 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -7,6 +7,11 @@ mod http; mod marshal; mod python_settings; mod routes; +#[allow( + dead_code, + reason = "secret-manager foundations await rollout activation" +)] +mod secrets; mod token_counter; #[pymodule(gil_used = true)] @@ -43,7 +48,7 @@ mod _native { let dict = module.dict(); dict.set_item("_CacheTestHandle", py.get_type::())?; dict.set_item("_CacheTestResolver", py.get_type::())?; - dict.set_item("_CacheTestBinding", py.get_type::()) + dict.set_item("_ResponseCacheRuntime", py.get_type::()) } } diff --git a/litellm-rust/crates/python-bridge/src/python_settings.rs b/litellm-rust/crates/python-bridge/src/python_settings.rs index bdc6d14356d..111ac3bc259 100644 --- a/litellm-rust/crates/python-bridge/src/python_settings.rs +++ b/litellm-rust/crates/python-bridge/src/python_settings.rs @@ -1,5 +1,7 @@ use pyo3::prelude::*; +use crate::coercion::{FieldSpec, ProjectionError}; + const MODULE: &str = "litellm.rust_bridge.settings"; #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -8,28 +10,39 @@ pub(crate) enum PythonSettings { UrlPolicy, ProviderDefaults, SecretManager, + SecretManagerBinding, +} + +pub(crate) struct Snapshot<'py> { + group: PythonSettings, + value: Bound<'py, PyAny>, +} + +impl Snapshot<'_> { + pub(crate) fn read(&self, spec: &FieldSpec) -> Result { + spec.read(&self.value, self.group.name()) + } } impl PythonSettings { - #[cfg(test)] - pub(crate) const ALL: [Self; 4] = [ - Self::Http, - Self::UrlPolicy, - Self::ProviderDefaults, - Self::SecretManager, - ]; - pub(crate) fn name(self) -> &'static str { match self { Self::Http => "http_settings", Self::UrlPolicy => "url_policy", Self::ProviderDefaults => "provider_defaults", Self::SecretManager => "secret_manager", + Self::SecretManagerBinding => "secret_manager_binding", } } - pub(crate) fn read(self, py: Python<'_>) -> PyResult> { - py.import(MODULE)?.getattr(self.name())?.call0() + pub(crate) fn read(self, py: Python<'_>) -> PyResult> { + let value = py.import(MODULE)?.getattr(self.name())?.call0()?; + Ok(Snapshot { group: self, value }) + } + + #[cfg(test)] + pub(crate) fn snapshot(self, value: Bound<'_, PyAny>) -> Snapshot<'_> { + Snapshot { group: self, value } } pub(crate) fn warn(py: Python<'_>, message: &str) -> PyResult<()> { @@ -38,209 +51,98 @@ impl PythonSettings { } } -#[cfg(test)] -pub(crate) const CONTRACT: &str = include_str!("../python_settings.json"); - #[cfg(test)] mod tests { - use super::{CONTRACT, PythonSettings}; - use pyo3::prelude::*; - use serde_json::{Value, json}; + use pyo3::{exceptions::PyRuntimeError, prelude::*, types::PyDict}; - struct SettingSpec { - group: &'static str, - name: &'static str, - adapter: &'static str, - precedence: &'static str, - sensitive: bool, - shapes: &'static [&'static str], - unsupported_live: Option<&'static str>, - } - - const SETTINGS: &[SettingSpec] = &[ - SettingSpec { - group: "http_settings", - name: "ssl_verify", - adapter: "SslVerifyInput", - precedence: "module_global", - sensitive: false, - shapes: &["none", "bool", "str"], - unsupported_live: Some("configuration_error"), - }, - SettingSpec { - group: "http_settings", - name: "ssl_certificate", - adapter: "OptionalStrictString", - precedence: "module_global", - sensitive: false, - shapes: &[], - unsupported_live: None, - }, - SettingSpec { - group: "http_settings", - name: "ssl_security_level", - adapter: "TuningString", - precedence: "module_global", - sensitive: false, - shapes: &[], - unsupported_live: None, - }, - SettingSpec { - group: "http_settings", - name: "ssl_ecdh_curve", - adapter: "TuningString", - precedence: "module_global", - sensitive: false, - shapes: &[], - unsupported_live: None, - }, - SettingSpec { - group: "http_settings", - name: "force_ipv4", - adapter: "Truthy", - precedence: "module_global", - sensitive: false, - shapes: &[], - unsupported_live: None, - }, - SettingSpec { - group: "http_settings", - name: "http2", - adapter: "ExactTrue", - precedence: "module_global", - sensitive: false, - shapes: &[], - unsupported_live: None, - }, - SettingSpec { - group: "http_settings", - name: "aiohttp_trust_env", - adapter: "Truthy", - precedence: "module_global", - sensitive: false, - shapes: &[], - unsupported_live: None, - }, - SettingSpec { - group: "http_settings", - name: "disable_aiohttp_trust_env", - adapter: "Truthy", - precedence: "module_global", - sensitive: false, - shapes: &[], - unsupported_live: None, - }, - SettingSpec { - group: "http_settings", - name: "disable_aiohttp_transport", - adapter: "ExactTrue", - precedence: "module_global", - sensitive: false, - shapes: &[], - unsupported_live: None, - }, - SettingSpec { - group: "http_settings", - name: "user_agent", - adapter: "StrictString", - precedence: "accessor", - sensitive: false, - shapes: &[], - unsupported_live: None, - }, - SettingSpec { - group: "url_policy", - name: "user_url_validation", - adapter: "Truthy", - precedence: "module_global", - sensitive: false, - shapes: &[], - unsupported_live: None, - }, - SettingSpec { - group: "url_policy", - name: "user_url_allowed_hosts", - adapter: "HostCollection", - precedence: "module_global", - sensitive: false, - shapes: &[], - unsupported_live: None, - }, - SettingSpec { - group: "provider_defaults", - name: "vertex_project", - adapter: "FalsyOptionalString", - precedence: "module_global", - sensitive: true, - shapes: &[], - unsupported_live: None, - }, - SettingSpec { - group: "provider_defaults", - name: "vertex_location", - adapter: "FalsyOptionalString", - precedence: "module_global", - sensitive: true, - shapes: &[], - unsupported_live: None, - }, - SettingSpec { - group: "provider_defaults", - name: "enable_azure_ad_token_refresh", - adapter: "ExactTrue", - precedence: "module_global", - sensitive: false, - shapes: &[], - unsupported_live: None, - }, - SettingSpec { - group: "secret_manager", - name: "readable", - adapter: "StrictBool", - precedence: "accessor", - sensitive: false, - shapes: &[], - unsupported_live: None, - }, - ]; + use super::PythonSettings; + use crate::coercion::FieldSpec; #[test] - fn settings_manifest_matches_the_semantic_contract() { - pyo3::Python::initialize(); - let manifest: Value = pyo3::Python::attach(|py| { - let value = py - .import("json") - .unwrap() - .call_method1("loads", (CONTRACT,)) - .unwrap(); - litellm_host_python::from_py(&value).unwrap() + fn declarations_select_the_decoder_and_read_only_the_requested_field() { + const TRUTHY: FieldSpec = FieldSpec::new("flag", |field| field.truthy()); + const EXACT: FieldSpec = FieldSpec::new("flag", |field| Ok(field.exact_true())); + Python::initialize(); + Python::attach(|py| { + let locals = PyDict::new(py); + py.run( + c" +reads = [] +class Settings: + value = 1 + @property + def flag(self): + reads.append('flag') + return self.value + @property + def unrelated(self): + raise AssertionError('unrequested field') +settings = Settings() +", + Some(&locals), + Some(&locals), + ) + .unwrap(); + let value = locals.get_item("settings").unwrap().unwrap(); + let snapshot = PythonSettings::Http.snapshot(value.clone()); + assert!(snapshot.read(&TRUTHY).unwrap()); + assert!(!snapshot.read(&EXACT).unwrap()); + value.setattr("value", true).unwrap(); + assert!(snapshot.read(&EXACT).unwrap()); + assert_eq!( + locals + .get_item("reads") + .unwrap() + .unwrap() + .extract::>() + .unwrap(), + ["flag", "flag", "flag"] + ); + }); + } + + #[test] + fn declared_reads_preserve_descriptor_and_decoder_failures_and_name_missing_fields() { + const FLAG: FieldSpec = FieldSpec::new("flag", |field| field.truthy()); + Python::initialize(); + Python::attach(|py| { + let locals = PyDict::new(py); + py.run( + c" +from types import SimpleNamespace +failure = AttributeError('read failed') +class Descriptor: + @property + def flag(self): raise failure +class Truth: + def __bool__(self): raise failure +values = (Descriptor(), SimpleNamespace(flag=Truth())) +", + Some(&locals), + Some(&locals), + ) + .unwrap(); + let failure = locals.get_item("failure").unwrap().unwrap(); + for value in locals + .get_item("values") + .unwrap() + .unwrap() + .try_iter() + .unwrap() + { + let snapshot = PythonSettings::Http.snapshot(value.unwrap()); + let error = PyErr::from(snapshot.read(&FLAG).unwrap_err()); + assert!(error.value(py).is(&failure)); + assert!(error.traceback(py).is_some()); + } + let missing = PythonSettings::Http.snapshot(py.eval(c"object()", None, None).unwrap()); + let error = PyErr::from(missing.read(&FLAG).unwrap_err()); + assert!(error.is_instance_of::(py)); + assert!( + error + .to_string() + .contains("http_settings.flag: missing snapshot field") + ); }); - let expected: serde_json::Map = PythonSettings::ALL - .into_iter() - .map(|group| { - let fields: serde_json::Map = SETTINGS - .iter() - .filter(|spec| spec.group == group.name()) - .map(|spec| { - ( - spec.name.to_owned(), - json!({ - "adapter": spec.adapter, - "required": true, - "precedence": spec.precedence, - "sensitive": spec.sensitive, - "shapes": spec.shapes, - "unsupported_live": spec.unsupported_live, - }), - ) - }) - .collect(); - ( - group.name().to_owned(), - json!({"version": 1, "fields": fields}), - ) - }) - .collect(); - assert_eq!(manifest, Value::Object(expected)); } } diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/host.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/host.rs index 77c8d5d6641..325377e5285 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/host.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/host.rs @@ -130,6 +130,11 @@ impl RouteHost for OcrRouteHost { } fn classify(&self, py: Python<'_>, error: Error) -> PyResult { + if let Error::Secret(source) = &error + && let Some(original) = crate::secrets::callback::python_error(py, source) + { + return Ok(original); + } Ok(self.map_failure(py, ocr_error_to_pyerr(error))) } diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs index d0b13e5056a..2dca6da66cd 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs @@ -10,16 +10,33 @@ use litellm_auth_gcp::VertexAuth; use litellm_callbacks_legacy_python::{LegacySurface, PublicCall, run_legacy_call}; use litellm_core::ocr::route::ocr_machine; use litellm_core_utils::settings::ProcessEnvironment; -use litellm_llms::base_llm::ocr::{ - handler::OcrClient, - settings::{OcrSettings, Secrets}, +use litellm_llms::base_llm::{ + inference::secrets::{EnvironmentSecrets, SecretSource}, + ocr::{handler::OcrClient, settings::OcrSettings}, }; use pyo3::{ prelude::*, types::{PyDict, PyTuple}, }; -use crate::{coercion::Field, errors::RustBridgeDeclined, http, python_settings::PythonSettings}; +use crate::{ + coercion::FieldSpec, + errors::RustBridgeDeclined, + http, + python_settings::{PythonSettings, Snapshot}, +}; + +const SECRET_MANAGER_READABLE: FieldSpec = + FieldSpec::new("readable", |field| field.schema_bool()); + +const VERTEX_PROJECT: FieldSpec> = + FieldSpec::new("vertex_project", |field| field.falsy_optional_string()); +const VERTEX_LOCATION: FieldSpec> = + FieldSpec::new("vertex_location", |field| field.falsy_optional_string()); +const ENABLE_AZURE_AD_TOKEN_REFRESH: FieldSpec = + FieldSpec::new("enable_azure_ad_token_refresh", |field| { + Ok(field.exact_true()) + }); const SURFACE: LegacySurface = LegacySurface { call_type: "ocr", @@ -62,33 +79,24 @@ fn run_ocr( ) } -fn process_environment_secrets(secret_manager: &Bound<'_, PyAny>) -> PyResult { - if Field::read(secret_manager, "secret_manager.readable")?.schema_bool()? { +fn process_environment_secrets(snapshot: &Snapshot<'_>) -> PyResult> { + if snapshot.read(&SECRET_MANAGER_READABLE)? { return Err(RustBridgeDeclined::new_err( "a readable secret manager is configured and the Rust route only reads the process environment", )); } - Ok(Arc::new(ProcessEnvironment)) + Ok(Arc::new(EnvironmentSecrets)) } fn ocr_settings(py: Python<'_>) -> PyResult { project_provider_defaults(&PythonSettings::ProviderDefaults.read(py)?) } -fn project_provider_defaults(value: &Bound<'_, PyAny>) -> PyResult { +fn project_provider_defaults(snapshot: &Snapshot<'_>) -> PyResult { Ok(OcrSettings { - vertex_project: Field::read(value, "provider_defaults.vertex_project")? - .falsy_optional_string()? - .0, - vertex_location: Field::read(value, "provider_defaults.vertex_location")? - .falsy_optional_string()? - .0, - enable_azure_ad_token_refresh: Field::read( - value, - "provider_defaults.enable_azure_ad_token_refresh", - )? - .exact_true() - .0, + vertex_project: snapshot.read(&VERTEX_PROJECT)?, + vertex_location: snapshot.read(&VERTEX_LOCATION)?, + enable_azure_ad_token_refresh: snapshot.read(&ENABLE_AZURE_AD_TOKEN_REFRESH)?, ..OcrSettings::from_environment(&ProcessEnvironment) }) } @@ -120,6 +128,8 @@ mod tests { use super::process_environment_secrets; use crate::errors::RustBridgeDeclined; + use crate::python_settings::PythonSettings; + fn secret_manager<'py>(py: Python<'py>, readable: bool) -> Bound<'py, PyAny> { let locals = PyDict::new(py); locals.set_item("readable", readable).unwrap(); @@ -132,12 +142,26 @@ mod tests { locals.get_item("manager").unwrap().unwrap() } + #[test] + fn a_readable_secret_manager_sends_the_call_back_to_python() { + Python::initialize(); + Python::attach(|py| { + let declined = process_environment_secrets( + &PythonSettings::SecretManager.snapshot(secret_manager(py, true)), + ) + .err() + .expect("the Rust route declines"); + assert!(declined.is_instance_of::(py)); + }); + } + #[test] fn provider_defaults_distinguish_falsey_values_and_exact_true() { Python::initialize(); Python::attach(|py| { let value = py.eval(c"__import__('types').SimpleNamespace(vertex_project=[], vertex_location=0, enable_azure_ad_token_refresh=1)", None, None).unwrap(); - let projected = super::project_provider_defaults(&value).unwrap(); + let snapshot = PythonSettings::ProviderDefaults.snapshot(value.clone()); + let projected = super::project_provider_defaults(&snapshot).unwrap(); assert_eq!(projected.vertex_project, None); assert_eq!(projected.vertex_location, None); assert!(!projected.enable_azure_ad_token_refresh); @@ -146,12 +170,12 @@ mod tests { value .setattr("enable_azure_ad_token_refresh", true) .unwrap(); - let next = super::project_provider_defaults(&value).unwrap(); + let next = super::project_provider_defaults(&snapshot).unwrap(); assert_eq!(next.vertex_project.as_deref(), Some("project")); assert_eq!(next.vertex_location.as_deref(), Some("region")); assert!(next.enable_azure_ad_token_refresh); value.setattr("vertex_project", 1).unwrap(); - let error = super::project_provider_defaults(&value).err().unwrap(); + let error = super::project_provider_defaults(&snapshot).err().unwrap(); assert!(error.is_instance_of::(py)); assert!( error @@ -160,28 +184,4 @@ mod tests { ); }); } - - #[test] - fn a_readable_secret_manager_sends_the_call_back_to_python() { - Python::initialize(); - Python::attach(|py| { - let declined = process_environment_secrets(&secret_manager(py, true)) - .err() - .expect("the Rust route declines"); - assert!(declined.is_instance_of::(py)); - }); - } - - #[test] - fn without_a_readable_secret_manager_secrets_are_the_process_environment() { - Python::initialize(); - Python::attach(|py| { - let secrets = process_environment_secrets(&secret_manager(py, false)).unwrap(); - assert_eq!( - secrets.get("LITELLM_RUST_BRIDGE_UNSET_VARIABLE_FOR_TEST"), - None - ); - assert_eq!(secrets.get("PATH"), std::env::var("PATH").ok()); - }); - } } diff --git a/litellm-rust/crates/python-bridge/src/secrets/callback.rs b/litellm-rust/crates/python-bridge/src/secrets/callback.rs new file mode 100644 index 00000000000..2b98081acef --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/secrets/callback.rs @@ -0,0 +1,349 @@ +use std::{fmt, future::Future, pin::Pin}; + +use litellm_core_utils::settings::Lookup; +use litellm_secrets::{ + Error, ExternalSecretManager, KeyManagementSettings, KeyManagementSystem, Secret, SecretValue, +}; +use pyo3::{exceptions::PyBaseException, prelude::*, types::PyDict}; + +const HANDLER_MODULE: &str = "litellm.secret_managers.secret_manager_handler"; + +struct PythonSecretError(Py); + +impl fmt::Debug for PythonSecretError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("PythonSecretError") + } +} + +impl fmt::Display for PythonSecretError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("Python secret manager failed") + } +} + +impl std::error::Error for PythonSecretError {} + +pub(crate) fn python_error(py: Python<'_>, error: &Error) -> Option { + let Error::ExternalManager(source) = error else { + return None; + }; + source + .downcast_ref::() + .map(|error| PyErr::from_value(error.0.clone_ref(py).into_bound(py).into_any())) +} + +/// A secret manager whose reads execute in Python: a custom manager, a legacy compatible +/// client, or a manually assigned SDK client. +pub(crate) struct PythonSecretManager { + client: Py, + system: Option, + /// The `key_manager` name Python's handler dispatches on. + key_manager: &'static str, + settings: Option>, +} + +impl PythonSecretManager { + pub(crate) fn new( + client: Py, + system: Option, + settings: Option>, + ) -> Self { + Self { + client, + system, + key_manager: system.map_or("local", python_name), + settings, + } + } + + fn read(&self, py: Python<'_>, name: &str) -> PyResult> { + let client = self.client.bind(py); + if self.system == Some(KeyManagementSystem::Custom) + || (self.system.is_none() && client.hasattr("sync_read_secret")?) + { + let kwargs = PyDict::new(py); + kwargs.set_item("secret_name", name)?; + if self.system == Some(KeyManagementSystem::Custom) { + let optional_params = self + .settings + .as_ref() + .map(|settings| settings.bind(py).call_method0("model_dump")) + .transpose()?; + kwargs.set_item("optional_params", optional_params)?; + } + return client + .call_method("sync_read_secret", (), Some(&kwargs))? + .extract(); + } + let kwargs = PyDict::new(py); + kwargs.set_item("client", client)?; + kwargs.set_item("key_manager", self.key_manager)?; + kwargs.set_item("secret_name", name)?; + kwargs.set_item( + "key_management_settings", + self.settings + .as_ref() + .map_or_else(|| py.None(), |settings| settings.clone_ref(py)), + )?; + py.import(HANDLER_MODULE)? + .getattr("get_secret_from_manager")? + .call((), Some(&kwargs))? + .extract() + } +} + +/// The `KeyManagementSystem` value as Python spells it. +fn python_name(system: KeyManagementSystem) -> &'static str { + match system { + KeyManagementSystem::GoogleKms => "google_kms", + KeyManagementSystem::AzureKeyVault => "azure_key_vault", + KeyManagementSystem::AwsSecretManager => "aws_secret_manager", + KeyManagementSystem::GoogleSecretManager => "google_secret_manager", + KeyManagementSystem::HashicorpVault => "hashicorp_vault", + KeyManagementSystem::Cyberark => "cyberark", + KeyManagementSystem::Local => "local", + KeyManagementSystem::AwsKms => "aws_kms", + KeyManagementSystem::Custom => "custom", + } +} + +impl ExternalSecretManager for PythonSecretManager { + fn system(&self) -> KeyManagementSystem { + self.system.unwrap_or(KeyManagementSystem::Custom) + } + + fn read_secret<'a>( + &'a self, + name: &'a str, + _settings: &'a KeyManagementSettings, + _environment: &'a (dyn Lookup + Send + Sync), + ) -> Pin, Error>> + Send + 'a>> { + Box::pin(async move { + Python::attach(|py| { + self.read(py, name) + .map(|value| value.map(SecretValue::new).map(Secret::String)) + .map_err(|error| { + Error::ExternalManager(Box::new(PythonSecretError(error.into_value(py)))) + }) + }) + }) + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use litellm_secrets::{ + FailurePolicy, KeyManagementSettings, KeyManagementSystem, OidcResolver, SecretManager, + SecretManagerState, SecretResolver, + }; + use pyo3::{prelude::*, types::PyDict}; + + use super::{HANDLER_MODULE, PythonSecretManager, python_error, python_name}; + + #[tokio::test] + async fn callback_failures_preserve_python_exceptions_even_with_environment_fallback() { + Python::initialize(); + for failure_type in ["ValueError", "asyncio.CancelledError"] { + for fallback in [None, Some("environment-key")] { + let (reader, locals) = Python::attach(|py| { + let locals = PyDict::new(py); + locals.set_item("failure_type", failure_type).unwrap(); + py.run( + c" +import asyncio +failure = eval(failure_type)('secret manager failed') +cause = RuntimeError('original cause') +context = RuntimeError('original context') +failure.__cause__ = cause +failure.__context__ = context +class Manager: + def sync_read_secret(self, secret_name): + raise failure +manager = Manager() +", + Some(&locals), + Some(&locals), + ) + .unwrap(); + let reader = PythonSecretManager::new( + locals.get_item("manager").unwrap().unwrap().unbind(), + None, + None, + ); + (reader, locals.unbind()) + }); + let resolver = SecretResolver::new( + Arc::new(SecretManagerState::new( + SecretManager::External(Arc::new(reader)), + KeyManagementSettings::default(), + )), + Arc::new(move |_: &str| fallback.map(str::to_owned)), + OidcResolver::default(), + ) + .with_failure_policy(FailurePolicy::EnvironmentFallback); + let error = resolver.get_secret("API_KEY", None).await.unwrap_err(); + Python::attach(|py| { + let original = python_error(py, &error).unwrap(); + let locals = locals.bind(py); + assert!( + original + .value(py) + .is(locals.get_item("failure").unwrap().unwrap()) + ); + for (attribute, name) in [("__cause__", "cause"), ("__context__", "context")] { + assert!( + original + .value(py) + .getattr(attribute) + .unwrap() + .is(locals.get_item(name).unwrap().unwrap()) + ); + } + assert!(original.traceback(py).is_some()); + }); + } + } + } + + /// Installs a fake `get_secret_from_manager` that records its kwargs, runs `body`, and + /// removes the fake modules again. + fn with_fake_handler<'py>(py: Python<'py>, body: impl FnOnce(&Bound<'py, PyDict>)) { + let locals = PyDict::new(py); + py.run( + c" +import sys, types +calls = [] +def get_secret_from_manager(**kwargs): + calls.append(kwargs) + return 'handled-' + kwargs['secret_name'] +handler = types.ModuleType('litellm.secret_managers.secret_manager_handler') +handler.get_secret_from_manager = get_secret_from_manager +installed = {} +for name in ('litellm', 'litellm.secret_managers'): + if name not in sys.modules: + sys.modules[name] = types.ModuleType(name) + installed[name] = True +sys.modules['litellm.secret_managers.secret_manager_handler'] = handler +", + Some(&locals), + Some(&locals), + ) + .unwrap(); + body(&locals); + py.run( + c" +sys.modules.pop('litellm.secret_managers.secret_manager_handler', None) +for name in installed: + sys.modules.pop(name, None) +", + Some(&locals), + Some(&locals), + ) + .unwrap(); + } + + #[test] + fn python_names_round_trip_through_serde() { + for system in [ + KeyManagementSystem::GoogleKms, + KeyManagementSystem::AzureKeyVault, + KeyManagementSystem::AwsSecretManager, + KeyManagementSystem::GoogleSecretManager, + KeyManagementSystem::HashicorpVault, + KeyManagementSystem::Cyberark, + KeyManagementSystem::Local, + KeyManagementSystem::AwsKms, + KeyManagementSystem::Custom, + ] { + assert_eq!( + serde_json::to_value(system).unwrap(), + serde_json::Value::String(python_name(system).to_owned()) + ); + } + } + + #[test] + fn custom_readers_without_a_system_are_called_directly() { + Python::initialize(); + Python::attach(|py| { + let locals = PyDict::new(py); + py.run( + c" +class Manager: + def __init__(self): + self.names = [] + def sync_read_secret(self, secret_name, optional_params=None, timeout=None): + self.names.append(secret_name) + return 'direct-' + secret_name +manager = Manager() +", + Some(&locals), + Some(&locals), + ) + .unwrap(); + let manager = locals.get_item("manager").unwrap().unwrap(); + let reader = PythonSecretManager::new(manager.clone().unbind(), None, None); + assert_eq!( + reader.read(py, "API_KEY").unwrap().as_deref(), + Some("direct-API_KEY") + ); + assert_eq!( + manager + .getattr("names") + .unwrap() + .extract::>() + .unwrap(), + ["API_KEY"] + ); + }); + } + + #[test] + fn configured_systems_dispatch_through_the_python_handler_with_the_original_settings() { + Python::initialize(); + Python::attach(|py| { + with_fake_handler(py, |locals| { + let client = py.eval(c"object()", None, None).unwrap(); + let settings = py.eval(c"object()", None, None).unwrap(); + let reader = PythonSecretManager::new( + client.clone().unbind(), + Some(KeyManagementSystem::AzureKeyVault), + Some(settings.clone().unbind()), + ); + assert_eq!( + reader.read(py, "API_KEY").unwrap().as_deref(), + Some("handled-API_KEY") + ); + assert!(py.import(HANDLER_MODULE).is_ok()); + let calls = locals.get_item("calls").unwrap().unwrap(); + let call = calls.get_item(0).unwrap().cast_into::().unwrap(); + assert!(call.get_item("client").unwrap().unwrap().is(&client)); + assert!( + call.get_item("key_management_settings") + .unwrap() + .unwrap() + .is(&settings) + ); + assert_eq!( + call.get_item("key_manager") + .unwrap() + .unwrap() + .extract::() + .unwrap(), + "azure_key_vault" + ); + assert_eq!( + call.get_item("secret_name") + .unwrap() + .unwrap() + .extract::() + .unwrap(), + "API_KEY" + ); + }); + }); + } +} diff --git a/litellm-rust/crates/python-bridge/src/secrets/config.rs b/litellm-rust/crates/python-bridge/src/secrets/config.rs new file mode 100644 index 00000000000..6fd380fe40c --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/secrets/config.rs @@ -0,0 +1,338 @@ +use std::sync::Arc; + +use litellm_secrets::{SecretManager, SecretManagerState}; +use litellm_secrets_types::{AccessMode, KeyManagementSettings, KeyManagementSystem, SecretValue}; +use pyo3::prelude::*; +use serde_json::Value; + +use super::callback::PythonSecretManager; +use crate::{ + coercion::{Field, FieldSpec, ProjectionError}, + python_settings::{PythonSettings, Snapshot}, +}; + +const SYSTEM: FieldSpec> = + FieldSpec::new("system", parse_optional_system); +const ACCESS_MODE: FieldSpec = FieldSpec::new("access_mode", parse_access_mode); +const HOSTED_KEYS: FieldSpec>> = + FieldSpec::new("hosted_keys", |field| field.optional_string_collection()); +const STORE_VIRTUAL_KEYS: FieldSpec = + FieldSpec::new("store_virtual_keys", |field| field.truthy()); +const PREFIX_FOR_STORED_VIRTUAL_KEYS: FieldSpec = + FieldSpec::new("prefix_for_stored_virtual_keys", |field| { + field.strict_string() + }); +const PRIMARY_SECRET_NAME: FieldSpec> = + FieldSpec::new("primary_secret_name", |field| field.falsy_optional_string()); +const KMS_KEY_ID: FieldSpec> = + FieldSpec::new("kms_key_id", |field| field.falsy_optional_string()); +const CUSTOM_SECRET_MANAGER: FieldSpec> = + FieldSpec::new("custom_secret_manager", |field| { + field.falsy_optional_string() + }); +const AWS_REGION_NAME: FieldSpec> = + FieldSpec::new("aws_region_name", |field| field.falsy_optional_string()); +const AWS_ROLE_NAME: FieldSpec> = + FieldSpec::new("aws_role_name", |field| field.falsy_optional_string()); +const AWS_SESSION_NAME: FieldSpec> = + FieldSpec::new("aws_session_name", |field| field.falsy_optional_string()); +const AWS_EXTERNAL_ID: FieldSpec> = + FieldSpec::new("aws_external_id", |field| field.falsy_optional_string()); +const AWS_PROFILE_NAME: FieldSpec> = + FieldSpec::new("aws_profile_name", |field| field.falsy_optional_string()); +const AWS_WEB_IDENTITY_TOKEN: FieldSpec> = + FieldSpec::new("aws_web_identity_token", |field| { + field.falsy_optional_string() + }); +const AWS_STS_ENDPOINT: FieldSpec> = + FieldSpec::new("aws_sts_endpoint", |field| field.falsy_optional_string()); +const REPLICA_REGIONS: FieldSpec>> = + FieldSpec::new("replica_regions", |field| { + field.optional_string_collection() + }); +const CLIENT: FieldSpec>> = + FieldSpec::new("client", |field| Ok(field.python_binding())); +const SETTINGS_OBJECT: FieldSpec>> = + FieldSpec::new("settings_object", |field| Ok(field.python_binding())); + +/// `litellm.secret_manager_client` as the bridge classifies it. +#[derive(Debug)] +pub(crate) enum SecretManagerClient { + /// `None`: reads come from the process environment. + Local, + /// A custom manager, legacy compatible client, or manually assigned SDK client that keeps + /// executing in Python. + PythonCallback(Py), +} + +/// One operation-local capture of the secret manager globals, taken while attached to Python. +#[derive(Debug)] +pub(crate) struct SecretManagerSnapshot { + pub(crate) client: SecretManagerClient, + pub(crate) system: Option, + /// Typed settings that drive native routing: access mode and hosted keys. + pub(crate) settings: KeyManagementSettings, + /// The original `KeyManagementSettings` object, handed back to Python callbacks unchanged. + pub(crate) settings_object: Option>, +} + +impl SecretManagerSnapshot { + pub(crate) fn into_state(self) -> Arc { + match self.client { + SecretManagerClient::Local => Arc::new(SecretManagerState::default()), + SecretManagerClient::PythonCallback(client) => Arc::new(SecretManagerState::new( + SecretManager::External(Arc::new(PythonSecretManager::new( + client, + self.system, + self.settings_object, + ))), + self.settings, + )), + } + } +} + +/// Reads and projects the secret manager settings group in one attached operation. +pub(crate) fn read(py: Python<'_>) -> PyResult { + Ok(project(&PythonSettings::SecretManagerBinding.read(py)?)?) +} + +pub(crate) fn project(snapshot: &Snapshot<'_>) -> Result { + let system = snapshot.read(&SYSTEM)?; + let access_mode = snapshot.read(&ACCESS_MODE)?; + let settings = KeyManagementSettings { + hosted_keys: snapshot.read(&HOSTED_KEYS)?, + store_virtual_keys: Some(snapshot.read(&STORE_VIRTUAL_KEYS)?), + prefix_for_stored_virtual_keys: snapshot.read(&PREFIX_FOR_STORED_VIRTUAL_KEYS)?, + access_mode, + primary_secret_name: snapshot.read(&PRIMARY_SECRET_NAME)?, + kms_key_id: snapshot.read(&KMS_KEY_ID)?, + custom_secret_manager: snapshot.read(&CUSTOM_SECRET_MANAGER)?, + aws_region_name: snapshot.read(&AWS_REGION_NAME)?, + aws_role_name: snapshot.read(&AWS_ROLE_NAME)?, + aws_session_name: snapshot.read(&AWS_SESSION_NAME)?, + aws_external_id: snapshot.read(&AWS_EXTERNAL_ID)?.map(SecretValue::new), + aws_profile_name: snapshot.read(&AWS_PROFILE_NAME)?, + aws_web_identity_token: snapshot + .read(&AWS_WEB_IDENTITY_TOKEN)? + .map(SecretValue::new), + aws_sts_endpoint: snapshot.read(&AWS_STS_ENDPOINT)?, + replica_regions: snapshot.read(&REPLICA_REGIONS)?, + ..KeyManagementSettings::default() + }; + let client = match snapshot.read(&CLIENT)? { + None => SecretManagerClient::Local, + Some(client) => SecretManagerClient::PythonCallback(client), + }; + Ok(SecretManagerSnapshot { + client, + system, + settings, + settings_object: snapshot.read(&SETTINGS_OBJECT)?, + }) +} + +fn parse_optional_system( + field: &Field<'_>, +) -> Result, ProjectionError> { + let Some(value) = field.falsy_optional_string()? else { + return Ok(None); + }; + serde_json::from_value(Value::String(value)) + .map(Some) + .map_err(|error| { + ProjectionError::InvalidConfiguration(format!("secret manager system: {error}")) + }) +} + +fn parse_access_mode(field: &Field<'_>) -> Result { + let value = field.strict_string()?; + serde_json::from_value(Value::String(value)).map_err(|error| { + ProjectionError::InvalidConfiguration(format!("secret manager access mode: {error}")) + }) +} + +#[cfg(test)] +mod tests { + use pyo3::{ + prelude::*, + types::{PyDict, PyTuple}, + }; + + use super::{SecretManagerClient, project}; + use crate::python_settings::PythonSettings; + + fn snapshot<'py>( + py: Python<'py>, + system: &str, + access_mode: &str, + store_virtual_keys: Bound<'py, PyAny>, + hosted_keys: Bound<'py, PyAny>, + ) -> crate::python_settings::Snapshot<'py> { + snapshot_with_client( + py, + system, + access_mode, + store_virtual_keys, + hosted_keys, + py.None().into_bound(py), + ) + } + + fn snapshot_with_client<'py>( + py: Python<'py>, + system: &str, + access_mode: &str, + store_virtual_keys: Bound<'py, PyAny>, + hosted_keys: Bound<'py, PyAny>, + client: Bound<'py, PyAny>, + ) -> crate::python_settings::Snapshot<'py> { + let locals = PyDict::new(py); + locals.set_item("client", client).unwrap(); + locals.set_item("system", system).unwrap(); + locals.set_item("access_mode", access_mode).unwrap(); + locals + .set_item("store_virtual_keys", store_virtual_keys) + .unwrap(); + locals.set_item("hosted_keys", hosted_keys).unwrap(); + py.run( + cr#" +from dataclasses import dataclass +from types import SimpleNamespace + +@dataclass(frozen=True, slots=True) +class SecretManager: + system: object + access_mode: object + hosted_keys: object + primary_secret_name: object + store_virtual_keys: object + prefix_for_stored_virtual_keys: object + kms_key_id: object + custom_secret_manager: object + aws_region_name: object + aws_role_name: object + aws_session_name: object + aws_external_id: object + aws_profile_name: object + aws_web_identity_token: object + aws_sts_endpoint: object + replica_regions: object + client: object + settings_object: object + +root = SimpleNamespace(secret_manager=SecretManager( + system=system, + access_mode=access_mode, + hosted_keys=hosted_keys, + primary_secret_name=None, + store_virtual_keys=store_virtual_keys, + prefix_for_stored_virtual_keys="litellm/", + kms_key_id=None, + custom_secret_manager=None, + aws_region_name=None, + aws_role_name=None, + aws_session_name=None, + aws_external_id=None, + aws_profile_name=None, + aws_web_identity_token=None, + aws_sts_endpoint=None, + replica_regions=None, + client=client, + settings_object=None, +)) +"#, + Some(&locals), + Some(&locals), + ) + .unwrap(); + PythonSettings::SecretManagerBinding.snapshot( + locals + .get_item("root") + .unwrap() + .unwrap() + .getattr("secret_manager") + .unwrap(), + ) + } + + #[rstest::rstest] + #[case::string_true(Some("true"), false, true)] + #[case::string_one(Some("1"), false, true)] + #[case::true_value(None, true, true)] + #[case::false_value(None, false, false)] + #[case::string_false(Some("false"), false, true)] + fn python_compatible_boolean_coercion( + #[case] string_value: Option<&str>, + #[case] bool_value: bool, + #[case] expected: bool, + ) { + Python::initialize(); + Python::attach(|py| { + let store_virtual_keys = match string_value { + Some(value) => value.into_pyobject(py).unwrap().into_any(), + None => bool_value.into_pyobject(py).unwrap().to_owned().into_any(), + }; + let hosted_keys = PyTuple::new(py, ["ONE"]).unwrap().into_any(); + let projected = project(&snapshot( + py, + "local", + "read_only", + store_virtual_keys, + hosted_keys, + )) + .unwrap(); + assert_eq!(projected.settings.store_virtual_keys, Some(expected)); + }); + } + + #[test] + fn unknown_system_is_rejected() { + Python::initialize(); + Python::attach(|py| { + let error = project(&snapshot( + py, + "unknown", + "read_only", + false.into_pyobject(py).unwrap().to_owned().into_any(), + PyTuple::empty(py).into_any(), + )) + .unwrap_err(); + let error: PyErr = error.into(); + assert!(error.is_instance_of::(py)); + }); + } + + #[test] + fn client_identity_selects_local_or_python_callback() { + Python::initialize(); + Python::attach(|py| { + let falsy = false.into_pyobject(py).unwrap().to_owned().into_any(); + let local = project(&snapshot( + py, + "local", + "read_only", + falsy.clone(), + PyTuple::empty(py).into_any(), + )) + .unwrap(); + assert!(matches!(local.client, SecretManagerClient::Local)); + assert!(local.settings_object.is_none()); + + let manager = py.eval(c"object()", None, None).unwrap(); + let custom = project(&snapshot_with_client( + py, + "custom", + "read_only", + falsy, + PyTuple::empty(py).into_any(), + manager.clone(), + )) + .unwrap(); + let SecretManagerClient::PythonCallback(client) = custom.client else { + panic!("a live client must stay a Python callback"); + }; + assert!(client.bind(py).is(&manager)); + }); + } +} diff --git a/litellm-rust/crates/python-bridge/src/secrets/mod.rs b/litellm-rust/crates/python-bridge/src/secrets/mod.rs new file mode 100644 index 00000000000..f6ca57b08d1 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/secrets/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod callback; +pub(crate) mod config; +pub(crate) mod resolved; diff --git a/litellm-rust/crates/python-bridge/src/secrets/resolved.rs b/litellm-rust/crates/python-bridge/src/secrets/resolved.rs new file mode 100644 index 00000000000..877c429169a --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/secrets/resolved.rs @@ -0,0 +1,252 @@ +use std::{collections::HashMap, sync::Arc}; + +use futures_util::{future::BoxFuture, future::try_join_all}; +use litellm_core_utils::settings::{Lookup, ProcessEnvironment}; +use litellm_llms::base_llm::inference::secrets::{SecretSource, Secrets}; +use litellm_secrets::{ + Error, FailurePolicy, OidcResolver, Secret, SecretManagerState, SecretResolver, +}; + +use super::config::SecretManagerSnapshot; + +pub(crate) struct ResolvedSecrets { + resolver: SecretResolver, +} + +impl ResolvedSecrets { + pub(crate) fn new(snapshot: SecretManagerSnapshot) -> Self { + Self::from_state(snapshot.into_state()) + } + + fn from_state(state: Arc) -> Self { + Self { + resolver: SecretResolver::new( + state, + Arc::new(ProcessEnvironment), + OidcResolver::default(), + ) + .with_failure_policy(FailurePolicy::EnvironmentFallback), + } + } +} + +impl SecretSource for ResolvedSecrets { + fn resolve<'a>(&'a self, names: &'a [&'static str]) -> BoxFuture<'a, Result> { + Box::pin(async move { + let values = try_join_all(names.iter().map(|name| async move { + self.resolver + .get_secret(name, None) + .await + .map(|secret| secret.map(|secret| ((*name).to_owned(), secret_value(secret)))) + })) + .await? + .into_iter() + .flatten() + .collect::>(); + Ok(Arc::new(ResolvedLookup { values }) as Secrets) + }) + } +} + +struct ResolvedLookup { + values: HashMap, +} + +impl Lookup for ResolvedLookup { + fn get(&self, name: &str) -> Option { + self.values + .get(name) + .cloned() + .or_else(|| ProcessEnvironment.get(name)) + } +} + +fn secret_value(secret: Secret) -> String { + match secret { + Secret::String(value) => value.expose().to_owned(), + Secret::Bool(value) => if value { "True" } else { "False" }.to_owned(), + Secret::Json(value) => value.to_string(), + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use aws_sdk_secretsmanager::Client; + use aws_sdk_secretsmanager::config::{ + BehaviorVersion, Credentials, Region, retry::RetryConfig, + }; + use litellm_secrets::{AccessMode, KeyManagementSettings, SecretManager, SecretManagerState}; + use litellm_secrets_aws::AwsSecretsManagerV2; + use serde_json::json; + use wiremock::{ + Mock, MockServer, ResponseTemplate, + matchers::{body_partial_json, header}, + }; + + use super::ResolvedSecrets; + use litellm_llms::base_llm::inference::secrets::SecretSource; + + fn state(server: &MockServer, settings: KeyManagementSettings) -> Arc { + let client = Client::from_conf( + aws_sdk_secretsmanager::Config::builder() + .behavior_version(BehaviorVersion::latest()) + .region(Region::new("us-east-1")) + .credentials_provider(Credentials::new("test", "test", None, None, "test")) + .endpoint_url(server.uri()) + .retry_config(RetryConfig::disabled()) + .build(), + ); + Arc::new(SecretManagerState::new( + SecretManager::AwsSecretsManagerV2(AwsSecretsManagerV2::new( + client, + (&settings).into(), + )), + settings, + )) + } + + async fn resolve(state: Arc, name: &'static str) -> Option { + ResolvedSecrets::from_state(state) + .resolve(&[name]) + .await + .unwrap() + .get(name) + } + + #[tokio::test] + async fn hosted_key_miss_falls_back_to_environment() { + let name = "LITELLM_RUST_BRIDGE_HOSTED_KEY_MISS"; + unsafe { std::env::set_var(name, "env-key") }; + let server = MockServer::start().await; + Mock::given(header("x-amz-target", "secretsmanager.GetSecretValue")) + .and(body_partial_json(json!({"SecretId": name}))) + .respond_with( + ResponseTemplate::new(200).set_body_json(json!({"SecretString": "manager-key"})), + ) + .expect(0) + .mount(&server) + .await; + let result = resolve( + state( + &server, + KeyManagementSettings { + hosted_keys: Some(vec!["OTHER".into()]), + ..Default::default() + }, + ), + name, + ) + .await; + unsafe { std::env::remove_var(name) }; + assert_eq!(result.as_deref(), Some("env-key")); + assert_eq!(server.received_requests().await.unwrap().len(), 0); + } + + #[tokio::test] + async fn manager_failure_falls_back_to_environment() { + let name = "LITELLM_RUST_BRIDGE_MANAGER_FAILURE"; + unsafe { std::env::set_var(name, "env-key") }; + let server = MockServer::start().await; + Mock::given(header("x-amz-target", "secretsmanager.GetSecretValue")) + .respond_with(ResponseTemplate::new(500)) + .expect(1) + .mount(&server) + .await; + let result = resolve(state(&server, KeyManagementSettings::default()), name).await; + unsafe { std::env::remove_var(name) }; + assert_eq!(result.as_deref(), Some("env-key")); + assert_eq!(server.received_requests().await.unwrap().len(), 1); + + let missing_server = MockServer::start().await; + Mock::given(header("x-amz-target", "secretsmanager.GetSecretValue")) + .respond_with(ResponseTemplate::new(500)) + .expect(1) + .mount(&missing_server) + .await; + let missing = + ResolvedSecrets::from_state(state(&missing_server, KeyManagementSettings::default())) + .resolve(&["LITELLM_RUST_BRIDGE_MANAGER_FAILURE_MISSING"]) + .await; + assert!(matches!(missing, Err(litellm_secrets::Error::Aws(_)))); + } + + #[tokio::test] + async fn write_only_mode_never_consults_the_manager() { + let name = "LITELLM_RUST_BRIDGE_WRITE_ONLY"; + unsafe { std::env::set_var(name, "env-key") }; + let server = MockServer::start().await; + Mock::given(header("x-amz-target", "secretsmanager.GetSecretValue")) + .respond_with( + ResponseTemplate::new(200).set_body_json(json!({"SecretString": "manager-key"})), + ) + .expect(0) + .mount(&server) + .await; + let result = resolve( + state( + &server, + KeyManagementSettings { + access_mode: AccessMode::WriteOnly, + ..Default::default() + }, + ), + name, + ) + .await; + unsafe { std::env::remove_var(name) }; + assert_eq!(result.as_deref(), Some("env-key")); + assert_eq!(server.received_requests().await.unwrap().len(), 0); + } + + #[tokio::test] + async fn read_only_mode_resolves_from_the_manager() { + let name = "LITELLM_RUST_BRIDGE_READ_ONLY"; + let server = MockServer::start().await; + Mock::given(header("x-amz-target", "secretsmanager.GetSecretValue")) + .and(body_partial_json(json!({"SecretId": name}))) + .respond_with( + ResponseTemplate::new(200).set_body_json(json!({"SecretString": "manager-key"})), + ) + .expect(1) + .mount(&server) + .await; + assert_eq!( + resolve(state(&server, KeyManagementSettings::default()), name) + .await + .as_deref(), + Some("manager-key") + ); + assert_eq!(server.received_requests().await.unwrap().len(), 1); + } + + #[tokio::test] + async fn oidc_failures_are_not_converted_to_missing_secrets() { + let result = ResolvedSecrets::from_state(Arc::new(SecretManagerState::default())) + .resolve(&["oidc/"]) + .await; + assert!(matches!(result, Err(litellm_secrets::Error::InvalidOidc))); + } + + #[tokio::test] + async fn undeclared_names_still_read_the_process_environment() { + let name = "LITELLM_RUST_BRIDGE_UNDECLARED"; + unsafe { std::env::set_var(name, "env-key") }; + let server = MockServer::start().await; + let result = resolve( + state( + &server, + KeyManagementSettings { + hosted_keys: Some(vec!["OTHER".into()]), + ..Default::default() + }, + ), + name, + ) + .await; + unsafe { std::env::remove_var(name) }; + assert_eq!(result.as_deref(), Some("env-key")); + assert_eq!(server.received_requests().await.unwrap().len(), 0); + } +} diff --git a/litellm-rust/crates/secrets-types/src/config.rs b/litellm-rust/crates/secrets-types/src/config.rs index 36d319311a3..44acf512224 100644 --- a/litellm-rust/crates/secrets-types/src/config.rs +++ b/litellm-rust/crates/secrets-types/src/config.rs @@ -4,7 +4,7 @@ use serde::{Deserialize, Serialize}; use crate::SecretValue; -#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)] #[serde(rename_all = "snake_case")] pub enum KeyManagementSystem { GoogleKms, @@ -18,7 +18,7 @@ pub enum KeyManagementSystem { Custom, } -#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, Hash, PartialEq, Serialize)] #[serde(rename_all = "snake_case")] pub enum AccessMode { #[default] @@ -33,7 +33,7 @@ impl AccessMode { } } -#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq, Hash)] #[serde(default)] pub struct KeyManagementSettings { pub hosted_keys: Option>, diff --git a/litellm-rust/crates/secrets/src/error.rs b/litellm-rust/crates/secrets/src/error.rs index 1be0adc2bf5..de325ff4981 100644 --- a/litellm-rust/crates/secrets/src/error.rs +++ b/litellm-rust/crates/secrets/src/error.rs @@ -24,6 +24,8 @@ pub enum Error { OidcFile, #[error("secret cannot be converted to {expected}")] TypeMismatch { expected: &'static str }, + #[error("external secret manager failed")] + ExternalManager(#[source] Box), #[cfg(feature = "aws")] #[error(transparent)] Aws(#[from] litellm_secrets_aws::Error), diff --git a/litellm-rust/crates/secrets/src/handler.rs b/litellm-rust/crates/secrets/src/handler.rs index 5ab2caa75d4..8762b8e7405 100644 --- a/litellm-rust/crates/secrets/src/handler.rs +++ b/litellm-rust/crates/secrets/src/handler.rs @@ -1,10 +1,24 @@ +use std::{future::Future, pin::Pin, sync::Arc}; + use litellm_core_utils::settings::Lookup; use crate::{Error, KeyManagementSettings, KeyManagementSystem, Secret, SecretValue}; +pub trait ExternalSecretManager: Send + Sync { + fn system(&self) -> KeyManagementSystem; + + fn read_secret<'a>( + &'a self, + name: &'a str, + settings: &'a KeyManagementSettings, + environment: &'a (dyn Lookup + Send + Sync), + ) -> Pin, Error>> + Send + 'a>>; +} + #[derive(Clone)] pub enum SecretManager { Local, + External(Arc), #[cfg(feature = "aws")] AwsKms(crate::aws::AwsKms), #[cfg(feature = "aws")] @@ -25,6 +39,7 @@ impl SecretManager { pub fn system(&self) -> KeyManagementSystem { match self { Self::Local => KeyManagementSystem::Local, + Self::External(manager) => manager.system(), #[cfg(feature = "aws")] Self::AwsKms(_) => KeyManagementSystem::AwsKms, #[cfg(feature = "aws")] @@ -54,6 +69,11 @@ pub async fn get_secret_from_manager( .get(secret_name) .map(SecretValue::new) .map(Secret::String)), + SecretManager::External(manager) => { + manager + .read_secret(secret_name, _settings, environment) + .await + } #[cfg(feature = "aws")] SecretManager::AwsKms(client) => { let ciphertext = environment diff --git a/litellm-rust/crates/secrets/src/lib.rs b/litellm-rust/crates/secrets/src/lib.rs index 1acb5269e66..58aba8494fd 100644 --- a/litellm-rust/crates/secrets/src/lib.rs +++ b/litellm-rust/crates/secrets/src/lib.rs @@ -7,7 +7,7 @@ mod resolver; mod state; pub use error::Error; -pub use handler::{SecretManager, get_secret_from_manager}; +pub use handler::{ExternalSecretManager, SecretManager, get_secret_from_manager}; pub use litellm_secrets_types::{ AccessMode, KeyManagementSettings, KeyManagementSystem, Secret, SecretValue, }; diff --git a/litellm-rust/crates/secrets/src/resolver.rs b/litellm-rust/crates/secrets/src/resolver.rs index 89439893852..597ca11b171 100644 --- a/litellm-rust/crates/secrets/src/resolver.rs +++ b/litellm-rust/crates/secrets/src/resolver.rs @@ -72,6 +72,7 @@ impl SecretResolver { Ok(value) => Ok(value .or_else(|| self.environment_secret(name)) .or(default_value)), + Err(error @ Error::ExternalManager(_)) => Err(error), Err(error) => match self.failure_policy { FailurePolicy::Propagate => Err(error), FailurePolicy::EnvironmentFallback => self diff --git a/litellm/chat_completions/dispatch.py b/litellm/chat_completions/dispatch.py index d36c0343988..8e4b7f82eb8 100644 --- a/litellm/chat_completions/dispatch.py +++ b/litellm/chat_completions/dispatch.py @@ -4,7 +4,7 @@ from types import MappingProxyType from typing import Final, TypeAlias, cast # noqa: TID251 # native binding selects a sync result or an async awaitable from litellm import main -from litellm.rust_bridge.catalog import Context, Delivery, Route +from litellm.rust_bridge.catalog import Delivery, Route, RouteContext from litellm.rust_bridge.chat_completions.entrypoints import ( NATIVE_ACOMPLETION, NATIVE_COMPLETION, @@ -72,8 +72,8 @@ def _public_request( ) -def _context(request: LiteLLMChatCompletionsRequest) -> Context: - return Context( +def _context(request: LiteLLMChatCompletionsRequest) -> RouteContext: + return RouteContext( Route.CHAT_COMPLETIONS, provider=request.custom_llm_provider, model=request.model, diff --git a/litellm/llms/bedrock/audio_transcription/__init__.py b/litellm/llms/bedrock/audio_transcription/__init__.py index 8f35b8eac7a..172316027a8 100644 --- a/litellm/llms/bedrock/audio_transcription/__init__.py +++ b/litellm/llms/bedrock/audio_transcription/__init__.py @@ -5,7 +5,7 @@ import httpx from litellm.litellm_core_utils.audio_utils.utils import process_audio_file from litellm.rust_bridge import runtime -from litellm.rust_bridge.catalog import Context, Route +from litellm.rust_bridge.catalog import Route, RouteContext from litellm.rust_bridge.timeouts import timeout_to_seconds from litellm.rust_bridge.transcription.native import ( NATIVE_ATRANSCRIPTION, @@ -74,7 +74,7 @@ class BedrockAudioTranscriptionRustDispatch: ) return runtime.run( - Context(Route.TRANSCRIPTION, provider=custom_llm_provider, model=model), + RouteContext(Route.TRANSCRIPTION, provider=custom_llm_provider, model=model), binding=NATIVE_TRANSCRIPTION, native=native, python=_no_python_implementation, @@ -107,7 +107,7 @@ class BedrockAudioTranscriptionRustDispatch: ) return await runtime.arun( - Context(Route.TRANSCRIPTION, provider=custom_llm_provider, model=model), + RouteContext(Route.TRANSCRIPTION, provider=custom_llm_provider, model=model), binding=NATIVE_ATRANSCRIPTION, native=native, python=_no_async_python_implementation, diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 2a105301521..18d4c27fec1 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -182,10 +182,10 @@ from litellm.utils import ( def _rust_responses_websocket_enabled( custom_llm_provider: str | None, ) -> bool: - from litellm.rust_bridge.catalog import Context, Delivery, Route, decision + from litellm.rust_bridge.catalog import Delivery, Route, RouteContext, decision from litellm.rust_bridge.configuration import Decision - context: Final = Context(Route.RESPONSES, provider=custom_llm_provider, delivery=Delivery.WEBSOCKET) + context: Final = RouteContext(Route.RESPONSES, provider=custom_llm_provider, delivery=Delivery.WEBSOCKET) return decision(context) is not Decision.PYTHON diff --git a/litellm/messages/dispatch.py b/litellm/messages/dispatch.py index c75f6564d1b..a0c791a136c 100644 --- a/litellm/messages/dispatch.py +++ b/litellm/messages/dispatch.py @@ -4,7 +4,7 @@ from types import MappingProxyType from typing import Final, TypeAlias, cast # noqa: TID251 # native binding selects a sync result or an async awaitable from litellm.llms.anthropic.experimental_pass_through.messages import handler as main -from litellm.rust_bridge.catalog import Context, Delivery, Route +from litellm.rust_bridge.catalog import Delivery, Route, RouteContext from litellm.rust_bridge.dispatch import PublicDispatch, call_hook from litellm.rust_bridge.messages.entrypoints import ( NATIVE_AMESSAGES, @@ -71,8 +71,8 @@ def _public_request( ) -def _context(request: LiteLLMMessagesRequest) -> Context: - return Context( +def _context(request: LiteLLMMessagesRequest) -> RouteContext: + return RouteContext( Route.MESSAGES, provider=request.custom_llm_provider, model=request.model, diff --git a/litellm/ocr/dispatch.py b/litellm/ocr/dispatch.py index 55b19458b7a..b26175c943b 100644 --- a/litellm/ocr/dispatch.py +++ b/litellm/ocr/dispatch.py @@ -6,7 +6,7 @@ import httpx from litellm.llms.base_llm.ocr.transformation import OCRResponse from litellm.ocr import main from litellm.ocr.main import convert_file_document_to_url_document, get_mime_type -from litellm.rust_bridge.catalog import Context, Route +from litellm.rust_bridge.catalog import Route, RouteContext from litellm.rust_bridge.dispatch import PublicDispatch, call_hook from litellm.rust_bridge.ocr.entrypoints import NATIVE_AOCR, NATIVE_OCR, LiteLLMOcrRequest @@ -52,10 +52,10 @@ _PYTHON_AOCR: Final = cast( # cast-ok: forward the original call shape through ) -def _context(request: LiteLLMOcrRequest) -> Context: +def _context(request: LiteLLMOcrRequest) -> RouteContext: prefix, separator, _ = request.model.partition("/") provider: Final = request.custom_llm_provider or (prefix if separator else None) - return Context(Route.OCR, provider=provider, model=request.model) + return RouteContext(Route.OCR, provider=provider, model=request.model) _DISPATCH: Final = PublicDispatch( diff --git a/litellm/responses/dispatch.py b/litellm/responses/dispatch.py index b2748fca4b6..d240356805c 100644 --- a/litellm/responses/dispatch.py +++ b/litellm/responses/dispatch.py @@ -5,7 +5,7 @@ from typing import Final, TypeAlias, cast # noqa: TID251 # native binding sele from litellm.responses import main from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator -from litellm.rust_bridge.catalog import Context, Delivery, Route +from litellm.rust_bridge.catalog import Delivery, Route, RouteContext from litellm.rust_bridge.dispatch import PublicDispatch, call_hook from litellm.rust_bridge.public_call import bind, optional_bool, optional_mapping, optional_str, signature from litellm.rust_bridge.responses.entrypoints import ( @@ -64,8 +64,8 @@ def _public_request( ) -def _context(request: LiteLLMResponsesRequest) -> Context: - return Context( +def _context(request: LiteLLMResponsesRequest) -> RouteContext: + return RouteContext( Route.RESPONSES, provider=request.custom_llm_provider, model=request.model, diff --git a/litellm/rust_bridge/_native.pyi b/litellm/rust_bridge/_native.pyi index 31623a53dd0..a033b89a6a8 100644 --- a/litellm/rust_bridge/_native.pyi +++ b/litellm/rust_bridge/_native.pyi @@ -94,7 +94,9 @@ class ResponsesWebSocketConnection: def close(self) -> Future[None]: ... @final -class _CacheTestBinding: +class _ResponseCacheRuntime: + @staticmethod + def from_cache(cache: object) -> _ResponseCacheRuntime: ... @property def kind(self) -> str: ... def lookup( @@ -218,7 +220,7 @@ class _CacheTestHandle: @final class _CacheTestResolver: def __new__(cls, namespace: object) -> _CacheTestResolver: ... - def resolve(self) -> _CacheTestBinding: ... + def resolve(self) -> _ResponseCacheRuntime: ... @final class TokenCounter: diff --git a/litellm/rust_bridge/catalog.py b/litellm/rust_bridge/catalog.py index 8794ff2db95..74ceb1ba123 100644 --- a/litellm/rust_bridge/catalog.py +++ b/litellm/rust_bridge/catalog.py @@ -1,9 +1,7 @@ -"""Declarative Rust/Python selection for routes with Rust integration. +"""Ordered rollout policy for routes, cache backends, and secret managers. -Rules are static data matched top to bottom; the first match wins and a -context with no matching rule stays on Python. Whether the Rust core can serve -a specific request body is not decided here: that is Rust admission, which -signals ``RustBridgeDeclined`` before any provider I/O. +The first matching rule wins; unmatched contexts stay on Python. Native +admission separately decides whether the selected implementation can execute. """ from __future__ import annotations @@ -14,6 +12,8 @@ from typing import Final, TypeAlias from litellm.rust_bridge.configuration import Decision, Rollout from litellm.rust_bridge.configuration import decision as _decision +from litellm.types.caching import LiteLLMCacheType +from litellm.types.secret_managers.main import KeyManagementSystem class Route(str, Enum): @@ -31,7 +31,7 @@ class Delivery(Enum): @dataclass(frozen=True, slots=True) -class Context: +class RouteContext: route: Route provider: str | None = None model: str | None = None @@ -39,7 +39,7 @@ class Context: @dataclass(frozen=True, slots=True) -class Rule: +class RouteRule: route: Route rollout: Rollout providers: frozenset[str] | None = None @@ -48,26 +48,76 @@ class Rule: def matches(self, context: Context) -> bool: return ( - context.route is self.route + isinstance(context, RouteContext) + and context.route is self.route and (self.providers is None or context.provider in self.providers) and (self.models is None or context.model in self.models) and (self.deliveries is None or context.delivery in self.deliveries) ) +@dataclass(frozen=True, slots=True) +class CacheContext: + backend: str + + +@dataclass(frozen=True, slots=True) +class CacheRule: + rollout: Rollout + backends: frozenset[str] | None = None + + def matches(self, context: Context) -> bool: + return isinstance(context, CacheContext) and (self.backends is None or context.backend in self.backends) + + +@dataclass(frozen=True, slots=True) +class SecretManagerContext: + system: str + + +@dataclass(frozen=True, slots=True) +class SecretManagerRule: + rollout: Rollout + systems: frozenset[str] | None = None + + def matches(self, context: Context) -> bool: + return isinstance(context, SecretManagerContext) and (self.systems is None or context.system in self.systems) + + +Context: TypeAlias = RouteContext | CacheContext | SecretManagerContext +Rule: TypeAlias = RouteRule | CacheRule | SecretManagerRule Rules: TypeAlias = tuple[Rule, ...] RULES: Final[Rules] = ( - Rule(Route.OCR, Rollout.RUST_REQUIRED, providers=frozenset({"aws_textract"})), - Rule(Route.OCR, Rollout.RUST_OPT_OUT), - Rule(Route.MESSAGES, Rollout.RUST_OPT_IN), - Rule(Route.TRANSCRIPTION, Rollout.RUST_REQUIRED, providers=frozenset({"bedrock"})), + RouteRule(Route.OCR, Rollout.RUST_REQUIRED, providers=frozenset({"aws_textract"})), + RouteRule(Route.OCR, Rollout.RUST_OPT_OUT), + RouteRule(Route.MESSAGES, Rollout.RUST_OPT_IN), + RouteRule(Route.TRANSCRIPTION, Rollout.RUST_REQUIRED, providers=frozenset({"bedrock"})), + CacheRule(Rollout.PYTHON_ONLY, backends=frozenset({LiteLLMCacheType.LOCAL})), + CacheRule(Rollout.PYTHON_ONLY, backends=frozenset({LiteLLMCacheType.REDIS})), + CacheRule(Rollout.PYTHON_ONLY, backends=frozenset({LiteLLMCacheType.REDIS_SEMANTIC})), + CacheRule(Rollout.PYTHON_ONLY, backends=frozenset({LiteLLMCacheType.VALKEY_SEMANTIC})), + CacheRule(Rollout.PYTHON_ONLY, backends=frozenset({LiteLLMCacheType.S3})), + CacheRule(Rollout.PYTHON_ONLY, backends=frozenset({LiteLLMCacheType.DISK})), + CacheRule(Rollout.PYTHON_ONLY, backends=frozenset({LiteLLMCacheType.QDRANT_SEMANTIC})), + CacheRule(Rollout.PYTHON_ONLY, backends=frozenset({LiteLLMCacheType.AZURE_BLOB})), + CacheRule(Rollout.PYTHON_ONLY, backends=frozenset({LiteLLMCacheType.GCS})), + SecretManagerRule(Rollout.PYTHON_ONLY, systems=frozenset({KeyManagementSystem.GOOGLE_KMS.value})), + SecretManagerRule(Rollout.PYTHON_ONLY, systems=frozenset({KeyManagementSystem.AZURE_KEY_VAULT.value})), + SecretManagerRule(Rollout.PYTHON_ONLY, systems=frozenset({KeyManagementSystem.AWS_SECRET_MANAGER.value})), + SecretManagerRule(Rollout.PYTHON_ONLY, systems=frozenset({KeyManagementSystem.GOOGLE_SECRET_MANAGER.value})), + SecretManagerRule(Rollout.PYTHON_ONLY, systems=frozenset({KeyManagementSystem.HASHICORP_VAULT.value})), + SecretManagerRule(Rollout.PYTHON_ONLY, systems=frozenset({KeyManagementSystem.CYBERARK.value})), + SecretManagerRule(Rollout.PYTHON_ONLY, systems=frozenset({KeyManagementSystem.LOCAL.value})), + SecretManagerRule(Rollout.PYTHON_ONLY, systems=frozenset({KeyManagementSystem.AWS_KMS.value})), + SecretManagerRule(Rollout.PYTHON_ONLY, systems=frozenset({KeyManagementSystem.CUSTOM.value})), ) -def rollout(context: Context, rules: Rules = RULES) -> Rollout: - return next((rule.rollout for rule in rules if rule.matches(context)), Rollout.PYTHON_ONLY) +def rollout(context: Context, rules: Rules | None = None) -> Rollout: + selected_rules: Final = RULES if rules is None else rules + return next((rule.rollout for rule in selected_rules if rule.matches(context)), Rollout.PYTHON_ONLY) -def decision(context: Context, rules: Rules = RULES) -> Decision: +def decision(context: Context, rules: Rules | None = None) -> Decision: return _decision(rollout(context, rules)) diff --git a/litellm/rust_bridge/configuration.py b/litellm/rust_bridge/configuration.py index 791e13a51d0..cd468c80655 100644 --- a/litellm/rust_bridge/configuration.py +++ b/litellm/rust_bridge/configuration.py @@ -84,7 +84,7 @@ def reset_rust_configuration() -> None: def rust(enabled: bool | None) -> None: """Set the process override for optional Rust paths. - ``PYTHON_ONLY`` and ``RUST_REQUIRED`` routes in the catalog ignore this switch, + ``PYTHON_ONLY`` and ``RUST_REQUIRED`` entries in the catalog ignore this switch, and an explicit ``LITELLM_RUST`` environment value wins over it. """ _CONFIGURATION.override = enabled diff --git a/litellm/rust_bridge/dispatch.py b/litellm/rust_bridge/dispatch.py index 7ddc903df58..076b7759c6d 100644 --- a/litellm/rust_bridge/dispatch.py +++ b/litellm/rust_bridge/dispatch.py @@ -6,7 +6,7 @@ from typing import Final, Generic, TypeVar from litellm.rust_bridge import catalog, runtime from litellm.rust_bridge.bindings import NativeBinding -from litellm.rust_bridge.catalog import Context, Route, Rules +from litellm.rust_bridge.catalog import Route, RouteContext, RouteRule, Rules from litellm.rust_bridge.configuration import Decision from litellm.rust_bridge.configuration import decision as rollout_decision @@ -30,12 +30,12 @@ def call_hook( class PublicDispatch(Generic[RequestT]): route: Route request: Callable[[tuple[object, ...], Mapping[str, object]], RequestT | None] - context: Callable[[RequestT], Context] + context: Callable[[RequestT], RouteContext] bypass: Callable[[RequestT], bool] | None = None def _requires_projection(self, rules: Rules) -> bool: for rule in rules: - if rule.route is not self.route: + if not isinstance(rule, RouteRule) or rule.route is not self.route: continue if rule.providers is not None or rule.models is not None or rule.deliveries is not None: if rollout_decision(rule.rollout) is not Decision.PYTHON: diff --git a/litellm/rust_bridge/response_cache.py b/litellm/rust_bridge/response_cache.py new file mode 100644 index 00000000000..82d27fce27b --- /dev/null +++ b/litellm/rust_bridge/response_cache.py @@ -0,0 +1,180 @@ +from __future__ import annotations + +import math +from collections.abc import Awaitable, Mapping, Sequence +from dataclasses import dataclass +from typing import Final, Protocol, cast + +from typing_extensions import ReadOnly, Required, TypedDict, assert_never + +from litellm.rust_bridge.bindings import NativeBinding, native_exception_types +from litellm.rust_bridge.catalog import CacheContext, Rules, decision +from litellm.rust_bridge.configuration import Decision + + +class CacheFacade(Protocol): + @property + def type(self) -> object: ... + + @property + def ttl(self) -> float | None: ... + + @property + def semantic_cache_scope(self) -> str: ... + + def get_cache_key(self, **kwargs: object) -> str: ... # kwargs-ok: mirrors the legacy cache facade contract + + +class NativeCacheKey(TypedDict): + preset: ReadOnly[str] + + +class NativeCacheRequest(TypedDict, total=False): + key: Required[ReadOnly[NativeCacheKey]] + ttl_seconds: ReadOnly[float | None] + max_age_seconds: ReadOnly[float | None] + messages: ReadOnly[object | None] + input: ReadOnly[object | None] + metadata: ReadOnly[object | None] + litellm_metadata: ReadOnly[object | None] + litellm_params: ReadOnly[object | None] + scope: ReadOnly[str] + + +class NativeResponseCacheRuntime(Protocol): + @property + def kind(self) -> str: ... + + def lookup(self, request: NativeCacheRequest) -> object: ... + def store(self, request: NativeCacheRequest, response: object) -> None: ... + def lookup_batch(self, requests: Sequence[NativeCacheRequest]) -> object: ... + def async_lookup(self, request: NativeCacheRequest) -> Awaitable[object]: ... + def async_store(self, request: NativeCacheRequest, response: object) -> Awaitable[None]: ... + def async_lookup_batch(self, requests: Sequence[NativeCacheRequest]) -> Awaitable[object]: ... + def async_store_batch( + self, + requests: Sequence[NativeCacheRequest], + responses: Sequence[object], + ) -> Awaitable[object]: ... + def async_flush(self) -> Awaitable[None]: ... + def ping(self) -> Awaitable[object]: ... + + +class NativeResponseCacheRuntimeFactory(Protocol): + @staticmethod + def from_cache(cache: CacheFacade) -> NativeResponseCacheRuntime: ... + + +def _runtime_factory(value: object) -> NativeResponseCacheRuntimeFactory | None: + return cast(NativeResponseCacheRuntimeFactory, value) if callable(getattr(value, "from_cache", None)) else None + + +_RUNTIME: Final = NativeBinding("_ResponseCacheRuntime", validate=_runtime_factory) + + +@dataclass(frozen=True, slots=True) +class ResponseCacheRuntime: + native: NativeResponseCacheRuntime + + @property + def kind(self) -> str: + return self.native.kind + + def request(self, cache: CacheFacade, kwargs: Mapping[str, object]) -> NativeCacheRequest | None: + key_value: Final = kwargs.get("cache_key") + key: Final = key_value if isinstance(key_value, str) else cache.get_cache_key(**dict(kwargs)) + if not key: + return None + control_value: Final = kwargs.get("cache") + control: Final = _string_mapping(control_value) + configured_ttl: Final = cache.ttl if cache.ttl is not None else _duration(kwargs.get("ttl")) + control_ttl: Final = _duration(control.get("ttl")) + current_max_age: Final = _duration(control.get("s-max-age")) + legacy_max_age: Final = _duration(control.get("s-maxage")) + ttl: Final = configured_ttl if control_ttl is None else control_ttl + max_age: Final = legacy_max_age if current_max_age is None else current_max_age + return NativeCacheRequest( + key=NativeCacheKey(preset=key), + ttl_seconds=ttl, + max_age_seconds=max_age, + messages=kwargs.get("messages"), + input=kwargs.get("input"), + metadata=kwargs.get("metadata"), + litellm_metadata=kwargs.get("litellm_metadata"), + litellm_params=kwargs.get("litellm_params"), + scope=cache.semantic_cache_scope, + ) + + def lookup(self, request: NativeCacheRequest) -> object: + return self.native.lookup(request) + + def store(self, request: NativeCacheRequest, response: object) -> None: + self.native.store(request, response) + + def lookup_batch(self, requests: Sequence[NativeCacheRequest]) -> object: + return self.native.lookup_batch(requests) + + async def async_lookup(self, request: NativeCacheRequest) -> object: + return await self.native.async_lookup(request) + + async def async_store(self, request: NativeCacheRequest, response: object) -> None: + await self.native.async_store(request, response) + + async def async_lookup_batch(self, requests: Sequence[NativeCacheRequest]) -> object: + return await self.native.async_lookup_batch(requests) + + async def async_store_batch( + self, + requests: Sequence[NativeCacheRequest], + responses: Sequence[object], + ) -> object: + return await self.native.async_store_batch(requests, responses) + + async def ping(self) -> object: + return await self.native.ping() + + async def async_flush(self) -> None: + await self.native.async_flush() + + +def resolve_response_cache( + cache: CacheFacade, + rules: Rules | None = None, +) -> ResponseCacheRuntime | None: + backend_value: Final = cache.type + backend: Final = str.__str__(backend_value) if isinstance(backend_value, str) else str(backend_value) + selected: Final = decision(CacheContext(backend=backend), rules) + match selected: + case Decision.PYTHON: + return None + case Decision.RUST_WITH_FALLBACK | Decision.RUST_REQUIRED: + factory: Final = _RUNTIME.load() + if factory is None: + if selected is Decision.RUST_REQUIRED: + raise RuntimeError("Rust response cache runtime is unavailable") + return None + try: + return ResponseCacheRuntime(factory.from_cache(cache)) + except Exception as error: + exceptions: Final = native_exception_types() + if exceptions is None or not isinstance(error, exceptions[0]): + raise + if selected is Decision.RUST_REQUIRED: + raise RuntimeError(f"Rust response cache runtime declined the cache: {error}") from error + return None + case _: + assert_never(selected) + + +def _duration(value: object) -> float | None: + if isinstance(value, bool) or not isinstance(value, int | float): + return None + duration: Final = float(value) + return duration if math.isfinite(duration) and duration >= 0 else None + + +def _string_mapping(value: object) -> Mapping[str, object]: + if not isinstance(value, Mapping): + return {} + source: Final = cast(Mapping[object, object], value) + return {key: item for key, item in source.items() if isinstance(key, str)} diff --git a/litellm/rust_bridge/runtime.py b/litellm/rust_bridge/runtime.py index 1fcde1bf555..cf02a33eab6 100644 --- a/litellm/rust_bridge/runtime.py +++ b/litellm/rust_bridge/runtime.py @@ -8,7 +8,7 @@ from typing_extensions import assert_never from litellm.exceptions import APIError from litellm.rust_bridge.bindings import NativeBinding, native_exception_types -from litellm.rust_bridge.catalog import RULES, Context, Rules, decision +from litellm.rust_bridge.catalog import RouteContext, Rules, decision from litellm.rust_bridge.configuration import Decision from litellm.rust_bridge.response_metadata import mark_rust_response @@ -42,14 +42,14 @@ class BridgeErrorContext: def run( - context: Context, + context: RouteContext, *, binding: NativeBinding[NativeT], native: Callable[[NativeT], ResultT], python: Callable[[], ResultT], rules: Rules | None = None, ) -> ResultT: - selected: Final = decision(context, RULES if rules is None else rules) + selected: Final = decision(context, rules) match selected: case Decision.PYTHON: return python() @@ -70,14 +70,14 @@ def run( async def arun( - context: Context, + context: RouteContext, *, binding: NativeBinding[NativeT], native: Callable[[NativeT], Awaitable[ResultT]], python: Callable[[], Awaitable[ResultT]], rules: Rules | None = None, ) -> ResultT: - selected: Final = decision(context, RULES if rules is None else rules) + selected: Final = decision(context, rules) match selected: case Decision.PYTHON: return await python() @@ -101,7 +101,7 @@ def _identity(value: ResultT) -> ResultT: return value -def _error_context(context: Context) -> BridgeErrorContext: +def _error_context(context: RouteContext) -> BridgeErrorContext: return BridgeErrorContext(route=context.route.value, provider=context.provider or "", model=context.model or "") diff --git a/litellm/rust_bridge/settings.py b/litellm/rust_bridge/settings.py index 9a5cf49f298..866f2fce989 100644 --- a/litellm/rust_bridge/settings.py +++ b/litellm/rust_bridge/settings.py @@ -1,6 +1,7 @@ from __future__ import annotations from dataclasses import dataclass +from typing import Final @dataclass(frozen=True, slots=True) @@ -35,6 +36,28 @@ class SecretManager: readable: bool +@dataclass(frozen=True, slots=True) +class SecretManagerBinding: + system: object + access_mode: object + hosted_keys: object + primary_secret_name: object + store_virtual_keys: object + prefix_for_stored_virtual_keys: object + kms_key_id: object + custom_secret_manager: object + aws_region_name: object + aws_role_name: object + aws_session_name: object + aws_external_id: object + aws_profile_name: object + aws_web_identity_token: object + aws_sts_endpoint: object + replica_regions: object + client: object + settings_object: object + + def warn(message: str) -> None: from litellm._logging import verbose_logger @@ -49,6 +72,42 @@ def secret_manager() -> SecretManager: return SecretManager(readable=_should_read_secret_from_secret_manager()) +def secret_manager_binding() -> SecretManagerBinding: + import litellm + from litellm.types.secret_managers.main import KeyManagementSettings + + configured_system: Final = ( + litellm._key_management_system # pyright: ignore[reportPrivateUsage] # canonical key management globals are private + ) + configured_settings: Final = ( + litellm._key_management_settings # pyright: ignore[reportPrivateUsage] # canonical key management globals are private + ) + settings: Final = configured_settings or KeyManagementSettings() + system: Final = ( + configured_system.value if litellm.secret_manager_client is not None and configured_system is not None else None + ) + return SecretManagerBinding( + system=system, + access_mode=settings.access_mode, + hosted_keys=settings.hosted_keys, + primary_secret_name=settings.primary_secret_name, + store_virtual_keys=settings.store_virtual_keys, + prefix_for_stored_virtual_keys=settings.prefix_for_stored_virtual_keys, + kms_key_id=settings.kms_key_id, + custom_secret_manager=settings.custom_secret_manager, + aws_region_name=settings.aws_region_name, + aws_role_name=settings.aws_role_name, + aws_session_name=settings.aws_session_name, + aws_external_id=settings.aws_external_id, + aws_profile_name=settings.aws_profile_name, + aws_web_identity_token=settings.aws_web_identity_token, + aws_sts_endpoint=settings.aws_sts_endpoint, + replica_regions=settings.replica_regions, + client=litellm.secret_manager_client, + settings_object=configured_settings, + ) + + def provider_defaults() -> ProviderDefaults: import litellm diff --git a/tests/test_litellm/responses/test_dispatch.py b/tests/test_litellm/responses/test_dispatch.py index 2990360d550..45cb5c4f1ad 100644 --- a/tests/test_litellm/responses/test_dispatch.py +++ b/tests/test_litellm/responses/test_dispatch.py @@ -13,7 +13,7 @@ from litellm.responses.dispatch import ( ) from litellm.rust_bridge import catalog from litellm.rust_bridge.bindings import NativeBinding -from litellm.rust_bridge.catalog import Route, Rule +from litellm.rust_bridge.catalog import Route, RouteRule from litellm.rust_bridge.configuration import Rollout from litellm.rust_bridge.responses.entrypoints import ( NATIVE_ARESPONSES, @@ -26,7 +26,7 @@ from litellm.types.llms.openai import ResponsesAPIResponse INPUT: Final = [{"role": "user", "content": "hi"}] PYTHON_RULES: Final = () -RUST_RULES: Final = (Rule(Route.RESPONSES, Rollout.RUST_REQUIRED),) +RUST_RULES: Final = (RouteRule(Route.RESPONSES, Rollout.RUST_REQUIRED),) def _response(model: str = "gpt-4o") -> ResponsesAPIResponse: @@ -102,7 +102,8 @@ async def test_async_python_route_forwards_original_call_shape() -> None: response: Final = _response() async def python( - *call_args: object, **call_kwargs: object # kwargs-ok: records call shape + *call_args: object, + **call_kwargs: object, # kwargs-ok: records call shape ) -> ResponsesAPIResponse: captured.append((call_args, call_kwargs)) return response @@ -143,9 +144,7 @@ def test_native_receives_normalized_request_and_original_call_shape() -> None: "custom_llm_provider": "anthropic", "litellm_metadata": metadata, } - captured: Final[ - list[tuple[LiteLLMResponsesRequest, tuple[object, ...], Mapping[str, object]]] - ] = [] + captured: Final[list[tuple[LiteLLMResponsesRequest, tuple[object, ...], Mapping[str, object]]]] = [] response: Final = _response("anthropic/claude-sonnet-4-5") def python(*call_args: object, **call_kwargs: object) -> ResponsesAPIResponse: # kwargs-ok: rejected fallback @@ -228,9 +227,7 @@ def test_internal_async_marker_bypasses_native() -> None: ((), {}), ), ) -def test_binding_errors_delegate_unchanged_to_python( - args: tuple[object, ...], kwargs: Mapping[str, object] -) -> None: +def test_binding_errors_delegate_unchanged_to_python(args: tuple[object, ...], kwargs: Mapping[str, object]) -> None: captured: Final[list[tuple[tuple[object, ...], Mapping[str, object]]]] = [] response: Final = _response() diff --git a/tests/test_litellm/rust_bridge/ocr/test_secrets.py b/tests/test_litellm/rust_bridge/ocr/test_secrets.py new file mode 100644 index 00000000000..085a42dd373 --- /dev/null +++ b/tests/test_litellm/rust_bridge/ocr/test_secrets.py @@ -0,0 +1,112 @@ +from __future__ import annotations + +from typing import Final + +import httpx +import pytest + +import litellm +from litellm.integrations.custom_secret_manager import CustomSecretManager +from litellm.llms.base_llm.ocr.transformation import OCRResponse +from litellm.rust_bridge import configuration +from litellm.types.secret_managers.main import KeyManagementSettings, KeyManagementSystem +from tests.test_litellm_rust.support.recording_server import ResponseSpec, recording_service + + +class _VaultSecrets(CustomSecretManager): + def __init__(self) -> None: + super().__init__(secret_manager_name="rust_bridge_ocr_test") + + async def async_read_secret( + self, + secret_name: str, + optional_params: dict[str, object] | None = None, + timeout: float | httpx.Timeout | None = None, + ) -> str | None: + return "vault-key" if secret_name == "MISTRAL_API_KEY" else None + + def sync_read_secret( + self, + secret_name: str, + optional_params: dict[str, object] | None = None, + timeout: float | httpx.Timeout | None = None, + ) -> str | None: + return "vault-key" if secret_name == "MISTRAL_API_KEY" else None + + +async def _call(asynchronous: bool, api_base: str) -> OCRResponse: + if asynchronous: + return await litellm.aocr( + model="mistral/mistral-ocr-latest", + document={"type": "document_url", "document_url": "https://example.com/document.pdf"}, + api_base=api_base, + ) + return litellm.ocr( + model="mistral/mistral-ocr-latest", + document={"type": "document_url", "document_url": "https://example.com/document.pdf"}, + api_base=api_base, + ) + + +_RESPONSE: Final = { + "pages": [{"index": 0, "markdown": "parsed document", "images": []}], + "model": "mistral-ocr-latest", + "usage_info": {"pages_processed": 1}, +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", (False, True)) +@pytest.mark.parametrize("rust_enabled", ("0", "1")) +@pytest.mark.parametrize("access_mode", ("read_only", "read_and_write")) +@pytest.mark.parametrize("system", (None, KeyManagementSystem.CUSTOM)) +async def test_readable_secret_managers_keep_python_ocr_fallback( + monkeypatch: pytest.MonkeyPatch, + asynchronous: bool, + rust_enabled: str, + access_mode: str, + system: KeyManagementSystem | None, +) -> None: + pytest.importorskip("litellm.rust_bridge._native") + monkeypatch.setenv("LITELLM_RUST", rust_enabled) + monkeypatch.setenv("MISTRAL_API_KEY", "environment-key") + monkeypatch.setattr(litellm, "secret_manager_client", _VaultSecrets()) + monkeypatch.setattr(litellm, "_key_management_system", system) + monkeypatch.setattr( + litellm, + "_key_management_settings", + KeyManagementSettings(access_mode=access_mode, hosted_keys=["MISTRAL_API_KEY"]), + ) + configuration.reset_rust_configuration() + + with recording_service() as server: + server.default_response = ResponseSpec(body=_RESPONSE) + result: Final = await _call(asynchronous, server.base_url) + + assert result.pages[0].markdown == "parsed document" + assert len(server.requests) == 1 + expected_key: Final = "vault-key" if system is KeyManagementSystem.CUSTOM else "environment-key" + assert server.requests[0].headers["authorization"] == f"Bearer {expected_key}" + assert "x-litellm-rust" not in result._hidden_params.get("additional_headers", {}) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", (False, True)) +async def test_no_secret_client_leaves_dormant_binding_settings_unread( + monkeypatch: pytest.MonkeyPatch, asynchronous: bool +) -> None: + pytest.importorskip("litellm.rust_bridge._native") + monkeypatch.setenv("LITELLM_RUST", "1") + monkeypatch.setenv("MISTRAL_API_KEY", "environment-key") + monkeypatch.setattr(litellm, "secret_manager_client", None) + monkeypatch.setattr(litellm, "_key_management_settings", object()) + configuration.reset_rust_configuration() + + with recording_service() as server: + server.default_response = ResponseSpec(body=_RESPONSE) + result: Final = await _call(asynchronous, server.base_url) + + assert result.pages[0].markdown == "parsed document" + assert len(server.requests) == 1 + assert server.requests[0].headers["authorization"] == "Bearer environment-key" + assert result._hidden_params["additional_headers"]["x-litellm-rust"] == "true" diff --git a/tests/test_litellm/rust_bridge/test_catalog.py b/tests/test_litellm/rust_bridge/test_catalog.py index 147e863baf5..3fa5f3de7ab 100644 --- a/tests/test_litellm/rust_bridge/test_catalog.py +++ b/tests/test_litellm/rust_bridge/test_catalog.py @@ -6,8 +6,21 @@ from typing import Final import pytest from litellm.rust_bridge import catalog, configuration -from litellm.rust_bridge.catalog import Context, Delivery, Route, Rule +from litellm.rust_bridge.catalog import ( + CacheContext, + CacheRule, + Context, + Delivery, + Route, + RouteContext, + RouteRule, + Rules, + SecretManagerContext, + SecretManagerRule, +) from litellm.rust_bridge.configuration import Decision, Rollout +from litellm.types.caching import LiteLLMCacheType +from litellm.types.secret_managers.main import KeyManagementSystem @pytest.fixture(autouse=True) @@ -34,7 +47,7 @@ def test_shipped_decisions( configuration.rust(process) if environment is not None: monkeypatch.setenv("LITELLM_RUST", environment) - context: Final = Context(route, provider=provider, model="test-model", delivery=delivery) + context: Final = RouteContext(route, provider=provider, model="test-model", delivery=delivery) if route is Route.OCR: enabled: Final = environment == "1" if environment is not None else process is not False @@ -57,31 +70,63 @@ def test_missing_rule_stays_on_python_even_when_rust_is_enabled(monkeypatch: pyt configuration.rust(True) monkeypatch.setenv("LITELLM_RUST", "1") - assert catalog.rollout(Context(route), rules=()) is Rollout.PYTHON_ONLY - assert catalog.decision(Context(route), rules=()) is Decision.PYTHON + assert catalog.rollout(RouteContext(route), rules=()) is Rollout.PYTHON_ONLY + assert catalog.decision(RouteContext(route), rules=()) is Decision.PYTHON + + +@pytest.mark.parametrize( + "context", + ( + *(CacheContext(backend.value) for backend in LiteLLMCacheType), + *(SecretManagerContext(system.value) for system in KeyManagementSystem), + CacheContext("custom"), + SecretManagerContext("unknown"), + ), +) +def test_backend_rollouts_stay_on_python_when_global_rust_is_enabled( + monkeypatch: pytest.MonkeyPatch, context: Context +) -> None: + configuration.rust(True) + monkeypatch.setenv("LITELLM_RUST", "1") + + assert catalog.rollout(context) is Rollout.PYTHON_ONLY + assert catalog.decision(context) is Decision.PYTHON + + +def test_response_cache_rules_select_the_whole_backend_runtime() -> None: + rules: Final = ( + CacheRule(Rollout.RUST_REQUIRED, backends=frozenset({"local"})), + CacheRule(Rollout.PYTHON_ONLY), + ) + + assert catalog.decision(CacheContext(backend="local"), rules) is Decision.RUST_REQUIRED + assert catalog.decision(CacheContext(backend="redis"), rules) is Decision.PYTHON @pytest.mark.parametrize( ("context", "expected"), ( - (Context(Route.RESPONSES, provider="openai", model="m", delivery=Delivery.WEBSOCKET), Decision.RUST_REQUIRED), - (Context(Route.RESPONSES, provider="openai", model="m"), Decision.PYTHON), - (Context(Route.RESPONSES, provider="openai", model="m", delivery=Delivery.STREAMING), Decision.PYTHON), - (Context(Route.RESPONSES, provider="openai", model="other", delivery=Delivery.WEBSOCKET), Decision.PYTHON), - (Context(Route.RESPONSES, provider="anthropic", model="m", delivery=Delivery.WEBSOCKET), Decision.PYTHON), - (Context(Route.MESSAGES, provider="openai", model="m", delivery=Delivery.WEBSOCKET), Decision.PYTHON), + ( + RouteContext(Route.RESPONSES, provider="openai", model="m", delivery=Delivery.WEBSOCKET), + Decision.RUST_REQUIRED, + ), + (RouteContext(Route.RESPONSES, provider="openai", model="m"), Decision.PYTHON), + (RouteContext(Route.RESPONSES, provider="openai", model="m", delivery=Delivery.STREAMING), Decision.PYTHON), + (RouteContext(Route.RESPONSES, provider="openai", model="other", delivery=Delivery.WEBSOCKET), Decision.PYTHON), + (RouteContext(Route.RESPONSES, provider="anthropic", model="m", delivery=Delivery.WEBSOCKET), Decision.PYTHON), + (RouteContext(Route.MESSAGES, provider="openai", model="m", delivery=Delivery.WEBSOCKET), Decision.PYTHON), ), ) -def test_first_matching_rule_respects_every_constraint(context: Context, expected: Decision) -> None: +def test_first_matching_rule_respects_every_constraint(context: RouteContext, expected: Decision) -> None: rules: Final = ( - Rule( + RouteRule( Route.RESPONSES, Rollout.RUST_REQUIRED, providers=frozenset({"openai"}), models=frozenset({"m"}), deliveries=frozenset({Delivery.WEBSOCKET}), ), - Rule(Route.RESPONSES, Rollout.PYTHON_ONLY), + RouteRule(Route.RESPONSES, Rollout.PYTHON_ONLY), ) assert catalog.decision(context, rules) is expected @@ -96,4 +141,78 @@ def test_textract_ocr_has_no_python_path_to_opt_out_to( if environment is not None: monkeypatch.setenv("LITELLM_RUST", environment) - assert catalog.decision(Context(Route.OCR, provider="aws_textract", model="m")) is Decision.RUST_REQUIRED + assert catalog.decision(RouteContext(Route.OCR, provider="aws_textract", model="m")) is Decision.RUST_REQUIRED + + +@pytest.mark.parametrize( + ("context", "expected"), + ( + (RouteContext(Route.OCR, provider="local"), Decision.RUST_REQUIRED), + (RouteContext(Route.OCR, provider="other"), Decision.PYTHON), + (RouteContext(Route.MESSAGES, provider="local"), Decision.PYTHON), + (CacheContext("local"), Decision.RUST_WITH_FALLBACK), + (CacheContext("other"), Decision.PYTHON), + (SecretManagerContext("local"), Decision.PYTHON), + (SecretManagerContext("other"), Decision.RUST_REQUIRED), + ), +) +def test_mixed_rules_select_only_the_matching_domain(context: Context, expected: Decision) -> None: + rules: Final[Rules] = ( + CacheRule(Rollout.RUST_OPT_OUT, backends=frozenset({"local"})), + CacheRule(Rollout.PYTHON_ONLY), + SecretManagerRule(Rollout.PYTHON_ONLY, systems=frozenset({"local"})), + SecretManagerRule(Rollout.RUST_REQUIRED), + RouteRule(Route.OCR, Rollout.RUST_REQUIRED, providers=frozenset({"local"})), + RouteRule(Route.OCR, Rollout.PYTHON_ONLY), + ) + + assert catalog.decision(context, rules) is expected + + +@pytest.mark.parametrize("context", (RouteContext(Route.OCR), CacheContext("local"), SecretManagerContext("local"))) +@pytest.mark.parametrize( + ("rollout", "process", "environment", "expected"), + ( + (Rollout.PYTHON_ONLY, True, "1", Decision.PYTHON), + (Rollout.RUST_REQUIRED, False, "0", Decision.RUST_REQUIRED), + (Rollout.RUST_OPT_IN, None, None, Decision.PYTHON), + (Rollout.RUST_OPT_OUT, None, None, Decision.RUST_WITH_FALLBACK), + (Rollout.RUST_OPT_IN, True, None, Decision.RUST_WITH_FALLBACK), + (Rollout.RUST_OPT_OUT, False, None, Decision.PYTHON), + (Rollout.RUST_OPT_IN, False, "1", Decision.RUST_WITH_FALLBACK), + (Rollout.RUST_OPT_OUT, True, "0", Decision.PYTHON), + ), +) +def test_all_domains_share_rollout_switches_and_first_match( + monkeypatch: pytest.MonkeyPatch, + context: Context, + rollout: Rollout, + process: bool | None, + environment: str | None, + expected: Decision, +) -> None: + configuration.rust(process) + if environment is not None: + monkeypatch.setenv("LITELLM_RUST", environment) + rules: Final[Rules] = ( + RouteRule(Route.OCR, rollout), + CacheRule(rollout), + SecretManagerRule(rollout), + RouteRule(Route.OCR, Rollout.RUST_REQUIRED), + CacheRule(Rollout.RUST_REQUIRED), + SecretManagerRule(Rollout.RUST_REQUIRED), + ) + + assert catalog.decision(context, rules) is expected + assert catalog.decision(context, ()) is Decision.PYTHON + + +@pytest.mark.parametrize("context", (RouteContext(Route.OCR), CacheContext("local"), SecretManagerContext("local"))) +def test_empty_constraints_match_nothing(context: Context) -> None: + rules: Final[Rules] = ( + RouteRule(Route.OCR, Rollout.RUST_REQUIRED, providers=frozenset()), + CacheRule(Rollout.RUST_REQUIRED, backends=frozenset()), + SecretManagerRule(Rollout.RUST_REQUIRED, systems=frozenset()), + ) + + assert catalog.decision(context, rules) is Decision.PYTHON diff --git a/tests/test_litellm/rust_bridge/test_dispatch.py b/tests/test_litellm/rust_bridge/test_dispatch.py index 66f8d114f7a..9a3793a772e 100644 --- a/tests/test_litellm/rust_bridge/test_dispatch.py +++ b/tests/test_litellm/rust_bridge/test_dispatch.py @@ -6,7 +6,7 @@ import pytest from litellm.rust_bridge import configuration from litellm.rust_bridge.bindings import NativeBinding -from litellm.rust_bridge.catalog import Context, Delivery, Route, Rule, Rules +from litellm.rust_bridge.catalog import CacheRule, Delivery, Route, RouteContext, RouteRule, Rules, SecretManagerRule from litellm.rust_bridge.configuration import Rollout from litellm.rust_bridge.dispatch import PublicDispatch @@ -22,14 +22,15 @@ def binding() -> NativeBinding[object]: return bound -def test_route_without_rules_forwards_before_request_projection() -> None: +@pytest.mark.parametrize("rules", ((), (CacheRule(Rollout.RUST_REQUIRED), SecretManagerRule(Rollout.RUST_REQUIRED)))) +def test_route_without_rules_forwards_before_request_projection(rules: Rules) -> None: stream: Final[Iterator[int]] = iter((1, 2)) def reject_request(args: tuple[object, ...], kwargs: Mapping[str, object]) -> Request: pytest.fail("Python-only routes must not project the request") dispatch: Final = PublicDispatch( - route=Route.CHAT_COMPLETIONS, request=reject_request, context=lambda _: Context(Route.CHAT_COMPLETIONS) + route=Route.CHAT_COMPLETIONS, request=reject_request, context=lambda _: RouteContext(Route.CHAT_COMPLETIONS) ) result: Final = dispatch.run( ("model",), @@ -37,15 +38,15 @@ def test_route_without_rules_forwards_before_request_projection() -> None: python=lambda *args, **kwargs: stream, binding=binding(), native=lambda hook, request, args, kwargs: pytest.fail("Python-only routes must not call native"), - rules=(), + rules=rules, ) assert result is stream def test_unconditional_python_rule_prevents_later_rust_rule_projection() -> None: rules: Final[Rules] = ( - Rule(Route.CHAT_COMPLETIONS, Rollout.PYTHON_ONLY), - Rule(Route.CHAT_COMPLETIONS, Rollout.RUST_REQUIRED), + RouteRule(Route.CHAT_COMPLETIONS, Rollout.PYTHON_ONLY), + RouteRule(Route.CHAT_COMPLETIONS, Rollout.RUST_REQUIRED), ) def reject_request(args: tuple[object, ...], kwargs: Mapping[str, object]) -> Request: @@ -54,7 +55,7 @@ def test_unconditional_python_rule_prevents_later_rust_rule_projection() -> None dispatch: Final = PublicDispatch( route=Route.CHAT_COMPLETIONS, request=reject_request, - context=lambda _: Context(Route.CHAT_COMPLETIONS), + context=lambda _: RouteContext(Route.CHAT_COMPLETIONS), ) expected: Final = object() result: Final = dispatch.run( @@ -69,12 +70,12 @@ def test_unconditional_python_rule_prevents_later_rust_rule_projection() -> None def test_disabled_optional_rust_rule_forwards_before_projection() -> None: - rules: Final[Rules] = (Rule(Route.OCR, Rollout.RUST_OPT_OUT),) + rules: Final[Rules] = (RouteRule(Route.OCR, Rollout.RUST_OPT_OUT),) def reject_request(args: tuple[object, ...], kwargs: Mapping[str, object]) -> Request: pytest.fail("Disabled optional Rust must not project the request") - dispatch: Final = PublicDispatch(route=Route.OCR, request=reject_request, context=lambda _: Context(Route.OCR)) + dispatch: Final = PublicDispatch(route=Route.OCR, request=reject_request, context=lambda _: RouteContext(Route.OCR)) expected: Final = object() configuration.rust(False) try: @@ -95,12 +96,14 @@ def test_native_stream_result_is_not_consumed_or_wrapped() -> None: request: Final = Request(model="streaming-model") stream: Final[Iterator[int]] = iter((1, 2)) rules: Final[Rules] = ( - Rule(Route.CHAT_COMPLETIONS, Rollout.RUST_REQUIRED, deliveries=frozenset({Delivery.STREAMING})), + CacheRule(Rollout.PYTHON_ONLY), + SecretManagerRule(Rollout.PYTHON_ONLY), + RouteRule(Route.CHAT_COMPLETIONS, Rollout.RUST_REQUIRED, deliveries=frozenset({Delivery.STREAMING})), ) dispatch: Final = PublicDispatch( route=Route.CHAT_COMPLETIONS, request=lambda args, kwargs: request, - context=lambda value: Context(Route.CHAT_COMPLETIONS, model=value.model, delivery=Delivery.STREAMING), + context=lambda value: RouteContext(Route.CHAT_COMPLETIONS, model=value.model, delivery=Delivery.STREAMING), ) def native(request: Request, args: tuple[object, ...], kwargs: Mapping[str, object]) -> Iterator[int]: @@ -122,7 +125,8 @@ def test_native_stream_result_is_not_consumed_or_wrapped() -> None: @pytest.mark.asyncio -async def test_async_route_without_rules_preserves_async_iterator_result() -> None: +@pytest.mark.parametrize("rules", ((), (CacheRule(Rollout.RUST_REQUIRED), SecretManagerRule(Rollout.RUST_REQUIRED)))) +async def test_async_route_without_rules_preserves_async_iterator_result(rules: Rules) -> None: async def chunks() -> AsyncGenerator[int, None]: yield 1 @@ -135,7 +139,7 @@ async def test_async_route_without_rules_preserves_async_iterator_result() -> No return stream dispatch: Final = PublicDispatch( - route=Route.RESPONSES, request=reject_request, context=lambda _: Context(Route.RESPONSES) + route=Route.RESPONSES, request=reject_request, context=lambda _: RouteContext(Route.RESPONSES) ) result: Final = await dispatch.arun( ("model",), @@ -143,7 +147,7 @@ async def test_async_route_without_rules_preserves_async_iterator_result() -> No python=python, binding=binding(), native=lambda hook, request, args, kwargs: pytest.fail("Python-only routes must not call native"), - rules=(), + rules=rules, ) assert result is stream await stream.aclose() @@ -152,11 +156,13 @@ async def test_async_route_without_rules_preserves_async_iterator_result() -> No @pytest.mark.asyncio async def test_async_dispatch_accepts_websocket_style_none_result() -> None: request: Final = Request(model="realtime-model") - rules: Final[Rules] = (Rule(Route.RESPONSES, Rollout.RUST_REQUIRED, deliveries=frozenset({Delivery.WEBSOCKET})),) + rules: Final[Rules] = ( + RouteRule(Route.RESPONSES, Rollout.RUST_REQUIRED, deliveries=frozenset({Delivery.WEBSOCKET})), + ) dispatch: Final = PublicDispatch( route=Route.RESPONSES, request=lambda args, kwargs: request, - context=lambda value: Context(Route.RESPONSES, model=value.model, delivery=Delivery.WEBSOCKET), + context=lambda value: RouteContext(Route.RESPONSES, model=value.model, delivery=Delivery.WEBSOCKET), ) async def python(*args: object, **kwargs: object) -> None: # kwargs-ok: public pass-through shape @@ -183,14 +189,14 @@ async def test_async_dispatch_accepts_websocket_style_none_result() -> None: def test_rules_for_other_routes_and_constrained_python_rules_skip_projection() -> None: rules: Final[Rules] = ( - Rule(Route.MESSAGES, Rollout.RUST_REQUIRED), - Rule(Route.OCR, Rollout.PYTHON_ONLY, providers=frozenset({"mistral"})), + RouteRule(Route.MESSAGES, Rollout.RUST_REQUIRED), + RouteRule(Route.OCR, Rollout.PYTHON_ONLY, providers=frozenset({"mistral"})), ) def reject_request(args: tuple[object, ...], kwargs: Mapping[str, object]) -> Request: pytest.fail("Rules that cannot select Rust must not project the request") - dispatch: Final = PublicDispatch(route=Route.OCR, request=reject_request, context=lambda _: Context(Route.OCR)) + dispatch: Final = PublicDispatch(route=Route.OCR, request=reject_request, context=lambda _: RouteContext(Route.OCR)) expected: Final = object() result: Final = dispatch.run( ("model",), @@ -206,11 +212,11 @@ def test_rules_for_other_routes_and_constrained_python_rules_skip_projection() - @pytest.mark.asyncio async def test_async_bypass_forwards_to_python_without_native() -> None: request: Final = Request(model="bypassed-model") - rules: Final[Rules] = (Rule(Route.RESPONSES, Rollout.RUST_REQUIRED),) + rules: Final[Rules] = (RouteRule(Route.RESPONSES, Rollout.RUST_REQUIRED),) dispatch: Final = PublicDispatch( route=Route.RESPONSES, request=lambda args, kwargs: request, - context=lambda value: Context(Route.RESPONSES, model=value.model), + context=lambda value: RouteContext(Route.RESPONSES, model=value.model), bypass=lambda value: value.model == "bypassed-model", ) expected: Final = object() diff --git a/tests/test_litellm/rust_bridge/test_runtime.py b/tests/test_litellm/rust_bridge/test_runtime.py index fa6c0b30413..bff7ded3114 100644 --- a/tests/test_litellm/rust_bridge/test_runtime.py +++ b/tests/test_litellm/rust_bridge/test_runtime.py @@ -10,7 +10,7 @@ from litellm.exceptions import APIError from litellm.llms.base_llm.ocr.transformation import OCRResponse from litellm.router_utils.add_retry_fallback_headers import get_hidden_params_dict from litellm.rust_bridge import bindings, configuration, runtime -from litellm.rust_bridge.catalog import Context, Delivery, Route, Rule +from litellm.rust_bridge.catalog import Delivery, Route, RouteContext, RouteRule from litellm.rust_bridge.configuration import Rollout @@ -39,7 +39,7 @@ class NativeFn(Protocol): def __call__(self) -> str: ... -CONTEXT: Final = Context(Route.MESSAGES, provider="anthropic", model="model") +CONTEXT: Final = RouteContext(Route.MESSAGES, provider="anthropic", model="model") RUST: Final = "rust" PYTHON: Final = "python" @@ -50,8 +50,8 @@ def binding(native: NativeFn | None) -> bindings.NativeBinding[NativeFn]: return bound -def rules(rollout: Rollout) -> tuple[Rule, ...]: - return (Rule(Route.MESSAGES, rollout, providers=frozenset({"anthropic"})),) +def rules(rollout: Rollout) -> tuple[RouteRule, ...]: + return (RouteRule(Route.MESSAGES, rollout, providers=frozenset({"anthropic"})),) class Recorder: @@ -74,7 +74,7 @@ def recorder(native_effect: BaseException | None = None) -> Recorder: return Recorder(native_effect) -def run(rollout: Rollout, calls: Recorder, *, native_missing: bool = False, context: Context = CONTEXT) -> str: +def run(rollout: Rollout, calls: Recorder, *, native_missing: bool = False, context: RouteContext = CONTEXT) -> str: return runtime.run( context, binding=binding(None if native_missing else calls.rust), @@ -146,8 +146,8 @@ def test_context_outside_rule_stays_on_python() -> None: calls: Final = recorder() configuration.rust(True) - assert run(Rollout.RUST_REQUIRED, calls, context=Context(Route.MESSAGES, provider="openai")) == "python" - assert run(Rollout.RUST_REQUIRED, calls, context=Context(Route.RESPONSES, provider="anthropic")) == "python" + assert run(Rollout.RUST_REQUIRED, calls, context=RouteContext(Route.MESSAGES, provider="openai")) == "python" + assert run(Rollout.RUST_REQUIRED, calls, context=RouteContext(Route.RESPONSES, provider="anthropic")) == "python" assert calls.calls == (PYTHON, PYTHON) @@ -155,20 +155,20 @@ def test_context_outside_rule_stays_on_python() -> None: @pytest.mark.parametrize( "context", ( - Context(Route.CHAT_COMPLETIONS, provider="anthropic"), - Context(Route.CHAT_COMPLETIONS, provider="bedrock"), - Context(Route.RESPONSES, provider="openai"), - Context(Route.TRANSCRIPTION, provider="openai"), + RouteContext(Route.CHAT_COMPLETIONS, provider="anthropic"), + RouteContext(Route.CHAT_COMPLETIONS, provider="bedrock"), + RouteContext(Route.RESPONSES, provider="openai"), + RouteContext(Route.TRANSCRIPTION, provider="openai"), ), ) @pytest.mark.parametrize("delivery", tuple(Delivery)) async def test_shipped_python_routes_never_load_native( - monkeypatch: pytest.MonkeyPatch, context: Context, delivery: Delivery + monkeypatch: pytest.MonkeyPatch, context: RouteContext, delivery: Delivery ) -> None: monkeypatch.setenv("LITELLM_RUST", "1") configuration.rust(True) calls: Final = recorder() - request: Final = Context(context.route, provider=context.provider, delivery=delivery) + request: Final = RouteContext(context.route, provider=context.provider, delivery=delivery) def reject_load(value: object) -> NativeFn | None: pytest.fail("Python-only dispatch must not load a native binding") diff --git a/tests/test_litellm/rust_bridge/test_settings.py b/tests/test_litellm/rust_bridge/test_settings.py index 023f02cffbb..3a86a69ed8b 100644 --- a/tests/test_litellm/rust_bridge/test_settings.py +++ b/tests/test_litellm/rust_bridge/test_settings.py @@ -1,12 +1,8 @@ -import dataclasses import logging -from pathlib import Path from typing import Final import httpx import pytest -from pydantic import TypeAdapter -from typing_extensions import ReadOnly, TypedDict import litellm from litellm.integrations.custom_secret_manager import CustomSecretManager @@ -15,34 +11,6 @@ from litellm.rust_bridge import settings from litellm.secret_managers.main import get_secret_str from litellm.types.secret_managers.main import KeyManagementSettings, KeyManagementSystem -CONTRACT_PATH: Final = Path(__file__).parents[3] / "litellm-rust/crates/python-bridge/python_settings.json" - - -class SettingSpec(TypedDict): - adapter: ReadOnly[str] - required: ReadOnly[bool] - precedence: ReadOnly[str] - sensitive: ReadOnly[bool] - shapes: ReadOnly[list[str]] - unsupported_live: ReadOnly[str | None] - - -class SettingsGroup(TypedDict): - version: ReadOnly[int] - fields: ReadOnly[dict[str, SettingSpec]] - - -def test_the_rust_contract_matches_the_returned_fields() -> None: - contract: Final = TypeAdapter(dict[str, SettingsGroup]).validate_json(CONTRACT_PATH.read_text()) - - assert {name: tuple(group["fields"]) for name, group in contract.items()} == { - "http_settings": tuple(field.name for field in dataclasses.fields(settings.http_settings())), - "url_policy": tuple(field.name for field in dataclasses.fields(settings.url_policy())), - "provider_defaults": tuple(field.name for field in dataclasses.fields(settings.provider_defaults())), - "secret_manager": tuple(field.name for field in dataclasses.fields(settings.secret_manager())), - } - - def test_url_policy_reads_the_litellm_globals(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(litellm, "user_url_validation", False) monkeypatch.setattr(litellm, "user_url_allowed_hosts", ["docs.internal:8443"]) @@ -140,6 +108,75 @@ def test_secret_manager_is_not_readable_without_a_client(monkeypatch: pytest.Mon assert settings.secret_manager() == settings.SecretManager(readable=False) +def test_secret_manager_projects_custom_settings(monkeypatch: pytest.MonkeyPatch) -> None: + manager_settings: Final = KeyManagementSettings( + access_mode="read_and_write", + hosted_keys=["MISTRAL_API_KEY"], + primary_secret_name="primary", + aws_region_name="us-east-1", + ) + client: Final = _VaultSecrets({"MISTRAL_API_KEY": "vault-key"}) + monkeypatch.setattr(litellm, "secret_manager_client", client) + monkeypatch.setattr(litellm, "_key_management_system", KeyManagementSystem.CUSTOM) + monkeypatch.setattr(litellm, "_key_management_settings", manager_settings) + + assert settings.secret_manager_binding() == settings.SecretManagerBinding( + system="custom", + access_mode="read_and_write", + hosted_keys=["MISTRAL_API_KEY"], + primary_secret_name="primary", + store_virtual_keys=manager_settings.store_virtual_keys, + prefix_for_stored_virtual_keys=manager_settings.prefix_for_stored_virtual_keys, + kms_key_id=manager_settings.kms_key_id, + custom_secret_manager=manager_settings.custom_secret_manager, + aws_region_name="us-east-1", + aws_role_name=manager_settings.aws_role_name, + aws_session_name=manager_settings.aws_session_name, + aws_external_id=manager_settings.aws_external_id, + aws_profile_name=manager_settings.aws_profile_name, + aws_web_identity_token=manager_settings.aws_web_identity_token, + aws_sts_endpoint=manager_settings.aws_sts_endpoint, + replica_regions=manager_settings.replica_regions, + client=client, + settings_object=manager_settings, + ) + + +def test_secret_manager_without_a_client_has_no_system(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "secret_manager_client", None) + + assert settings.secret_manager_binding().system is None + + +def test_secret_manager_uses_key_management_defaults(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "secret_manager_client", None) + monkeypatch.setattr(litellm, "_key_management_settings", None) + + defaults: Final = KeyManagementSettings() + result: Final = settings.secret_manager_binding() + + assert result == settings.SecretManagerBinding( + system=None, + access_mode=defaults.access_mode, + hosted_keys=defaults.hosted_keys, + primary_secret_name=defaults.primary_secret_name, + store_virtual_keys=defaults.store_virtual_keys, + prefix_for_stored_virtual_keys=defaults.prefix_for_stored_virtual_keys, + kms_key_id=defaults.kms_key_id, + custom_secret_manager=defaults.custom_secret_manager, + aws_region_name=defaults.aws_region_name, + aws_role_name=defaults.aws_role_name, + aws_session_name=defaults.aws_session_name, + aws_external_id=defaults.aws_external_id, + aws_profile_name=defaults.aws_profile_name, + aws_web_identity_token=defaults.aws_web_identity_token, + aws_sts_endpoint=defaults.aws_sts_endpoint, + replica_regions=defaults.replica_regions, + client=None, + settings_object=None, + ) + + def test_provider_defaults_read_the_litellm_globals(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(litellm, "vertex_project", "configured-project") monkeypatch.setattr(litellm, "vertex_location", "europe-west4") diff --git a/tests/test_litellm_rust/ocr/test_requests.py b/tests/test_litellm_rust/ocr/test_requests.py index 51815651eb4..0d3b8ba472d 100644 --- a/tests/test_litellm_rust/ocr/test_requests.py +++ b/tests/test_litellm_rust/ocr/test_requests.py @@ -593,7 +593,7 @@ def test_native_projection_errors_never_select_python( import ssl from litellm.rust_bridge import runtime, settings - from litellm.rust_bridge.catalog import Context, Route, Rule + from litellm.rust_bridge.catalog import Route, RouteContext, RouteRule from litellm.rust_bridge.configuration import Rollout from litellm.rust_bridge.ocr.entrypoints import NATIVE_OCR, LiteLLMOcrRequest @@ -621,11 +621,11 @@ def test_native_projection_errors_never_select_python( with pytest.raises(RuntimeError if failure == "schema" else ValueError, match="http_settings"): runtime.run( - Context(Route.OCR, provider="mistral"), + RouteContext(Route.OCR, provider="mistral"), binding=NATIVE_OCR, native=lambda native: native(request, (), {}), python=python_fallback, - rules=(Rule(Route.OCR, Rollout.RUST_REQUIRED if required else Rollout.RUST_OPT_OUT),), + rules=(RouteRule(Route.OCR, Rollout.RUST_REQUIRED if required else Rollout.RUST_OPT_OUT),), ) assert ocr_server.requests == [] diff --git a/tests/test_litellm_rust/test_cache.py b/tests/test_litellm_rust/test_cache.py index 48ca6f5e165..0f389edaa27 100644 --- a/tests/test_litellm_rust/test_cache.py +++ b/tests/test_litellm_rust/test_cache.py @@ -38,6 +38,9 @@ from litellm.caching.redis_cluster_cache import RedisClusterCache from litellm.caching.redis_semantic_cache import RedisSemanticCache from litellm.caching.s3_cache import S3Cache from litellm.rust_bridge import _native +from litellm.rust_bridge.catalog import CacheRule, Route, RouteRule, SecretManagerRule +from litellm.rust_bridge.configuration import Rollout +from litellm.rust_bridge.response_cache import ResponseCacheRuntime, resolve_response_cache from litellm.types.caching import LiteLLMCacheType from litellm.types.llms.custom_llm import CustomLLMItem from litellm.types.utils import EmbeddingResponse @@ -189,6 +192,7 @@ def test_existing_constructor_and_global_are_unchanged() -> None: facade: Final = Cache(type=LiteLLMCacheType.LOCAL) assert type(facade.cache) is InMemoryCache assert "_native_cache_handle" not in vars(facade) + assert resolve_response_cache(facade) is None with rebound(litellm, "cache", facade): resolver: Final = _CacheTestResolver(litellm) assert resolver.resolve().kind == "python_callback" @@ -196,6 +200,42 @@ def test_existing_constructor_and_global_are_unchanged() -> None: assert cast(CacheLookup, facade).get_cache(cache_key="key") == {"answer": 7} +async def test_catalog_constructs_native_runtime_from_public_cache_configuration() -> None: + rules: Final = ( + RouteRule(Route.OCR, Rollout.PYTHON_ONLY), + SecretManagerRule(Rollout.PYTHON_ONLY, systems=frozenset({"local"})), + CacheRule(Rollout.RUST_REQUIRED, backends=frozenset({"local"})), + ) + facade: Final = Cache(type=LiteLLMCacheType.LOCAL) + runtime: Final = resolve_response_cache(facade, rules) + assert isinstance(runtime, ResponseCacheRuntime) + assert runtime.kind == "native" + + sync_request: Final = runtime.request(facade, {"cache_key": "sync"}) + assert sync_request is not None + runtime.store(sync_request, {"answer": 1}) + assert runtime.lookup(sync_request) == {"answer": 1} + assert facade.cache.get_cache("sync") is None + + async_request: Final = runtime.request(facade, {"cache_key": "async"}) + assert async_request is not None + await runtime.async_store(async_request, {"answer": 2}) + assert await runtime.async_lookup(async_request) == {"answer": 2} + assert await facade.cache.async_get_cache("async") is None + + requests: Final = (sync_request, async_request) + expected: Final = { + "values": [{"answer": 1}, {"answer": 2}], + "missing_indices": [], + } + assert runtime.lookup_batch(requests) == expected + assert await runtime.async_lookup_batch(requests) == expected + + await runtime.async_flush() + assert runtime.lookup(sync_request) is None + assert await runtime.async_lookup(async_request) is None + + def test_existing_global_lifecycle_remains_the_resolver_source_of_truth() -> None: resolver: Final = _CacheTestResolver(litellm) @@ -410,9 +450,7 @@ async def test_memory_size_policy_is_applied_by_the_native_host() -> None: await binding.async_store(request("large"), {"answer": "x" * 256}) assert binding.lookup(request("large")) is None assert binding.lookup(request("small")) == small - disabled: Final = _CacheTestResolver( - SimpleNamespace(cache=_CacheTestHandle.memory(capacity=0)) - ).resolve() + disabled: Final = _CacheTestResolver(SimpleNamespace(cache=_CacheTestHandle.memory(capacity=0))).resolve() await disabled.async_store(request(), small) assert await disabled.async_lookup(request()) is None @@ -456,9 +494,7 @@ async def test_python_batch_callbacks_use_the_builtin_cache_api() -> None: ) -> object: return result, kwargs - binding: Final = _CacheTestResolver( - SimpleNamespace(cache=CustomCache(type=LiteLLMCacheType.LOCAL)) - ).resolve() + binding: Final = _CacheTestResolver(SimpleNamespace(cache=CustomCache(type=LiteLLMCacheType.LOCAL))).resolve() assert binding.kind == "python_callback" requests: Final = [request("first"), request("second")] kwargs: Final = [{"cache_key": "first"}, {"cache_key": "second"}] @@ -1212,10 +1248,7 @@ def _semantic_embedding(prompt: str) -> list[float]: base: Final = _base_embedding(prompt.replace(PARAPHRASE_MARKER, "").strip()) pivot: Final = min(range(8), key=lambda index: abs(base[index])) direction: Final = _normalized( - [ - (1.0 - base[pivot] * base[pivot]) if index == pivot else -base[index] * base[pivot] - for index in range(8) - ] + [(1.0 - base[pivot] * base[pivot]) if index == pivot else -base[index] * base[pivot] for index in range(8)] ) # Rotating an orthogonal unit direction by 0.329 produces ~0.05 cosine distance return _normalized([base[index] + 0.329 * direction[index] for index in range(8)]) @@ -1311,9 +1344,7 @@ def semantic_embedding() -> Generator[DeterministicEmbedding]: [*litellm._custom_providers, "semantic-test"], # pyright: ignore[reportPrivateUsage] # no public provider-registration hook ) ) - stack.enter_context( - rebound(litellm, "provider_list", [*litellm.provider_list, "semantic-test"]) - ) + stack.enter_context(rebound(litellm, "provider_list", [*litellm.provider_list, "semantic-test"])) yield handler @@ -1441,9 +1472,7 @@ async def test_redis_semantic_async_paths_and_store_batch_share_one_layout( binding: Final = _CacheTestResolver(SimpleNamespace(cache=facade)).resolve() client: Final = redis.Redis.from_url(url) - await binding.async_store( - semantic_request("async", "name a primary color"), {"answer": "blue"} - ) + await binding.async_store(semantic_request("async", "name a primary color"), {"answer": "blue"}) hash_key: Final = f"{index}:{semantic_entry_id('name a primary color', 'async')}" decoded: Final = cast(dict[str, object], json.loads(cast(bytes, client.hget(hash_key, "response")))) python_read: Final = await cast(RedisSemanticCache, facade.cache).async_get_cache( # pyright: ignore[reportUnknownMemberType] # **kwargs stays unknown on the backend class @@ -1459,9 +1488,7 @@ async def test_redis_semantic_async_paths_and_store_batch_share_one_layout( [{"answer": 1}, {"answer": 2}], ) expected: Final = { - key: json.loads( - cast(bytes, client.hget(f"{index}:{semantic_entry_id(prompt, key)}", "response")) - ) + key: json.loads(cast(bytes, client.hget(f"{index}:{semantic_entry_id(prompt, key)}", "response"))) for key, prompt in ( ("batch-one", "first batch prompt"), ("batch-two", "second batch prompt"), @@ -1471,18 +1498,19 @@ async def test_redis_semantic_async_paths_and_store_batch_share_one_layout( ("batch-one", "first batch prompt"), ("batch-two", "second batch prompt"), ): - assert cast(RedisSemanticCache, facade.cache).get_cache( # pyright: ignore[reportUnknownMemberType] # **kwargs stays unknown on the backend class - key, messages=semantic_messages(prompt) - ) == expected[key], key + assert ( + cast(RedisSemanticCache, facade.cache).get_cache( # pyright: ignore[reportUnknownMemberType] # **kwargs stays unknown on the backend class + key, messages=semantic_messages(prompt) + ) + == expected[key] + ), key cast(RedisSemanticCache, facade.cache).set_cache( # pyright: ignore[reportUnknownMemberType] # **kwargs stays unknown on the backend class "async-python", json.dumps({"timestamp": 1700000000.0, "response": {"answer": "python"}}), messages=semantic_messages("python written prompt"), ) - assert await binding.async_lookup( - semantic_request("async-python", "python written prompt") - ) == {"answer": "python"} + assert await binding.async_lookup(semantic_request("async-python", "python written prompt")) == {"answer": "python"} client.close() @@ -1497,13 +1525,9 @@ async def test_native_semantic_async_embedding_runs_inline_in_the_callers_task( SEMANTIC_CONTEXT.set("caller-sentinel") response: Final = {"choices": [{"text": "paris"}]} - await binding.async_store( - semantic_request("inline", "what is the capital of france"), response - ) + await binding.async_store(semantic_request("inline", "what is the capital of france"), response) assert ( - await binding.async_lookup( - semantic_request("inline", f"what is the capital of france{PARAPHRASE_MARKER}") - ) + await binding.async_lookup(semantic_request("inline", f"what is the capital of france{PARAPHRASE_MARKER}")) == response ) assert await binding.async_lookup(semantic_request("inline", "python written prompt")) is None @@ -1540,9 +1564,7 @@ async def test_native_semantic_cancellation_during_embedding_skips_the_backend( semantic_embedding.gate = asyncio.Event() async def lookup() -> object: - return await binding.async_lookup( - semantic_request("cancel", "cancelled prompt") - ) + return await binding.async_lookup(semantic_request("cancel", "cancelled prompt")) task: Final = asyncio.create_task(lookup()) await semantic_embedding.entered.wait() @@ -1587,9 +1609,7 @@ def test_redis_semantic_ttl_is_written_only_when_requested( binding: Final = _CacheTestResolver(SimpleNamespace(cache=facade)).resolve() client: Final = redis.Redis.from_url(url) - binding.store( - {**semantic_request("ttl", "ttl prompt"), "ttl_seconds": 12.0}, {"answer": 1} - ) + binding.store({**semantic_request("ttl", "ttl prompt"), "ttl_seconds": 12.0}, {"answer": 1}) expiring: Final = f"{index}:{semantic_entry_id('ttl prompt', 'ttl')}" assert 0 < client.ttl(expiring) <= 12 @@ -1737,16 +1757,13 @@ def test_redis_semantic_handle_rejects_wrong_backends( redis_semantic_cache_index_name=index, ) subclassed_facade.cache = CustomSemanticCache( # pyright: ignore[reportAttributeAccessIssue] # facade backend slot is not declared - redis_url=url, similarity_threshold=0.8, embedding_model=SEMANTIC_EMBEDDING_MODEL, index_name=index, ) with pytest.raises(TypeError): - _CacheTestHandle.redis_semantic( - subclassed_facade.cache - )._bind_facade(subclassed_facade) + _CacheTestHandle.redis_semantic(subclassed_facade.cache)._bind_facade(subclassed_facade) replacement_facade: Final = Cache( type=LiteLLMCacheType.REDIS_SEMANTIC, @@ -1770,9 +1787,7 @@ def qdrant_facade(qdrant_url: str, collection_name: str) -> Cache: ) -def test_qdrant_semantic_facade_binds_native_and_shares_entries( - qdrant_url: str, fake_embedding_endpoint: str -) -> None: +def test_qdrant_semantic_facade_binds_native_and_shares_entries(qdrant_url: str, fake_embedding_endpoint: str) -> None: del fake_embedding_endpoint messages: Final = [{"role": "user", "content": "shared prompt"}] collection: Final = f"cache_{uuid4().hex}" @@ -1803,9 +1818,7 @@ def test_qdrant_semantic_facade_binds_native_and_shares_entries( assert facade.cache.get_cache("different-key", messages=messages) is None -async def test_qdrant_semantic_async_parity( - qdrant_url: str, fake_embedding_endpoint: str -) -> None: +async def test_qdrant_semantic_async_parity(qdrant_url: str, fake_embedding_endpoint: str) -> None: del fake_embedding_endpoint messages: Final = [{"role": "user", "content": "async prompt"}] collection: Final = f"cache_{uuid4().hex}" @@ -1905,9 +1918,7 @@ async def test_qdrant_semantic_malformed_entries_and_unsupported_operations( await binding.ping() -def test_qdrant_semantic_ignores_request_expiry( - qdrant_url: str, fake_embedding_endpoint: str -) -> None: +def test_qdrant_semantic_ignores_request_expiry(qdrant_url: str, fake_embedding_endpoint: str) -> None: del fake_embedding_endpoint messages: Final = [{"role": "user", "content": "persistent prompt"}] collection: Final = f"cache_{uuid4().hex}" @@ -1928,9 +1939,7 @@ def test_qdrant_semantic_ignores_request_expiry( assert python_value["response"] == {"id": "persistent"} -def test_qdrant_semantic_mutation_and_projection_fallback( - qdrant_url: str, fake_embedding_endpoint: str -) -> None: +def test_qdrant_semantic_mutation_and_projection_fallback(qdrant_url: str, fake_embedding_endpoint: str) -> None: del fake_embedding_endpoint collection: Final = f"cache_{uuid4().hex}" facade: Final = qdrant_facade(qdrant_url, collection) diff --git a/tests/unit/chat_completions/test_dispatch.py b/tests/unit/chat_completions/test_dispatch.py index 63821c74208..2807ed7f8f7 100644 --- a/tests/unit/chat_completions/test_dispatch.py +++ b/tests/unit/chat_completions/test_dispatch.py @@ -10,7 +10,7 @@ from litellm.chat_completions.dispatch import ( ) from litellm.rust_bridge import catalog from litellm.rust_bridge.bindings import NativeBinding -from litellm.rust_bridge.catalog import Route, Rule +from litellm.rust_bridge.catalog import Route, RouteRule from litellm.rust_bridge.chat_completions.entrypoints import ( NATIVE_ACOMPLETION, NATIVE_COMPLETION, @@ -23,7 +23,7 @@ from litellm.types.utils import ModelResponse MESSAGES: Final = [{"role": "user", "content": "hi"}] PYTHON_RULES: Final = () -RUST_RULES: Final = (Rule(Route.CHAT_COMPLETIONS, Rollout.RUST_REQUIRED),) +RUST_RULES: Final = (RouteRule(Route.CHAT_COMPLETIONS, Rollout.RUST_REQUIRED),) def completion_binding(native: NativeCompletion | None) -> NativeBinding[NativeCompletion]: @@ -117,9 +117,7 @@ def test_native_receives_bound_request_and_original_call_shape() -> None: "custom_llm_provider": "anthropic", "metadata": metadata, } - captured: Final[ - list[tuple[LiteLLMChatCompletionsRequest, tuple[object, ...], Mapping[str, object]]] - ] = [] + captured: Final[list[tuple[LiteLLMChatCompletionsRequest, tuple[object, ...], Mapping[str, object]]]] = [] def python(*call_args: object, **call_kwargs: object) -> ModelResponse: # kwargs-ok: rejected Rust fallback pytest.fail("Required Rust dispatch must not call Python") diff --git a/tests/unit/messages/test_dispatch.py b/tests/unit/messages/test_dispatch.py index 586b77d9a25..48eb1adbf51 100644 --- a/tests/unit/messages/test_dispatch.py +++ b/tests/unit/messages/test_dispatch.py @@ -12,7 +12,7 @@ from litellm.messages.dispatch import ( ) from litellm.rust_bridge import catalog from litellm.rust_bridge.bindings import NativeBinding -from litellm.rust_bridge.catalog import Route, Rule, Rules +from litellm.rust_bridge.catalog import Route, RouteRule, Rules from litellm.rust_bridge.configuration import Rollout from litellm.rust_bridge.messages.entrypoints import ( NATIVE_AMESSAGES, @@ -25,7 +25,7 @@ from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMe MESSAGES: Final = [{"role": "user", "content": "hi"}] PYTHON_RULES: Final[Rules] = () -RUST_RULES: Final[Rules] = (Rule(Route.MESSAGES, Rollout.RUST_REQUIRED),) +RUST_RULES: Final[Rules] = (RouteRule(Route.MESSAGES, Rollout.RUST_REQUIRED),) def messages_binding(native: NativeMessages | None) -> NativeBinding[NativeMessages]: diff --git a/tests/unit/ocr/test_dispatch.py b/tests/unit/ocr/test_dispatch.py index e54d4070ba8..2727ff23449 100644 --- a/tests/unit/ocr/test_dispatch.py +++ b/tests/unit/ocr/test_dispatch.py @@ -12,7 +12,7 @@ from litellm.ocr.dispatch import ( ) from litellm.rust_bridge import catalog from litellm.rust_bridge.bindings import NativeBinding -from litellm.rust_bridge.catalog import Route, Rule, Rules +from litellm.rust_bridge.catalog import Route, RouteRule, Rules from litellm.rust_bridge.configuration import Rollout from litellm.rust_bridge.ocr.entrypoints import ( NATIVE_AOCR, @@ -22,8 +22,8 @@ from litellm.rust_bridge.ocr.entrypoints import ( NativeOcr, ) -PYTHON_RULES: Final[Rules] = (Rule(Route.OCR, Rollout.PYTHON_ONLY),) -RUST_RULES: Final[Rules] = (Rule(Route.OCR, Rollout.RUST_REQUIRED),) +PYTHON_RULES: Final[Rules] = (RouteRule(Route.OCR, Rollout.PYTHON_ONLY),) +RUST_RULES: Final[Rules] = (RouteRule(Route.OCR, Rollout.RUST_REQUIRED),) def ocr_binding(native: NativeOcr | None) -> NativeBinding[NativeOcr]: @@ -403,8 +403,8 @@ def test_provider_scoped_rule_sees_the_provider_named_by_the_model_prefix( model: str, custom_llm_provider: str | None, expected: str ) -> None: rules: Final[Rules] = ( - Rule(Route.OCR, Rollout.RUST_REQUIRED, providers=frozenset({"aws_textract"})), - Rule(Route.OCR, Rollout.PYTHON_ONLY), + RouteRule(Route.OCR, Rollout.RUST_REQUIRED, providers=frozenset({"aws_textract"})), + RouteRule(Route.OCR, Rollout.PYTHON_ONLY), ) document: Final[Mapping[str, object]] = {"type": "image_url", "image_url": "data:image/png;base64,YQ=="} kwargs: Final[Mapping[str, object]] = ( From 25ebb9458c29843438a9da9e2d2cc4af03fbef56 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 20:52:41 -0700 Subject: [PATCH 04/15] fix(proxy): surface a database outage from the user read as 503 no_db_connection (#42399) get_user_object wrapped every failed read, a refused connection included, in ValueError("User doesn't exist in db ..."), so JWT callers got a 401 naming a missing user while Postgres was down and virtual-key callers got 503 no_db_connection for the same outage. A connection or transport error now propagates as-is and the auth exception mapper answers 503 no_db_connection; a genuinely missing row and query-level errors still answer 401. The MCP auth and token-exchange docstrings and the exception-chain helper's docstring described the old wrap and are updated to the new contract. Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../mcp_server/auth/user_api_key_auth_mcp.py | 14 +-- .../mcp_server/bridge_token_flow.py | 6 +- .../mcp_server/idp_token_exchange.py | 5 +- litellm/proxy/auth/auth_checks.py | 12 ++- litellm/proxy/db/exception_handler.py | 7 +- .../test_user_api_key_auth.py | 19 ++-- .../auth/test_user_api_key_auth_mcp.py | 13 +-- .../proxy/auth/test_auth_checks.py | 90 ++++++++++++++----- 8 files changed, 107 insertions(+), 59 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py index 2e5cad29b0a..60dc91a69cc 100644 --- a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py +++ b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py @@ -966,10 +966,11 @@ class MCPRequestHandler: on top of these direct grants, each source bounded by ITS OWN org, so a user spanning organizations cannot leak one org's servers past another's ceiling. - Error handling: ``get_user_object`` catches every DB failure and re-raises a bare ``ValueError``, so a - missing user and a real outage look identical (the cause survives only as ``__context__``). - ``_raise_503_if_db_unavailable`` walks the cause chain so an outage stays a retryable 503 while any - other failure fails closed as 401, not an opaque 500; the object-permission load shares that boundary.""" + Error handling: ``get_user_object`` lets a database outage propagate as-is and re-raises every other + DB failure as a bare ``ValueError`` (the cause surviving only as ``__context__``). + ``_raise_503_if_db_unavailable`` walks the cause chain so an outage stays a retryable 503 whichever + shape it arrives in, while any other failure fails closed as 401, not an opaque 500; the + object-permission load shares that boundary.""" from litellm.proxy.auth.auth_checks import get_object_permission, get_user_object from litellm.proxy.proxy_server import prisma_client, user_api_key_cache @@ -1116,9 +1117,8 @@ class MCPRequestHandler: (401) or surface as an opaque 500; the caller retries. Mirrors ``UserAPIKeyAuthExceptionHandler``, which renders a service-unavailable database error as 503 on the standard pipeline. - Classifies across the ``__cause__``/``__context__`` chain, not just ``e`` itself: ``get_user_object`` - re-raises every DB failure as a bare ``ValueError``, so a type-based check on the top exception - would miss a real outage wrapped inside it.""" + Classifies across the ``__cause__``/``__context__`` chain, not just ``e`` itself, so an outage a + caller re-raised inside a domain exception is still recognized.""" from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler outage: Final = PrismaDBExceptionHandler.find_database_service_unavailable_error_in_chain(e) diff --git a/litellm/proxy/_experimental/mcp_server/bridge_token_flow.py b/litellm/proxy/_experimental/mcp_server/bridge_token_flow.py index 37a893973e3..6ae33cc1629 100644 --- a/litellm/proxy/_experimental/mcp_server/bridge_token_flow.py +++ b/litellm/proxy/_experimental/mcp_server/bridge_token_flow.py @@ -276,9 +276,9 @@ async def load_active_user_by_id( database-service-unavailable error is a retryable outage (``unavailable``). Everything else fails closed as ``no_active_key`` (the caller maps it to invalid_grant): a ``ProxyException`` / ``HTTPException``, a SCIM-deactivated user, and, unlike the key path, a missing user. ``get_user_object`` - catches every DB failure and re-raises a bare ``ValueError`` (a deleted user and a real outage look - identical, the original error surviving only as ``__context__``), so the outage check walks the cause - chain, and a missing user falls through to ``no_active_key`` rather than an opaque gateway fault. + lets a real outage propagate as-is and re-raises any other DB failure as a bare ``ValueError`` (the + original error surviving only as ``__context__``), so the outage check walks the cause chain, and a + missing user falls through to ``no_active_key`` rather than an opaque gateway fault. ``source="database"`` reads the row from the database, never the cache, so the credential mint refuses a user that a writer deactivated or deleted without evicting the cached row, and it leaves the fresh row in the cache for the requests the credential makes next. Every other caller keeps the cache read, diff --git a/litellm/proxy/_experimental/mcp_server/idp_token_exchange.py b/litellm/proxy/_experimental/mcp_server/idp_token_exchange.py index 80868296b50..a437df17e6a 100644 --- a/litellm/proxy/_experimental/mcp_server/idp_token_exchange.py +++ b/litellm/proxy/_experimental/mcp_server/idp_token_exchange.py @@ -196,9 +196,8 @@ def _check_unavailable_description(outage: GatewayOutage) -> str: def _gateway_could_not_verify(denied: Exception) -> GatewayOutage | None: - """A database fault anywhere in the chain (``get_user_object`` wraps prisma failures in a - bare ``ValueError``) or a 5xx from JWT auth (the IdP's JWKS unreachable with no cached - copy) is the gateway failing, not the token. A fault retrying cannot clear (a missing or + """A database fault anywhere in the chain or a 5xx from JWT auth (the IdP's JWKS + unreachable with no cached copy) is the gateway failing, not the token. A fault retrying cannot clear (a missing or version-skewed query engine) is named as such, the way the mint path words it, so the client is not told to wait on a deployment that needs repair.""" fault: Final = PrismaDBExceptionHandler.find_database_service_unavailable_error_in_chain(denied) diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index b3efde7f05f..f0588b3fa79 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -2697,9 +2697,15 @@ async def get_user_object( raise except Exception as e: _log_budget_lookup_failure("user", e) - raise ValueError( - f"User doesn't exist in db. 'user_id'={user_id}. Create user via `/user/new` call. Got error - {e}" - ) + raise _user_read_failure(user_id=user_id, error=e) + + +def _user_read_failure(user_id: str, error: Exception) -> Exception: + if PrismaDBExceptionHandler.is_database_service_unavailable_error(error): + return error + return ValueError( + f"User doesn't exist in db. 'user_id'={user_id}. Create user via `/user/new` call. Got error - {error}" + ) async def _cache_management_object( diff --git a/litellm/proxy/db/exception_handler.py b/litellm/proxy/db/exception_handler.py index f25c2787252..99167c1275c 100644 --- a/litellm/proxy/db/exception_handler.py +++ b/litellm/proxy/db/exception_handler.py @@ -398,11 +398,8 @@ class PrismaDBExceptionHandler: ``is_database_service_unavailable_error`` classifies a single exception by type, which a caller that catches a raw DB failure and re-raises a - domain exception of a different type defeats. ``get_user_object`` in - ``litellm/proxy/auth/auth_checks.py`` is the concrete case: it wraps - every DB error, a genuine outage included, in a bare ``ValueError`` - whose original error survives only as ``__context__``. A type check on - the ``ValueError`` misses the outage, so the caller would mistake an + domain exception of a different type defeats. A type check on the + wrapper misses the outage, so the caller would mistake an infrastructure fault for an auth failure. Walking the chain recovers the real signal, which is the PEP 3134 way to inspect a wrapped cause. diff --git a/tests/proxy_unit_tests/test_user_api_key_auth.py b/tests/proxy_unit_tests/test_user_api_key_auth.py index f5e8d861d79..9cdac341b1f 100644 --- a/tests/proxy_unit_tests/test_user_api_key_auth.py +++ b/tests/proxy_unit_tests/test_user_api_key_auth.py @@ -1540,18 +1540,17 @@ async def test_user_budget_lookup_is_also_unenforced_when_the_database_is_down() """ KNOWN LIMITATION, pinned deliberately rather than discovered later. - `get_user_object` cannot tell "row absent" from "database unreachable": the - absent case raises inside its own try (auth_checks.py:2177) and the handler - at :2213 rewrites every exception into the same - `ValueError("User doesn't exist in db...")`. A connection error, a query - timeout and a malformed row all reach us as that one type and message. + `get_user_object` lets a connection-level outage propagate as-is and rewrites + every other read failure (a query-level Prisma error, a malformed row) into + the same `ValueError("User doesn't exist in db...")` as an absent row, and + `_read_user_model_max_budget` swallows every exception either way. - So tolerating the absent case, which the test above requires, unavoidably - tolerates an outage too, and a user who DOES have a per-model budget goes + So tolerating the absent case, which the test above requires, also + tolerates an outage, and a user who DOES have a per-model budget goes unenforced while the DB is unreachable. This is pre-existing behaviour of - `get_user_object` that the virtual-key path inherits identically; it is not - introduced here. Distinguishing them needs a dedicated exception type for - the absent case and a change to both auth paths. + the virtual-key path; it is not introduced here. Distinguishing them needs + `_read_user_model_max_budget` to let an outage through the way the JWT + path does. """ from litellm.caching.dual_cache import DualCache from litellm.proxy.auth.user_api_key_auth import _read_user_model_max_budget diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py b/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py index 5c885a3168d..35e7055bbc0 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py @@ -5936,12 +5936,13 @@ class TestMCPDcrBridgeDelegateAdmission: @staticmethod def _wrapped_user_lookup_error(original: BaseException) -> ValueError: - """Reproduce get_user_object's real exception contract (litellm/proxy/auth/auth_checks.py): it - catches every DB failure in a broad ``except`` and re-raises a bare ``ValueError``, so the - original error (a missing-user Exception or a real outage) survives only as ``__context__``. - Injecting a raw ConnectionError/Exception instead would exercise a shape production never - produces and let a chain-blind outage classifier pass. That wrapping fidelity is itself pinned by - test_get_user_object_wraps_db_outage_as_valueerror_preserving_context in test_auth_checks.""" + """Reproduce get_user_object's exception contract (litellm/proxy/auth/auth_checks.py): a read + failure that is not a database outage is re-raised as a bare ``ValueError`` with the original + error only as ``__context__``, while an outage propagates raw (pinned by + test_get_user_object_surfaces_a_db_outage_as_503_not_as_a_missing_user and + test_get_user_object_still_reports_a_non_outage_read_failure_as_a_missing_user in + test_auth_checks). The wrapped shape is the harder one for the outage classifier, so injecting + it here keeps a chain-blind classifier from passing.""" try: raise original except BaseException: diff --git a/tests/test_litellm/proxy/auth/test_auth_checks.py b/tests/test_litellm/proxy/auth/test_auth_checks.py index 0cab6535f3d..14b739e60ba 100644 --- a/tests/test_litellm/proxy/auth/test_auth_checks.py +++ b/tests/test_litellm/proxy/auth/test_auth_checks.py @@ -75,6 +75,8 @@ from litellm.constants import ( ) from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper +from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler +from prisma.errors import DataError from litellm.proxy.common_utils.user_api_key_cache import ( END_USER_RESTRICTED_REGISTRY_OVERFLOW_SENTINEL, TAG_REGISTRY_OVERFLOW_SENTINEL, @@ -892,36 +894,80 @@ async def test_get_user_object_upsert_sets_budget_reset_at(monkeypatch, has_budg assert "budget_reset_at" not in creation_args -@pytest.mark.asyncio -async def test_get_user_object_wraps_db_outage_as_valueerror_preserving_context(): - """Pin get_user_object's exception contract: it catches every DB failure in a broad except and - re-raises a bare ValueError, so a real outage survives only as __context__ rather than as the - exception type. The MCP dcr_bridge admission and refresh paths depend on this to tell a transient - outage (retry, 503) from a missing user (fail closed), which is why they classify across the cause - chain instead of the top exception's type. If this wrapping ever changes, that classification must - change with it, so this test guards the contract the callers rely on.""" - from unittest.mock import AsyncMock, MagicMock, patch +def _user_read_raising(error: Exception) -> tuple[MagicMock, MagicMock]: + prisma_client = MagicMock() + prisma_client.db.litellm_usertable.find_unique = AsyncMock(side_effect=error) + cache = MagicMock() + cache.async_get_cache = AsyncMock(return_value=None) + cache.async_set_cache = AsyncMock() + return prisma_client, cache - mock_prisma_client = MagicMock() - mock_prisma_client.db = AsyncMock() - mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock( - side_effect=ConnectionError("can't reach database server") - ) - mock_cache = MagicMock() - mock_cache.async_get_cache = AsyncMock(return_value=None) - mock_cache.async_set_cache = AsyncMock() + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "outage", + [ + httpx.ConnectError("All connection attempts failed"), + httpx.ReadTimeout("timed out"), + DataError( + data={ + "user_facing_error": { + "message": "Can't reach database server at `127.0.0.1:41071`", + "error_code": "P1001", + } + } + ), + ], + ids=["connect_error", "read_timeout", "p1001_as_data_error"], +) +async def test_get_user_object_surfaces_a_db_outage_as_503_not_as_a_missing_user(outage): + from litellm.proxy.auth.auth_exception_handler import _as_proxy_exception + + prisma_client, cache = _user_read_raising(outage) with patch("litellm.proxy.auth.auth_checks._should_check_db", return_value=True): - with pytest.raises(ValueError, match="User doesn't exist in db\\.") as exc_info: + with pytest.raises(type(outage)) as raised: await get_user_object( - user_id="outage-contract-probe-user", - prisma_client=mock_prisma_client, - user_api_key_cache=mock_cache, + user_id="outage-probe-user", + prisma_client=prisma_client, + user_api_key_cache=cache, user_id_upsert=False, proxy_logging_obj=None, ) - assert isinstance(exc_info.value.__context__, ConnectionError) + assert raised.value is outage + assert PrismaDBExceptionHandler.find_database_service_unavailable_error_in_chain(raised.value) is outage + surfaced = _as_proxy_exception(raised.value) + assert (surfaced.code, surfaced.type) == ("503", ProxyErrorTypes.no_db_connection) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "failure", + [ + DataError(data={"user_facing_error": {"message": "invalid byte sequence for encoding UTF8: 0x00"}}), + RuntimeError("row validation failed"), + ], + ids=["query_level_data_error", "runtime_error"], +) +async def test_get_user_object_still_reports_a_non_outage_read_failure_as_a_missing_user(failure): + from litellm.proxy.auth.auth_exception_handler import _as_proxy_exception + + prisma_client, cache = _user_read_raising(failure) + + with patch("litellm.proxy.auth.auth_checks._should_check_db", return_value=True): + with pytest.raises(ValueError, match="User doesn't exist in db\\.") as raised: + await get_user_object( + user_id="data-error-probe-user", + prisma_client=prisma_client, + user_api_key_cache=cache, + user_id_upsert=False, + proxy_logging_obj=None, + ) + + assert raised.value.__context__ is failure + surfaced = _as_proxy_exception(raised.value) + assert (surfaced.code, surfaced.type) == ("401", ProxyErrorTypes.auth_error) @pytest.mark.asyncio From 571e5797d01dedd9f094e2f6a3e39fe83a141ea8 Mon Sep 17 00:00:00 2001 From: Techboy bebop <142545999+kumarpriyanshu09@users.noreply.github.com> Date: Mon, 21 Sep 2026 23:58:11 -0400 Subject: [PATCH 05/15] test(pricing): assert cache-priced vertex grok rows advertise supports_prompt_caching (#41526) * test(pricing): lock supports_prompt_caching on cache-priced grok rows Check the catalog field itself so a price sync cannot drop the Vertex Grok flag while the helper still passes via the xai/ fallback. Also cover get_model_info on the full vertex_ai/xai/grok-4.6 key and keep the backup map in lockstep. Co-authored-by: Techboy bebop * test(pricing): assert grok cache rows via get_model_info The helper can still pass via the bare xai/ grok row, so lock the catalog flag through get_model_info instead of reading the JSON files Co-authored-by: Techboy bebop * test(pricing): assert grok cache flags via get_model_info Address Greptile P2 by checking supports_prompt_caching through get_model_info on full catalog keys instead of raw JSON fields, so the check cannot pass via the bare xai/grok-* helper path. Keep backup/primary parity for vertex_ai/xai/grok-* entries. * test(pricing): assert grok cache flags via runtime APIs only --------- Co-authored-by: Cursor Agent Co-authored-by: Techboy bebop --- ...tex_ai_xai_grok_prompt_caching_metadata.py | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/tests/test_litellm/test_vertex_ai_xai_grok_prompt_caching_metadata.py b/tests/test_litellm/test_vertex_ai_xai_grok_prompt_caching_metadata.py index 72e98711f0c..2d5936fabdb 100644 --- a/tests/test_litellm/test_vertex_ai_xai_grok_prompt_caching_metadata.py +++ b/tests/test_litellm/test_vertex_ai_xai_grok_prompt_caching_metadata.py @@ -13,16 +13,18 @@ GROK_KEY_PREFIXES: Final = ("vertex_ai/xai/grok-", "azure_ai/grok-", "xai/grok-" @pytest.mark.usefixtures("local_model_cost_map") def test_grok_models_with_cache_read_price_advertise_prompt_caching() -> None: - cached_grok_models = tuple( + cached_grok_models: Final = tuple( key for key, entry in litellm.model_cost.items() if key.startswith(GROK_KEY_PREFIXES) and entry.get("cache_read_input_token_cost") ) assert cached_grok_models, "expected at least one grok model with a cache read price" - missing_flag = tuple(key for key in cached_grok_models if supports_prompt_caching(model=key) is not True) + missing_flag: Final = tuple( + key for key in cached_grok_models if get_model_info(model=key).get("supports_prompt_caching") is not True + ) assert missing_flag == (), ( - f"grok models with cache_read_input_token_cost fail supports_prompt_caching: {missing_flag}" + f"grok models with cache_read_input_token_cost fail get_model_info supports_prompt_caching: {missing_flag}" ) @@ -31,8 +33,14 @@ def test_vertex_ai_grok_4_6_supports_prompt_caching_via_get_model_info() -> None routed_model, provider, _, _ = get_llm_provider(model=MODEL) assert (routed_model, provider) == ("xai/grok-4.6", "vertex_ai") - info = get_model_info(model=routed_model, custom_llm_provider=provider) - assert info["litellm_provider"] == "vertex_ai" - assert info.get("supports_prompt_caching") is True + routed_info: Final = get_model_info(model=routed_model, custom_llm_provider=provider) + assert routed_info["litellm_provider"] == "vertex_ai" + assert routed_info.get("supports_prompt_caching") is True + assert routed_info.get("cache_read_input_token_cost") + + catalog_info: Final = get_model_info(model=MODEL) + assert catalog_info["key"] == MODEL + assert catalog_info.get("supports_prompt_caching") is True + assert catalog_info.get("cache_read_input_token_cost") assert supports_prompt_caching(model=MODEL) is True From 418561991b06324d14d7b0076aa43514ac5035bd Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 20:59:36 -0700 Subject: [PATCH 06/15] fix(bedrock): price bedrock/mantle/ deployments from the base model row (#42402) Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/llms/bedrock/common_utils.py | 3 +- .../llms/bedrock/test_bedrock_common_utils.py | 10 +++++ tests/test_litellm/test_cost_calculator.py | 39 +++++++++++++++++++ 3 files changed, 51 insertions(+), 1 deletion(-) diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index f0816566aa7..d9fc813a594 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -787,7 +787,7 @@ def is_bedrock_application_inference_profile_arn(model: str) -> bool: def strip_bedrock_routing_prefix(model: str) -> str: """Strip LiteLLM routing prefixes from model name.""" - for prefix in ["bedrock/", "converse/", "invoke/", "openai/", "nova-2/", "nova/"]: + for prefix in ["bedrock/", "converse/", "invoke/", "openai/", "mantle/", "nova-2/", "nova/"]: if model.startswith(prefix): model = model.split("/", 1)[1] return model @@ -850,6 +850,7 @@ def get_bedrock_base_model(model: str) -> str: Handle model names like: - "us.meta.llama3-2-11b-instruct-v1:0" -> "meta.llama3-2-11b-instruct-v1" - "bedrock/converse/model" -> "model" + - "bedrock/mantle/anthropic.claude-sonnet-5" -> "anthropic.claude-sonnet-5" - "anthropic.claude-3-5-sonnet-20241022-v2:0:51k" -> "anthropic.claude-3-5-sonnet-20241022-v2:0" - "bedrock/nova-2/arn:aws:..." -> "amazon.nova-2-custom" - "bedrock/nova/arn:aws:..." -> "amazon.nova-custom" diff --git a/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py b/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py index 87321cc2e65..117814a41ff 100644 --- a/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py +++ b/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py @@ -279,6 +279,16 @@ def test_context_window_suffix_stripped_for_cost_lookup(): ) +def test_legacy_mantle_route_prefix_stripped_for_cost_lookup(): + """The mantle/ route token is a routing prefix like openai/, so a bedrock/mantle/ + deployment must resolve the bare Bedrock model for cost lookup while still routing to Mantle.""" + from litellm.llms.bedrock.common_utils import get_bedrock_base_model, strip_bedrock_routing_prefix + + assert strip_bedrock_routing_prefix("mantle/anthropic.claude-sonnet-5") == "anthropic.claude-sonnet-5" + assert get_bedrock_base_model("bedrock/mantle/anthropic.claude-sonnet-5") == "anthropic.claude-sonnet-5" + assert BedrockModelInfo.get_bedrock_route("bedrock/mantle/anthropic.claude-sonnet-5") == "mantle" + + def test_output_config_effort_normalization_uses_model_info_ceiling(monkeypatch): import litellm.llms.bedrock.common_utils as mod diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index 1d6c229f9ce..84c60e029cb 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -3574,6 +3574,45 @@ def test_completion_cost_mantle_native_messages_prices_haiku_from_the_mantle_row ) == pytest.approx(expected), model +def test_completion_cost_legacy_mantle_route_prices_after_router_registration(local_model_cost_map): + """The proxy registers every deployment under its provider-prefixed key at boot. A + bedrock/mantle/ deployment must resolve to the bare Bedrock row there, otherwise the boot + entry is a cost-less capability rule that shadows the priced row and every call on the deployment, + /v1/chat/completions and /v1/messages alike, bills $0.""" + from litellm import Router + + Router( + model_list=[ + { + "model_name": "claude-sonnet-5", + "litellm_params": { + "model": "bedrock/mantle/anthropic.claude-sonnet-5", + "aws_region_name": "us-east-1", + }, + } + ] + ) + assert "bedrock/mantle/anthropic.claude-sonnet-5" not in litellm.model_cost + + response = litellm.ModelResponse( + id="msg_x", + choices=[{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}], + model="claude-sonnet-5", + usage={"prompt_tokens": 16, "completion_tokens": 4, "total_tokens": 20}, + ) + row = litellm.model_cost["anthropic.claude-sonnet-5"] + expected = 16 * row["input_cost_per_token"] + 4 * row["output_cost_per_token"] + assert expected > 0 + + for call_type in ("completion", "anthropic_messages"): + assert litellm.completion_cost( + completion_response=response, + model="mantle/anthropic.claude-sonnet-5", + custom_llm_provider="bedrock", + call_type=call_type, + ) == pytest.approx(expected), call_type + + def test_select_model_name_keeps_base_model_free_of_region(_local_model_cost_map): """An explicit base_model keeps pricing on that model's own key even when the request carries a region with different regional rates, so the private provider model never widens region pricing.""" From 7956dd6e8c72ccf88af02bc7693d3d8e28246bf4 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 21:08:28 -0700 Subject: [PATCH 07/15] fix(anthropic_adapter): keep reasoning_effort a string for targets that stay on chat completions (#42401) * fix(anthropic_adapter): keep reasoning_effort a string for targets that stay on chat completions * fix(anthropic_adapter): judge the summary bridge with the deployment's api_base The adapter's bridge check now resolves the provider and base the same way completion() does, passing the deployment's api_base and api_key into get_llm_provider and the resolved base into the bridge check, so a Foundry deployment lands on the same route in both places and a bare model name routed by its api_base still gets the plain tier. A litellm_proxy target keeps the dict, since the upstream gateway makes its own bridge decision and needs the summary to make it. * refactor(anthropic_adapter): return the plain effort instead of writing it inside the helper --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../adapters/handler.py | 51 ++++++++ ..._handler_reasoning_effort_normalization.py | 122 ++++++++++++++++++ 2 files changed, 173 insertions(+) diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py b/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py index 54d10837d74..116f96cf00c 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py @@ -27,6 +27,7 @@ from litellm.llms.anthropic.experimental_pass_through.utils import ( from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, ) +from litellm.types.llms.openai import OpenAIWebSearchOptions from litellm.types.utils import ModelResponse from litellm.utils import get_model_info @@ -383,6 +384,49 @@ class LiteLLMMessagesToCompletionTransformationHandler: updated_reasoning_effort["summary"] = effective_summary completion_kwargs["reasoning_effort"] = updated_reasoning_effort + @staticmethod + def _plain_effort_for_chat_target( + completion_kwargs: _CompletionKwargs, + *, + thinking: Mapping[str, object] | None, + ) -> str | None: + reasoning_effort: Final = completion_kwargs.get("reasoning_effort") + if not thinking or not isinstance(reasoning_effort, dict) or "summary" not in reasoning_effort: + return None + effort: Final = reasoning_effort.get("effort") + model: Final = completion_kwargs.get("model") + if not isinstance(effort, str) or not isinstance(model, str) or not model: + return None + custom_llm_provider: Final = completion_kwargs.get("custom_llm_provider") + api_base: Final = completion_kwargs.get("api_base") + api_key: Final = completion_kwargs.get("api_key") + try: + local_model, resolved_provider, _, resolved_api_base = litellm.utils.get_llm_provider( + model=model, + custom_llm_provider=custom_llm_provider if isinstance(custom_llm_provider, str) else None, + api_base=api_base if isinstance(api_base, str) else None, + api_key=api_key if isinstance(api_key, str) else None, + ) + except Exception: + return None + if resolved_provider == "litellm_proxy": + return None + from litellm.main import responses_api_bridge_check + + web_search_options: Final = completion_kwargs.get("web_search_options") + tools: Final = completion_kwargs.get("tools") + model_info, _ = responses_api_bridge_check( + model=local_model, + custom_llm_provider=resolved_provider, + web_search_options=( + cast(OpenAIWebSearchOptions, web_search_options) if isinstance(web_search_options, dict) else None + ), + tools=cast("list[dict[str, object]]", tools) if isinstance(tools, list) else None, + reasoning_effort=reasoning_effort, + api_base=resolved_api_base, + ) + return None if model_info.get("mode") == "responses" else effort + @staticmethod def _normalize_reasoning_effort( completion_kwargs: _CompletionKwargs, @@ -547,6 +591,13 @@ class LiteLLMMessagesToCompletionTransformationHandler: thinking=thinking, ) + plain_effort: Final = LiteLLMMessagesToCompletionTransformationHandler._plain_effort_for_chat_target( + completion_kwargs, + thinking=thinking, + ) + if plain_effort is not None: + completion_kwargs["reasoning_effort"] = plain_effort + return completion_kwargs, tool_name_mapping @staticmethod diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_handler_reasoning_effort_normalization.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_handler_reasoning_effort_normalization.py index af7befecc33..895b3b57f7b 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_handler_reasoning_effort_normalization.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_handler_reasoning_effort_normalization.py @@ -6,6 +6,8 @@ regression they guard is the one a caller sees: a tier the proxy advertises has leaves the adapter, in the shape the target expects. """ +from typing import Final + import pytest from litellm.llms.anthropic.experimental_pass_through.adapters.handler import ( @@ -36,6 +38,126 @@ def _reasoning_effort_sent(model: str, provider: str, reasoning_effort: object) return completion_kwargs.get("reasoning_effort") +def _reasoning_effort_sent_for_thinking( + model: str, + provider: str | None, + thinking: dict[str, object], + *, + tools: list[dict[str, object]] | None = None, + api_base: str | None = None, +) -> object: + extra_kwargs: Final = { + key: value for key, value in (("custom_llm_provider", provider), ("api_base", api_base)) if value is not None + } + completion_kwargs, _ = LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs( + max_tokens=1024, + messages=MESSAGES, + model=model, + metadata=None, + stop_sequences=None, + stream=False, + system=None, + temperature=None, + thinking=thinking, + tool_choice=None, + tools=tools, + top_k=None, + top_p=None, + output_format=None, + extra_kwargs=extra_kwargs, + ) + return completion_kwargs.get("reasoning_effort") + + +SUMMARIZED_THINKING = {"type": "enabled", "budget_tokens": 4096, "summary": "auto"} +PLAIN_THINKING = {"type": "enabled", "budget_tokens": 4096} +MULTIPLY_TOOL = { + "name": "multiply", + "description": "Multiply two integers", + "input_schema": {"type": "object", "properties": {"a": {"type": "integer"}, "b": {"type": "integer"}}}, +} + + +class TestTheSummaryWrappingOnlyRidesTheResponsesBridge: + """Only the Responses API takes ``reasoning_effort`` as a dict. Databricks answered the wrapped + ``{"effort", "summary"}`` with ``field 'reasoning_effort' expects input with json type 'string' + but got 'object'``, so a target that stays on chat completions has to get the plain tier and a + target the bridge picks up has to keep the summary it can honor.""" + + @pytest.mark.parametrize( + "model, provider", + [ + ("databricks/databricks-qwen35-122b-a10b", "databricks"), + ("databricks-qwen35-122b-a10b", "databricks"), + ("fireworks_ai/kimi-k3", "fireworks_ai"), + ], + ) + def test_a_chat_target_gets_the_plain_tier(self, local_model_cost_map: None, model: str, provider: str) -> None: + assert _reasoning_effort_sent_for_thinking(model, provider, SUMMARIZED_THINKING) == "high" + + def test_auto_summary_stays_a_plain_tier_on_a_chat_target( + self, local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("LITELLM_REASONING_AUTO_SUMMARY", "true") + + sent = _reasoning_effort_sent_for_thinking("databricks/databricks-qwen35-122b-a10b", "databricks", PLAIN_THINKING) + + assert sent == "high" + + @pytest.mark.parametrize( + "model, provider", + [ + ("azure/responses/gpt-5-mini", "azure"), + ("gpt-5-mini", "openai"), + ("databricks/databricks-gpt-5-5", "databricks"), + ], + ) + def test_a_bridged_target_keeps_the_summary(self, local_model_cost_map: None, model: str, provider: str) -> None: + sent = _reasoning_effort_sent_for_thinking(model, provider, SUMMARIZED_THINKING) + + assert sent == {"effort": "high", "summary": "auto"} + + def test_auto_summary_still_reaches_a_bridged_target( + self, local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("LITELLM_REASONING_AUTO_SUMMARY", "true") + + sent = _reasoning_effort_sent_for_thinking("azure/responses/gpt-5-mini", "azure", PLAIN_THINKING) + + assert sent == {"effort": "high", "summary": "detailed"} + + @pytest.mark.parametrize( + "api_base, expected", + [ + ("https://foo.services.ai.azure.com/openai/v1", "high"), + ("https://foo.eastus.models.ai.azure.com", {"effort": "high", "summary": "auto"}), + ], + ) + def test_a_foundry_deployment_is_judged_by_its_api_base( + self, local_model_cost_map: None, api_base: str, expected: object + ) -> None: + """``completion()`` keeps a gpt-5.5 deployment with function tools on Foundry's chat route when + its ``api_base`` is a Foundry OpenAI host, and bridges it to Responses when the base makes it an + Azure OpenAI deployment. The adapter has to read the same ``api_base`` to land on the same call.""" + sent = _reasoning_effort_sent_for_thinking( + "azure_ai/gpt-5.5", "azure_ai", SUMMARIZED_THINKING, tools=[MULTIPLY_TOOL], api_base=api_base + ) + + assert sent == expected + + def test_a_provider_resolved_from_the_api_base_gets_the_plain_tier(self, local_model_cost_map: None) -> None: + sent = _reasoning_effort_sent_for_thinking( + "kimi-k3", None, SUMMARIZED_THINKING, api_base="https://api.together.xyz/v1" + ) + + assert sent == "high" + + def test_a_chained_gateway_keeps_the_dict_for_its_own_bridge(self, local_model_cost_map: None) -> None: + sent = _reasoning_effort_sent_for_thinking("litellm_proxy/gpt-5.4", "litellm_proxy", SUMMARIZED_THINKING) + + assert sent == {"effort": "high", "summary": "auto"} + + class TestTheNormalizedTierIsTheTierSent: """The bug in the caller's terms: a proxy advertising kimi-k3 ``max`` accepted the request and then put ``high`` on the wire. Every spelling of the entry has to survive the adapter, including From dfd8ffc5451f86067ca6b97696dc34d85f92d91e Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 21:11:22 -0700 Subject: [PATCH 08/15] chore(prices): sync OpenRouter prices: 2 models, 2 deprecated (#42381) openrouter/nex-agi/nex-n2.5-mini:free: deprecation_date openrouter/nex-agi/nex-n2.5-pro:free: deprecation_date Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 2 ++ model_prices_and_context_window.json | 2 ++ 2 files changed, 4 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 70c54b91cbd..08cb2ddb8f7 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -75143,6 +75143,7 @@ "supports_web_search": false }, "openrouter/nex-agi/nex-n2.5-mini:free": { + "deprecation_date": "2026-09-25", "input_cost_per_token": 0.0, "litellm_provider": "openrouter", "max_input_tokens": 262144, @@ -75162,6 +75163,7 @@ "supports_web_search": false }, "openrouter/nex-agi/nex-n2.5-pro:free": { + "deprecation_date": "2026-09-25", "input_cost_per_token": 0.0, "litellm_provider": "openrouter", "max_input_tokens": 262144, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 70c54b91cbd..08cb2ddb8f7 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -75143,6 +75143,7 @@ "supports_web_search": false }, "openrouter/nex-agi/nex-n2.5-mini:free": { + "deprecation_date": "2026-09-25", "input_cost_per_token": 0.0, "litellm_provider": "openrouter", "max_input_tokens": 262144, @@ -75162,6 +75163,7 @@ "supports_web_search": false }, "openrouter/nex-agi/nex-n2.5-pro:free": { + "deprecation_date": "2026-09-25", "input_cost_per_token": 0.0, "litellm_provider": "openrouter", "max_input_tokens": 262144, From 19c13ac971ca5324e21f410e4c4783375a5cdbda Mon Sep 17 00:00:00 2001 From: Zachary Lyon Date: Mon, 21 Sep 2026 21:21:43 -0700 Subject: [PATCH 09/15] feat(proxy): add TinyFish Agent API passthrough with per-step billing (#41099) * feat(proxy): add TinyFish Agent API passthrough with per-step billing * chore(ui): regenerate dashboard API types for /tinyfish passthrough * fix(proxy): satisfy strict lint budget for tinyfish passthrough * style: ruff format tinyfish passthrough handler * test(proxy): exercise tinyfish route through the app with a faked upstream * refactor(proxy): make cross-module tinyfish billing hooks public * fix(proxy): tolerate transient tinyfish poll failures instead of dropping the charge * fix(proxy): defer billing for disconnected tinyfish SSE runs to the background poller * Revert "fix(proxy): defer billing for disconnected tinyfish SSE runs to the background poller" This reverts commit ef0bcfb4a0b9189eda0f8f43619949c4d50accd2. * fix(proxy): bill tinyfish SSE runs via detached poller and only COMPLETED runs Disconnected run-sse clients previously left completed runs unbilled: the stream-end handler saw a still-RUNNING run and logged $0. The poller now spawns from the streaming path on the first run_id frame, outlives the disconnect, and writes the one spend row when the run turns terminal; the stream-end path only logs the $0 fallback for run_id-less streams. Costs now apply only to COMPLETED runs ($0 for FAILED/CANCELLED, matching the upstream invoice), spend rows carry the request's litellm_call_id (previously NULL request_ids collided and were silently dropped), and the GET /v1/runs listing is blocked so callers behind the shared key cannot discover each other's runs. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Pxz11TQKYvCXvKawNUa2g4 * fix(proxy): drop GET /v1/runs from the tinyfish allowlist error message Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Pxz11TQKYvCXvKawNUa2g4 * chore(proxy): sync openapi artifacts for tinyfish docstring, suppress LIT011 on flag write Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Pxz11TQKYvCXvKawNUa2g4 * style(proxy): ruff-format the sse poller flag write Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Pxz11TQKYvCXvKawNUa2g4 * style(proxy): keep the rebind-ok suppression on the flag write's own line Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Pxz11TQKYvCXvKawNUa2g4 * fix(proxy): harden tinyfish billing paths from review findings Skip failure dispatch when the SSE poller owns billing (a failure row collided with the poller's billed row on request_id and dropped the charge), late-spawn the poller for run_ids that arrive in unterminated frames instead of mispricing RUNNING runs at $0, thread litellm_params into poller-billed standard logging objects so SLO consumers see attribution, untype the run error field so upstream error-shape drift cannot void a billable run, normalize a schemeless TINYFISH_AGENT_API_BASE, extend the poll budget to cover queue wait (3600s) with ~60s outage tolerance, and log poller cancellation. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Pxz11TQKYvCXvKawNUa2g4 * chore(proxy): satisfy ratcheted BLE001/LIT002 budgets from main in tinyfish handler Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Pxz11TQKYvCXvKawNUa2g4 * style(proxy): drop stray blank line from merge resolution Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Pxz11TQKYvCXvKawNUa2g4 * fix(proxy): reject passthrough envelope controls on tinyfish route, raise blocking-run timeout The generic passthrough unwraps a caller-supplied custom_body as the forwarded request and honors a caller stream flag, so custom_body.use_vault bypassed the credentialed-run 403 and stream: true flipped a blocking run into the streaming pipeline. The route now 400s the envelope fields (custom_body, stream, query_params); streaming comes from the endpoint. Blocking runs also get a 1500s default timeout covering the upstream 1200s run cap, unless the operator configured pass_through_request_timeout. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Pxz11TQKYvCXvKawNUa2g4 * style(proxy): resolve operator timeout without a dict-literal default Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Pxz11TQKYvCXvKawNUa2g4 * test(passthrough): list the TinyFish route among protocol-constrained pass-through routes * chore(proxy): regenerate the lazy OpenAPI snapshot after merging main * chore(proxy): keep the lazy OpenAPI snapshot as CI's Python 3.12 renders it * fix(proxy): reject TinyFish POST bodies that are not a JSON object A form-encoded or text body carried stream and use_vault past both field gates, because the gates only saw fields the body parsed to as JSON. The route now checks the content type before reading the body and answers 400 for anything that is not a JSON object. * fix(tinyfish): reject submit paths with extra slashes so run-async always bills The allowlist dropped empty path segments, so POST /v1/automation/run-async/ was forwarded upstream while the billing dispatch only recognises the exact path and would have logged the submit at $0 without starting the poller. Any path with a trailing or doubled slash now returns 403 before forwarding. --------- Co-authored-by: Claude Fable 5 Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- gateway/routes/allowlist.py | 1 + litellm/proxy/_lazy_features.py | 1 + litellm/proxy/_lazy_openapi_snapshot.json | 90 ++++ litellm/proxy/_types.py | 1 + .../llm_passthrough_endpoints.py | 139 ++++++ .../tinyfish_passthrough_logging_handler.py | 425 ++++++++++++++++++ .../pass_through_endpoints.py | 5 + .../streaming_handler.py | 52 +++ .../pass_through_endpoints/success_handler.py | 38 ++ .../pass_through_endpoints.py | 1 + .../types/passthrough_endpoints/tinyfish.py | 55 +++ .../test_pass_through_unit_tests.py | 1 + ...st_tinyfish_passthrough_logging_handler.py | 400 +++++++++++++++++ .../test_llm_pass_through_endpoints.py | 197 ++++++++ .../test_streaming_handler_interrupt.py | 220 +++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 120 +++++ 16 files changed, 1746 insertions(+) create mode 100644 litellm/proxy/pass_through_endpoints/llm_provider_handlers/tinyfish_passthrough_logging_handler.py create mode 100644 litellm/types/passthrough_endpoints/tinyfish.py create mode 100644 tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_tinyfish_passthrough_logging_handler.py diff --git a/gateway/routes/allowlist.py b/gateway/routes/allowlist.py index 01ba9da3364..fe58e2dd58c 100644 --- a/gateway/routes/allowlist.py +++ b/gateway/routes/allowlist.py @@ -108,6 +108,7 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = ( "/cursor/", "/milvus/", "/openai_passthrough/", + "/tinyfish/", # Dynamic provider / toolset passthrough (path templates) "/{provider}/", "/toolset/", diff --git a/litellm/proxy/_lazy_features.py b/litellm/proxy/_lazy_features.py index c53f7625550..5be87a8bf4d 100644 --- a/litellm/proxy/_lazy_features.py +++ b/litellm/proxy/_lazy_features.py @@ -211,6 +211,7 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = ( "/nvidia_nim/", "/openai/", "/openai_passthrough/", + "/tinyfish/", "/transcribe", "/typesafe/", "/openrouter/", diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 3985feff08a..03122f25870 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -21163,6 +21163,96 @@ ] } }, + "/tinyfish/{endpoint}": { + "get": { + "description": "Pass-through for the TinyFish Agent API (goal-based web automation).\n\nForwarded endpoints:\n- POST /v1/automation/run \u2014 run to completion (blocking)\n- POST /v1/automation/run-async \u2014 submit a run, poll GET /v1/runs/{id} for the result\n- POST /v1/automation/run-sse \u2014 run with SSE progress events\n- GET /v1/runs/{id} \u2014 run status / result\n- POST /v1/runs/{id}/cancel \u2014 cancel a run\n\nEvery other Agent API endpoint (vault, wallet, browser profiles, and the GET /v1/runs\nlisting, which would let any caller discover other callers' run ids) returns 403: all\nproxy callers share one upstream key.\n\nCredential lookup order:\n1. passthrough_endpoint_router (config.yaml deployments with use_in_pass_through)\n2. TINYFISH_API_KEY environment variable\n\n[Docs](https://docs.litellm.ai/docs/pass_through/tinyfish)", + "operationId": "tinyfish_proxy_route_tinyfish__endpoint__get", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Tinyfish Proxy Route", + "tags": [ + "llm_passthrough" + ] + }, + "post": { + "description": "Pass-through for the TinyFish Agent API (goal-based web automation).\n\nForwarded endpoints:\n- POST /v1/automation/run \u2014 run to completion (blocking)\n- POST /v1/automation/run-async \u2014 submit a run, poll GET /v1/runs/{id} for the result\n- POST /v1/automation/run-sse \u2014 run with SSE progress events\n- GET /v1/runs/{id} \u2014 run status / result\n- POST /v1/runs/{id}/cancel \u2014 cancel a run\n\nEvery other Agent API endpoint (vault, wallet, browser profiles, and the GET /v1/runs\nlisting, which would let any caller discover other callers' run ids) returns 403: all\nproxy callers share one upstream key.\n\nCredential lookup order:\n1. passthrough_endpoint_router (config.yaml deployments with use_in_pass_through)\n2. TINYFISH_API_KEY environment variable\n\n[Docs](https://docs.litellm.ai/docs/pass_through/tinyfish)", + "operationId": "tinyfish_proxy_route_tinyfish__endpoint__post", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Tinyfish Proxy Route", + "tags": [ + "llm_passthrough" + ] + } + }, "/transcribe": { "post": { "description": "AWS-SDK-shaped pass-through for Amazon Transcribe: point the SDK's `endpoint_url`\nat `/transcribe` and the operation is read from the `X-Amz-Target` header, per the\nAWS JSON 1.1 protocol.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/transcribe)", diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 2a6b15e4097..8b2c81fea77 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -491,6 +491,7 @@ class LiteLLMRoutes(enum.Enum): "/openai_passthrough", "/assemblyai", "/eu.assemblyai", + "/tinyfish", "/vllm", "/mistral", "/typesafe", diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 74caa1050bb..b1960b9a046 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -14,6 +14,7 @@ import json import os import posixpath import re +import sys from collections.abc import AsyncGenerator, Callable, Mapping, Sequence from dataclasses import dataclass from functools import partial @@ -101,6 +102,12 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import ( LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY, LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY, ) +from litellm.types.passthrough_endpoints.tinyfish import ( + TINYFISH_AUTHENTICATED_RUN_FIELDS, + TINYFISH_PASSTHROUGH_TIMEOUT_SECONDS, + TINYFISH_REJECTED_ENVELOPE_FIELDS, + is_allowed_tinyfish_endpoint, +) from litellm.types.passthrough_endpoints.vertex_ai import VertexPassThroughCredentials from litellm.types.router import LiteLLMParamsTypedDict from litellm.types.utils import LlmProviders @@ -3357,6 +3364,138 @@ async def cursor_proxy_route( return received_value +TINYFISH_JSON_OBJECT_BODY_DETAIL: Final = ( + "TinyFish requests must be a JSON object body sent with Content-Type: application/json." +) + + +async def _tinyfish_json_object_field_names(request: Request) -> frozenset[str] | None: + content_type: Final = request.headers.get("content-type", "") + if content_type and not is_json_content_type(content_type): + return None + raw_body: Final = await request.body() + if not raw_body: + return frozenset() + try: + parsed: Final[object] = json.loads(raw_body) # any-ok: json.loads -> Any + except (json.JSONDecodeError, UnicodeDecodeError): + return None + return frozenset(parsed) if isinstance(parsed, dict) else None + + +def _tinyfish_route_timeout() -> float | None: + # only raise the 600s default to cover legal 1200s runs; an operator's configured timeout still wins + proxy_server: Final = sys.modules.get("litellm.proxy.proxy_server") + operator_settings: Final = getattr(proxy_server, "general_settings", None) + operator_timeout: Final = ( + operator_settings.get("pass_through_request_timeout") if isinstance(operator_settings, Mapping) else None + ) + return None if operator_timeout is not None else TINYFISH_PASSTHROUGH_TIMEOUT_SECONDS + + +@router.api_route( + "/tinyfish/{endpoint:path}", + methods=["GET", "POST"], # mutable-ok: fastapi api_route requires List[str] + tags=["TinyFish Pass-through", "pass-through"], # mutable-ok: fastapi api_route requires a list +) +async def tinyfish_proxy_route( + endpoint: str, + request: Request, + fastapi_response: Response, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), # noqa: B008 # FastAPI dependency injection +) -> Response: + """ + Pass-through for the TinyFish Agent API (goal-based web automation). + + Forwarded endpoints: + - POST /v1/automation/run — run to completion (blocking) + - POST /v1/automation/run-async — submit a run, poll GET /v1/runs/{id} for the result + - POST /v1/automation/run-sse — run with SSE progress events + - GET /v1/runs/{id} — run status / result + - POST /v1/runs/{id}/cancel — cancel a run + + Every other Agent API endpoint (vault, wallet, browser profiles, and the GET /v1/runs + listing, which would let any caller discover other callers' run ids) returns 403: all + proxy callers share one upstream key. + + Credential lookup order: + 1. passthrough_endpoint_router (config.yaml deployments with use_in_pass_through) + 2. TINYFISH_API_KEY environment variable + + [Docs](https://docs.litellm.ai/docs/pass_through/tinyfish) + """ + from .llm_provider_handlers.tinyfish_passthrough_logging_handler import ( + resolve_tinyfish_agent_api_base, + ) + + raw_endpoint_path: Final = httpx.URL(endpoint).path + encoded_endpoint: Final = raw_endpoint_path if raw_endpoint_path.startswith("/") else f"/{raw_endpoint_path}" + + if not is_allowed_tinyfish_endpoint(request.method, encoded_endpoint): + raise HTTPException( + status_code=403, + detail=f"{request.method} {encoded_endpoint} is not an allowed TinyFish Agent passthrough endpoint. " + "Allowed: POST /v1/automation/run, POST /v1/automation/run-async, POST /v1/automation/run-sse, " + "GET /v1/runs/{id}, POST /v1/runs/{id}/cancel.", + ) + + if request.method == "POST": + body_fields: Final = await _tinyfish_json_object_field_names(request) + if body_fields is None: + raise HTTPException(status_code=400, detail=TINYFISH_JSON_OBJECT_BODY_DETAIL) + envelope_fields: Final = tuple(sorted(body_fields & TINYFISH_REJECTED_ENVELOPE_FIELDS)) + if envelope_fields: + raise HTTPException( + status_code=400, + detail=f"Request fields [{', '.join(envelope_fields)}] are LiteLLM pass-through envelope controls " + "and are not accepted on the TinyFish route. Send the native TinyFish request body; streaming is " + "determined by the endpoint.", + ) + blocked_fields: Final = tuple(sorted(body_fields & TINYFISH_AUTHENTICATED_RUN_FIELDS)) + if ( + blocked_fields + and encoded_endpoint.startswith("/v1/automation/") + and str_to_bool(os.getenv("TINYFISH_ALLOW_AUTHENTICATED_RUNS")) is not True + ): + raise HTTPException( + status_code=403, + detail=f"Request fields [{', '.join(blocked_fields)}] run with the shared TinyFish account's saved " + "credentials and are disabled on this proxy. Ask the proxy admin to set " + "TINYFISH_ALLOW_AUTHENTICATED_RUNS=true to allow them.", + ) + + tinyfish_api_key: Final = passthrough_endpoint_router.get_credentials( + custom_llm_provider="tinyfish", + region_name=None, + ) + if tinyfish_api_key is None: + raise HTTPException( + status_code=401, + detail="TinyFish API key not found. Set the TINYFISH_API_KEY environment variable or add a " + "deployment with use_in_pass_through: true.", + ) + + base_url: Final = httpx.URL(resolve_tinyfish_agent_api_base()) + updated_url: Final = base_url.copy_with( + path=HttpPassThroughEndpointHelpers.join_base_and_endpoint_path(base_url, encoded_endpoint) + ) + + endpoint_func: Final = create_pass_through_route( + endpoint=endpoint, + target=str(updated_url), + custom_headers=MappingProxyType({"X-API-Key": tinyfish_api_key}), + custom_llm_provider="tinyfish", + timeout=_tinyfish_route_timeout(), + ) + received_value: Final = await endpoint_func( + request, + fastapi_response, + user_api_key_dict, + ) + + return received_value + + VERTEX_LIVE_UNCONFIGURED_CLOSE_REASON: Final = ( "Vertex AI auth failed: set a use_in_pass_through vertex model, default_vertex_config, or DEFAULT_VERTEXAI_* env" ) diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/tinyfish_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/tinyfish_passthrough_logging_handler.py new file mode 100644 index 00000000000..a6c3cb669a6 --- /dev/null +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/tinyfish_passthrough_logging_handler.py @@ -0,0 +1,425 @@ +import asyncio +import json +import os +import time +import urllib.parse +from collections.abc import Mapping, Sequence +from datetime import datetime +from types import MappingProxyType +from typing import Final, NamedTuple +from urllib.parse import urlparse + +import httpx +from pydantic import TypeAdapter, ValidationError + +from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.litellm_logging import ( + get_standard_logging_object_payload, +) +from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + get_async_httpx_client, +) +from litellm.proxy._types import PassThroughEndpointLoggingTypedDict +from litellm.types.llms.custom_http import httpxSpecialProvider +from litellm.types.passthrough_endpoints.tinyfish import ( + TINYFISH_AGENT_DEFAULT_API_BASE, + TINYFISH_DEFAULT_COST_PER_STEP, + TINYFISH_MAX_CONSECUTIVE_POLL_FAILURES, + TINYFISH_MAX_POLLING_SECONDS, + TINYFISH_MODEL_NAME, + TINYFISH_POLLING_INTERVAL_SECONDS, + TINYFISH_TERMINAL_RUN_STATUSES, + TinyfishRun, +) +from litellm.types.utils import StandardPassThroughResponseObject + +_RUN_ADAPTER: Final = TypeAdapter(TinyfishRun) + +_EMPTY_KWARGS: Final[Mapping[str, object]] = MappingProxyType({}) + + +class _TinyfishLoggingPayload(NamedTuple): + result: StandardPassThroughResponseObject + kwargs: Mapping[str, object] + + def as_handler_result(self) -> PassThroughEndpointLoggingTypedDict: + handler_result: Final[PassThroughEndpointLoggingTypedDict] = { + "result": self.result, + "kwargs": {**self.kwargs}, + } + return handler_result + + +# asyncio tasks are weakly referenced by the loop; hold them until done or they can vanish mid-poll +_BACKGROUND_BILLING_TASKS: Final[set["asyncio.Task[None]"]] = set() # mutable-ok: task registry + + +def _register_billing_task(task: "asyncio.Task[None]") -> None: + _BACKGROUND_BILLING_TASKS.add(task) + task.add_done_callback(_BACKGROUND_BILLING_TASKS.discard) + task.add_done_callback(_warn_if_cancelled) + + +def _warn_if_cancelled(task: "asyncio.Task[None]") -> None: + # CancelledError bypasses the poller's exception handler, so shutdown-time charge loss must be logged here + if task.cancelled(): + verbose_proxy_logger.warning("TinyFish passthrough: billing poller cancelled mid-poll; the run may go unbilled") + + +_SSE_POLLER_SPAWNED_KEY: Final = "tinyfish_sse_poller_spawned" + + +def mark_sse_poller_spawned(logging_obj: LiteLLMLoggingObj) -> None: + logging_obj.model_call_details[_SSE_POLLER_SPAWNED_KEY] = True # rebind-ok: request-scoped scratch dict + + +def sse_poller_spawned(logging_obj: LiteLLMLoggingObj) -> bool: + return logging_obj.model_call_details.get(_SSE_POLLER_SPAWNED_KEY) is True + + +def run_id_from_sse_frames(frames: bytes) -> str | None: + return _run_id_from_sse_chunks(frames.decode("utf-8", errors="replace").splitlines()) + + +def resolve_tinyfish_agent_api_base() -> str: + raw: Final = (os.getenv("TINYFISH_AGENT_API_BASE") or TINYFISH_AGENT_DEFAULT_API_BASE).rstrip("/") + # a schemeless override would silently break both routing and billing (urlparse hostname becomes None) + return raw if "://" in raw else f"https://{raw}" + + +def resolve_tinyfish_cost_per_step() -> float: + raw: Final = os.getenv("TINYFISH_COST_PER_STEP") + if raw is None: + return TINYFISH_DEFAULT_COST_PER_STEP + try: + return float(raw) + except ValueError: + verbose_proxy_logger.warning( + "TINYFISH_COST_PER_STEP=%r is not a number; using the default rate %s", + raw, + TINYFISH_DEFAULT_COST_PER_STEP, + ) + return TINYFISH_DEFAULT_COST_PER_STEP + + +def is_tinyfish_agent_url(url: str) -> bool: + hostname: Final = urlparse(url).hostname + return hostname is not None and hostname == urlparse(resolve_tinyfish_agent_api_base()).hostname + + +def _parse_run(payload: object) -> TinyfishRun | None: + try: + return _RUN_ADAPTER.validate_python(payload) + except ValidationError as e: + verbose_proxy_logger.warning("TinyFish passthrough: unexpected run object shape: %s", e) + return None + + +def _run_cost(run: TinyfishRun | None) -> float | None: + if run is None: + return None + # TinyFish only invoices COMPLETED runs, so FAILED/CANCELLED runs must charge the team $0 + if run.get("status") != "COMPLETED": + return None + num_of_steps: Final = run.get("num_of_steps") + if num_of_steps is None: + return None + return num_of_steps * resolve_tinyfish_cost_per_step() + + +class TinyFishPassthroughLoggingHandler: + @staticmethod + def should_log_request(request_method: str, url_route: str) -> bool: + """Only run submissions are billed; GET /v1/runs* polling and cancels never write spend rows.""" + return request_method == "POST" and "/v1/automation/" in urlparse(url_route).path + + @staticmethod + def is_run_async_route(url_route: str) -> bool: + return urlparse(url_route).path.endswith("/v1/automation/run-async") + + @staticmethod + def tinyfish_passthrough_handler( + httpx_response: httpx.Response, + response_body: Mapping[str, object] | None, + logging_obj: LiteLLMLoggingObj, + url_route: str, + result: str, + start_time: datetime, + end_time: datetime, + cache_hit: bool, + request_body: Mapping[str, object], + **kwargs: object, # kwargs-ok: the passthrough logging dispatch forwards shared logging kwargs to every handler + ) -> PassThroughEndpointLoggingTypedDict: + """Bill a blocking POST /v1/automation/run: the response is the terminal run object.""" + try: + run: Final = _parse_run(response_body) if response_body is not None else None + handler_payload: Final = TinyFishPassthroughLoggingHandler._build_logging_payload( + run=run, + logging_obj=logging_obj, + result=result, + start_time=start_time, + end_time=end_time, + kwargs=kwargs, + ).as_handler_result() + except Exception as e: # noqa: BLE001 # billing/logging must never break the relayed request + verbose_proxy_logger.exception("Error in TinyFish passthrough logging handler: %s", e) + fallback_payload: Final[PassThroughEndpointLoggingTypedDict] = { + "result": StandardPassThroughResponseObject(response=result), + "kwargs": kwargs, + } + return fallback_payload + return handler_payload + + @staticmethod + def start_async_run_billing( + response_body: Mapping[str, object] | None, + logging_obj: LiteLLMLoggingObj, + result: str, + start_time: datetime, + cache_hit: bool, + **kwargs: object, # kwargs-ok: shared logging kwargs, replayed into _handle_logging when the run finishes + ) -> None: + """Bill POST /v1/automation/run-async once, when the polled run turns terminal.""" + submitted: Final = _parse_run(response_body) if response_body is not None else None + run_id: Final = submitted.get("run_id") if submitted is not None else None + if not run_id: + verbose_proxy_logger.warning( + "TinyFish passthrough: run-async response carried no run_id; logging the request without cost" + ) + task: Final = asyncio.create_task( + TinyFishPassthroughLoggingHandler._poll_and_log( + run_id=run_id, + logging_obj=logging_obj, + result=result, + start_time=start_time, + cache_hit=cache_hit, + kwargs=kwargs, + ) + ) + _register_billing_task(task) + + @staticmethod + def start_sse_run_billing( + run_id: str, + litellm_logging_obj: LiteLLMLoggingObj, + start_time: datetime, + client: AsyncHTTPHandler | None = None, + ) -> None: + """Bill POST /v1/automation/run-sse once via a detached poller that outlives client disconnects.""" + mark_sse_poller_spawned(litellm_logging_obj) + task: Final = asyncio.create_task( + TinyFishPassthroughLoggingHandler._poll_and_log( + run_id=run_id, + logging_obj=litellm_logging_obj, + result="", + start_time=start_time, + cache_hit=litellm_logging_obj.model_call_details.get("cache_hit") is True, + kwargs=_EMPTY_KWARGS, + client=client, + ) + ) + _register_billing_task(task) + + @staticmethod + async def _poll_and_log( + run_id: str | None, + logging_obj: LiteLLMLoggingObj, + result: str, + start_time: datetime, + cache_hit: bool, + kwargs: Mapping[str, object], + client: AsyncHTTPHandler | None = None, + ) -> None: + from ..pass_through_endpoints import pass_through_endpoint_logging + + try: + run: Final = ( + await TinyFishPassthroughLoggingHandler._poll_until_terminal(run_id, client) if run_id else None + ) + run_end_time: Final = datetime.now() # noqa: DTZ005 # naive to match the start_time stamped by pass_through_request + payload: Final = TinyFishPassthroughLoggingHandler._build_logging_payload( + run=run, + logging_obj=logging_obj, + result=result, + start_time=start_time, + end_time=run_end_time, + kwargs=kwargs, + ) + await pass_through_endpoint_logging._handle_logging( # pyright: ignore[reportPrivateUsage] # shared passthrough logging dispatcher, same access as the assemblyai handler + logging_obj=logging_obj, + standard_logging_response_object=payload.result, + result=result, + start_time=start_time, + end_time=run_end_time, + cache_hit=cache_hit, + **payload.kwargs, + ) + except Exception as e: # noqa: BLE001 # billing/logging must never break the relayed request + verbose_proxy_logger.exception("[Non blocking logging error] TinyFish run-async billing failed: %s", e) + + @staticmethod + async def _poll_until_terminal( + run_id: str, + client: AsyncHTTPHandler | None = None, + poll_interval_seconds: float = TINYFISH_POLLING_INTERVAL_SECONDS, + ) -> TinyfishRun | None: + deadline: Final = time.monotonic() + TINYFISH_MAX_POLLING_SECONDS + last_run: TinyfishRun | None = None # rebind-ok: poll-loop state + consecutive_failures = 0 # rebind-ok: poll-loop state + while time.monotonic() < deadline: + run = await TinyFishPassthroughLoggingHandler._fetch_run(run_id, client) + if run is None: + # a single transient poll failure must not drop the run's charge + consecutive_failures += 1 + if consecutive_failures >= TINYFISH_MAX_CONSECUTIVE_POLL_FAILURES: + verbose_proxy_logger.warning( + "TinyFish passthrough: giving up on run %s after %s consecutive poll failures; " + "logging the request without cost", + run_id, + consecutive_failures, + ) + return last_run + else: + consecutive_failures = 0 + last_run = run + if (run.get("status") or "") in TINYFISH_TERMINAL_RUN_STATUSES: + return run + await asyncio.sleep(poll_interval_seconds) + verbose_proxy_logger.warning( + "TinyFish passthrough: run %s not terminal after %ss; logging the request without cost", + run_id, + TINYFISH_MAX_POLLING_SECONDS, + ) + return last_run + + @staticmethod + async def _fetch_run(run_id: str, client: AsyncHTTPHandler | None = None) -> TinyfishRun | None: + from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( + passthrough_endpoint_router, + ) + + api_key: Final = passthrough_endpoint_router.get_credentials(custom_llm_provider="tinyfish", region_name=None) + if api_key is None: + verbose_proxy_logger.warning("TinyFish passthrough: no API key available to poll run %s", run_id) + return None + if any(c in run_id for c in ("/", "\\", "#", "?")) or ".." in run_id: + verbose_proxy_logger.warning("TinyFish passthrough: invalid run_id %r", run_id) + return None + safe_run_id: Final = urllib.parse.quote(run_id, safe="") + resolved_client: Final = client or get_async_httpx_client( + llm_provider=httpxSpecialProvider.PassThroughEndpoint, + params={"timeout": 30.0}, # mutable-ok: get_async_httpx_client takes a plain dict of client params + ) + try: + # screenshots=none keeps the poll payload small (no per-step screenshot URLs needed) + response: Final = await resolved_client.get( + f"{resolve_tinyfish_agent_api_base()}/v1/runs/{safe_run_id}?screenshots=none", + headers={"X-API-Key": api_key}, # mutable-ok: httpx headers= takes a plain dict + ) + if not (200 <= response.status_code < 300): + verbose_proxy_logger.warning( + "TinyFish passthrough: GET /v1/runs/%s returned %s", safe_run_id, response.status_code + ) + return None + payload: Final[object] = response.json() # any-ok: httpx Response.json() -> Any + return _parse_run(payload) + except Exception as e: # noqa: BLE001 # billing/logging must never break the relayed request + verbose_proxy_logger.warning("[Non blocking logging error] TinyFish run fetch failed: %s", e) + return None + + @staticmethod + async def handle_logging_tinyfish_collected_chunks( + litellm_logging_obj: LiteLLMLoggingObj, + url_route: str, + start_time: datetime, + all_chunks: Sequence[str], + end_time: datetime, + client: AsyncHTTPHandler | None = None, + ) -> PassThroughEndpointLoggingTypedDict: + """Fallback for run-sse streams with no poller: logs the request, pricing via one GET if a run_id parses.""" + try: + run_id: Final = _run_id_from_sse_chunks(all_chunks) + if run_id is None: + verbose_proxy_logger.warning( + "TinyFish passthrough: no run_id in SSE stream; logging the request without cost" + ) + run: Final = await TinyFishPassthroughLoggingHandler._fetch_run(run_id, client) if run_id else None + payload: Final = TinyFishPassthroughLoggingHandler._build_logging_payload( + run=run, + logging_obj=litellm_logging_obj, + result="", + start_time=start_time, + end_time=end_time, + kwargs=_EMPTY_KWARGS, + ).as_handler_result() + except Exception as e: # noqa: BLE001 # billing/logging must never break the relayed request + verbose_proxy_logger.exception("Error in TinyFish SSE passthrough logging handler: %s", e) + fallback_payload: Final[PassThroughEndpointLoggingTypedDict] = { + "result": StandardPassThroughResponseObject(response=""), + "kwargs": {}, + } + return fallback_payload + return payload + + @staticmethod + def _build_logging_payload( + run: TinyfishRun | None, + logging_obj: LiteLLMLoggingObj, + result: str, + start_time: datetime, + end_time: datetime, + kwargs: Mapping[str, object], + ) -> _TinyfishLoggingPayload: + response_cost: Final = _run_cost(run) + updated_kwargs: Final = { # mutable-ok: the logging pipeline requires a plain kwargs dict + **kwargs, + "model": TINYFISH_MODEL_NAME, + "custom_llm_provider": "tinyfish", + "response_cost": response_cost, + # spend rows key on this as request_id; without it every poller-billed row is a NULL-key collision + "litellm_call_id": logging_obj.litellm_call_id, + # the poller paths pass no request kwargs, so SLO attribution (key hash, team, tags) needs the stored params + "litellm_params": kwargs.get("litellm_params") + or logging_obj.model_call_details.get("litellm_params") + or {}, # mutable-ok: the logging pipeline requires a plain kwargs dict + } + logging_obj.model_call_details.update( + model=TINYFISH_MODEL_NAME, + custom_llm_provider="tinyfish", + response_cost=response_cost, + ) + + logged_response: Final = StandardPassThroughResponseObject( + response=json.dumps(run) if run is not None else result + ) + standard_logging_object: Final = get_standard_logging_object_payload( + kwargs=updated_kwargs, + init_response_obj=logged_response, + start_time=start_time, + end_time=end_time, + logging_obj=logging_obj, + status="success", + ) + return _TinyfishLoggingPayload( + result=logged_response, + kwargs=MappingProxyType({**updated_kwargs, "standard_logging_object": standard_logging_object}), + ) + + +def _run_id_from_sse_chunks(all_chunks: Sequence[str]) -> str | None: + for line in all_chunks: + if not line.startswith("data:"): + continue + try: + event_payload: object = json.loads(line[5:].strip()) # any-ok: json.loads -> Any + except json.JSONDecodeError: + continue + event = _parse_run(event_payload) + if event is None: + continue + run_id = event.get("run_id") + if run_id: + return run_id + return None diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 8902e599788..c2874ac948f 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -113,6 +113,9 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import ( ) from litellm.types.utils import TRUSTED_CALLBACK_VARS_FIELD, Usage +from .llm_provider_handlers.tinyfish_passthrough_logging_handler import ( + is_tinyfish_agent_url, +) from .streaming_handler import PassThroughStreamingHandler from .success_handler import PassThroughEndpointLogging from .upstream_usage_headers import ( @@ -383,6 +386,8 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils): or (parsed_url.hostname and "openai.com" in parsed_url.hostname) ): return EndpointType.OPENAI + elif is_tinyfish_agent_url(url): + return EndpointType.TINYFISH return EndpointType.GENERIC @staticmethod diff --git a/litellm/proxy/pass_through_endpoints/streaming_handler.py b/litellm/proxy/pass_through_endpoints/streaming_handler.py index fae26b5a72d..e1f13f2bee0 100644 --- a/litellm/proxy/pass_through_endpoints/streaming_handler.py +++ b/litellm/proxy/pass_through_endpoints/streaming_handler.py @@ -27,6 +27,11 @@ from .llm_provider_handlers.gemini_passthrough_logging_handler import ( from .llm_provider_handlers.openai_passthrough_logging_handler import ( OpenAIPassthroughLoggingHandler, ) +from .llm_provider_handlers.tinyfish_passthrough_logging_handler import ( + TinyFishPassthroughLoggingHandler, + run_id_from_sse_frames, + sse_poller_spawned, +) from .llm_provider_handlers.vertex_passthrough_logging_handler import ( VertexPassthroughLoggingHandler, ) @@ -70,6 +75,9 @@ class PassThroughStreamingHandler: exception: Exception, stream_context: PassThroughStreamContext | None = None, ) -> None: + # the tinyfish poller writes the one authoritative row; a failure row here would collide on its request_id + if endpoint_type == EndpointType.TINYFISH and sse_poller_spawned(litellm_logging_obj): + return await asyncify(PassThroughStreamingHandler._record_partial_usage_for_failure)( litellm_logging_obj=litellm_logging_obj, endpoint_type=endpoint_type, @@ -179,12 +187,25 @@ class PassThroughStreamingHandler: ) ) ) + # TinyFish SSE bills via a detached poller spawned on the first run_id frame, so disconnects can't lose the charge + tinyfish_scan_active = endpoint_type == EndpointType.TINYFISH # rebind-ok: scan stops once the poller spawns + tinyfish_pending = b"" # rebind-ok: SSE frame reassembly buffer across transport chunks try: if not cost_injection_active: # Hot path: just buffer for end-of-stream logging and forward. async for chunk in response.aiter_bytes(): raw_bytes.append(chunk) PassThroughStreamingHandler._stamp_first_chunk_if_needed(litellm_logging_obj) + if tinyfish_scan_active: + complete_frames, tinyfish_pending = split_complete_sse_frames(tinyfish_pending + chunk) + run_id = run_id_from_sse_frames(complete_frames) if b"run_id" in complete_frames else None + if run_id: + TinyFishPassthroughLoggingHandler.start_sse_run_billing( + run_id=run_id, + litellm_logging_obj=litellm_logging_obj, + start_time=start_time, + ) + tinyfish_scan_active = False yield chunk else: # ``cost_injection_active`` already requires ``model_name`` to @@ -294,6 +315,37 @@ class PassThroughStreamingHandler: and not _is_provider_error_chunk(complete_frames) ) try: + # TinyFish billing is owned by the detached poller; the $0 fallback below is only for streams with no run_id + if endpoint_type == EndpointType.TINYFISH: + if sse_poller_spawned(litellm_logging_obj): + return + late_run_id: Final = run_id_from_sse_frames(b"".join(raw_bytes)) + if late_run_id: + # the run_id arrived in an unterminated frame; poll to terminal instead of mispricing a RUNNING run + TinyFishPassthroughLoggingHandler.start_sse_run_billing( + run_id=late_run_id, + litellm_logging_obj=litellm_logging_obj, + start_time=start_time, + ) + return + tinyfish_payload: Final = ( + await TinyFishPassthroughLoggingHandler.handle_logging_tinyfish_collected_chunks( + litellm_logging_obj=litellm_logging_obj, + url_route=url_route, + start_time=start_time, + all_chunks=PassThroughStreamingHandler._convert_raw_bytes_to_str_lines(raw_bytes), + end_time=end_time, + ) + ) + await litellm_logging_obj.dispatch_success_handlers( + result=tinyfish_payload["result"], + start_time=start_time, + end_time=end_time, + cache_hit=litellm_logging_obj.model_call_details.get("cache_hit") is True, + prefer_async_handlers=True, + **tinyfish_payload["kwargs"], + ) + return ( standard_logging_response_object, kwargs, diff --git a/litellm/proxy/pass_through_endpoints/success_handler.py b/litellm/proxy/pass_through_endpoints/success_handler.py index d1e4da2e47c..6bba879b6c1 100644 --- a/litellm/proxy/pass_through_endpoints/success_handler.py +++ b/litellm/proxy/pass_through_endpoints/success_handler.py @@ -35,6 +35,10 @@ from .llm_provider_handlers.fal_ai_passthrough_logging_handler import ( from .llm_provider_handlers.gemini_passthrough_logging_handler import ( GeminiPassthroughLoggingHandler, ) +from .llm_provider_handlers.tinyfish_passthrough_logging_handler import ( + TinyFishPassthroughLoggingHandler, + is_tinyfish_agent_url, +) from .llm_provider_handlers.transcribe_passthrough_logging_handler import ( TRANSCRIBE_CUSTOM_LLM_PROVIDER, PassThroughLogDispatch, @@ -281,6 +285,22 @@ class PassThroughEndpointLogging: ) standard_logging_response_object = comprehend_medical_handler_result["result"] # rebind-ok: elif-chain kwargs = comprehend_medical_handler_result["kwargs"] # rebind-ok: elif-chain contract + elif self.is_tinyfish_route(url_route, custom_llm_provider): + tinyfish_handler_result: Final = TinyFishPassthroughLoggingHandler.tinyfish_passthrough_handler( + httpx_response=httpx_response, + response_body=response_body if isinstance(response_body, dict) else None, + logging_obj=logging_obj, + url_route=url_route, + result=result, + start_time=start_time, + end_time=end_time, + cache_hit=cache_hit, + request_body=request_body, + **kwargs, + ) + standard_logging_response_object = tinyfish_handler_result["result"] # rebind-ok: elif-chain + kwargs = tinyfish_handler_result["kwargs"] # rebind-ok: elif-chain contract + elif self.is_azure_speech_route(custom_llm_provider): from .llm_provider_handlers.azure_speech_passthrough_logging_handler import ( AzureSpeechPassthroughLoggingHandler, @@ -336,6 +356,7 @@ class PassThroughEndpointLogging: ) standard_logging_response_object = typesafe_handler_result["result"] kwargs = typesafe_handler_result["kwargs"] + elif self.is_vertex_ai_live_route(url_route): from .llm_provider_handlers.vertex_ai_live_passthrough_logging_handler import ( VertexAILivePassthroughLoggingHandler, @@ -405,6 +426,20 @@ class PassThroughEndpointLogging: ): standard_logging_response_object: PassThroughEndpointLoggingResultValues | None = None logging_obj.model_call_details["passthrough_logging_payload"] = passthrough_logging_payload + if self.is_tinyfish_route(url_route, custom_llm_provider): + # polls and cancels never write spend rows; run-async bills once from the background poller + if not TinyFishPassthroughLoggingHandler.should_log_request(httpx_response.request.method, url_route): + return + if TinyFishPassthroughLoggingHandler.is_run_async_route(url_route): + TinyFishPassthroughLoggingHandler.start_async_run_billing( + response_body=response_body if isinstance(response_body, dict) else None, + logging_obj=logging_obj, + result=result, + start_time=start_time, + cache_hit=cache_hit, + **kwargs, + ) + return if self.is_assemblyai_route(url_route) and not self.is_azure_speech_route(custom_llm_provider): if AssemblyAIPassthroughLoggingHandler._should_log_request(httpx_response.request.method) is not True: return @@ -512,6 +547,9 @@ class PassThroughEndpointLogging: def is_comprehend_medical_route(self, custom_llm_provider: str | None) -> bool: return custom_llm_provider == "comprehendmedical" + def is_tinyfish_route(self, url_route: str, custom_llm_provider: str | None) -> bool: + return custom_llm_provider == "tinyfish" or is_tinyfish_agent_url(url_route) + def is_azure_speech_route(self, custom_llm_provider: str | None) -> bool: return custom_llm_provider == AZURE_SPEECH_CUSTOM_LLM_PROVIDER diff --git a/litellm/types/passthrough_endpoints/pass_through_endpoints.py b/litellm/types/passthrough_endpoints/pass_through_endpoints.py index e47acf9d68b..619001a5791 100644 --- a/litellm/types/passthrough_endpoints/pass_through_endpoints.py +++ b/litellm/types/passthrough_endpoints/pass_through_endpoints.py @@ -28,6 +28,7 @@ class EndpointType(str, Enum): GEMINI = "gemini" ANTHROPIC = "anthropic" OPENAI = "openai" + TINYFISH = "tinyfish" GENERIC = "generic" diff --git a/litellm/types/passthrough_endpoints/tinyfish.py b/litellm/types/passthrough_endpoints/tinyfish.py new file mode 100644 index 00000000000..365eaef77a6 --- /dev/null +++ b/litellm/types/passthrough_endpoints/tinyfish.py @@ -0,0 +1,55 @@ +from typing import Final + +from typing_extensions import ReadOnly, TypedDict + +TINYFISH_AGENT_DEFAULT_API_BASE: Final = "https://agent.tinyfish.ai" +TINYFISH_AGENT_DOCS_URL: Final = "https://docs.tinyfish.ai/agent-api" +# TinyFish's published Agent API rate (USD per run step); override with env TINYFISH_COST_PER_STEP +TINYFISH_DEFAULT_COST_PER_STEP: Final = 0.016 +TINYFISH_MODEL_NAME: Final = "tinyfish/automation-run" +TINYFISH_POLLING_INTERVAL_SECONDS: Final = 5.0 +# the Agent API caps runs at 1200s but queue wait extends wall time, so billing polls with generous headroom +TINYFISH_MAX_POLLING_SECONDS: Final = 3600.0 +# at the 5s interval this tolerates a ~60s upstream outage before abandoning the charge +TINYFISH_MAX_CONSECUTIVE_POLL_FAILURES: Final = 12 + +TINYFISH_TERMINAL_RUN_STATUSES: Final = frozenset({"COMPLETED", "FAILED", "CANCELLED"}) + +# these fields use the shared account's saved logins/vault, so they 403 unless TINYFISH_ALLOW_AUTHENTICATED_RUNS=true +TINYFISH_AUTHENTICATED_RUN_FIELDS: Final = frozenset({"use_profile", "profile_id", "use_vault", "credential_item_ids"}) + +# litellm's pass-through envelope controls; rejected here or custom_body smuggles past the field gate and +# a caller stream flag flips the billing mode away from what the endpoint dictates +TINYFISH_REJECTED_ENVELOPE_FIELDS: Final = frozenset({"custom_body", "stream", "query_params"}) + +# covers the upstream 1200s max run duration plus response headroom for blocking runs +TINYFISH_PASSTHROUGH_TIMEOUT_SECONDS: Final = 1500.0 + +_RUN_SUBMIT_PATHS: Final = frozenset( + {("v1", "automation", "run"), ("v1", "automation", "run-async"), ("v1", "automation", "run-sse")} +) + + +class TinyfishRun(TypedDict, total=False): + """Run objects are null-heavy until terminal, so every field must tolerate None.""" + + run_id: ReadOnly[str | None] + status: ReadOnly[str | None] + num_of_steps: ReadOnly[int | None] + result: ReadOnly[object] + # left untyped on purpose: a strict error shape would fail whole-run validation on upstream drift and drop the charge + error: ReadOnly[object] + type: ReadOnly[str | None] + + +def is_allowed_tinyfish_endpoint(method: str, path: str) -> bool: + """The host also serves vault/wallet/profile management under the same key, so only run endpoints forward.""" + segments: Final = tuple(path.split("/")[1:]) + if not path.startswith("/") or any(segment in ("", ".", "..") for segment in segments): + return False + if method == "POST" and segments in _RUN_SUBMIT_PATHS: + return True + # no GET /v1/runs listing: run ids are unguessable, so blocking the list keeps teams out of each other's runs + if method == "GET" and len(segments) == 3 and segments[:2] == ("v1", "runs"): + return True + return method == "POST" and len(segments) == 4 and segments[:2] == ("v1", "runs") and segments[3] == "cancel" diff --git a/tests/pass_through_unit_tests/test_pass_through_unit_tests.py b/tests/pass_through_unit_tests/test_pass_through_unit_tests.py index 1d4e13474a7..6c57e59f7e3 100644 --- a/tests/pass_through_unit_tests/test_pass_through_unit_tests.py +++ b/tests/pass_through_unit_tests/test_pass_through_unit_tests.py @@ -413,6 +413,7 @@ PROTOCOL_CONSTRAINED_PASS_THROUGH_ROUTES = { "/comprehendmedical/{operation}": {"POST"}, "/transcribe": {"POST"}, "/transcribe/{operation}": {"POST"}, + "/tinyfish/{endpoint:path}": {"GET", "POST"}, } diff --git a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_tinyfish_passthrough_logging_handler.py b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_tinyfish_passthrough_logging_handler.py new file mode 100644 index 00000000000..1b83c5140ca --- /dev/null +++ b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_tinyfish_passthrough_logging_handler.py @@ -0,0 +1,400 @@ +import asyncio +import json +from datetime import datetime +from unittest.mock import AsyncMock, MagicMock + +import httpx +import pytest + +from litellm.proxy.pass_through_endpoints.llm_provider_handlers.tinyfish_passthrough_logging_handler import ( + _BACKGROUND_BILLING_TASKS, + TinyFishPassthroughLoggingHandler, + is_tinyfish_agent_url, + resolve_tinyfish_agent_api_base, + resolve_tinyfish_cost_per_step, + run_id_from_sse_frames, + sse_poller_spawned, +) +from litellm.proxy.pass_through_endpoints.success_handler import ( + PassThroughEndpointLogging, +) +from litellm.types.passthrough_endpoints.tinyfish import is_allowed_tinyfish_endpoint + +RUN_URL = "https://agent.tinyfish.ai/v1/automation/run" +RUN_ASYNC_URL = "https://agent.tinyfish.ai/v1/automation/run-async" + + +def _make_logging_obj() -> MagicMock: + logging_obj = MagicMock() + logging_obj.litellm_call_id = "test-call-id" + logging_obj.model_call_details = {} + return logging_obj + + +def _make_response(method: str, url: str, body: dict) -> httpx.Response: + request = httpx.Request(method, url) + return httpx.Response(200, request=request, text=json.dumps(body)) + + +class _FakeClient: + """Payload items are dicts served with status_code, or (status, dict) tuples for scripted failures.""" + + def __init__(self, payloads: list, status_code: int = 200): + self.payloads = payloads + self.status_code = status_code + self.requested_urls: list[str] = [] + + async def get(self, url: str, headers: dict) -> httpx.Response: + self.requested_urls.append(url) + item = self.payloads[min(len(self.requested_urls) - 1, len(self.payloads) - 1)] + status, payload = item if isinstance(item, tuple) else (self.status_code, item) + return httpx.Response(status, text=json.dumps(payload), request=httpx.Request("GET", url)) + + +@pytest.fixture +def tinyfish_env(monkeypatch): + monkeypatch.setenv("TINYFISH_API_KEY", "sk-tf-test") + monkeypatch.delenv("TINYFISH_COST_PER_STEP", raising=False) + monkeypatch.delenv("TINYFISH_AGENT_API_BASE", raising=False) + + +class TestCostResolution: + def test_default_rate(self, tinyfish_env): + assert resolve_tinyfish_cost_per_step() == pytest.approx(0.016) + + def test_env_override(self, tinyfish_env, monkeypatch): + monkeypatch.setenv("TINYFISH_COST_PER_STEP", "0.02") + assert resolve_tinyfish_cost_per_step() == pytest.approx(0.02) + + def test_invalid_env_falls_back_to_default(self, tinyfish_env, monkeypatch): + monkeypatch.setenv("TINYFISH_COST_PER_STEP", "free") + assert resolve_tinyfish_cost_per_step() == pytest.approx(0.016) + + +class TestBillingGate: + @pytest.mark.parametrize( + "method,url,expected", + [ + ("POST", RUN_URL, True), + ("POST", RUN_ASYNC_URL, True), + ("POST", "https://agent.tinyfish.ai/v1/automation/run-sse", True), + ("GET", "https://agent.tinyfish.ai/v1/runs", False), + ("GET", "https://agent.tinyfish.ai/v1/runs/run-123?screenshots=none", False), + ("POST", "https://agent.tinyfish.ai/v1/runs/run-123/cancel", False), + ], + ) + def test_only_run_submissions_are_billed(self, method, url, expected): + assert TinyFishPassthroughLoggingHandler.should_log_request(method, url) is expected + + def test_polling_writes_no_spend_row(self, tinyfish_env): + logging_obj = _make_logging_obj() + logging_obj.dispatch_success_handlers = AsyncMock() + poll_url = "https://agent.tinyfish.ai/v1/runs/run-123" + + asyncio.run( + PassThroughEndpointLogging().pass_through_async_success_handler( + httpx_response=_make_response("GET", poll_url, {"run_id": "run-123", "status": "RUNNING"}), + response_body={"run_id": "run-123", "status": "RUNNING"}, + logging_obj=logging_obj, + url_route=poll_url, + result="", + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + request_body={}, + passthrough_logging_payload={"url": poll_url}, + custom_llm_provider="tinyfish", + ) + ) + + logging_obj.dispatch_success_handlers.assert_not_awaited() + + +class TestBlockingRunBilling: + def _handle(self, response_body: dict, logging_obj: MagicMock): + return TinyFishPassthroughLoggingHandler.tinyfish_passthrough_handler( + httpx_response=_make_response("POST", RUN_URL, response_body), + response_body=response_body, + logging_obj=logging_obj, + url_route=RUN_URL, + result=json.dumps(response_body), + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + request_body={"url": "https://scrapeme.live/shop", "goal": "extract products"}, + ) + + def test_bills_steps_times_rate(self, tinyfish_env): + logging_obj = _make_logging_obj() + run = {"run_id": "run-1", "status": "COMPLETED", "num_of_steps": 3, "result": {"products": []}} + + handler_result = self._handle(run, logging_obj) + + assert handler_result["kwargs"]["model"] == "tinyfish/automation-run" + assert handler_result["kwargs"]["custom_llm_provider"] == "tinyfish" + assert handler_result["kwargs"]["response_cost"] == pytest.approx(0.048) + assert "standard_logging_object" in handler_result["kwargs"] + assert logging_obj.model_call_details["response_cost"] == pytest.approx(0.048) + + def test_env_rate_override_applies(self, tinyfish_env, monkeypatch): + monkeypatch.setenv("TINYFISH_COST_PER_STEP", "0.5") + run = {"run_id": "run-1", "status": "COMPLETED", "num_of_steps": 2} + + handler_result = self._handle(run, _make_logging_obj()) + + assert handler_result["kwargs"]["response_cost"] == pytest.approx(1.0) + + def test_failed_run_logs_without_cost(self, tinyfish_env): + run = {"run_id": "run-1", "status": "FAILED", "num_of_steps": 2, "error": {"code": "AGENT_FAILURE"}} + + handler_result = self._handle(run, _make_logging_obj()) + + assert handler_result["kwargs"]["response_cost"] is None + + def test_cancelled_run_logs_without_cost(self, tinyfish_env): + run = {"run_id": "run-1", "status": "CANCELLED", "num_of_steps": 2} + + handler_result = self._handle(run, _make_logging_obj()) + + assert handler_result["kwargs"]["response_cost"] is None + + def test_null_steps_logs_without_cost(self, tinyfish_env): + run = {"run_id": "run-1", "status": "RUNNING", "num_of_steps": None} + + handler_result = self._handle(run, _make_logging_obj()) + + assert handler_result["kwargs"]["response_cost"] is None + + def test_unexpected_error_shape_still_bills(self, tinyfish_env): + run = {"run_id": "run-1", "status": "COMPLETED", "num_of_steps": 2, "error": {"retry_after": "5s"}} + + handler_result = self._handle(run, _make_logging_obj()) + + assert handler_result["kwargs"]["response_cost"] == pytest.approx(0.032) + + +class TestRunAsyncBilling: + def test_poll_and_log_bills_once_terminal(self, tinyfish_env): + logging_obj = _make_logging_obj() + logging_obj.dispatch_success_handlers = AsyncMock() + fake_client = _FakeClient( + payloads=[{"run_id": "run-9", "status": "COMPLETED", "num_of_steps": 4, "result": "ok"}] + ) + + asyncio.run( + TinyFishPassthroughLoggingHandler._poll_and_log( + run_id="run-9", + logging_obj=logging_obj, + result="", + start_time=datetime.now(), + cache_hit=False, + kwargs={}, + client=fake_client, + ) + ) + + logging_obj.dispatch_success_handlers.assert_awaited_once() + awaited_kwargs = logging_obj.dispatch_success_handlers.await_args.kwargs + assert awaited_kwargs["response_cost"] == pytest.approx(0.064) + assert awaited_kwargs["model"] == "tinyfish/automation-run" + assert fake_client.requested_urls == ["https://agent.tinyfish.ai/v1/runs/run-9?screenshots=none"] + assert logging_obj.model_call_details["response_cost"] == pytest.approx(0.064) + + def test_transient_poll_failure_keeps_polling(self, tinyfish_env): + fake_client = _FakeClient( + payloads=[(500, {}), {"run_id": "run-9", "status": "COMPLETED", "num_of_steps": 3}] + ) + + run = asyncio.run( + TinyFishPassthroughLoggingHandler._poll_until_terminal("run-9", fake_client, poll_interval_seconds=0.0) + ) + + assert run is not None + assert run["num_of_steps"] == 3 + assert len(fake_client.requested_urls) == 2 + + def test_gives_up_after_consecutive_poll_failures(self, tinyfish_env): + fake_client = _FakeClient(payloads=[(500, {})]) + + run = asyncio.run( + TinyFishPassthroughLoggingHandler._poll_until_terminal("run-9", fake_client, poll_interval_seconds=0.0) + ) + + assert run is None + assert len(fake_client.requested_urls) == 12 + + def test_traversal_run_id_is_rejected(self, tinyfish_env): + fake_client = _FakeClient(payloads=[{}]) + + run = asyncio.run(TinyFishPassthroughLoggingHandler._fetch_run("../vault/items", fake_client)) + + assert run is None + assert fake_client.requested_urls == [] + + def test_upstream_error_status_returns_none(self, tinyfish_env): + fake_client = _FakeClient(payloads=[{"error": {"code": "NOT_FOUND"}}], status_code=404) + + run = asyncio.run(TinyFishPassthroughLoggingHandler._fetch_run("run-1", fake_client)) + + assert run is None + + +class TestRunCostStatusGate: + def test_poller_bills_zero_for_terminal_failed_run(self, tinyfish_env): + logging_obj = _make_logging_obj() + logging_obj.dispatch_success_handlers = AsyncMock() + fake_client = _FakeClient(payloads=[{"run_id": "run-9", "status": "FAILED", "num_of_steps": 4}]) + + asyncio.run( + TinyFishPassthroughLoggingHandler._poll_and_log( + run_id="run-9", + logging_obj=logging_obj, + result="", + start_time=datetime.now(), + cache_hit=False, + kwargs={}, + client=fake_client, + ) + ) + + logging_obj.dispatch_success_handlers.assert_awaited_once() + assert logging_obj.dispatch_success_handlers.await_args.kwargs["response_cost"] is None + assert len(fake_client.requested_urls) == 1 + + +class TestRunIdFromSseFrames: + def test_finds_run_id_in_first_frame(self): + frames = b'data: {"run_id": "run-7", "event": "INITIALIZED"}\n\ndata: {"run_id": "run-7", "event": "ACTION"}\n\n' + assert run_id_from_sse_frames(frames) == "run-7" + + def test_skips_frames_without_run_id(self): + frames = b': keepalive\n\ndata: not-json\n\ndata: {"event": "HEARTBEAT"}\n\n' + assert run_id_from_sse_frames(frames) is None + + +class TestStartSseRunBilling: + def test_spawns_detached_poller_that_bills_once(self, tinyfish_env): + logging_obj = _make_logging_obj() + logging_obj.dispatch_success_handlers = AsyncMock() + logging_obj.model_call_details["litellm_params"] = {"metadata": {"user_api_key_hash": "hash-team-a"}} + fake_client = _FakeClient( + payloads=[{"run_id": "run-7", "status": "COMPLETED", "num_of_steps": 5, "result": "done"}] + ) + + async def _run() -> None: + tasks_before = set(_BACKGROUND_BILLING_TASKS) + TinyFishPassthroughLoggingHandler.start_sse_run_billing( + run_id="run-7", + litellm_logging_obj=logging_obj, + start_time=datetime.now(), + client=fake_client, + ) + assert sse_poller_spawned(logging_obj) + await asyncio.gather(*(_BACKGROUND_BILLING_TASKS - tasks_before)) + + asyncio.run(_run()) + + logging_obj.dispatch_success_handlers.assert_awaited_once() + awaited_kwargs = logging_obj.dispatch_success_handlers.await_args.kwargs + assert awaited_kwargs["response_cost"] == pytest.approx(0.08) + # a missing call id makes every poller row a NULL request_id primary-key collision + assert awaited_kwargs["standard_logging_object"]["id"] == "test-call-id" + # SLO consumers (Prometheus, Langfuse) must see the caller's attribution despite the empty poller kwargs + assert awaited_kwargs["standard_logging_object"]["metadata"]["user_api_key_hash"] == "hash-team-a" + assert fake_client.requested_urls == ["https://agent.tinyfish.ai/v1/runs/run-7?screenshots=none"] + + def test_flag_defaults_to_not_spawned(self): + assert not sse_poller_spawned(_make_logging_obj()) + def test_collected_chunks_price_via_run_fetch(self, tinyfish_env): + logging_obj = _make_logging_obj() + chunks = [ + 'data: {"type": "STARTED", "run_id": "run-7", "status": "RUNNING"}', + 'data: {"type": "PROGRESS", "run_id": "run-7"}', + 'data: {"type": "COMPLETE", "run_id": "run-7", "status": "COMPLETED", "result": "done"}', + ] + fake_client = _FakeClient( + payloads=[{"run_id": "run-7", "status": "COMPLETED", "num_of_steps": 5, "result": "done"}] + ) + + payload = asyncio.run( + TinyFishPassthroughLoggingHandler.handle_logging_tinyfish_collected_chunks( + litellm_logging_obj=logging_obj, + url_route="https://agent.tinyfish.ai/v1/automation/run-sse", + start_time=datetime.now(), + all_chunks=chunks, + end_time=datetime.now(), + client=fake_client, + ) + ) + + assert payload["kwargs"]["response_cost"] == pytest.approx(0.08) + assert payload["kwargs"]["model"] == "tinyfish/automation-run" + assert fake_client.requested_urls == ["https://agent.tinyfish.ai/v1/runs/run-7?screenshots=none"] + + def test_stream_without_run_id_logs_without_cost(self, tinyfish_env): + fake_client = _FakeClient(payloads=[{}]) + + payload = asyncio.run( + TinyFishPassthroughLoggingHandler.handle_logging_tinyfish_collected_chunks( + litellm_logging_obj=_make_logging_obj(), + url_route="https://agent.tinyfish.ai/v1/automation/run-sse", + start_time=datetime.now(), + all_chunks=["data: not-json", ": keepalive"], + end_time=datetime.now(), + client=fake_client, + ) + ) + + assert payload["kwargs"]["response_cost"] is None + assert fake_client.requested_urls == [] + + +class TestRouteDetection: + def test_provider_tag_claims_route(self): + assert PassThroughEndpointLogging().is_tinyfish_route("https://example.com/x", "tinyfish") + + def test_agent_host_claims_route(self): + assert PassThroughEndpointLogging().is_tinyfish_route("https://agent.tinyfish.ai/v1/runs", None) + + def test_other_providers_do_not_claim(self): + assert not PassThroughEndpointLogging().is_tinyfish_route("https://api.openai.com/v1", "openai") + + def test_env_base_override_claims_route(self, monkeypatch): + monkeypatch.setenv("TINYFISH_AGENT_API_BASE", "https://agent.staging.tinyfish.ai") + assert is_tinyfish_agent_url("https://agent.staging.tinyfish.ai/v1/runs/x") + assert not is_tinyfish_agent_url("https://agent.tinyfish.ai/v1/runs/x") + + def test_schemeless_env_base_is_normalized(self, monkeypatch): + monkeypatch.setenv("TINYFISH_AGENT_API_BASE", "agent.staging.tinyfish.ai") + assert resolve_tinyfish_agent_api_base() == "https://agent.staging.tinyfish.ai" + assert is_tinyfish_agent_url("https://agent.staging.tinyfish.ai/v1/runs/x") + + +class TestEndpointAllowlist: + @pytest.mark.parametrize( + "method,path,expected", + [ + ("POST", "/v1/automation/run", True), + ("POST", "/v1/automation/run-async", True), + ("POST", "/v1/automation/run-sse", True), + ("GET", "/v1/runs", False), + ("GET", "/v1/runs/run-abc-123", True), + ("POST", "/v1/runs/run-abc-123/cancel", True), + ("GET", "/v1/vault/items", False), + ("GET", "/v1/wallet", False), + ("POST", "/v1/browser-profiles", False), + ("DELETE", "/v1/runs/run-abc-123", False), + ("GET", "/v1/automation/run", False), + ("POST", "/v1/runs", False), + ("GET", "/v1/runs/..", False), + ("POST", "/v1/runs/../automation/run/cancel", False), + ("POST", "/v1/automation/run/", False), + ("POST", "/v1/automation/run-async/", False), + ("POST", "/v1/automation/run-sse/", False), + ("POST", "/v1//automation/run-async", False), + ("GET", "/v1/runs/run-abc-123/", False), + ("POST", "/v1/runs/run-abc-123/cancel/", False), + ], + ) + def test_allowlist(self, method, path, expected): + assert is_allowed_tinyfish_endpoint(method, path) is expected diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index dcc3bd2b690..fd81fcc8e72 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -7371,3 +7371,200 @@ class TestOpenRouterPassthroughRoute: ) assert create_route.call_args.kwargs["target"] == f"{expected_root}/{endpoint}" + + +class TestTinyFishProxyRoute: + """Tests for the TinyFish Agent pass-through route, faking the upstream HTTP boundary.""" + + RUN_BODY = {"url": "https://scrapeme.live/shop", "goal": "Extract the first 2 product names. Return JSON."} + + @pytest.fixture + def tinyfish_client(self, monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]: + from litellm.proxy.proxy_server import app + + monkeypatch.setenv("TINYFISH_API_KEY", "sk-tf-upstream") + monkeypatch.delenv("TINYFISH_AGENT_API_BASE", raising=False) + monkeypatch.delenv("TINYFISH_ALLOW_AUTHENTICATED_RUNS", raising=False) + monkeypatch.delenv("SERVER_ROOT_PATH", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, lambda: UserAPIKeyAuth(api_key="sk-virtual")) + yield TestClient(app) + + def test_forwards_run_with_server_key_not_callers(self, tinyfish_client: TestClient) -> None: + with respx.mock(assert_all_called=True) as upstream: + route = upstream.post("https://agent.tinyfish.ai/v1/automation/run").mock( + return_value=httpx.Response(200, json={"run_id": "run-1", "status": "COMPLETED", "num_of_steps": 2}) + ) + response = tinyfish_client.post( + "/tinyfish/v1/automation/run", json=self.RUN_BODY, headers={"X-API-Key": "sk-callers-virtual-key"} + ) + + assert (response.status_code, response.json()["run_id"]) == (200, "run-1") + assert route.calls.last.request.headers["x-api-key"] == "sk-tf-upstream" + + @pytest.mark.parametrize( + "method,path", + [ + ("GET", "/tinyfish/v1/vault/items"), + ("GET", "/tinyfish/v1/wallet"), + ("POST", "/tinyfish/v1/browser-profiles"), + ("GET", "/tinyfish/v1/automation/run"), + ("GET", "/tinyfish/v1/runs"), + ], + ) + def test_blocks_endpoints_outside_allowlist(self, tinyfish_client: TestClient, method: str, path: str) -> None: + with respx.mock: + response = tinyfish_client.request(method, path) + + assert response.status_code == 403 + assert "not an allowed TinyFish Agent passthrough endpoint" in response.json()["detail"] + + @pytest.mark.parametrize( + "path", + [ + "/tinyfish/v1/automation/run/", + "/tinyfish/v1/automation/run-async/", + "/tinyfish/v1/automation/run-sse/", + "/tinyfish/v1//automation/run-async", + ], + ) + def test_submit_paths_with_extra_slashes_are_rejected_before_forwarding( + self, tinyfish_client: TestClient, path: str + ) -> None: + with respx.mock(assert_all_called=False) as upstream: + upstream.post(url__regex=r"https://agent\.tinyfish\.ai/.*").mock( + return_value=httpx.Response(200, json={"run_id": "run-slash", "status": "PENDING"}) + ) + response = tinyfish_client.post(path, json=self.RUN_BODY) + + assert response.status_code == 403 + assert "not an allowed TinyFish Agent passthrough endpoint" in response.json()["detail"] + assert upstream.calls.call_count == 0 + + def test_rejects_authenticated_run_fields_by_default(self, tinyfish_client: TestClient) -> None: + with respx.mock: + response = tinyfish_client.post("/tinyfish/v1/automation/run", json={**self.RUN_BODY, "use_vault": True}) + + assert response.status_code == 403 + assert "use_vault" in response.json()["detail"] + + def test_env_opt_in_allows_authenticated_run_fields( + self, tinyfish_client: TestClient, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("TINYFISH_ALLOW_AUTHENTICATED_RUNS", "true") + + with respx.mock(assert_all_called=True) as upstream: + route = upstream.post("https://agent.tinyfish.ai/v1/automation/run").mock( + return_value=httpx.Response(200, json={"run_id": "run-2", "status": "COMPLETED", "num_of_steps": 1}) + ) + response = tinyfish_client.post("/tinyfish/v1/automation/run", json={**self.RUN_BODY, "use_vault": True}) + + assert response.status_code == 200 + assert json.loads(route.calls.last.request.content)["use_vault"] is True + + def test_returns_401_on_missing_api_key( + self, tinyfish_client: TestClient, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.delenv("TINYFISH_API_KEY") + + with respx.mock: + response = tinyfish_client.get("/tinyfish/v1/runs/run-123") + + assert response.status_code == 401 + assert "TINYFISH_API_KEY" in response.json()["detail"] + + def test_env_base_override_changes_target( + self, tinyfish_client: TestClient, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("TINYFISH_AGENT_API_BASE", "https://agent.staging.tinyfish.ai") + + with respx.mock(assert_all_called=True) as upstream: + upstream.get("https://agent.staging.tinyfish.ai/v1/runs/run-123").mock( + return_value=httpx.Response(200, json={"run_id": "run-123", "status": "RUNNING"}) + ) + response = tinyfish_client.get("/tinyfish/v1/runs/run-123") + + assert (response.status_code, response.json()["status"]) == (200, "RUNNING") + + @pytest.mark.parametrize( + "body", + [ + {"custom_body": {"url": "https://scrapeme.live/shop", "goal": "g", "use_vault": True}}, + {"url": "https://scrapeme.live/shop", "goal": "g", "stream": True}, + {"url": "https://scrapeme.live/shop", "goal": "g", "query_params": {"x": "1"}}, + ], + ) + def test_rejects_passthrough_envelope_controls(self, tinyfish_client: TestClient, body: dict) -> None: + """custom_body smuggled vault fields past the 403 gate and a stream flag flipped the + billing mode, because the generic passthrough honors both from the caller's body.""" + with respx.mock as upstream: + route = upstream.post("https://agent.tinyfish.ai/v1/automation/run").mock( + return_value=httpx.Response(200, json={"run_id": "run-1", "status": "COMPLETED", "num_of_steps": 1}) + ) + response = tinyfish_client.post("/tinyfish/v1/automation/run", json=body) + + assert response.status_code == 400 + assert "envelope" in response.json()["detail"] + assert not route.called + + def test_rejects_envelope_stream_on_cancel(self, tinyfish_client: TestClient) -> None: + with respx.mock as upstream: + route = upstream.post("https://agent.tinyfish.ai/v1/runs/run-1/cancel").mock( + return_value=httpx.Response(200, json={"run_id": "run-1", "status": "CANCELLED"}) + ) + response = tinyfish_client.post("/tinyfish/v1/runs/run-1/cancel", json={"stream": True}) + + assert response.status_code == 400 + assert not route.called + + @pytest.mark.parametrize( + "content,content_type", + [ + ("url=https%3A%2F%2Fscrapeme.live%2Fshop&goal=g&stream=true", "application/x-www-form-urlencoded"), + ("url=https%3A%2F%2Fscrapeme.live%2Fshop&goal=g&use_vault=true", "application/x-www-form-urlencoded"), + ('{"url": "https://scrapeme.live/shop", "goal": "g", "use_vault": true}', "text/plain"), + ('[{"url": "https://scrapeme.live/shop", "goal": "g", "stream": true}]', "application/json"), + ], + ) + def test_rejects_bodies_that_are_not_json_objects( + self, tinyfish_client: TestClient, content: str, content_type: str + ) -> None: + """A form-encoded body carried stream and use_vault past both field gates, because + the gates only saw fields the body parsed to as JSON.""" + with respx.mock as upstream: + route = upstream.post("https://agent.tinyfish.ai/v1/automation/run").mock( + return_value=httpx.Response(200, json={"run_id": "run-1", "status": "COMPLETED", "num_of_steps": 1}) + ) + response = tinyfish_client.post( + "/tinyfish/v1/automation/run", content=content, headers={"Content-Type": content_type} + ) + + assert response.status_code == 400 + assert "JSON object" in response.json()["detail"] + assert not route.called + + def test_cancel_without_body_forwards(self, tinyfish_client: TestClient) -> None: + with respx.mock(assert_all_called=True) as upstream: + upstream.post("https://agent.tinyfish.ai/v1/runs/run-1/cancel").mock( + return_value=httpx.Response(200, json={"run_id": "run-1", "status": "CANCELLED"}) + ) + response = tinyfish_client.post("/tinyfish/v1/runs/run-1/cancel") + + assert (response.status_code, response.json()["status"]) == (200, "CANCELLED") + + +class TestTinyFishRouteTimeout: + def test_default_covers_upstream_run_cap(self, monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy import proxy_server + from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import _tinyfish_route_timeout + + monkeypatch.setattr(proxy_server, "general_settings", {}, raising=False) + assert _tinyfish_route_timeout() == 1500.0 + + def test_operator_configured_timeout_wins(self, monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy import proxy_server + from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import _tinyfish_route_timeout + + monkeypatch.setattr(proxy_server, "general_settings", {"pass_through_request_timeout": 30}, raising=False) + assert _tinyfish_route_timeout() is None diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_streaming_handler_interrupt.py b/tests/test_litellm/proxy/pass_through_endpoints/test_streaming_handler_interrupt.py index 88b82349c83..00a3606bbf5 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_streaming_handler_interrupt.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_streaming_handler_interrupt.py @@ -7,8 +7,14 @@ from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest +import respx import litellm +import litellm.proxy.pass_through_endpoints.llm_provider_handlers.tinyfish_passthrough_logging_handler as tinyfish_handler_module +from litellm.proxy.pass_through_endpoints.llm_provider_handlers.tinyfish_passthrough_logging_handler import ( + mark_sse_poller_spawned, + sse_poller_spawned, +) from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER @@ -840,6 +846,220 @@ async def test_chunk_processor_bills_partial_google_usage_on_mid_stream_exceptio assert isinstance(recorder.failure_kwargs[0]["exception"], httpx.ReadTimeout) +class TestTinyFishStreamBilling: + """SSE billing is owned by the detached poller spawned on the first run_id frame; it must + survive gen.aclose() (client disconnect) and the stream-end path must not double-bill.""" + + RUNS_URL = "https://agent.tinyfish.ai/v1/runs/run-sse-1?screenshots=none" + SSE_ROUTE = "https://agent.tinyfish.ai/v1/automation/run-sse" + + @pytest.fixture + def tinyfish_env(self, monkeypatch): + monkeypatch.setenv("TINYFISH_API_KEY", "sk-tf-test") + monkeypatch.delenv("TINYFISH_COST_PER_STEP", raising=False) + monkeypatch.delenv("TINYFISH_AGENT_API_BASE", raising=False) + # aiohttp transport bypasses respx; force plain httpx and drop any cached aiohttp client + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + + def _tinyfish_logging_obj(self): + obj = _unarmed_logging_obj() + obj.model_call_details = {} + obj.dispatch_success_handlers = AsyncMock() + return obj + + def _spawned_since(self, tasks_before): + return tinyfish_handler_module._BACKGROUND_BILLING_TASKS - tasks_before + + @pytest.mark.asyncio + async def test_run_id_split_across_chunks_spawns_one_poller(self, tinyfish_env): + chunks = [ + b'data: {"run_id": "run-s', + b'se-1", "event": "INITIALIZED"}\n\n', + b'data: {"run_id": "run-sse-1", "event": "COMPLETE"}\n\n', + ] + logging_obj = self._tinyfish_logging_obj() + tasks_before = set(tinyfish_handler_module._BACKGROUND_BILLING_TASKS) + + with respx.mock(assert_all_called=True) as upstream: + upstream.get(self.RUNS_URL).respond( + json={"run_id": "run-sse-1", "status": "COMPLETED", "num_of_steps": 2, "result": "ok"} + ) + received = [] + async for chunk in PassThroughStreamingHandler.chunk_processor( + response=_make_streaming_response(chunks), + request_body={"url": "https://scrapeme.live/shop", "goal": "extract"}, + litellm_logging_obj=logging_obj, + endpoint_type=EndpointType.TINYFISH, + start_time=datetime.now(), + passthrough_success_handler_obj=MagicMock(), + url_route=self.SSE_ROUTE, + route_streaming_logging=AsyncMock(), + ): + received.append(chunk) + + assert received == chunks + assert sse_poller_spawned(logging_obj) + spawned = self._spawned_since(tasks_before) + assert len(spawned) == 1 + await asyncio.gather(*spawned) + + logging_obj.dispatch_success_handlers.assert_awaited_once() + assert logging_obj.dispatch_success_handlers.await_args.kwargs["response_cost"] == pytest.approx(0.032) + + @pytest.mark.asyncio + async def test_poller_survives_client_disconnect_and_bills(self, tinyfish_env): + chunks = [ + b'data: {"run_id": "run-sse-1", "event": "INITIALIZED"}\n\n', + b'data: {"run_id": "run-sse-1", "event": "ACTION"}\n\n', + ] + logging_obj = self._tinyfish_logging_obj() + tasks_before = set(tinyfish_handler_module._BACKGROUND_BILLING_TASKS) + + with respx.mock(assert_all_called=True) as upstream: + upstream.get(self.RUNS_URL).respond( + json={"run_id": "run-sse-1", "status": "COMPLETED", "num_of_steps": 2, "result": "ok"} + ) + gen = PassThroughStreamingHandler.chunk_processor( + response=_make_streaming_response(chunks), + request_body={"url": "https://scrapeme.live/shop", "goal": "extract"}, + litellm_logging_obj=logging_obj, + endpoint_type=EndpointType.TINYFISH, + start_time=datetime.now(), + passthrough_success_handler_obj=MagicMock(), + url_route=self.SSE_ROUTE, + route_streaming_logging=AsyncMock(), + ) + await gen.__anext__() + spawned = self._spawned_since(tasks_before) + assert len(spawned) == 1 + await gen.aclose() + + task = next(iter(spawned)) + assert not task.cancelled() + await task + + logging_obj.dispatch_success_handlers.assert_awaited_once() + assert logging_obj.dispatch_success_handlers.await_args.kwargs["response_cost"] == pytest.approx(0.032) + + @pytest.mark.asyncio + async def test_stream_without_run_id_spawns_nothing(self, tinyfish_env): + logging_obj = self._tinyfish_logging_obj() + tasks_before = set(tinyfish_handler_module._BACKGROUND_BILLING_TASKS) + + async for _ in PassThroughStreamingHandler.chunk_processor( + response=_make_streaming_response([b": keepalive\n\n", b'data: {"event": "HEARTBEAT"}\n\n']), + request_body={}, + litellm_logging_obj=logging_obj, + endpoint_type=EndpointType.TINYFISH, + start_time=datetime.now(), + passthrough_success_handler_obj=MagicMock(), + url_route=self.SSE_ROUTE, + route_streaming_logging=AsyncMock(), + ): + pass + + assert not sse_poller_spawned(logging_obj) + assert self._spawned_since(tasks_before) == set() + + @pytest.mark.asyncio + async def test_stream_end_skips_dispatch_when_poller_owns_billing(self, tinyfish_env): + logging_obj = self._tinyfish_logging_obj() + mark_sse_poller_spawned(logging_obj) + + await PassThroughStreamingHandler._route_streaming_logging_to_handler( + litellm_logging_obj=logging_obj, + passthrough_success_handler_obj=MagicMock(), + url_route=self.SSE_ROUTE, + request_body={}, + endpoint_type=EndpointType.TINYFISH, + start_time=datetime.now(), + raw_bytes=[b'data: {"run_id": "run-sse-1", "event": "COMPLETE"}\n\n'], + end_time=datetime.now(), + ) + + logging_obj.dispatch_success_handlers.assert_not_awaited() + + @pytest.mark.asyncio + async def test_stream_end_fallback_still_logs_when_no_poller_spawned(self, tinyfish_env): + logging_obj = self._tinyfish_logging_obj() + + await PassThroughStreamingHandler._route_streaming_logging_to_handler( + litellm_logging_obj=logging_obj, + passthrough_success_handler_obj=MagicMock(), + url_route=self.SSE_ROUTE, + request_body={}, + endpoint_type=EndpointType.TINYFISH, + start_time=datetime.now(), + raw_bytes=[b": keepalive\n\n"], + end_time=datetime.now(), + ) + + logging_obj.dispatch_success_handlers.assert_awaited_once() + assert logging_obj.dispatch_success_handlers.await_args.kwargs["response_cost"] is None + + @pytest.mark.asyncio + async def test_upstream_error_after_spawn_skips_failure_dispatch(self, tinyfish_env): + logging_obj = self._tinyfish_logging_obj() + logging_obj.dispatch_failure_handlers = MagicMock() + tasks_before = set(tinyfish_handler_module._BACKGROUND_BILLING_TASKS) + + async def _aiter_bytes(): + yield b'data: {"run_id": "run-sse-1", "event": "INITIALIZED"}\n\n' + raise httpx.ReadTimeout("upstream died") + + response = MagicMock(spec=httpx.Response) + response.status_code = 200 + response.aiter_bytes = _aiter_bytes + + async def _consume(): + async for _ in PassThroughStreamingHandler.chunk_processor( + response=response, + request_body={}, + litellm_logging_obj=logging_obj, + endpoint_type=EndpointType.TINYFISH, + start_time=datetime.now(), + passthrough_success_handler_obj=MagicMock(), + url_route=self.SSE_ROUTE, + route_streaming_logging=AsyncMock(), + ): + pass + + with pytest.raises(httpx.ReadTimeout): + await _consume() + + spawned = self._spawned_since(tasks_before) + assert len(spawned) == 1 + # the poller owns the single row; a failure dispatch would collide on its request_id + logging_obj.dispatch_failure_handlers.assert_not_called() + for task in spawned: + task.cancel() + + @pytest.mark.asyncio + async def test_unterminated_run_id_frame_late_spawns_poller(self, tinyfish_env): + logging_obj = self._tinyfish_logging_obj() + tasks_before = set(tinyfish_handler_module._BACKGROUND_BILLING_TASKS) + + await PassThroughStreamingHandler._route_streaming_logging_to_handler( + litellm_logging_obj=logging_obj, + passthrough_success_handler_obj=MagicMock(), + url_route=self.SSE_ROUTE, + request_body={}, + endpoint_type=EndpointType.TINYFISH, + start_time=datetime.now(), + raw_bytes=[b'data: {"run_id": "run-sse-1", "event": "INITIALIZED"}'], + end_time=datetime.now(), + ) + + spawned = self._spawned_since(tasks_before) + assert len(spawned) == 1 + assert sse_poller_spawned(logging_obj) + # the poller polls to terminal instead of the old single fetch that mispriced a RUNNING run at $0 + logging_obj.dispatch_success_handlers.assert_not_awaited() + for task in spawned: + task.cancel() + + @pytest.mark.asyncio @pytest.mark.parametrize( "deferred_dispatch_armed", diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index bd0de11e9fb..6a184d0765d 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -16600,6 +16600,64 @@ export interface paths { patch?: never; trace?: never; }; + "/tinyfish/{endpoint}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Tinyfish Proxy Route + * @description Pass-through for the TinyFish Agent API (goal-based web automation). + * + * Forwarded endpoints: + * - POST /v1/automation/run — run to completion (blocking) + * - POST /v1/automation/run-async — submit a run, poll GET /v1/runs/{id} for the result + * - POST /v1/automation/run-sse — run with SSE progress events + * - GET /v1/runs/{id} — run status / result + * - POST /v1/runs/{id}/cancel — cancel a run + * + * Every other Agent API endpoint (vault, wallet, browser profiles, and the GET /v1/runs + * listing, which would let any caller discover other callers' run ids) returns 403: all + * proxy callers share one upstream key. + * + * Credential lookup order: + * 1. passthrough_endpoint_router (config.yaml deployments with use_in_pass_through) + * 2. TINYFISH_API_KEY environment variable + * + * [Docs](https://docs.litellm.ai/docs/pass_through/tinyfish) + */ + get: operations["tinyfish_proxy_route_tinyfish__endpoint__get"]; + put?: never; + /** + * Tinyfish Proxy Route + * @description Pass-through for the TinyFish Agent API (goal-based web automation). + * + * Forwarded endpoints: + * - POST /v1/automation/run — run to completion (blocking) + * - POST /v1/automation/run-async — submit a run, poll GET /v1/runs/{id} for the result + * - POST /v1/automation/run-sse — run with SSE progress events + * - GET /v1/runs/{id} — run status / result + * - POST /v1/runs/{id}/cancel — cancel a run + * + * Every other Agent API endpoint (vault, wallet, browser profiles, and the GET /v1/runs + * listing, which would let any caller discover other callers' run ids) returns 403: all + * proxy callers share one upstream key. + * + * Credential lookup order: + * 1. passthrough_endpoint_router (config.yaml deployments with use_in_pass_through) + * 2. TINYFISH_API_KEY environment variable + * + * [Docs](https://docs.litellm.ai/docs/pass_through/tinyfish) + */ + post: operations["tinyfish_proxy_route_tinyfish__endpoint__post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/token": { parameters: { query?: never; @@ -63022,6 +63080,68 @@ export interface operations { }; }; }; + tinyfish_proxy_route_tinyfish__endpoint__get: { + parameters: { + query?: never; + header?: never; + path: { + endpoint: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + tinyfish_proxy_route_tinyfish__endpoint__post: { + parameters: { + query?: never; + header?: never; + path: { + endpoint: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; token_endpoint_token_post: { parameters: { query?: { From 0132f34356907d312273991deba2bba487d9db72 Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 21:40:25 -0700 Subject: [PATCH 10/15] chore(prices): sync OpenRouter prices: 2 models, 2 new (#42407) openrouter/nex-agi/nex-n2.5-mini: max_tokens, supports_vision, max_input_tokens, max_output_tokens, supports_pdf_input, supports_reasoning, supports_web_search, supports_audio_input, supports_tool_choice, supports_prompt_caching, supports_response_schema, supports_function_calling, input_cost_per_token, output_cost_per_token, cache_read_input_token_cost openrouter/nex-agi/nex-n2.5-pro: max_tokens, supports_vision, max_input_tokens, max_output_tokens, supports_pdf_input, supports_reasoning, supports_web_search, supports_audio_input, supports_tool_choice, supports_prompt_caching, supports_response_schema, supports_function_calling, input_cost_per_token, output_cost_per_token, cache_read_input_token_cost Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 40 +++++++++++++++++++ model_prices_and_context_window.json | 40 +++++++++++++++++++ 2 files changed, 80 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 08cb2ddb8f7..77cada25d25 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -77528,5 +77528,45 @@ "input_cost_per_image_token": 1.25e-06, "supports_prompt_caching": true, "supports_response_schema": true + }, + "openrouter/nex-agi/nex-n2.5-mini": { + "cache_read_input_token_cost": 2.5e-09, + "input_cost_per_token": 2.5e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 235929, + "max_tokens": 235929, + "mode": "chat", + "output_cost_per_token": 1e-07, + "source": "https://openrouter.ai/api/v1/models", + "supports_audio_input": false, + "supports_function_calling": false, + "supports_pdf_input": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": false, + "supports_tool_choice": false, + "supports_vision": true, + "supports_web_search": false + }, + "openrouter/nex-agi/nex-n2.5-pro": { + "cache_read_input_token_cost": 1.5e-08, + "input_cost_per_token": 7.5e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 235929, + "max_tokens": 235929, + "mode": "chat", + "output_cost_per_token": 2.5e-07, + "source": "https://openrouter.ai/api/v1/models", + "supports_audio_input": false, + "supports_function_calling": false, + "supports_pdf_input": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": false, + "supports_tool_choice": false, + "supports_vision": true, + "supports_web_search": false } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 08cb2ddb8f7..77cada25d25 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -77528,5 +77528,45 @@ "input_cost_per_image_token": 1.25e-06, "supports_prompt_caching": true, "supports_response_schema": true + }, + "openrouter/nex-agi/nex-n2.5-mini": { + "cache_read_input_token_cost": 2.5e-09, + "input_cost_per_token": 2.5e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 235929, + "max_tokens": 235929, + "mode": "chat", + "output_cost_per_token": 1e-07, + "source": "https://openrouter.ai/api/v1/models", + "supports_audio_input": false, + "supports_function_calling": false, + "supports_pdf_input": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": false, + "supports_tool_choice": false, + "supports_vision": true, + "supports_web_search": false + }, + "openrouter/nex-agi/nex-n2.5-pro": { + "cache_read_input_token_cost": 1.5e-08, + "input_cost_per_token": 7.5e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 235929, + "max_tokens": 235929, + "mode": "chat", + "output_cost_per_token": 2.5e-07, + "source": "https://openrouter.ai/api/v1/models", + "supports_audio_input": false, + "supports_function_calling": false, + "supports_pdf_input": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": false, + "supports_tool_choice": false, + "supports_vision": true, + "supports_web_search": false } } From a550b95d70a7c6e538aac9cdc7ebfb949c38051e Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 21:40:48 -0700 Subject: [PATCH 11/15] ci: skip cost map file checks on PRs that leave the cost map untouched (#42406) Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- ci_cd/cost_map_guard.py | 41 +++++++++--- tests/test_litellm/test_cost_map_guard.py | 78 +++++++++++++++++++++++ 2 files changed, 110 insertions(+), 9 deletions(-) diff --git a/ci_cd/cost_map_guard.py b/ci_cd/cost_map_guard.py index 50aa40ba220..5842cf6f1ac 100644 --- a/ci_cd/cost_map_guard.py +++ b/ci_cd/cost_map_guard.py @@ -1,8 +1,11 @@ """Guard the cost map on pull requests. -Every pull request gets the file checks: the three cost map files parse, the backup copy matches the root file, -and the JSON schema is in sync and validates the map. Pull requests from the cost map sync bot (branches named -litellm_cost_map_sync_*) additionally may only touch those three files and may only add or update models. +Every pull request whose diff against its merge base touches one of the three cost map files gets the file +checks: the files parse, the backup copy matches the root file, and the JSON schema is in sync and validates the +map. A pull request that leaves all three untouched skips them, since merging it keeps the base branch's copies +and its head tree only carries whatever state the branch was cut from. Pull requests from the cost map sync bot +(branches named litellm_cost_map_sync_*) always get the file checks and additionally may only touch those three +files and may only add or update models. """ from __future__ import annotations @@ -108,20 +111,37 @@ def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str ) +def touches_cost_map(changed_files: Sequence[str]) -> bool: + return any(path in GUARDED_PATHS for path in changed_files) + + +def contract_for(bot: bool, changed_files: Sequence[str]) -> str: + if bot: + return "bot contract enforced" + return "human PR, file checks only" if touches_cost_map(changed_files) else "human PR, cost map untouched" + + def guard_failures(base: Snapshot, head: Snapshot, changed_files: Sequence[str], bot: bool) -> tuple[str, ...]: + if not bot and not touches_cost_map(changed_files): + return () head_map: Final = _parse_object(head.cost_map, COST_MAP_PATH) if isinstance(head_map, str): return (head_map,) return (*_file_failures(head, head_map), *(_bot_failures(base, head_map, changed_files) if bot else ())) -def _git(*args: str) -> str: +def _git(*args: str) -> str | None: result: Final = subprocess.run(("git", *args), check=False, capture_output=True, text=True) - return result.stdout if result.returncode == 0 else "" + return result.stdout if result.returncode == 0 else None def snapshot(revision: str) -> Snapshot: - return Snapshot(*(_git("show", f"{revision}:{path}") for path in GUARDED_PATHS)) + return Snapshot(*(_git("show", f"{revision}:{path}") or "" for path in GUARDED_PATHS)) + + +def changed_files(base: str, head: str) -> tuple[str, ...] | None: + diff: Final = _git("diff", "--name-only", "--no-renames", base, head) + return None if diff is None else tuple(diff.splitlines()) def main(argv: Sequence[str]) -> int: @@ -131,9 +151,12 @@ def main(argv: Sequence[str]) -> int: parser.add_argument("--head-ref", required=True, help="head branch name of the pull request") args: Final = parser.parse_args(argv) bot: Final = args.head_ref.startswith(BOT_BRANCH_PREFIX) - changed_files: Final = tuple(_git("diff", "--name-only", args.base, args.head).splitlines()) - failures: Final = guard_failures(snapshot(args.base), snapshot(args.head), changed_files, bot) - contract: Final = "bot contract enforced" if bot else "human PR, file checks only" + changed: Final = changed_files(args.base, args.head) + if changed is None: + print(f"cost map guard failed: git diff {args.base} {args.head} failed, so the changed files are unknown") + return 1 + failures: Final = guard_failures(snapshot(args.base), snapshot(args.head), changed, bot) + contract: Final = contract_for(bot, changed) if failures: print(f"cost map guard failed ({contract}):") print("\n".join(f"- {failure}" for failure in failures)) diff --git a/tests/test_litellm/test_cost_map_guard.py b/tests/test_litellm/test_cost_map_guard.py index 1b4330ed62c..1595bd9864f 100644 --- a/tests/test_litellm/test_cost_map_guard.py +++ b/tests/test_litellm/test_cost_map_guard.py @@ -114,6 +114,40 @@ def test_unclassified_entry_key_is_reported() -> None: assert "Unclassified keys" in failure and "weird_thing" in failure +STALE_HEAD: Final = _snapshot(BASE_MAP, backup=_serialize({**BASE_MAP, "openrouter/b": _entry(3e-06)}), schema="{}") +CODE_ONLY: Final = ( + "litellm/utils.py", + "tests/test_litellm/test_utils.py", + "docs/model_prices_and_context_window.json", +) + + +def test_human_pr_that_leaves_the_cost_map_alone_skips_the_file_checks() -> None: + unparseable: Final = guard.Snapshot(cost_map="{not json", backup="", schema="") + assert _failures(STALE_HEAD, changed_files=CODE_ONLY, bot=False) == () + assert _failures(STALE_HEAD, changed_files=(), bot=False) == () + assert _failures(unparseable, changed_files=CODE_ONLY, bot=False) == () + + +@pytest.mark.parametrize("guarded_path", guard.GUARDED_PATHS) +def test_touching_any_cost_map_file_keeps_the_file_checks(guarded_path: str) -> None: + failures: Final = _failures(STALE_HEAD, changed_files=(*CODE_ONLY, guarded_path), bot=False) + assert [failure for failure in failures if failure.startswith(guard.BACKUP_PATH)] + assert [failure for failure in failures if failure.startswith(guard.SCHEMA_PATH)] + + +def test_bot_pr_always_gets_the_file_checks() -> None: + failures: Final = _failures(STALE_HEAD, changed_files=CODE_ONLY, bot=True) + assert [failure for failure in failures if failure.startswith(guard.BACKUP_PATH)] + assert [failure for failure in failures if failure.startswith(guard.SCHEMA_PATH)] + + +def test_contract_names_the_skip() -> None: + assert guard.contract_for(False, CODE_ONLY) == "human PR, cost map untouched" + assert guard.contract_for(False, (*CODE_ONLY, guard.SCHEMA_PATH)) == "human PR, file checks only" + assert guard.contract_for(True, CODE_ONLY) == "bot contract enforced" + + def test_bot_may_only_touch_the_cost_map_files() -> None: changed = (*guard.GUARDED_PATHS, "litellm/utils.py", ".github/workflows/cost-map-guard.yml") assert _failures(BASE, changed_files=changed, bot=False) == () @@ -147,6 +181,16 @@ def _commit(repo: Path, cost_map: dict[str, object], message: str) -> str: (repo / guard.BACKUP_PATH).parent.mkdir(exist_ok=True) (repo / guard.BACKUP_PATH).write_text(text) (repo / guard.SCHEMA_PATH).write_text(schema_module.render(schema_module.build_schema(cost_map))) + return _git_commit(repo, message) + + +def _commit_code_only(repo: Path, message: str) -> str: + (repo / "litellm").mkdir(exist_ok=True) + (repo / "litellm" / "utils.py").write_text(f"print('{message}')\n") + return _git_commit(repo, message) + + +def _git_commit(repo: Path, message: str) -> str: subprocess.run(("git", "add", "-A"), cwd=repo, check=True) subprocess.run( ("git", "-c", "user.name=t", "-c", "user.email=t@example.com", "commit", "-q", "-m", message), @@ -186,6 +230,40 @@ def test_main_reads_both_revisions_from_git( assert expected_line in result.stdout.splitlines() +def test_main_skips_the_file_checks_on_a_stale_base_the_pr_never_touched(tmp_path: Path) -> None: + subprocess.run(("git", "init", "-q", str(tmp_path)), check=True) + _commit(tmp_path, BASE_MAP, "base") + (tmp_path / guard.BACKUP_PATH).write_text(_serialize({**BASE_MAP, "openrouter/b": _entry(3e-06)})) + (tmp_path / guard.SCHEMA_PATH).write_text("{}") + stale_base: Final = _commit_code_only(tmp_path, "stale base with drifted backup and schema") + head: Final = _commit_code_only(tmp_path, "code change on the stale base") + human: Final = _run_guard(tmp_path, stale_base, head, "litellm_fix_pricing") + assert human.returncode == 0, human.stdout + human.stderr + assert "cost map guard passed (human PR, cost map untouched)" in human.stdout.splitlines() + bot: Final = _run_guard(tmp_path, stale_base, head, BOT_REF) + assert bot.returncode == 1 + backup_failure: Final = f"- {guard.BACKUP_PATH} differs from {guard.COST_MAP_PATH}; copy the root file over it" + assert backup_failure in bot.stdout.splitlines() + + +def test_main_keeps_the_file_checks_when_a_cost_map_file_is_renamed(tmp_path: Path) -> None: + subprocess.run(("git", "init", "-q", str(tmp_path)), check=True) + base: Final = _commit(tmp_path, BASE_MAP, "base") + subprocess.run(("git", "mv", guard.COST_MAP_PATH, "renamed.json"), cwd=tmp_path, check=True) + head: Final = _git_commit(tmp_path, "rename the cost map") + result: Final = _run_guard(tmp_path, base, head, "litellm_fix_pricing") + assert result.returncode == 1, result.stdout + result.stderr + assert "cost map guard failed (human PR, file checks only):" in result.stdout.splitlines() + + +def test_main_fails_when_the_changed_files_cannot_be_read(tmp_path: Path) -> None: + subprocess.run(("git", "init", "-q", str(tmp_path)), check=True) + head: Final = _commit(tmp_path, BASE_MAP, "head") + result: Final = _run_guard(tmp_path, "0" * 40, head, "litellm_fix_pricing") + assert result.returncode == 1, result.stdout + result.stderr + assert result.stdout.startswith("cost map guard failed: git diff ") + + def test_main_rejects_a_bot_pr_that_edits_code(tmp_path: Path) -> None: subprocess.run(("git", "init", "-q", str(tmp_path)), check=True) base = _commit(tmp_path, BASE_MAP, "base") From 0abd9267c106c8b6a276b1e68824cc7688e19435 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 04:41:11 +0000 Subject: [PATCH 12/15] feat(tokenizer): preserve Python defaults with opt-in Rust dispatch (#42174) * ci: benchmark and gate an installed release wheel Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * ci: simplify installed-wheel benchmark check Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(rust): add native tokenizer codec Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(tokenizer): route Python tokenization through the Rust extension Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(lint): format tokenizer call Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(packaging): restore runtime dependencies and native images Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(tokenizer): preserve Python SDK behavior with Rust tokenizers * fix(tokenizer): restore compatibility paths Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(tokenizer): count custom tokenizers directly Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(tokenizer): preserve caller-supplied Python tokenizer counts * fix(tokenizer): reuse packaged vocabularies in the native wheel * refactor(rust_bridge): route token counting through the catalog as RUST_OPT_IN Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(spend_tracking): compare tokenizer groups by value Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * chore(deps): re-resolve filelock under the <4.0 pin Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(llms): align transformation override signatures with base configs Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * build(rust): use fat LTO to keep the native wheel under the 35 MB limit Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(tokenizer): preserve Python defaults with opt-in Rust dispatch * test(proxy): tolerate missing litellm.utils.Tokenizer when patching it Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): patch the tokenizer dispatch function instead of the removed alias Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(tokenizer): give the Rust wrappers the tiktoken and tokenizers surface Callers of litellm.encoding and litellm.create_tokenizer must see the same read-only API whichever backend the catalog selects. - OpenAIEncoding mirrors tiktoken.Encoding: n_vocab, max_token_value, token_byte_values, encode_single_token, encode_with_unstable, encode_to_numpy, decode_with_offsets, is_special_token, repr; the Rust tiktoken crate keeps a Vocabulary beside each CoreBPE and reports the requested encoding name (gpt2 stays gpt2). - HuggingFaceTokenizer mirrors the read-only tokenizers.Tokenizer surface (token_to_id, id_to_token, get_vocab, get_vocab_size, get_added_tokens_decoder, num_special_tokens_to_add, padding, truncation, encode_special_tokens, from_buffer); HuggingFaceEncoding gains the char/word/token lookups, pad, truncate, set_sequence_id and merge. Mutators stay on the Python tokenizer. - from_json/from_pretrained claim the fork gate only when the huggingface feature is compiled in; the surrogate fallback matches on the Codec. - Tokenizer caching is keyed on the same catalog Context the dispatch runs on; rust_tokenizer reads the encoding name without loading an encoding; LITELLM_RUST parsing is cached. - Drop the unused tiktoken_encoding_for_model export and Error::Download. Co-Authored-By: Claude Fable 5.1 * fix(tokenizer): close the exhaustive matches with assert_never CodeQL reads a `match` over a Literal with no default arm as an implicit `None` return. `assert_never` makes the exhaustiveness explicit for both the HuggingFace tokenizer loader and the Rust token-counter factory. Co-Authored-By: Claude Fable 5.1 * feat(tokenizer): derive the fast counter from the shared tokenizer The count-only counter (`fast` feature) and the codec each parsed the same artifact: TokenCounter took the Anthropic JSON and the tiktoken rank files from Python while Tokenizer loaded them again. One parse now serves both. - FastTokenizer builds from a model another loader holds: `from_shared` takes the Arc the HF codec keeps, and `from_*_pairs` take the ranks the tiktoken vocabulary already parsed. - `FastCounter::fast_counter` in the core crate derives it from either codec; encodings the fast scanner does not reproduce are refused. - Native `Tokenizer.count(text, fast=False)` opts into that counter, built once per tokenizer on first use; `TokenCounter.from_tokenizer(tokenizer, fast=False)` replaces the JSON and rank-file constructors. - The Python route counts over the native tokenizers the codec path shares (`native_encoding`, `native_anthropic`) and no longer reads rank files; the packaged Anthropic tokenizer has one loader, `tokenizer_dispatch.anthropic`. - Public wrappers gain `count(text, fast=False)`. Co-Authored-By: Claude Fable 5.1 --------- Co-authored-by: Yujong Lee Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Claude Fable 5.1 --- .github/actions/cache-cargo-build/action.yml | 10 +- .github/scripts/verify_linux_native_wheel.py | 14 +- .github/workflows/codspeed.yml | 36 +- backend/Dockerfile | 2 + litellm-rust/Cargo.lock | 4 + litellm-rust/Cargo.toml | 2 +- .../crates/host-python/src/execution.rs | 11 +- litellm-rust/crates/host-python/src/lib.rs | 2 +- litellm-rust/crates/python-bridge/Cargo.toml | 2 +- litellm-rust/crates/python-bridge/src/lib.rs | 12 +- .../crates/python-bridge/src/routes/mod.rs | 1 + .../python-bridge/src/routes/token_counter.rs | 87 +++ .../crates/python-bridge/src/token_counter.rs | 158 ---- .../crates/python-bridge/src/tokenizer.rs | 713 ++++++++++++++++++ .../crates/token-counter-fast/src/lib.rs | 41 +- .../crates/token-counter-fast/src/scanner.rs | 13 +- .../crates/token-counter-fast/src/tiktoken.rs | 24 +- .../token-counter-huggingface/Cargo.toml | 1 + .../token-counter-huggingface/src/error.rs | 2 + .../token-counter-huggingface/src/lib.rs | 231 +++++- .../crates/token-counter-tiktoken/Cargo.toml | 3 + .../crates/token-counter-tiktoken/src/lib.rs | 188 ++++- .../token-counter-tiktoken/src/ranks.rs | 340 +++++++++ litellm-rust/crates/token-counter/README.md | 10 +- .../crates/token-counter/src/error.rs | 2 + litellm-rust/crates/token-counter/src/fast.rs | 88 ++- .../crates/token-counter/src/huggingface.rs | 23 +- litellm-rust/crates/token-counter/src/lib.rs | 2 +- .../crates/token-counter/src/tiktoken.rs | 29 +- .../crates/token-counter/src/tokenizer.rs | 6 + litellm/_lazy_imports.py | 24 +- litellm/litellm_core_utils/README.md | 3 +- .../litellm_core_utils/default_encoding.py | 15 - litellm/litellm_core_utils/token_counter.py | 33 +- litellm/litellm_core_utils/tokenizer.py | 402 ++++++++++ litellm/llms/a2a/chat/transformation.py | 5 +- .../aiml/image_generation/transformation.py | 5 +- .../aiohttp_openai/chat/transformation.py | 5 +- .../llms/amazon_nova/chat/transformation.py | 4 +- .../llms/anthropic/batches/transformation.py | 5 +- litellm/llms/anthropic/chat/transformation.py | 5 +- .../anthropic/completion/transformation.py | 4 +- litellm/llms/azure/chat/gpt_transformation.py | 5 +- .../llms/azure_ai/agents/transformation.py | 5 +- .../azure_model_router/transformation.py | 4 +- litellm/llms/azure_ai/chat/transformation.py | 4 +- .../image_generation/mai_transformation.py | 5 +- .../audio_transcription/transformation.py | 5 +- .../bridges/completion_transformation.py | 4 +- litellm/llms/base_llm/chat/transformation.py | 5 +- .../base_llm/completion/transformation.py | 5 +- .../llms/base_llm/embedding/transformation.py | 5 +- litellm/llms/base_llm/files/transformation.py | 5 +- .../image_generation/transformation.py | 5 +- .../image_variations/transformation.py | 9 +- .../bedrock/chat/agentcore/transformation.py | 5 +- .../bedrock/chat/converse_transformation.py | 4 +- .../chat/invoke_agent/transformation.py | 5 +- .../amazon_deepseek_transformation.py | 4 +- .../amazon_moonshot_transformation.py | 5 +- .../amazon_nova_transformation.py | 4 +- .../amazon_qwen2_transformation.py | 4 +- .../amazon_qwen3_transformation.py | 4 +- ...mazon_twelvelabs_pegasus_transformation.py | 5 +- .../anthropic_claude3_transformation.py | 5 +- .../base_invoke_transformation.py | 5 +- .../image_generation/transformation.py | 5 +- litellm/llms/brave/search/__init__.py | 14 +- litellm/llms/bytez/chat/transformation.py | 5 +- litellm/llms/clarifai/chat/transformation.py | 5 +- litellm/llms/cohere/chat/transformation.py | 5 +- litellm/llms/cohere/chat/v2_transformation.py | 5 +- litellm/llms/cohere/embed/handler.py | 6 +- .../image_generation/transformation.py | 5 +- .../llms/compactifai/chat/transformation.py | 5 +- litellm/llms/custom_httpx/aiohttp_handler.py | 5 +- litellm/llms/custom_httpx/llm_http_handler.py | 6 +- .../image_generation/transformation.py | 5 +- .../llms/databricks/chat/transformation.py | 7 +- .../llms/deprecated_providers/aleph_alpha.py | 7 +- litellm/llms/edenai/chat/transformation.py | 5 +- .../edenai/image_generation/transformation.py | 5 +- .../image_generation/bria_transformation.py | 5 +- .../flux_pro_v11_ultra_transformation.py | 5 +- .../ideogram_v3_transformation.py | 5 +- .../imagen4_transformation.py | 5 +- .../recraft_v3_transformation.py | 5 +- .../stable_diffusion_transformation.py | 5 +- .../fal_ai/image_generation/transformation.py | 5 +- .../llms/fireworks_ai/chat/transformation.py | 4 +- .../gemini/image_generation/transformation.py | 5 +- litellm/llms/gigachat/chat/transformation.py | 5 +- litellm/llms/groq/chat/transformation.py | 4 +- litellm/llms/huggingface/embedding/handler.py | 5 +- .../huggingface/embedding/transformation.py | 5 +- litellm/llms/langflow/chat/transformation.py | 5 +- litellm/llms/langgraph/chat/transformation.py | 5 +- litellm/llms/lemonade/chat/transformation.py | 4 +- litellm/llms/mistral/chat/transformation.py | 4 +- litellm/llms/nlp_cloud/chat/transformation.py | 5 +- litellm/llms/oci/chat/transformation.py | 5 +- litellm/llms/ollama/chat/transformation.py | 5 +- .../llms/ollama/completion/transformation.py | 5 +- litellm/llms/oobabooga/chat/transformation.py | 5 +- .../llms/openai/chat/gpt_transformation.py | 5 +- .../dall_e_2_transformation.py | 5 +- .../dall_e_3_transformation.py | 5 +- .../image_generation/gpt_transformation.py | 5 +- .../openai/image_variations/transformation.py | 6 +- litellm/llms/openai/openai.py | 5 +- .../llms/openai_like/chat/transformation.py | 5 +- .../llms/openrouter/chat/transformation.py | 5 +- .../image_generation/transformation.py | 5 +- .../llms/perplexity/chat/transformation.py | 4 +- .../llms/petals/completion/transformation.py | 4 +- litellm/llms/predibase/chat/transformation.py | 5 +- .../image_generation/transformation.py | 5 +- litellm/llms/replicate/chat/transformation.py | 5 +- .../image_generation/transformation.py | 7 +- .../sagemaker/completion/transformation.py | 5 +- litellm/llms/sap/chat/transformation.py | 9 +- .../image_generation/transformation.py | 5 +- .../topaz/image_variations/transformation.py | 6 +- .../llms/triton/completion/transformation.py | 8 +- .../vertex_ai/agent_engine/transformation.py | 5 +- .../vertex_gemini_transformation.py | 5 +- .../vertex_imagen_transformation.py | 5 +- .../anthropic/transformation.py | 4 +- .../llama3/transformation.py | 4 +- .../vertex_gemma_models/transformation.py | 7 +- .../llms/watsonx/completion/transformation.py | 5 +- litellm/main.py | 21 +- .../spend_tracking/budget_reservation.py | 127 +--- litellm/proxy/spend_tracking/input_tokens.py | 173 +++++ litellm/rust_bridge/_native.pyi | 107 ++- litellm/rust_bridge/catalog.py | 4 + litellm/rust_bridge/configuration.py | 3 + litellm/rust_bridge/token_counter.py | 70 +- litellm/rust_bridge/tokenizer.py | 108 +++ litellm/types/utils.py | 4 +- litellm/utils.py | 92 +-- migrations/Dockerfile | 2 + pyproject.toml | 13 +- tests/benchmarks/conftest.py | 18 + .../test_custom_tokenizer_bug.py | 8 +- .../test_decode_special_tokens.py | 20 +- .../litellm_core_utils/test_token_counter.py | 24 +- .../litellm_core_utils/test_tokenizer.py | 403 ++++++++++ .../spend_tracking/test_budget_reservation.py | 40 +- .../proxy/spend_tracking/test_input_tokens.py | 191 +++++ .../proxy/test_budget_reservation.py | 6 +- tests/test_litellm/proxy/test_proxy_server.py | 8 +- .../test_litellm/rust_bridge/test_catalog.py | 2 +- .../rust_bridge/test_token_counter.py | 159 ++-- .../rust_bridge/test_tokenizer.py | 134 ++++ .../test_verify_linux_native_wheel.py | 17 +- tests/test_litellm_rust/test_fork_guard.py | 74 ++ tests/test_litellm_rust/test_tokenizer.py | 130 ++++ uv.lock | 119 ++- 159 files changed, 4151 insertions(+), 954 deletions(-) create mode 100644 litellm-rust/crates/python-bridge/src/routes/token_counter.rs delete mode 100644 litellm-rust/crates/python-bridge/src/token_counter.rs create mode 100644 litellm-rust/crates/python-bridge/src/tokenizer.rs create mode 100644 litellm-rust/crates/token-counter-tiktoken/src/ranks.rs create mode 100644 litellm/litellm_core_utils/tokenizer.py create mode 100644 litellm/proxy/spend_tracking/input_tokens.py create mode 100644 litellm/rust_bridge/tokenizer.py create mode 100644 tests/test_litellm/litellm_core_utils/test_tokenizer.py create mode 100644 tests/test_litellm/proxy/spend_tracking/test_input_tokens.py create mode 100644 tests/test_litellm/rust_bridge/test_tokenizer.py create mode 100644 tests/test_litellm_rust/test_tokenizer.py diff --git a/.github/actions/cache-cargo-build/action.yml b/.github/actions/cache-cargo-build/action.yml index c3b8ce22c68..222fad637fb 100644 --- a/.github/actions/cache-cargo-build/action.yml +++ b/.github/actions/cache-cargo-build/action.yml @@ -15,6 +15,12 @@ description: >- cache the same directory for different workloads, and a shared key would let whichever ran first deny the others a save. +inputs: + profile: + description: "Cargo profile the build uses (dev or release)" + required: false + default: "dev" + runs: using: composite steps: @@ -25,6 +31,6 @@ runs: ~/.cargo/registry ~/.cargo/git litellm-rust/target - key: ${{ runner.os }}-maturin-dev-${{ hashFiles('litellm-rust/Cargo.lock') }} + key: ${{ runner.os }}-maturin-${{ inputs.profile }}-${{ hashFiles('litellm-rust/Cargo.lock') }} restore-keys: | - ${{ runner.os }}-maturin-dev- + ${{ runner.os }}-maturin-${{ inputs.profile }}- diff --git a/.github/scripts/verify_linux_native_wheel.py b/.github/scripts/verify_linux_native_wheel.py index ea6d2401084..f2b82f86b47 100644 --- a/.github/scripts/verify_linux_native_wheel.py +++ b/.github/scripts/verify_linux_native_wheel.py @@ -134,7 +134,16 @@ def main( uncompressed_wheel_size: Final = sum(member.file_size for member in wheel_members) native_path: Final = wheel.parent / "native" / Path(native_member.filename).name native_path.parent.mkdir(parents=True, exist_ok=True) - native_path.write_bytes(archive.read(native_member)) + native_bytes: Final = archive.read(native_member) + native_path.write_bytes(native_bytes) + duplicated_vocabularies: Final = tuple( + member.filename + for member in wheel_members + if member.filename.startswith("litellm/litellm_core_utils/tokenizers/") + and re.fullmatch(r"[0-9a-f]{40}", PurePosixPath(member.filename).name) + and member.file_size > 0 + and archive.read(member) in native_bytes + ) wheel_metadata_tags_match: Final = ( len(wheel_metadata_tags) == len(expanded_filename_tags) @@ -205,7 +214,7 @@ def main( native_module: Final = load_native_module(native_path) native_module_loads: Final = native_module is not None panic_test_hook_absent: Final = native_module is not None and not hasattr(native_module, "_panic_for_test") - native_size_limit: Final = 40_000_000 + native_size_limit: Final = 35_000_000 native_size_within_limit: Final = native_member.file_size <= native_size_limit validations: Final = ( (f"Python tag is {EXPECTED_PYTHON_TAG}", python_tag == EXPECTED_PYTHON_TAG), @@ -223,6 +232,7 @@ def main( ("Native module loads", native_module_loads), ("Production module omits the panic test hook", panic_test_hook_absent), (f"Native extension does not exceed {native_size_limit / 1_000_000:.0f} MB", native_size_within_limit), + ("Tokenizer vocabularies are not duplicated in the native extension", not duplicated_vocabularies), ("Wheel contents are valid", not unexpected_members), ) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index fd7513a3937..ec7e211faa1 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -13,6 +13,7 @@ on: - ".github/workflows/codspeed.yml" - ".github/actions/setup-uv-with-retries/**" - ".github/actions/cache-cargo-build/**" + - ".github/scripts/uv_sync_with_retries.sh" pull_request: branches: - main @@ -25,6 +26,7 @@ on: - ".github/workflows/codspeed.yml" - ".github/actions/setup-uv-with-retries/**" - ".github/actions/cache-cargo-build/**" + - ".github/scripts/uv_sync_with_retries.sh" # Allow CodSpeed to trigger backtest performance analysis # in order to generate initial data workflow_dispatch: @@ -59,19 +61,27 @@ jobs: - name: Cache the Rust build uses: ./.github/actions/cache-cargo-build + with: + profile: release # Build the wheel and resolve every dependency outside the CodSpeed # runner: the same maturin build took 42 minutes inside `codspeed run` # versus under 3 minutes as a plain step (LIT-6183) - - name: Build environment + - name: Build the release wheel + run: uv build --wheel --out-dir dist + + - name: Install the wheel into the benchmark environment + run: | + UV_PROJECT_ENVIRONMENT="${RUNNER_TEMP}/benchmark-venv" .github/scripts/uv_sync_with_retries.sh --frozen --no-default-groups --group benchmarks --no-install-project --python 3.12 + uv pip install --python "${RUNNER_TEMP}/benchmark-venv/bin/python" --no-deps dist/*.whl + + - name: Collect benchmarks + env: + PYTEST_DISABLE_PLUGIN_AUTOLOAD: "1" + LITELLM_REQUIRE_INSTALLED_WHEEL: "1" run: > - env PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 - uv run --frozen --no-default-groups - --with pytest==8.3.5 - --with pytest-codspeed==4.3.0 - --with "mcp>=2.2.0,<3.0" - --with "a2a-sdk>=1.1.0,<2.0" - pytest + "${RUNNER_TEMP}/benchmark-venv/bin/python" -I -m pytest + --import-mode=importlib -p pytest_codspeed.plugin tests/benchmarks/ --codspeed @@ -82,13 +92,9 @@ jobs: with: mode: simulation run: > - env PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 - uv run --frozen --no-default-groups - --with pytest==8.3.5 - --with pytest-codspeed==4.3.0 - --with "mcp>=2.2.0,<3.0" - --with "a2a-sdk>=1.1.0,<2.0" - pytest + env PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 LITELLM_REQUIRE_INSTALLED_WHEEL=1 + "${RUNNER_TEMP}/benchmark-venv/bin/python" -I -m pytest + --import-mode=importlib -p pytest_codspeed.plugin tests/benchmarks/ --codspeed diff --git a/backend/Dockerfile b/backend/Dockerfile index 622fedcd70d..57e0a43a98d 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -61,6 +61,8 @@ RUN --mount=type=cache,target=/root/.cache/uv \ --extra saml \ --python python3.13 +RUN cp "$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"/litellm/rust_bridge/_native*.so litellm/rust_bridge/ + RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \ npm_config_cache=/root/.npm \ prisma generate --schema=./schema.prisma diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 03e0dabbc17..37384cbfa53 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -3253,6 +3253,7 @@ dependencies = [ name = "litellm-token-counter-huggingface" version = "0.1.0" dependencies = [ + "serde_json", "thiserror 2.0.19", "tokenizers", ] @@ -3261,6 +3262,9 @@ dependencies = [ name = "litellm-token-counter-tiktoken" version = "0.1.0" dependencies = [ + "base64 0.22.1", + "once_cell", + "rustc-hash", "thiserror 2.0.19", "tiktoken-rs", ] diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index 0e8941cb8e6..813d0713128 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -89,7 +89,7 @@ veil = "0.3.0" [profile.release] opt-level = 3 -lto = "thin" +lto = "fat" codegen-units = 1 panic = "unwind" debug = false diff --git a/litellm-rust/crates/host-python/src/execution.rs b/litellm-rust/crates/host-python/src/execution.rs index 083c184e37e..b435bf241d2 100644 --- a/litellm-rust/crates/host-python/src/execution.rs +++ b/litellm-rust/crates/host-python/src/execution.rs @@ -29,7 +29,7 @@ pyo3::create_exception!( static FORK_GATE: ForkGate = ForkGate::new(); -/// Whether this process has started the Tokio runtime. +/// Whether this process has entered process-bound native execution. pub fn runtime_started() -> bool { FORK_GATE.started(std::process::id()) } @@ -40,9 +40,8 @@ pub fn reserve_process_for_forking() -> Result<(), RuntimeAlreadyStarted> { FORK_GATE.reserve(std::process::id()) } -/// The only door to the Tokio runtime: every route reaches it through this module, which is -/// what lets the gate speak for the whole extension. `clippy.toml` disallows going around it. -fn enter_runtime() -> PyResult<()> { +/// Claims process-bound native state before runtime startup or tokenizer execution. +pub fn enter_native() -> PyResult<()> { FORK_GATE .enter(std::process::id()) .map_err(|refused| match refused { @@ -60,7 +59,7 @@ fn enter_runtime() -> PyResult<()> { #[expect(clippy::disallowed_methods, reason = "this is the gated door")] fn runtime() -> PyResult<&'static Runtime> { - enter_runtime()?; + enter_native()?; Ok(pyo3_async_runtimes::tokio::get_runtime()) } @@ -70,7 +69,7 @@ where F: Future> + Send + 'static, T: for<'py> IntoPyObject<'py> + Send + 'static, { - enter_runtime()?; + enter_native()?; pyo3_async_runtimes::tokio::future_into_py(py, future) } diff --git a/litellm-rust/crates/host-python/src/lib.rs b/litellm-rust/crates/host-python/src/lib.rs index 7d164ab7535..4a33975a918 100644 --- a/litellm-rust/crates/host-python/src/lib.rs +++ b/litellm-rust/crates/host-python/src/lib.rs @@ -20,7 +20,7 @@ pub use argument::lookup; pub use callable::wrap_failure; pub use driver::run_call; pub use execution::{ - ForkedAfterNativeRuntimeStarted, ProcessReservedForForking, poll_async_value, + ForkedAfterNativeRuntimeStarted, ProcessReservedForForking, enter_native, poll_async_value, reserve_process_for_forking, run_async, run_async_value, run_sync, run_sync_value, runtime_started, }; diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml index 4e6c510d104..7846beef28a 100644 --- a/litellm-rust/crates/python-bridge/Cargo.toml +++ b/litellm-rust/crates/python-bridge/Cargo.toml @@ -10,7 +10,7 @@ name = "_native" crate-type = ["cdylib"] [features] -default = ["abi3", "fast"] +default = ["abi3", "fast", "huggingface", "tiktoken"] abi3 = ["pyo3/abi3-py310"] extension-module = ["pyo3/extension-module"] panic-test = [] diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index ed9bc90f650..b1fc5244d6f 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -12,7 +12,7 @@ mod routes; reason = "secret-manager foundations await rollout activation" )] mod secrets; -mod token_counter; +mod tokenizer; #[pymodule(gil_used = true)] mod _native { @@ -37,7 +37,12 @@ mod _native { #[pymodule_export] use crate::routes::responses::ResponsesWebSocketConnection; #[pymodule_export] - use crate::token_counter::TokenCounter; + use crate::routes::token_counter::TokenCounter; + #[cfg(feature = "huggingface")] + #[pymodule_export] + use crate::tokenizer::HuggingFaceEncoding; + #[pymodule_export] + use crate::tokenizer::Tokenizer; #[pymodule_export] use litellm_host_python::{ForkedAfterNativeRuntimeStarted, ProcessReservedForForking}; use pyo3::{prelude::*, types::PyModule}; @@ -83,10 +88,13 @@ mod tests { "achat_completions", "ResponsesWebSocketConnection", "TokenCounter", + "Tokenizer", "gil_stats", "process_state_started", "reserve_process_for_forking", ]; + #[cfg(feature = "huggingface")] + expected.push("HuggingFaceEncoding"); expected.sort_unstable(); let mut public_names: Vec = native_module(py) diff --git a/litellm-rust/crates/python-bridge/src/routes/mod.rs b/litellm-rust/crates/python-bridge/src/routes/mod.rs index 2d6b849a6b1..8a78a26423d 100644 --- a/litellm-rust/crates/python-bridge/src/routes/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/mod.rs @@ -3,6 +3,7 @@ pub(crate) mod chat_completions; pub(crate) mod messages; pub(crate) mod ocr; pub(crate) mod responses; +pub(crate) mod token_counter; #[cfg(test)] mod tests { diff --git a/litellm-rust/crates/python-bridge/src/routes/token_counter.rs b/litellm-rust/crates/python-bridge/src/routes/token_counter.rs new file mode 100644 index 00000000000..168c4883b0b --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/routes/token_counter.rs @@ -0,0 +1,87 @@ +use std::sync::Arc; +use std::{num::NonZero, thread::available_parallelism}; + +use litellm_host_python::{enter_native, run_async}; +use litellm_token_counter::{ + CountableRequest, Error, InputTokenCount, TokenCounter as CoreTokenCounter, +}; +use pyo3::{ + exceptions::{PyRuntimeError, PyValueError}, + prelude::*, + types::PyAny, +}; +use tokio::sync::Semaphore; + +use crate::errors::RustBridgeDeclined; +use crate::tokenizer::Tokenizer; + +/// Counts the input tokens of a raw request body off the Python event loop with +/// the GIL released. Python owns which requests get here and what to do with +/// the count. At most one encode per core runs at a time; the rest wait in the +/// async task, where a cancelled Python awaiter drops them before any blocking +/// work is scheduled. +#[pyclass(frozen)] +pub(crate) struct TokenCounter { + inner: Arc, + encode_slots: Arc, +} + +#[pymethods] +impl TokenCounter { + #[staticmethod] + #[pyo3(signature = (tokenizer, fast = false))] + fn from_tokenizer(py: Python<'_>, tokenizer: &Tokenizer, fast: bool) -> PyResult { + enter_native()?; + let inner = CoreTokenCounter::new(tokenizer.counter(py, fast)); + Ok(Self { + inner: Arc::new(inner), + encode_slots: Arc::new(Semaphore::new(encode_parallelism())), + }) + } + + fn acount_request<'py>(&self, py: Python<'py>, body: &[u8]) -> PyResult> { + let counter = Arc::clone(&self.inner); + let encode_slots = Arc::clone(&self.encode_slots); + let body = body.to_vec(); + run_async( + py, + async move { + let _slot = encode_slots + .acquire_owned() + .await + .map_err(|error| Error::Task(error.to_string()))?; + tokio::task::spawn_blocking(move || count_body(&counter, &body)) + .await + .map_err(|error| Error::Task(error.to_string()))? + }, + token_count_error_to_pyerr, + ) + } +} + +fn encode_parallelism() -> usize { + available_parallelism().map_or(1, NonZero::get) +} + +fn count_body(counter: &CoreTokenCounter, body: &[u8]) -> Result { + let request = CountableRequest::parse(body)?; + counter.count_request(&request) +} + +pub(crate) fn token_count_error_to_pyerr(error: Error) -> PyErr { + let message = error.to_string(); + match error { + Error::Load(_) + | Error::Ranks(_) + | Error::UnicodeClasses + | Error::UnsupportedTokenizer(_) => PyValueError::new_err(message), + Error::RequestParse(_) + | Error::MissingInput + | Error::FloatText + | Error::ContentBlock + | Error::ArrayItems + | Error::JsonSerialization(_) + | Error::JsonUtf8(_) => RustBridgeDeclined::new_err(message), + Error::Encode(_) | Error::Decode(_) | Error::Task(_) => PyRuntimeError::new_err(message), + } +} diff --git a/litellm-rust/crates/python-bridge/src/token_counter.rs b/litellm-rust/crates/python-bridge/src/token_counter.rs deleted file mode 100644 index 244401e6696..00000000000 --- a/litellm-rust/crates/python-bridge/src/token_counter.rs +++ /dev/null @@ -1,158 +0,0 @@ -use std::sync::Arc; - -#[cfg(any(feature = "fast", feature = "huggingface", feature = "tiktoken"))] -use std::{num::NonZero, thread::available_parallelism}; - -#[cfg(any(feature = "fast", feature = "huggingface", feature = "tiktoken"))] -use litellm_host_python::release_gil; -use litellm_host_python::run_async; -use litellm_token_counter::{ - CountableRequest, Error, InputTokenCount, TokenCounter as CoreTokenCounter, -}; -use pyo3::{ - exceptions::{PyRuntimeError, PyValueError}, - prelude::*, - types::PyAny, -}; -use tokio::sync::Semaphore; - -use crate::errors::RustBridgeDeclined; - -/// Counts the input tokens of a raw request body off the Python event loop with -/// the GIL released. Python owns which requests get here and what to do with -/// the count. At most one encode per core runs at a time; the rest wait in the -/// async task, where a cancelled Python awaiter drops them before any blocking -/// work is scheduled. -#[pyclass(frozen)] -pub(crate) struct TokenCounter { - inner: Arc, - encode_slots: Arc, -} - -#[pymethods] -impl TokenCounter { - #[new] - fn new(py: Python<'_>, tokenizer_json: &str) -> PyResult { - #[cfg(feature = "fast")] - { - Self::load(py, || CoreTokenCounter::from_json_fast(tokenizer_json)) - } - #[cfg(all(not(feature = "fast"), feature = "huggingface"))] - { - Self::load(py, || CoreTokenCounter::from_json(tokenizer_json)) - } - #[cfg(not(any(feature = "fast", feature = "huggingface")))] - { - let _ = (py, tokenizer_json); - Err(RustBridgeDeclined::new_err( - "tokenizer backend requires the fast or huggingface feature", - )) - } - } - - #[staticmethod] - fn from_cl100k_ranks(py: Python<'_>, rank_file: &str) -> PyResult { - #[cfg(feature = "fast")] - { - Self::load(py, || CoreTokenCounter::from_cl100k_ranks(rank_file)) - } - #[cfg(not(feature = "fast"))] - { - let _ = (py, rank_file); - Err(RustBridgeDeclined::new_err( - "tokenizer backend requires the fast feature", - )) - } - } - - #[staticmethod] - fn from_o200k_ranks(py: Python<'_>, rank_file: &str) -> PyResult { - #[cfg(feature = "fast")] - { - Self::load(py, || CoreTokenCounter::from_o200k_ranks(rank_file)) - } - #[cfg(not(feature = "fast"))] - { - let _ = (py, rank_file); - Err(RustBridgeDeclined::new_err( - "tokenizer backend requires the fast feature", - )) - } - } - - #[staticmethod] - fn from_tiktoken(py: Python<'_>, encoding: &str) -> PyResult { - #[cfg(feature = "tiktoken")] - { - Self::load(py, || CoreTokenCounter::from_tiktoken(encoding)) - } - #[cfg(not(feature = "tiktoken"))] - { - let _ = (py, encoding); - Err(RustBridgeDeclined::new_err( - "tokenizer backend requires the tiktoken feature", - )) - } - } - - fn acount_request<'py>(&self, py: Python<'py>, body: &[u8]) -> PyResult> { - let counter = Arc::clone(&self.inner); - let encode_slots = Arc::clone(&self.encode_slots); - let body = body.to_vec(); - run_async( - py, - async move { - let _slot = encode_slots - .acquire_owned() - .await - .map_err(|error| Error::Task(error.to_string()))?; - tokio::task::spawn_blocking(move || count_body(&counter, &body)) - .await - .map_err(|error| Error::Task(error.to_string()))? - }, - token_count_error_to_pyerr, - ) - } -} - -impl TokenCounter { - #[cfg(any(feature = "fast", feature = "huggingface", feature = "tiktoken"))] - fn load( - py: Python<'_>, - load: impl FnOnce() -> Result + Send, - ) -> PyResult { - let inner = release_gil(py, load).map_err(token_count_error_to_pyerr)?; - Ok(Self { - inner: Arc::new(inner), - encode_slots: Arc::new(Semaphore::new(encode_parallelism())), - }) - } -} - -#[cfg(any(feature = "fast", feature = "huggingface", feature = "tiktoken"))] -fn encode_parallelism() -> usize { - available_parallelism().map_or(1, NonZero::get) -} - -fn count_body(counter: &CoreTokenCounter, body: &[u8]) -> Result { - let request = CountableRequest::parse(body)?; - counter.count_request(&request) -} - -fn token_count_error_to_pyerr(error: Error) -> PyErr { - let message = error.to_string(); - match error { - Error::Load(_) - | Error::Ranks(_) - | Error::UnicodeClasses - | Error::UnsupportedTokenizer(_) => PyValueError::new_err(message), - Error::RequestParse(_) - | Error::MissingInput - | Error::FloatText - | Error::ContentBlock - | Error::ArrayItems - | Error::JsonSerialization(_) - | Error::JsonUtf8(_) => RustBridgeDeclined::new_err(message), - Error::Encode(_) | Error::Task(_) => PyRuntimeError::new_err(message), - } -} diff --git a/litellm-rust/crates/python-bridge/src/tokenizer.rs b/litellm-rust/crates/python-bridge/src/tokenizer.rs new file mode 100644 index 00000000000..df219d55eb6 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/tokenizer.rs @@ -0,0 +1,713 @@ +//! The Python face of the text codecs: one `Tokenizer` class over the tiktoken and Hugging +//! Face backends, carrying the read-only surface of `tiktoken.Encoding` and +//! `tokenizers.Tokenizer` that `litellm/litellm_core_utils/tokenizer.py` wraps. +use std::borrow::Cow; +#[cfg(any(feature = "tiktoken", feature = "huggingface"))] +use std::collections::HashMap; +use std::sync::Arc; +#[cfg(feature = "fast")] +use std::sync::OnceLock; + +use litellm_host_python::{enter_native, release_gil}; +#[cfg(feature = "fast")] +use litellm_token_counter::fast::{FastCounter, FastTokenizer}; +use litellm_token_counter::{Error, TextCodec}; +use pyo3::{exceptions::PyUnicodeEncodeError, prelude::*, types::PyString}; + +#[cfg(any(feature = "tiktoken", feature = "huggingface"))] +use pyo3::exceptions::PyValueError; +#[cfg(feature = "huggingface")] +use pyo3::{exceptions::PyIOError, types::PyDict}; +#[cfg(feature = "tiktoken")] +use pyo3::{ + exceptions::{PyKeyError, PyRuntimeError}, + types::PyBytes, +}; + +#[cfg(not(all(feature = "tiktoken", feature = "huggingface")))] +use crate::errors::RustBridgeDeclined; +use crate::routes::token_counter::token_count_error_to_pyerr; + +#[cfg(feature = "huggingface")] +use litellm_token_counter::huggingface::{ + EncodeInput, Encoding, HuggingFaceTokenizer, InputSequence, PaddingDirection, PaddingStrategy, + TruncationDirection, encoding_from_json, encoding_to_json, +}; +#[cfg(feature = "tiktoken")] +use litellm_token_counter::tiktoken::{TiktokenTokenizer, Vocabulary}; + +#[cfg(feature = "tiktoken")] +pub(crate) fn load_tiktoken(py: Python<'_>, encoding: &str) -> PyResult { + enter_native()?; + let resource: std::path::PathBuf = + PyModule::import(py, "litellm.litellm_core_utils.tokenizers")? + .getattr("__file__")? + .extract()?; + release_gil(py, || { + TiktokenTokenizer::from_cached_ranks(encoding, |file| { + std::fs::read_to_string(resource.with_file_name(file)) + }) + }) + .map_err(|error| token_count_error_to_pyerr(error.into())) +} + +pub(crate) enum Codec { + #[cfg(feature = "tiktoken")] + Tiktoken(TiktokenTokenizer), + #[cfg(feature = "huggingface")] + HuggingFace(HuggingFaceTokenizer), +} + +impl Codec { + pub(crate) fn codec(&self) -> &dyn TextCodec { + match *self { + #[cfg(feature = "tiktoken")] + Self::Tiktoken(ref tokenizer) => tokenizer, + #[cfg(feature = "huggingface")] + Self::HuggingFace(ref tokenizer) => tokenizer, + } + } + + #[cfg(feature = "fast")] + fn fast_counter(&self) -> Option { + match *self { + #[cfg(feature = "tiktoken")] + Self::Tiktoken(ref tokenizer) => tokenizer.fast_counter(), + #[cfg(feature = "huggingface")] + Self::HuggingFace(ref tokenizer) => tokenizer.fast_counter(), + } + } +} + +/// The loaded model is shared: `TokenCounter::from_tokenizer` counts with the same parse, +/// and the opt-in count-only counter is derived from it once, on first use. +#[pyclass(frozen, module = "litellm.rust_bridge._native")] +pub(crate) struct Tokenizer { + inner: Arc, + #[cfg(feature = "fast")] + fast: OnceLock>>, +} + +#[pymethods] +impl Tokenizer { + #[staticmethod] + fn from_tiktoken(py: Python<'_>, encoding: &str) -> PyResult { + #[cfg(feature = "tiktoken")] + { + let tokenizer = load_tiktoken(py, encoding)?; + Ok(Self::new(Codec::Tiktoken(tokenizer))) + } + #[cfg(not(feature = "tiktoken"))] + { + let _ = (py, encoding); + Err(RustBridgeDeclined::new_err( + "tokenizer backend requires the tiktoken feature", + )) + } + } + + #[staticmethod] + fn from_json(py: Python<'_>, tokenizer_json: &str) -> PyResult { + #[cfg(feature = "huggingface")] + { + enter_native()?; + let tokenizer = release_gil(py, || HuggingFaceTokenizer::from_json(tokenizer_json)) + .map_err(|error| token_count_error_to_pyerr(error.into()))?; + Ok(Self::new(Codec::HuggingFace(tokenizer))) + } + #[cfg(not(feature = "huggingface"))] + { + let _ = (py, tokenizer_json); + Err(RustBridgeDeclined::new_err( + "tokenizer backend requires the huggingface feature", + )) + } + } + + #[staticmethod] + #[pyo3(signature = (identifier, revision = "main", token = None))] + fn from_pretrained( + py: Python<'_>, + identifier: &str, + revision: &str, + token: Option<&str>, + ) -> PyResult { + #[cfg(feature = "huggingface")] + { + enter_native()?; + let kwargs = PyDict::new(py); + kwargs.set_item("repo_id", identifier)?; + kwargs.set_item("filename", "tokenizer.json")?; + kwargs.set_item("revision", revision)?; + kwargs.set_item("token", token)?; + let path: String = PyModule::import(py, "huggingface_hub")? + .getattr("hf_hub_download")? + .call((), Some(&kwargs))? + .extract()?; + let json = + release_gil(py, || std::fs::read_to_string(path)).map_err(PyIOError::new_err)?; + Self::from_json(py, &json) + } + #[cfg(not(feature = "huggingface"))] + { + let _ = (py, identifier, revision, token); + Err(RustBridgeDeclined::new_err( + "tokenizer backend requires the huggingface feature", + )) + } + } + + fn encode(&self, py: Python<'_>, text: &Bound<'_, PyString>) -> PyResult> { + enter_native()?; + let text = self.text(text)?; + release_gil(py, || self.inner.codec().encode(&text)).map_err(token_count_error_to_pyerr) + } + + #[pyo3(signature = (ids, skip_special_tokens = true))] + fn decode(&self, py: Python<'_>, ids: Vec, skip_special_tokens: bool) -> PyResult { + enter_native()?; + release_gil(py, || self.inner.codec().decode(&ids, skip_special_tokens)) + .map_err(token_count_error_to_pyerr) + } + + #[pyo3(signature = (text, fast = false))] + fn count(&self, py: Python<'_>, text: &Bound<'_, PyString>, fast: bool) -> PyResult { + enter_native()?; + let text = self.text(text)?; + let counter = self.counter(py, fast); + release_gil(py, || { + litellm_token_counter::Tokenizer::count_tokens(&counter, &text) + }) + .map_err(token_count_error_to_pyerr) + } + + #[getter] + fn name(&self) -> &str { + self.inner.codec().name() + } + + // ---- tiktoken: the `tiktoken.Encoding` surface ------------------------------------------ + + #[cfg(feature = "tiktoken")] + fn encode_special( + &self, + py: Python<'_>, + text: &Bound<'_, PyString>, + allowed: Vec, + ) -> PyResult> { + enter_native()?; + let tokenizer = self.tiktoken()?; + let text = self.text(text)?; + release_gil(py, || tokenizer.encode_special(&text, &allowed)) + .map_err(PyRuntimeError::new_err) + } + + /// tiktoken's `encode_with_unstable`: `(stable_tokens, completions)`. + #[cfg(feature = "tiktoken")] + fn encode_with_unstable( + &self, + py: Python<'_>, + text: &Bound<'_, PyString>, + allowed: Vec, + ) -> PyResult<(Vec, Vec>)> { + enter_native()?; + let tokenizer = self.tiktoken()?; + let text = self.text(text)?; + Ok(release_gil(py, || { + tokenizer.encode_with_unstable(&text, &allowed) + })) + } + + /// The special tokens by text: tiktoken's `_special_tokens`. + #[cfg(feature = "tiktoken")] + fn special_tokens(&self) -> PyResult> { + Ok(self + .vocabulary()? + .special_tokens() + .map(|(token, rank)| (token.to_owned(), rank)) + .collect()) + } + + #[cfg(feature = "tiktoken")] + fn max_token_value(&self) -> PyResult { + Ok(self.vocabulary()?.max_token_value()) + } + + #[cfg(feature = "tiktoken")] + fn is_special_token(&self, token: u32) -> PyResult { + Ok(self.vocabulary()?.is_special_token(token)) + } + + /// Every mergeable token's bytes, sorted bytewise like tiktoken's `token_byte_values`. + #[cfg(feature = "tiktoken")] + fn token_byte_values<'py>(&self, py: Python<'py>) -> PyResult>> { + let vocabulary = self.vocabulary()?; + let values = release_gil(py, || vocabulary.token_byte_values()); + Ok(values.iter().map(|value| PyBytes::new(py, value)).collect()) + } + + /// The token of one whole piece; `KeyError` when it is not in the vocabulary. + #[cfg(feature = "tiktoken")] + fn encode_single_token(&self, py: Python<'_>, piece: Vec) -> PyResult { + self.vocabulary()? + .encode_single_token(&piece) + .ok_or_else(|| PyKeyError::new_err(PyBytes::new(py, &piece).unbind())) + } + + #[cfg(feature = "tiktoken")] + fn decode_bytes<'py>(&self, py: Python<'py>, ids: Vec) -> PyResult> { + enter_native()?; + let tokenizer = self.tiktoken()?; + let bytes = + release_gil(py, || tokenizer.decode_bytes(&ids)).map_err(PyKeyError::new_err)?; + Ok(PyBytes::new(py, &bytes)) + } + + // ---- Hugging Face: the `tokenizers.Tokenizer` surface ----------------------------------- + + #[cfg(feature = "huggingface")] + #[pyo3(signature = (sequence, pair = None, is_pretokenized = false, add_special_tokens = true, fast = false))] + fn encode_huggingface( + &self, + py: Python<'_>, + sequence: Sequence, + pair: Option, + is_pretokenized: bool, + add_special_tokens: bool, + fast: bool, + ) -> PyResult { + enter_native()?; + let tokenizer = self.huggingface()?; + let sequence = sequence.input(is_pretokenized)?; + let input = match pair { + Some(pair) => EncodeInput::Dual(sequence, pair.input(is_pretokenized)?), + None => EncodeInput::Single(sequence), + }; + release_gil(py, || { + tokenizer.encode_result(input, add_special_tokens, fast) + }) + .map(|inner| HuggingFaceEncoding { inner }) + .map_err(|error| token_count_error_to_pyerr(Error::from(error))) + } + + #[cfg(feature = "huggingface")] + #[pyo3(signature = (inputs, is_pretokenized = false, add_special_tokens = true, fast = false))] + fn encode_batch_huggingface( + &self, + py: Python<'_>, + inputs: Vec<(Sequence, Option)>, + is_pretokenized: bool, + add_special_tokens: bool, + fast: bool, + ) -> PyResult> { + enter_native()?; + let tokenizer = self.huggingface()?; + let inputs = inputs + .into_iter() + .map(|(sequence, pair)| { + let sequence = sequence.input(is_pretokenized)?; + match pair { + Some(pair) => Ok(EncodeInput::Dual(sequence, pair.input(is_pretokenized)?)), + None => Ok(EncodeInput::Single(sequence)), + } + }) + .collect::>>()?; + release_gil(py, || { + tokenizer.encode_batch_result(inputs, add_special_tokens, fast) + }) + .map(|encodings| { + encodings + .into_iter() + .map(|inner| HuggingFaceEncoding { inner }) + .collect() + }) + .map_err(|error| token_count_error_to_pyerr(Error::from(error))) + } + + #[cfg(feature = "huggingface")] + #[pyo3(signature = (pretty = false))] + fn to_json(&self, py: Python<'_>, pretty: bool) -> PyResult { + enter_native()?; + let tokenizer = self.huggingface()?; + release_gil(py, || tokenizer.to_json(pretty)) + .map_err(|error| token_count_error_to_pyerr(Error::from(error))) + } + + #[cfg(feature = "huggingface")] + fn token_to_id(&self, token: &str) -> PyResult> { + Ok(self.huggingface()?.token_to_id(token)) + } + + #[cfg(feature = "huggingface")] + fn id_to_token(&self, id: u32) -> PyResult> { + Ok(self.huggingface()?.id_to_token(id)) + } + + #[cfg(feature = "huggingface")] + #[pyo3(signature = (with_added_tokens = true))] + fn get_vocab(&self, py: Python<'_>, with_added_tokens: bool) -> PyResult> { + let tokenizer = self.huggingface()?; + Ok(release_gil(py, || tokenizer.vocab(with_added_tokens))) + } + + #[cfg(feature = "huggingface")] + #[pyo3(signature = (with_added_tokens = true))] + fn get_vocab_size(&self, with_added_tokens: bool) -> PyResult { + Ok(self.huggingface()?.vocab_size(with_added_tokens)) + } + + /// The added tokens by id as `(id, (content, single_word, lstrip, rstrip, normalized, + /// special))`, for Python to rebuild as `tokenizers.AddedToken`. + #[cfg(feature = "huggingface")] + fn added_tokens_decoder(&self) -> PyResult> { + Ok(self + .huggingface()? + .added_tokens_decoder() + .into_iter() + .map(|(id, token)| { + ( + id, + ( + token.content, + token.single_word, + token.lstrip, + token.rstrip, + token.normalized, + token.special, + ), + ) + }) + .collect()) + } + + /// The padding parameters as `tokenizers.Tokenizer.padding` reports them. + #[cfg(feature = "huggingface")] + fn padding<'py>(&self, py: Python<'py>) -> PyResult>> { + let Some(params) = self.huggingface()?.padding() else { + return Ok(None); + }; + let padding = PyDict::new(py); + padding.set_item( + "length", + match params.strategy { + PaddingStrategy::BatchLongest => None, + PaddingStrategy::Fixed(length) => Some(length), + }, + )?; + padding.set_item("pad_to_multiple_of", params.pad_to_multiple_of)?; + padding.set_item("pad_id", params.pad_id)?; + padding.set_item("pad_type_id", params.pad_type_id)?; + padding.set_item("pad_token", ¶ms.pad_token)?; + padding.set_item("direction", params.direction.as_ref())?; + Ok(Some(padding)) + } + + /// The truncation parameters as `tokenizers.Tokenizer.truncation` reports them. + #[cfg(feature = "huggingface")] + fn truncation<'py>(&self, py: Python<'py>) -> PyResult>> { + let Some(params) = self.huggingface()?.truncation() else { + return Ok(None); + }; + let truncation = PyDict::new(py); + truncation.set_item("max_length", params.max_length)?; + truncation.set_item("stride", params.stride)?; + truncation.set_item("strategy", params.strategy.as_ref())?; + truncation.set_item("direction", params.direction.as_ref())?; + Ok(Some(truncation)) + } + + #[cfg(feature = "huggingface")] + fn num_special_tokens_to_add(&self, is_pair: bool) -> PyResult { + Ok(self.huggingface()?.num_special_tokens_to_add(is_pair)) + } + + #[cfg(feature = "huggingface")] + fn encode_special_tokens(&self) -> PyResult { + Ok(self.huggingface()?.encode_special_tokens()) + } +} + +#[cfg(feature = "huggingface")] +type AddedTokenFields = (String, bool, bool, bool, bool, bool); + +impl Tokenizer { + fn new(inner: Codec) -> Self { + Self { + inner: Arc::new(inner), + #[cfg(feature = "fast")] + fast: OnceLock::new(), + } + } + + pub(crate) fn counter(&self, py: Python<'_>, fast: bool) -> SharedCounter { + #[cfg(feature = "fast")] + if fast { + let counter = self.fast.get().unwrap_or_else(|| { + release_gil(py, || { + self.fast + .get_or_init(|| self.inner.fast_counter().map(Arc::new)) + }) + }); + if let Some(counter) = counter { + return SharedCounter::Fast(Arc::clone(counter)); + } + } + #[cfg(not(feature = "fast"))] + let _ = (py, fast); + SharedCounter::Codec(Arc::clone(&self.inner)) + } + + /// A Python `str` as UTF-8. tiktoken replaces lone surrogates the way its Python `encode` + /// does; `tokenizers` rejects them, so that backend keeps the encode error. + fn text<'a>(&self, text: &'a Bound<'_, PyString>) -> PyResult> { + match text.to_cow() { + Ok(text) => Ok(text), + Err(error) => match *self.inner { + #[cfg(feature = "tiktoken")] + Codec::Tiktoken(_) if error.is_instance_of::(text.py()) => { + text.call_method1("encode", ("utf-16", "surrogatepass"))? + .call_method1("decode", ("utf-16", "replace"))? + .extract::() + .map(Cow::Owned) + } + _ => Err(error), + }, + } + } + + #[cfg(feature = "tiktoken")] + fn tiktoken(&self) -> PyResult<&TiktokenTokenizer> { + match *self.inner { + Codec::Tiktoken(ref tokenizer) => Ok(tokenizer), + #[cfg(feature = "huggingface")] + Codec::HuggingFace(_) => Err(PyValueError::new_err("requires a tiktoken encoding")), + } + } + + #[cfg(feature = "tiktoken")] + fn vocabulary(&self) -> PyResult<&Vocabulary> { + self.tiktoken()?.vocabulary().ok_or_else(|| { + PyRuntimeError::new_err("this encoding was built without its vocabulary") + }) + } + + #[cfg(feature = "huggingface")] + fn huggingface(&self) -> PyResult<&HuggingFaceTokenizer> { + match *self.inner { + Codec::HuggingFace(ref tokenizer) => Ok(tokenizer), + #[cfg(feature = "tiktoken")] + Codec::Tiktoken(_) => Err(PyValueError::new_err("requires a Hugging Face tokenizer")), + } + } +} + +pub(crate) enum SharedCounter { + Codec(Arc), + #[cfg(feature = "fast")] + Fast(Arc), +} + +impl litellm_token_counter::Tokenizer for SharedCounter { + fn count_tokens(&self, text: &str) -> Result { + match self { + Self::Codec(codec) => codec.codec().count_tokens(text), + #[cfg(feature = "fast")] + Self::Fast(counter) => counter.count_tokens(text).map_err(Error::from), + } + } +} + +#[cfg(feature = "huggingface")] +#[derive(FromPyObject)] +pub(crate) enum Sequence { + Text(String), + Words(Vec), +} + +#[cfg(feature = "huggingface")] +impl Sequence { + fn input(self, is_pretokenized: bool) -> PyResult> { + match (self, is_pretokenized) { + (Self::Text(text), false) => Ok(text.into()), + (Self::Words(words), true) => Ok(words.into()), + _ => Err(pyo3::exceptions::PyTypeError::new_err( + "input must match is_pretokenized", + )), + } + } +} + +#[cfg(feature = "huggingface")] +fn direction(value: &str, left: T, right: T, what: &str) -> PyResult { + match value { + "left" => Ok(left), + "right" => Ok(right), + other => Err(PyValueError::new_err(format!( + "invalid {what} direction {other:?}: expected 'left' or 'right'" + ))), + } +} + +/// `tokenizers.Encoding`, mutable like the original: `pad`, `truncate` and `set_sequence_id` +/// change it in place. +#[cfg(feature = "huggingface")] +#[pyclass(module = "litellm.rust_bridge._native")] +pub(crate) struct HuggingFaceEncoding { + inner: Encoding, +} + +#[cfg(feature = "huggingface")] +#[pymethods] +impl HuggingFaceEncoding { + #[new] + #[pyo3(signature = (json = None))] + fn new(json: Option<&str>) -> PyResult { + let inner = match json { + Some(json) => encoding_from_json(json) + .map_err(|error| PyValueError::new_err(error.to_string()))?, + None => Encoding::default(), + }; + Ok(Self { inner }) + } + + #[staticmethod] + #[pyo3(signature = (encodings, growing_offsets = true))] + fn merge(encodings: Vec>, growing_offsets: bool) -> Self { + Self { + inner: Encoding::merge( + encodings.iter().map(|encoding| encoding.inner.clone()), + growing_offsets, + ), + } + } + + fn __reduce__<'py>( + &self, + py: Python<'py>, + ) -> PyResult<(Bound<'py, pyo3::types::PyType>, (String,))> { + let json = encoding_to_json(&self.inner) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + Ok((py.get_type::(), (json,))) + } + + fn __repr__(&self) -> String { + format!( + "Encoding(num_tokens={}, attributes=[ids, type_ids, tokens, offsets, \ + attention_mask, special_tokens_mask, overflowing])", + self.inner.len() + ) + } + + fn __len__(&self) -> usize { + self.inner.len() + } + #[getter] + fn ids(&self) -> Vec { + self.inner.get_ids().to_vec() + } + #[getter] + fn tokens(&self) -> Vec { + self.inner.get_tokens().to_vec() + } + #[getter] + fn offsets(&self) -> Vec<(usize, usize)> { + self.inner.get_offsets().to_vec() + } + #[getter] + fn type_ids(&self) -> Vec { + self.inner.get_type_ids().to_vec() + } + #[getter] + fn attention_mask(&self) -> Vec { + self.inner.get_attention_mask().to_vec() + } + #[getter] + fn special_tokens_mask(&self) -> Vec { + self.inner.get_special_tokens_mask().to_vec() + } + #[getter] + fn word_ids(&self) -> Vec> { + self.inner.get_word_ids().to_vec() + } + #[getter] + fn sequence_ids(&self) -> Vec> { + self.inner.get_sequence_ids() + } + #[getter] + fn overflowing(&self) -> Vec { + self.inner + .get_overflowing() + .iter() + .cloned() + .map(|inner| Self { inner }) + .collect() + } + #[getter] + fn n_sequences(&self) -> usize { + self.inner.n_sequences() + } + + #[pyo3(signature = (word_index, sequence_index = 0))] + fn word_to_tokens(&self, word_index: u32, sequence_index: usize) -> Option<(usize, usize)> { + self.inner.word_to_tokens(word_index, sequence_index) + } + #[pyo3(signature = (word_index, sequence_index = 0))] + fn word_to_chars(&self, word_index: u32, sequence_index: usize) -> Option<(usize, usize)> { + self.inner.word_to_chars(word_index, sequence_index) + } + fn token_to_sequence(&self, token_index: usize) -> Option { + self.inner.token_to_sequence(token_index) + } + fn token_to_chars(&self, token_index: usize) -> Option<(usize, usize)> { + self.inner + .token_to_chars(token_index) + .map(|(_, offsets)| offsets) + } + fn token_to_word(&self, token_index: usize) -> Option { + self.inner.token_to_word(token_index).map(|(_, word)| word) + } + #[pyo3(signature = (char_pos, sequence_index = 0))] + fn char_to_token(&self, char_pos: usize, sequence_index: usize) -> Option { + self.inner.char_to_token(char_pos, sequence_index) + } + #[pyo3(signature = (char_pos, sequence_index = 0))] + fn char_to_word(&self, char_pos: usize, sequence_index: usize) -> Option { + self.inner.char_to_word(char_pos, sequence_index) + } + + fn set_sequence_id(&mut self, sequence_id: usize) { + self.inner.set_sequence_id(sequence_id); + } + + #[pyo3(signature = (length, direction = "right", pad_id = 0, pad_type_id = 0, pad_token = "[PAD]"))] + fn pad( + &mut self, + length: usize, + direction: &str, + pad_id: u32, + pad_type_id: u32, + pad_token: &str, + ) -> PyResult<()> { + let direction = self::direction( + direction, + PaddingDirection::Left, + PaddingDirection::Right, + "padding", + )?; + self.inner + .pad(length, pad_id, pad_type_id, pad_token, direction); + Ok(()) + } + + #[pyo3(signature = (max_length, stride = 0, direction = "right"))] + fn truncate(&mut self, max_length: usize, stride: usize, direction: &str) -> PyResult<()> { + let direction = self::direction( + direction, + TruncationDirection::Left, + TruncationDirection::Right, + "truncation", + )?; + self.inner.truncate(max_length, stride, direction); + Ok(()) + } +} diff --git a/litellm-rust/crates/token-counter-fast/src/lib.rs b/litellm-rust/crates/token-counter-fast/src/lib.rs index ce91af642ea..157ee847658 100644 --- a/litellm-rust/crates/token-counter-fast/src/lib.rs +++ b/litellm-rust/crates/token-counter-fast/src/lib.rs @@ -8,6 +8,8 @@ mod scanner; mod tiktoken; mod unicode_classes; +use std::sync::Arc; + use byte_level::ByteLevelCounter; use scanner::{SplitPattern, TiktokenCounter}; @@ -15,22 +17,29 @@ pub use error::Error; enum Encoder { HuggingFace { - tokenizer: Box, + tokenizer: Arc, byte_level: Option, }, Tiktoken(TiktokenCounter), } +/// A count-only tokenizer. Its model tables are immutable, so one built from an already +/// loaded model (`from_shared`, `from_*_pairs`) adds only the count-specific tables. pub struct FastTokenizer(Encoder); impl FastTokenizer { pub fn from_json(json: &str) -> Result { let tokenizer = json.parse::().map_err(Error::Load)?; + Ok(Self::from_shared(Arc::new(tokenizer))) + } + + /// Counts with a Hugging Face model another codec already holds; nothing is re-parsed. + pub fn from_shared(tokenizer: Arc) -> Self { let byte_level = ByteLevelCounter::detect(&tokenizer); - Ok(Self(Encoder::HuggingFace { - tokenizer: Box::new(tokenizer), + Self(Encoder::HuggingFace { + tokenizer, byte_level, - })) + }) } pub fn from_cl100k_ranks(ranks: &str) -> Result { @@ -41,12 +50,36 @@ impl FastTokenizer { Self::from_ranks(SplitPattern::O200k, ranks) } + /// `cl100k_base` from ranks another loader already parsed. + pub fn from_cl100k_pairs<'a>( + pairs: impl IntoIterator, + ) -> Result { + Self::from_pairs(SplitPattern::Cl100k, pairs) + } + + /// `o200k_base` (and `o200k_harmony`, whose ordinary tokens are the same) from ranks + /// another loader already parsed. + pub fn from_o200k_pairs<'a>( + pairs: impl IntoIterator, + ) -> Result { + Self::from_pairs(SplitPattern::O200k, pairs) + } + fn from_ranks(split: SplitPattern, ranks: &str) -> Result { TiktokenCounter::from_ranks(split, ranks) .map(Encoder::Tiktoken) .map(Self) } + fn from_pairs<'a>( + split: SplitPattern, + pairs: impl IntoIterator, + ) -> Result { + TiktokenCounter::from_pairs(split, pairs) + .map(Encoder::Tiktoken) + .map(Self) + } + pub fn count_tokens(&self, text: &str) -> Result { match &self.0 { Encoder::Tiktoken(counter) => Ok(counter.count(text)), diff --git a/litellm-rust/crates/token-counter-fast/src/scanner.rs b/litellm-rust/crates/token-counter-fast/src/scanner.rs index c2c3057aeeb..882ea91db81 100644 --- a/litellm-rust/crates/token-counter-fast/src/scanner.rs +++ b/litellm-rust/crates/token-counter-fast/src/scanner.rs @@ -39,8 +39,19 @@ pub(super) struct TiktokenCounter { impl TiktokenCounter { pub(super) fn from_ranks(split: SplitPattern, rank_file: &str) -> Result { + Self::new(split, MergeRanks::parse(rank_file)?) + } + + pub(super) fn from_pairs<'a>( + split: SplitPattern, + pairs: impl IntoIterator, + ) -> Result { + Self::new(split, MergeRanks::from_pairs(pairs)?) + } + + fn new(split: SplitPattern, ranks: MergeRanks) -> Result { Ok(Self { - ranks: MergeRanks::parse(rank_file)?, + ranks, piece_len: split.piece_len(), unicode_classes: UnicodeClasses::get().ok_or(Error::UnicodeClasses)?, }) diff --git a/litellm-rust/crates/token-counter-fast/src/tiktoken.rs b/litellm-rust/crates/token-counter-fast/src/tiktoken.rs index 16172b7a688..68f09b14a25 100644 --- a/litellm-rust/crates/token-counter-fast/src/tiktoken.rs +++ b/litellm-rust/crates/token-counter-fast/src/tiktoken.rs @@ -22,11 +22,25 @@ pub(super) struct MergeRanks(FxHashMap, Rank>); impl MergeRanks { pub(super) fn parse(text: &str) -> Result { - let ranks = text - .lines() - .filter(|line| !line.is_empty()) - .map(parse_line) - .collect::, _>>()?; + Self::from_entries(text.lines().filter(|line| !line.is_empty()).map(parse_line)) + } + + /// The same table from ranks another loader already parsed. + pub(super) fn from_pairs<'a>( + pairs: impl IntoIterator, + ) -> Result { + Self::from_entries(pairs.into_iter().map(|(bytes, rank)| { + if rank == NO_RANK { + return Err(Error::Ranks(format!("rank {rank} is reserved"))); + } + Ok((Box::from(bytes), rank)) + })) + } + + fn from_entries( + entries: impl Iterator, Rank), Error>>, + ) -> Result { + let ranks = entries.collect::, _>>()?; if let Some(byte) = (0..=u8::MAX).find(|byte| !ranks.contains_key(&[*byte][..])) { return Err(Error::Ranks(format!("byte 0x{byte:02X} has no token"))); } diff --git a/litellm-rust/crates/token-counter-huggingface/Cargo.toml b/litellm-rust/crates/token-counter-huggingface/Cargo.toml index 6d8cb85e524..a5c2b1bb160 100644 --- a/litellm-rust/crates/token-counter-huggingface/Cargo.toml +++ b/litellm-rust/crates/token-counter-huggingface/Cargo.toml @@ -6,5 +6,6 @@ license.workspace = true repository.workspace = true [dependencies] +serde_json.workspace = true thiserror.workspace = true tokenizers.workspace = true diff --git a/litellm-rust/crates/token-counter-huggingface/src/error.rs b/litellm-rust/crates/token-counter-huggingface/src/error.rs index adc4551886f..e7f6260321b 100644 --- a/litellm-rust/crates/token-counter-huggingface/src/error.rs +++ b/litellm-rust/crates/token-counter-huggingface/src/error.rs @@ -6,4 +6,6 @@ pub enum Error { Load(#[source] tokenizers::Error), #[error("tokenization failed: {0}")] Encode(#[source] tokenizers::Error), + #[error("token decoding failed: {0}")] + Decode(#[source] tokenizers::Error), } diff --git a/litellm-rust/crates/token-counter-huggingface/src/lib.rs b/litellm-rust/crates/token-counter-huggingface/src/lib.rs index 8e05c2cca46..170a36aea05 100644 --- a/litellm-rust/crates/token-counter-huggingface/src/lib.rs +++ b/litellm-rust/crates/token-counter-huggingface/src/lib.rs @@ -2,22 +2,243 @@ mod error; -pub use error::Error; +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; -pub struct HuggingFaceTokenizer(Box); +pub use error::Error; +use tokenizers::PostProcessor; +pub use tokenizers::{ + AddedToken, EncodeInput, Encoding, InputSequence, PaddingDirection, PaddingParams, + PaddingStrategy, TruncationDirection, TruncationParams, +}; + +pub fn encoding_from_json(json: &str) -> Result { + serde_json::from_str(json).map_err(|error| Error::Load(error.into())) +} + +pub fn encoding_to_json(encoding: &Encoding) -> Result { + serde_json::to_string(encoding).map_err(|error| Error::Load(error.into())) +} + +pub struct HuggingFaceTokenizer { + tokenizer: Arc, + special_token_ids: HashSet, +} impl HuggingFaceTokenizer { pub fn from_json(json: &str) -> Result { json.parse::() - .map(Box::new) - .map(Self) + .map(Self::new) .map_err(Error::Load) } + fn new(tokenizer: tokenizers::Tokenizer) -> Self { + let special_token_ids: HashSet = tokenizer + .get_added_tokens_decoder() + .into_iter() + .filter_map(|(id, token)| token.special.then_some(id)) + .collect(); + Self { + tokenizer: Arc::new(tokenizer), + special_token_ids, + } + } + + /// The parsed model, for a count-only counter to share instead of parsing it again. + pub fn shared(&self) -> Arc { + Arc::clone(&self.tokenizer) + } + pub fn count_tokens(&self, text: &str) -> Result { - self.0 + self.tokenizer .encode_fast(text, true) .map(|encoding| encoding.len()) .map_err(Error::Encode) } + + pub fn encode(&self, text: &str) -> Result, Error> { + self.tokenizer + .encode_fast(text, true) + .map(|encoding| encoding.get_ids().to_vec()) + .map_err(Error::Encode) + } + + pub fn encode_result<'a>( + &self, + input: EncodeInput<'a>, + add_special_tokens: bool, + fast: bool, + ) -> Result { + if fast { + return self + .tokenizer + .encode_fast(input, add_special_tokens) + .map_err(Error::Encode); + } + self.tokenizer + .encode_char_offsets(input, add_special_tokens) + .map_err(Error::Encode) + } + + pub fn encode_batch_result<'a>( + &self, + inputs: Vec>, + add_special_tokens: bool, + fast: bool, + ) -> Result, Error> { + if fast { + return self + .tokenizer + .encode_batch_fast(inputs, add_special_tokens) + .map_err(Error::Encode); + } + self.tokenizer + .encode_batch_char_offsets(inputs, add_special_tokens) + .map_err(Error::Encode) + } + + pub fn to_json(&self, pretty: bool) -> Result { + self.tokenizer.to_string(pretty).map_err(Error::Load) + } + + pub fn token_to_id(&self, token: &str) -> Option { + self.tokenizer.token_to_id(token) + } + + pub fn id_to_token(&self, id: u32) -> Option { + self.tokenizer.id_to_token(id) + } + + pub fn vocab(&self, with_added_tokens: bool) -> HashMap { + self.tokenizer.get_vocab(with_added_tokens) + } + + pub fn vocab_size(&self, with_added_tokens: bool) -> usize { + self.tokenizer.get_vocab_size(with_added_tokens) + } + + /// The added tokens by id, in id order. + pub fn added_tokens_decoder(&self) -> Vec<(u32, AddedToken)> { + let mut added: Vec<(u32, AddedToken)> = self + .tokenizer + .get_added_tokens_decoder() + .into_iter() + .collect(); + added.sort_unstable_by_key(|(id, _)| *id); + added + } + + pub fn padding(&self) -> Option<&PaddingParams> { + self.tokenizer.get_padding() + } + + pub fn truncation(&self) -> Option<&TruncationParams> { + self.tokenizer.get_truncation() + } + + /// How many special tokens the post-processor adds to a single sequence or a pair. + pub fn num_special_tokens_to_add(&self, is_pair: bool) -> usize { + self.tokenizer + .get_post_processor() + .map_or(0, |processor| processor.added_tokens(is_pair)) + } + + pub fn encode_special_tokens(&self) -> bool { + self.tokenizer.get_encode_special_tokens() + } + + pub fn decode(&self, ids: &[u32], skip_special_tokens: bool) -> Result { + if !skip_special_tokens { + return self.tokenizer.decode(ids, false).map_err(Error::Decode); + } + let filtered_ids: Vec = ids + .iter() + .copied() + .filter(|id| !self.special_token_ids.contains(id)) + .collect(); + self.tokenizer + .decode(&filtered_ids, true) + .map_err(Error::Decode) + } + + pub fn name(&self) -> &str { + "huggingface" + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn codecs_round_trip_and_skip_special_tokens() { + let json = include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../../litellm/litellm_core_utils/tokenizers/anthropic_tokenizer.json" + )); + let tokenizer = HuggingFaceTokenizer::from_json(json).unwrap(); + let ids = tokenizer.encode("hello").unwrap(); + + assert!(tokenizer.decode(&ids, false).unwrap().contains("")); + assert_eq!(tokenizer.decode(&ids, true).unwrap(), "hello"); + } + + #[test] + fn decode_filters_special_added_tokens() { + let json = r#"{ + "version": "1.0", + "truncation": null, + "padding": null, + "added_tokens": [ + { + "id": 1, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + } + ], + "normalizer": null, + "pre_tokenizer": {"type": "Whitespace"}, + "post_processor": null, + "decoder": null, + "model": { + "type": "WordLevel", + "vocab": {"": 0, "": 1, "hello": 2}, + "unk_token": "" + } + }"#; + let tokenizer = HuggingFaceTokenizer::from_json(json).unwrap(); + + assert!(!tokenizer.decode(&[1, 2], true).unwrap().contains("")); + assert!(tokenizer.decode(&[1, 2], false).unwrap().contains("")); + } + + #[test] + fn vocabulary_lookups_mirror_the_tokenizers_api() { + let json = include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../../litellm/litellm_core_utils/tokenizers/anthropic_tokenizer.json" + )); + let tokenizer = HuggingFaceTokenizer::from_json(json).unwrap(); + let ids = tokenizer.encode("hello").unwrap(); + + let token = tokenizer.id_to_token(ids[0]).unwrap(); + assert_eq!(tokenizer.token_to_id(&token), Some(ids[0])); + assert_eq!(tokenizer.id_to_token(u32::MAX), None); + assert_eq!(tokenizer.vocab(true).len(), tokenizer.vocab_size(true)); + assert!(tokenizer.vocab_size(true) >= tokenizer.vocab_size(false)); + let added = tokenizer.added_tokens_decoder(); + assert!(added.windows(2).all(|pair| pair[0].0 < pair[1].0)); + assert!(added.iter().any(|(_, token)| token.special)); + assert!(tokenizer.padding().is_none()); + assert!(tokenizer.truncation().is_none()); + assert!(!tokenizer.encode_special_tokens()); + assert_eq!( + tokenizer.num_special_tokens_to_add(false), + tokenizer.encode("").unwrap().len() + ); + } } diff --git a/litellm-rust/crates/token-counter-tiktoken/Cargo.toml b/litellm-rust/crates/token-counter-tiktoken/Cargo.toml index 494a9233e69..2fb3103e0c8 100644 --- a/litellm-rust/crates/token-counter-tiktoken/Cargo.toml +++ b/litellm-rust/crates/token-counter-tiktoken/Cargo.toml @@ -6,5 +6,8 @@ license.workspace = true repository.workspace = true [dependencies] +base64.workspace = true +once_cell = "1.21.3" +rustc-hash = "2.1.3" thiserror.workspace = true tiktoken-rs.workspace = true diff --git a/litellm-rust/crates/token-counter-tiktoken/src/lib.rs b/litellm-rust/crates/token-counter-tiktoken/src/lib.rs index ecdb3946eee..f049e90a1cb 100644 --- a/litellm-rust/crates/token-counter-tiktoken/src/lib.rs +++ b/litellm-rust/crates/token-counter-tiktoken/src/lib.rs @@ -1,27 +1,125 @@ #![forbid(unsafe_code)] mod error; +mod ranks; + +use std::collections::HashSet; pub use error::UnsupportedTokenizer; +pub use ranks::{LoadError, Vocabulary}; -pub struct TiktokenTokenizer(&'static tiktoken_rs::CoreBPE); +pub struct TiktokenTokenizer { + encoder: &'static tiktoken_rs::CoreBPE, + /// Present for encodings built from a rank file; the embedded tiktoken-rs singletons + /// behind [`from_name`](Self::from_name) keep their ranks private. + vocabulary: Option<&'static Vocabulary>, + name: &'static str, +} impl TiktokenTokenizer { + /// Builds `name` from its packaged rank file (read through `load`), once per process. + /// The tokenizer reports the requested name, so `gpt2` stays `gpt2` like tiktoken does. + pub fn from_cached_ranks( + name: &str, + load: impl FnOnce(&str) -> std::io::Result, + ) -> Result { + let (loaded, name) = ranks::load(name, load)?; + Ok(Self { + encoder: &loaded.bpe, + vocabulary: Some(&loaded.vocabulary), + name, + }) + } + + /// The encodings tiktoken-rs embeds, for hosts without the packaged rank files. pub fn from_name(name: &str) -> Result { - let tokenizer = match name { - "cl100k_base" => tiktoken_rs::cl100k_base_singleton(), - "o200k_base" => tiktoken_rs::o200k_base_singleton(), - "o200k_harmony" => tiktoken_rs::o200k_harmony_singleton(), - "p50k_base" => tiktoken_rs::p50k_base_singleton(), - "p50k_edit" => tiktoken_rs::p50k_edit_singleton(), - "r50k_base" | "gpt2" => tiktoken_rs::r50k_base_singleton(), + let (encoder, name) = match name { + "cl100k_base" => (tiktoken_rs::cl100k_base_singleton(), "cl100k_base"), + "o200k_base" => (tiktoken_rs::o200k_base_singleton(), "o200k_base"), + "o200k_harmony" => (tiktoken_rs::o200k_harmony_singleton(), "o200k_harmony"), + "p50k_base" => (tiktoken_rs::p50k_base_singleton(), "p50k_base"), + "p50k_edit" => (tiktoken_rs::p50k_edit_singleton(), "p50k_edit"), + "r50k_base" => (tiktoken_rs::r50k_base_singleton(), "r50k_base"), + "gpt2" => (tiktoken_rs::r50k_base_singleton(), "gpt2"), _ => return Err(UnsupportedTokenizer(name.to_owned())), }; - Ok(Self(tokenizer)) + Ok(Self { + encoder, + vocabulary: None, + name, + }) + } + + pub fn vocabulary(&self) -> Option<&Vocabulary> { + self.vocabulary } pub fn count_tokens(&self, text: &str) -> usize { - self.0.count_ordinary(text) + self.encoder.count_ordinary(text) + } + + pub fn encode(&self, text: &str) -> Vec { + self.encoder.encode_ordinary(text) + } + + pub fn encode_special(&self, text: &str, allowed: &[String]) -> Result, String> { + let allowed = allowed.iter().map(String::as_str).collect(); + self.encoder + .encode(text, &allowed) + .map(|(ids, _)| ids) + .map_err(|error| error.to_string()) + } + + pub fn special_tokens(&self) -> HashSet { + self.encoder + .special_tokens() + .into_iter() + .map(str::to_owned) + .collect() + } + + /// tiktoken's `encode_with_unstable`: the stable prefix of `text`'s tokens and every + /// token sequence the unstable tail could still become, sorted for a stable order. + pub fn encode_with_unstable( + &self, + text: &str, + allowed: &[String], + ) -> (Vec, Vec>) { + let allowed = allowed.iter().map(String::as_str).collect(); + let (stable, completions) = self.encoder._encode_unstable_native(text, &allowed); + let mut completions: Vec> = completions.into_iter().collect(); + completions.sort_unstable(); + (stable, completions) + } + + pub fn decode_bytes(&self, ids: &[u32]) -> Result, String> { + self.encoder + .decode_bytes(ids) + .map_err(|error| error.to_string()) + } + + pub fn decode(&self, ids: &[u32]) -> Result { + self.encoder + .decode_bytes(ids) + .map(|bytes| String::from_utf8_lossy(&bytes).into_owned()) + .map_err(|error| error.to_string()) + } + + pub fn name(&self) -> &str { + self.name + } +} + +pub fn encoding_for_model(model: &str) -> Option<&'static str> { + match tiktoken_rs::tokenizer::get_tokenizer(model)? { + tiktoken_rs::tokenizer::Tokenizer::Cl100kBase => Some("cl100k_base"), + tiktoken_rs::tokenizer::Tokenizer::O200kBase => Some("o200k_base"), + tiktoken_rs::tokenizer::Tokenizer::O200kHarmony => Some("o200k_harmony"), + tiktoken_rs::tokenizer::Tokenizer::P50kBase => Some("p50k_base"), + tiktoken_rs::tokenizer::Tokenizer::P50kEdit => Some("p50k_edit"), + tiktoken_rs::tokenizer::Tokenizer::R50kBase | tiktoken_rs::tokenizer::Tokenizer::Gpt2 => { + Some("r50k_base") + } } } @@ -66,5 +164,75 @@ mod tests { panic!("unknown encoding must be rejected"); }; assert_eq!(name, "unknown-encoding"); + assert_eq!(TiktokenTokenizer::from_name("gpt2").unwrap().name(), "gpt2"); + } + + #[test] + fn codecs_round_trip_named_encodings() { + let encodings = [ + "cl100k_base", + "o200k_base", + "o200k_harmony", + "p50k_base", + "p50k_edit", + "r50k_base", + "gpt2", + ]; + let texts = ["hello world", "café 漢字 مرحبا 🙂", "line one\nline two"]; + for name in encodings { + let tokenizer = TiktokenTokenizer::from_name(name).unwrap(); + for text in texts { + assert_eq!( + tokenizer.decode(&tokenizer.encode(text)).unwrap(), + text, + "{name}: {text:?}", + ); + } + } + } + + #[test] + fn decoding_token_prefixes_replaces_incomplete_utf8() { + let tokenizer = TiktokenTokenizer::from_name("cl100k_base").unwrap(); + let reference = tiktoken_rs::cl100k_base_singleton(); + let ids = tokenizer.encode("🙂漢字"); + for end in 1..=ids.len() { + let bytes = reference.decode_bytes(&ids[..end]).unwrap(); + assert_eq!( + tokenizer.decode(&ids[..end]).unwrap(), + String::from_utf8_lossy(&bytes), + ); + } + assert!(tokenizer.decode(&[u32::MAX]).is_err()); + } + + #[test] + fn unstable_encoding_prefixes_stay_consistent_with_full_encoding() { + let tokenizer = TiktokenTokenizer::from_name("cl100k_base").unwrap(); + let text = "hello fanta"; + let (stable, completions) = tokenizer.encode_with_unstable(text, &[]); + assert!( + text.as_bytes() + .starts_with(&tokenizer.decode_bytes(&stable).unwrap()) + ); + assert!(!completions.is_empty()); + for completion in &completions { + let mut ids = stable.clone(); + ids.extend(completion); + assert!( + tokenizer + .decode_bytes(&ids) + .unwrap() + .starts_with(text.as_bytes()) + ); + } + assert!(completions.windows(2).all(|pair| pair[0] < pair[1])); + } + + #[test] + fn encoding_for_model_maps_known_models() { + assert_eq!(encoding_for_model("gpt-4o"), Some("o200k_base")); + assert_eq!(encoding_for_model("text-davinci-003"), Some("p50k_base")); + assert_eq!(encoding_for_model("unknown-model"), None); } } diff --git a/litellm-rust/crates/token-counter-tiktoken/src/ranks.rs b/litellm-rust/crates/token-counter-tiktoken/src/ranks.rs new file mode 100644 index 00000000000..1f5e5262de5 --- /dev/null +++ b/litellm-rust/crates/token-counter-tiktoken/src/ranks.rs @@ -0,0 +1,340 @@ +use base64::{Engine, engine::general_purpose::STANDARD}; +use once_cell::sync::OnceCell; +use rustc_hash::FxHashMap; +use thiserror::Error; +use tiktoken_rs::{CoreBPE, O200K_BASE_PAT_STR, Rank}; + +use crate::UnsupportedTokenizer; + +const CL100K: &str = "9b5ad71b2ce5302211f9c61530b329a4922fc6a4"; +const O200K: &str = "fb374d419588a4632f3f557e76b4b70aebbca790"; +const P50K: &str = "ec7223a39ce59f226a68acc30dc1af2788490e15"; +const LEGACY_PATTERN: &str = + r"'(?:[sdmt]|ll|ve|re)| ?\p{L}++| ?\p{N}++| ?[^\s\p{L}\p{N}]++|\s++$|\s+(?!\S)|\s"; +const CL100K_PATTERN: &str = r"'(?i:[sdmt]|ll|ve|re)|[^\r\n\p{L}\p{N}]?+\p{L}++|\p{N}{1,3}+| ?[^\s\p{L}\p{N}]++[\r\n]*+|\s++$|\s*[\r\n]|\s+(?!\S)|\s"; + +static CL100K_ENCODER: OnceCell = OnceCell::new(); +static O200K_ENCODER: OnceCell = OnceCell::new(); +static HARMONY_ENCODER: OnceCell = OnceCell::new(); +static P50K_ENCODER: OnceCell = OnceCell::new(); +static EDIT_ENCODER: OnceCell = OnceCell::new(); +static R50K_ENCODER: OnceCell = OnceCell::new(); + +/// One encoding built from a rank file: the BPE engine plus the vocabulary it was built +/// from, kept because `CoreBPE` does not expose its ranks and tiktoken's Python API does +/// (`token_byte_values`, `encode_single_token`, `max_token_value`, `_special_tokens`). +pub(super) struct Loaded { + pub(super) bpe: CoreBPE, + pub(super) vocabulary: Vocabulary, +} + +/// The byte-level vocabulary of a tiktoken encoding. +pub struct Vocabulary { + ranks: FxHashMap, Rank>, + special_tokens: FxHashMap, + max_token_value: Rank, +} + +impl Vocabulary { + /// Every mergeable token's bytes, sorted bytewise like tiktoken's `token_byte_values`. + pub fn token_byte_values(&self) -> Vec> { + let mut values: Vec> = self.ranks.keys().cloned().collect(); + values.sort_unstable(); + values + } + + /// The rank of one whole token: a mergeable piece first, then a special token's text. + pub fn encode_single_token(&self, piece: &[u8]) -> Option { + if let Some(rank) = self.ranks.get(piece) { + return Some(*rank); + } + std::str::from_utf8(piece) + .ok() + .and_then(|text| self.special_tokens.get(text).copied()) + } + + pub fn max_token_value(&self) -> Rank { + self.max_token_value + } + + /// Every mergeable token with its rank, for building other tables from one parse. + pub fn ranks(&self) -> impl Iterator + '_ { + self.ranks + .iter() + .map(|(bytes, rank)| (bytes.as_slice(), *rank)) + } + + /// The special tokens with their ranks, tiktoken's `_special_tokens`. + pub fn special_tokens(&self) -> impl Iterator + '_ { + self.special_tokens + .iter() + .map(|(token, rank)| (token.as_str(), *rank)) + } + + pub fn is_special_token(&self, rank: Rank) -> bool { + self.special_tokens.values().any(|special| *special == rank) + } +} + +#[derive(Debug, Error)] +pub enum LoadError { + #[error(transparent)] + Unsupported(#[from] UnsupportedTokenizer), + #[error("failed to load tiktoken ranks: {0}")] + Ranks(String), +} + +/// Loads `name` once per process. The returned name is the one requested (`gpt2` stays +/// `gpt2`, as `tiktoken.get_encoding("gpt2").name` does), while `gpt2` and `r50k_base` share +/// one cached encoder. +pub(super) fn load( + name: &str, + load_file: impl FnOnce(&str) -> std::io::Result, +) -> Result<(&'static Loaded, &'static str), LoadError> { + let (requested, canonical, file, cache) = match name { + "cl100k_base" => ("cl100k_base", "cl100k_base", CL100K, &CL100K_ENCODER), + "o200k_base" => ("o200k_base", "o200k_base", O200K, &O200K_ENCODER), + "o200k_harmony" => ("o200k_harmony", "o200k_harmony", O200K, &HARMONY_ENCODER), + "p50k_base" => ("p50k_base", "p50k_base", P50K, &P50K_ENCODER), + "p50k_edit" => ("p50k_edit", "p50k_edit", P50K, &EDIT_ENCODER), + "r50k_base" => ("r50k_base", "r50k_base", P50K, &R50K_ENCODER), + "gpt2" => ("gpt2", "r50k_base", P50K, &R50K_ENCODER), + _ => return Err(UnsupportedTokenizer(name.to_owned()).into()), + }; + let loaded = cache.get_or_try_init(|| { + let ranks = load_file(file).map_err(|error| LoadError::Ranks(error.to_string()))?; + build(canonical, &ranks) + })?; + Ok((loaded, requested)) +} + +fn build(name: &str, ranks: &str) -> Result { + let parsed = ranks + .lines() + .map(parse_rank) + .collect::, _>>()?; + let encoder: FxHashMap<_, _> = parsed + .into_iter() + .filter(|(_, rank)| name != "r50k_base" || *rank < 50256) + .collect(); + if encoder + .values() + .collect::>() + .len() + != encoder.len() + || (0..=u8::MAX).any(|byte| !encoder.contains_key(&[byte][..])) + { + return Err(LoadError::Ranks("invalid vocabulary ranks".into())); + } + let (pattern, specials): (&str, &[(&str, Rank)]) = match name { + "cl100k_base" => ( + CL100K_PATTERN, + &[ + ("<|endoftext|>", 100257), + ("<|fim_prefix|>", 100258), + ("<|fim_middle|>", 100259), + ("<|fim_suffix|>", 100260), + ("<|endofprompt|>", 100276), + ], + ), + "o200k_base" => ( + O200K_BASE_PAT_STR, + &[("<|endoftext|>", 199999), ("<|endofprompt|>", 200018)], + ), + "o200k_harmony" => ( + O200K_BASE_PAT_STR, + &[ + ("<|startoftext|>", 199998), + ("<|endoftext|>", 199999), + ("<|reserved_200000|>", 200000), + ("<|reserved_200001|>", 200001), + ("<|return|>", 200002), + ("<|constrain|>", 200003), + ("<|reserved_200004|>", 200004), + ("<|channel|>", 200005), + ("<|start|>", 200006), + ("<|end|>", 200007), + ("<|message|>", 200008), + ("<|reserved_200009|>", 200009), + ("<|reserved_200010|>", 200010), + ("<|reserved_200011|>", 200011), + ("<|call|>", 200012), + ], + ), + "p50k_edit" => ( + LEGACY_PATTERN, + &[ + ("<|endoftext|>", 50256), + ("<|fim_prefix|>", 50281), + ("<|fim_middle|>", 50282), + ("<|fim_suffix|>", 50283), + ], + ), + _ => (LEGACY_PATTERN, &[("<|endoftext|>", 50256)]), + }; + let reserved = (200013..=201087) + .filter(|_| name == "o200k_harmony") + .map(|rank| (format!("<|reserved_{rank}|>"), rank)); + let special_tokens: FxHashMap = specials + .iter() + .map(|(token, rank)| ((*token).to_owned(), *rank)) + .chain(reserved) + .collect(); + let max_token_value = encoder + .values() + .chain(special_tokens.values()) + .copied() + .max() + .ok_or_else(|| LoadError::Ranks("empty vocabulary".into()))?; + let bpe = CoreBPE::new(encoder.clone(), special_tokens.clone(), pattern) + .map_err(|error| LoadError::Ranks(error.to_string()))?; + Ok(Loaded { + bpe, + vocabulary: Vocabulary { + ranks: encoder, + special_tokens, + max_token_value, + }, + }) +} + +fn parse_rank(line: &str) -> Result<(Vec, Rank), LoadError> { + let (token, rank) = line + .split_once(' ') + .ok_or_else(|| LoadError::Ranks("missing rank".into()))?; + let bytes = STANDARD + .decode(token) + .map_err(|error| LoadError::Ranks(error.to_string()))?; + let rank = rank + .parse() + .map_err(|error: std::num::ParseIntError| LoadError::Ranks(error.to_string()))?; + Ok((bytes, rank)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TiktokenTokenizer; + + fn read_packaged_ranks(file: &str) -> std::io::Result { + std::fs::read_to_string( + std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../../litellm/litellm_core_utils/tokenizers") + .join(file), + ) + } + + #[test] + fn packaged_encodings_match_embedded_encodings_and_reuse_successful_loads() { + for name in [ + "cl100k_base", + "o200k_base", + "o200k_harmony", + "p50k_base", + "p50k_edit", + "r50k_base", + "gpt2", + ] { + if name != "gpt2" { + assert!( + TiktokenTokenizer::from_cached_ranks(name, |_| { + Err(std::io::Error::other("unreadable vocabulary")) + }) + .is_err() + ); + } + let loads = std::sync::atomic::AtomicUsize::new(0); + let barrier = std::sync::Barrier::new(4); + let encoders = std::thread::scope(|scope| { + let tasks: Vec<_> = (0..4) + .map(|_| { + scope.spawn(|| { + barrier.wait(); + TiktokenTokenizer::from_cached_ranks(name, |file| { + loads.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + read_packaged_ranks(file) + }) + .unwrap() + }) + }) + .collect(); + tasks + .into_iter() + .map(|task| task.join().unwrap()) + .collect::>() + }); + assert_eq!(loads.into_inner(), usize::from(name != "gpt2")); + let actual = &encoders[0]; + let expected = TiktokenTokenizer::from_name(name).unwrap(); + assert_eq!(actual.special_tokens(), expected.special_tokens()); + let specials: Vec<_> = expected.special_tokens().into_iter().collect(); + let special_text = specials.join(" "); + assert_eq!( + actual.encode_special(&special_text, &specials).unwrap(), + expected.encode_special(&special_text, &specials).unwrap() + ); + for text in [ + "", + "café 漢字 ع 🙂", + "a\r\nb\t ", + " hello 123456789", + &special_text, + ] { + let ids = expected.encode(text); + assert_eq!(actual.encode(text), ids, "{name}: {text:?}"); + assert_eq!(actual.count_tokens(text), ids.len(), "{name}: {text:?}"); + assert_eq!( + actual.decode_bytes(&ids).unwrap(), + expected.decode_bytes(&ids).unwrap() + ); + } + let cached = TiktokenTokenizer::from_cached_ranks(name, |_| { + panic!("reloaded cached vocabulary") + }) + .unwrap(); + assert_eq!(cached.encode("cached"), expected.encode("cached")); + assert_eq!(cached.name(), name); + assert!(expected.vocabulary().is_none()); + assert_vocabulary_lookups(name, actual); + } + } + + /// The token-level lookups tiktoken's Python `Encoding` exposes, checked against the + /// encoder itself and against the known vocabulary sizes. + fn assert_vocabulary_lookups(name: &str, tokenizer: &TiktokenTokenizer) { + let max_token_value = match name { + "cl100k_base" => 100_276, + "o200k_base" => 200_018, + "o200k_harmony" => 201_087, + "p50k_base" => 50_280, + "p50k_edit" => 50_283, + "r50k_base" | "gpt2" => 50_256, + _ => unreachable!("{name}"), + }; + let vocabulary = tokenizer.vocabulary().unwrap(); + assert_eq!(vocabulary.max_token_value(), max_token_value, "{name}"); + let values = vocabulary.token_byte_values(); + assert!(values.windows(2).all(|pair| pair[0] < pair[1]), "{name}"); + for piece in values.iter().step_by(997) { + let rank = vocabulary.encode_single_token(piece).unwrap(); + assert_eq!(tokenizer.decode_bytes(&[rank]).unwrap(), *piece, "{name}"); + assert!(!vocabulary.is_special_token(rank), "{name}"); + } + for (token, rank) in vocabulary.special_tokens() { + assert_eq!(vocabulary.encode_single_token(token.as_bytes()), Some(rank)); + assert!(vocabulary.is_special_token(rank), "{name}: {token}"); + } + assert_eq!(vocabulary.encode_single_token(b"<|not-a-token|>"), None); + } + + #[test] + fn malformed_ranks_return_errors_instead_of_panicking() { + for ranks in ["", "IQ==", "IQ== x", "!!! 1", "IQ== 1"] { + assert!(build("cl100k_base", ranks).is_err()); + } + let repeated_rank = (0..=u8::MAX) + .map(|byte| format!("{} 0\n", STANDARD.encode([byte]))) + .collect::(); + assert!(build("cl100k_base", &repeated_rank).is_err()); + } +} diff --git a/litellm-rust/crates/token-counter/README.md b/litellm-rust/crates/token-counter/README.md index a6b2b50aac0..3c6381fcea7 100644 --- a/litellm-rust/crates/token-counter/README.md +++ b/litellm-rust/crates/token-counter/README.md @@ -1,6 +1,10 @@ # Token counting -`Tokenizer` is the text-counting interface. `TokenCounter` applies LiteLLM request, message, and tool accounting using any implementation of that interface +`Tokenizer` is the text-counting interface. `TextCodec` adds encoding, decoding, and a name. `TokenCounter` applies LiteLLM request, message, and tool accounting using any `Tokenizer` + +Counts follow the codec: tiktoken treats special-token spellings as ordinary text, while Hugging Face applies its added tokens, post-processing, padding, and truncation. `fast=True` preserves those semantics and requests acceleration where available. Unsupported configurations use the normal codec, including tiktoken encodings without a scanner and builds without the `fast` feature. Invalid input and process-guard errors still propagate. Runtime request counting currently uses the normal codec; the custom accelerator is retained for explicit use and testing + +`FastCounter: TextCodec` exposes an optional accelerator over a loaded codec. `None` means callers should use that codec. The Python bridge caches this selection per immutable tokenizer, shares it with request counters, and initializes it with the GIL released. Hugging Face can also choose the full encoder per input when added tokens require it The `fast` feature provides `fast::FastTokenizer` from `litellm-token-counter-fast`. `TokenCounter::from_json_fast` uses this implementation @@ -8,7 +12,9 @@ The `huggingface` feature provides `huggingface::HuggingFaceTokenizer` through t The `tiktoken` feature provides `tiktoken::TiktokenTokenizer` through `tiktoken-rs`. Select an encoding with `TokenCounter::from_tiktoken`. The supported names are `cl100k_base`, `o200k_base`, `o200k_harmony`, `p50k_base`, `p50k_edit`, `r50k_base`, and `gpt2` -All three backends are enabled by default. The Python extension builds with `fast` only, which keeps the wheel at the size it had before the split. With `default-features = false`, callers can supply their own `Tokenizer` to `TokenCounter::new` without compiling a built-in backend +All three backends are enabled by default in this crate and the Python extension. With `default-features = false`, Rust callers can supply their own `Tokenizer` to `TokenCounter::new` without compiling a built-in backend + +Python `tiktoken` and `tokenizers` remain runtime dependencies and the default implementations. The catalog independently selects the tokenizer and request-counting routes. Enabling Rust changes factory dispatch; existing tokenizer objects keep their backend. Native Hugging Face wrappers provide an immutable encoding and decoding API, while training and mutable configuration remain available through the Python backend Budget checks, cost calculation, and the `max_tokens` adjustment policy belong to `litellm-core-utils`. The counter does not own prices, budgets, or request limits diff --git a/litellm-rust/crates/token-counter/src/error.rs b/litellm-rust/crates/token-counter/src/error.rs index b05ce007e46..6a94b0e39b8 100644 --- a/litellm-rust/crates/token-counter/src/error.rs +++ b/litellm-rust/crates/token-counter/src/error.rs @@ -32,6 +32,8 @@ pub enum Error { JsonUtf8(#[source] FromUtf8Error), #[error("tokenization failed: {0}")] Encode(#[source] Box), + #[error("token decoding failed: {0}")] + Decode(String), #[error("token counting task failed: {0}")] Task(String), } diff --git a/litellm-rust/crates/token-counter/src/fast.rs b/litellm-rust/crates/token-counter/src/fast.rs index de3f86abd68..f601c3cbcca 100644 --- a/litellm-rust/crates/token-counter/src/fast.rs +++ b/litellm-rust/crates/token-counter/src/fast.rs @@ -1,7 +1,31 @@ use litellm_token_counter_fast::Error as BackendError; pub use litellm_token_counter_fast::FastTokenizer; -use crate::{Error, TokenCounter, Tokenizer}; +use crate::{Error, TextCodec, TokenCounter, Tokenizer}; + +pub trait FastCounter: TextCodec { + fn fast_counter(&self) -> Option; +} + +#[cfg(feature = "huggingface")] +impl FastCounter for crate::huggingface::HuggingFaceTokenizer { + fn fast_counter(&self) -> Option { + Some(FastTokenizer::from_shared(self.shared())) + } +} + +#[cfg(feature = "tiktoken")] +impl FastCounter for crate::tiktoken::TiktokenTokenizer { + fn fast_counter(&self) -> Option { + let vocabulary = self.vocabulary()?; + match self.name() { + "cl100k_base" => FastTokenizer::from_cl100k_pairs(vocabulary.ranks()), + "o200k_base" | "o200k_harmony" => FastTokenizer::from_o200k_pairs(vocabulary.ranks()), + _ => return None, + } + .ok() + } +} impl TokenCounter { pub fn from_json_fast(tokenizer_json: &str) -> Result { @@ -39,3 +63,65 @@ impl From for Error { } } } + +#[cfg(all(test, feature = "huggingface", feature = "tiktoken"))] +mod tests { + use super::*; + use crate::huggingface::HuggingFaceTokenizer; + use crate::tiktoken::TiktokenTokenizer; + + const TEXTS: [&str; 4] = [ + "", + "hello world <|endoftext|>", + "café 漢字 ع 🙂 line\r\n indented 123456789", + "system a\u{301} fi", + ]; + + fn packaged(file: &str) -> String { + std::fs::read_to_string(format!( + "{}/../../../litellm/litellm_core_utils/tokenizers/{file}", + env!("CARGO_MANIFEST_DIR") + )) + .unwrap() + } + + #[test] + fn fast_counters_derived_from_codecs_count_like_the_codecs() { + let huggingface = + HuggingFaceTokenizer::from_json(&packaged("anthropic_tokenizer.json")).unwrap(); + let fast = huggingface.fast_counter().unwrap(); + for text in TEXTS { + assert_eq!( + fast.count_tokens(text).unwrap(), + Tokenizer::count_tokens(&huggingface, text).unwrap(), + "{text:?}" + ); + } + + for name in ["cl100k_base", "o200k_base", "o200k_harmony"] { + let tiktoken = + TiktokenTokenizer::from_cached_ranks(name, |file| Ok(packaged(file))).unwrap(); + let fast = tiktoken.fast_counter().unwrap(); + for text in TEXTS { + assert_eq!( + fast.count_tokens(text).unwrap(), + tiktoken.count_tokens(text), + "{name}: {text:?}" + ); + } + } + } + + #[test] + fn encodings_without_a_fast_scanner_keep_the_codec() { + let tiktoken = + TiktokenTokenizer::from_cached_ranks("p50k_base", |file| Ok(packaged(file))).unwrap(); + assert!(tiktoken.fast_counter().is_none()); + assert!( + TiktokenTokenizer::from_name("cl100k_base") + .unwrap() + .fast_counter() + .is_none() + ); + } +} diff --git a/litellm-rust/crates/token-counter/src/huggingface.rs b/litellm-rust/crates/token-counter/src/huggingface.rs index fb7683b373e..43613fac8ea 100644 --- a/litellm-rust/crates/token-counter/src/huggingface.rs +++ b/litellm-rust/crates/token-counter/src/huggingface.rs @@ -1,7 +1,11 @@ use litellm_token_counter_huggingface::Error as BackendError; -pub use litellm_token_counter_huggingface::HuggingFaceTokenizer; +pub use litellm_token_counter_huggingface::{ + AddedToken, EncodeInput, Encoding, HuggingFaceTokenizer, InputSequence, PaddingDirection, + PaddingParams, PaddingStrategy, TruncationDirection, TruncationParams, encoding_from_json, + encoding_to_json, +}; -use crate::{Error, TokenCounter, Tokenizer}; +use crate::{Error, TextCodec, TokenCounter, Tokenizer}; impl TokenCounter { pub fn from_json(tokenizer_json: &str) -> Result { @@ -17,11 +21,26 @@ impl Tokenizer for HuggingFaceTokenizer { } } +impl TextCodec for HuggingFaceTokenizer { + fn encode(&self, text: &str) -> Result, Error> { + HuggingFaceTokenizer::encode(self, text).map_err(Error::from) + } + + fn decode(&self, ids: &[u32], skip_special_tokens: bool) -> Result { + HuggingFaceTokenizer::decode(self, ids, skip_special_tokens).map_err(Error::from) + } + + fn name(&self) -> &str { + HuggingFaceTokenizer::name(self) + } +} + impl From for Error { fn from(error: BackendError) -> Self { match error { BackendError::Load(source) => Self::Load(source), BackendError::Encode(source) => Self::Encode(source), + BackendError::Decode(source) => Self::Decode(source.to_string()), } } } diff --git a/litellm-rust/crates/token-counter/src/lib.rs b/litellm-rust/crates/token-counter/src/lib.rs index 446c91049de..84bf35ca2ba 100644 --- a/litellm-rust/crates/token-counter/src/lib.rs +++ b/litellm-rust/crates/token-counter/src/lib.rs @@ -20,5 +20,5 @@ pub mod tiktoken; pub use counter::{InputTokenCount, TokenCounter}; pub use error::Error; -pub use tokenizer::Tokenizer; +pub use tokenizer::{TextCodec, Tokenizer}; pub use types::CountableRequest; diff --git a/litellm-rust/crates/token-counter/src/tiktoken.rs b/litellm-rust/crates/token-counter/src/tiktoken.rs index 07c1c9f5b73..5883a629ea5 100644 --- a/litellm-rust/crates/token-counter/src/tiktoken.rs +++ b/litellm-rust/crates/token-counter/src/tiktoken.rs @@ -1,7 +1,7 @@ -pub use litellm_token_counter_tiktoken::TiktokenTokenizer; -use litellm_token_counter_tiktoken::UnsupportedTokenizer; +use litellm_token_counter_tiktoken::{LoadError, UnsupportedTokenizer}; +pub use litellm_token_counter_tiktoken::{TiktokenTokenizer, Vocabulary, encoding_for_model}; -use crate::{Error, TokenCounter, Tokenizer}; +use crate::{Error, TextCodec, TokenCounter, Tokenizer}; impl TokenCounter { pub fn from_tiktoken(encoding: &str) -> Result { @@ -17,8 +17,31 @@ impl Tokenizer for TiktokenTokenizer { } } +impl TextCodec for TiktokenTokenizer { + fn encode(&self, text: &str) -> Result, Error> { + Ok(TiktokenTokenizer::encode(self, text)) + } + + fn decode(&self, ids: &[u32], _skip_special_tokens: bool) -> Result { + TiktokenTokenizer::decode(self, ids).map_err(|error| Error::Decode(error.to_string())) + } + + fn name(&self) -> &str { + TiktokenTokenizer::name(self) + } +} + impl From for Error { fn from(error: UnsupportedTokenizer) -> Self { Self::UnsupportedTokenizer(error.0) } } + +impl From for Error { + fn from(error: LoadError) -> Self { + match error { + LoadError::Unsupported(error) => error.into(), + LoadError::Ranks(message) => Self::Ranks(message), + } + } +} diff --git a/litellm-rust/crates/token-counter/src/tokenizer.rs b/litellm-rust/crates/token-counter/src/tokenizer.rs index 88c29c672a7..146ac5b4d0d 100644 --- a/litellm-rust/crates/token-counter/src/tokenizer.rs +++ b/litellm-rust/crates/token-counter/src/tokenizer.rs @@ -4,6 +4,12 @@ pub trait Tokenizer: Send + Sync { fn count_tokens(&self, text: &str) -> Result; } +pub trait TextCodec: Tokenizer { + fn encode(&self, text: &str) -> Result, Error>; + fn decode(&self, ids: &[u32], skip_special_tokens: bool) -> Result; + fn name(&self) -> &str; +} + #[cfg(test)] mod tests { use super::*; diff --git a/litellm/_lazy_imports.py b/litellm/_lazy_imports.py index fe3c7c264ee..29fb46fa125 100644 --- a/litellm/_lazy_imports.py +++ b/litellm/_lazy_imports.py @@ -58,7 +58,8 @@ from ._lazy_imports_registry import ( if TYPE_CHECKING: import httpx - from tiktoken import Encoding + + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer def get_litellm_globals() -> dict[str, object]: @@ -89,26 +90,11 @@ def _get_module_level_client_timeout(litellm_globals: Mapping[str, Any]) -> "flo # These are special lazy loaders for things that are used internally # They're separate from the main lazy import system because they have specific use cases -# Lazy loader for default encoding - avoids importing heavy tiktoken library at startup -_default_encoding: "Encoding | None" = None +def _get_default_encoding() -> "Tokenizer": + from litellm.rust_bridge.tokenizer import get_encoding -def _get_default_encoding() -> "Encoding": - """ - Lazily load and cache the default OpenAI encoding. - - This avoids importing `litellm.litellm_core_utils.default_encoding` (and thus tiktoken) - at `litellm` import time. The encoding is cached after the first import. - - This is used internally by utils.py functions that need the encoding but shouldn't - trigger its import during module load. - """ - global _default_encoding - if _default_encoding is None: - from litellm.litellm_core_utils.default_encoding import encoding - - _default_encoding = encoding - return _default_encoding + return get_encoding("cl100k_base") # Lazy loader for get_modified_max_tokens to avoid importing token_counter at module import time diff --git a/litellm/litellm_core_utils/README.md b/litellm/litellm_core_utils/README.md index b61c8982762..a5f5e8326b9 100644 --- a/litellm/litellm_core_utils/README.md +++ b/litellm/litellm_core_utils/README.md @@ -6,8 +6,9 @@ Core files: - `streaming_handler.py`: The core streaming logic + streaming related helper utils - `core_helpers.py`: code used in `types/` - e.g. `map_finish_reason`. - `exception_mapping_utils.py`: utils for mapping exceptions to openai-compatible error types. -- `default_encoding.py`: code for loading the default encoding (tiktoken) +- `default_encoding.py`: code for loading the default Python tokenizer and bundled cache - `get_llm_provider_logic.py`: code for inferring the LLM provider from a given model name. - `duration_parser.py`: code for parsing durations - e.g. "1d", "1mo", "10s" - `api_route_to_call_types.py`: mapping of API routes to their corresponding CallTypes (e.g., `/chat/completions` -> [acompletion, completion]) +Tokenizer factories return Python tokenizer objects by default. Set `LITELLM_RUST=1` or call `litellm.rust(True)` before constructing tokenizers to select the Rust backend through `Route.TOKENIZER` in the Rust catalog. Missing native bindings or unsupported native features fall back to Python. Existing tokenizer objects keep their selected backend. Rust-backed tokenizer objects carry the read-only `tiktoken.Encoding` / `tokenizers.Tokenizer` surface and are immutable: `enable_padding`, `enable_truncation` and `add_tokens` stay on the Python tokenizer. diff --git a/litellm/litellm_core_utils/default_encoding.py b/litellm/litellm_core_utils/default_encoding.py index 71b30614d8d..c3b6a008411 100644 --- a/litellm/litellm_core_utils/default_encoding.py +++ b/litellm/litellm_core_utils/default_encoding.py @@ -1,5 +1,4 @@ import os -from pathlib import Path from typing import Final import litellm @@ -15,20 +14,6 @@ except (ImportError, AttributeError): filename = pkg_resources.resource_filename(__name__, "litellm_core_utils/tokenizers") -CL100K_BASE_RANK_FILE: Final = "9b5ad71b2ce5302211f9c61530b329a4922fc6a4" -O200K_BASE_RANK_FILE: Final = "fb374d419588a4632f3f557e76b4b70aebbca790" - - -def cl100k_base_rank_file() -> str: - """The vendored tiktoken `cl100k_base` rank file (`base64(token) rank` lines).""" - return Path(filename, CL100K_BASE_RANK_FILE).read_text(encoding="ascii") - - -def o200k_base_rank_file() -> str: - """The vendored tiktoken `o200k_base` rank file (`base64(token) rank` lines).""" - return Path(filename, O200K_BASE_RANK_FILE).read_text(encoding="ascii") - - # Always default TIKTOKEN_CACHE_DIR to the bundled tokenizers directory # unless the user explicitly overrides it via CUSTOM_TIKTOKEN_CACHE_DIR. # This keeps tiktoken fully offline-capable by default (see #1071). diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index bf37b1be2e4..5d7956059e4 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -10,7 +10,6 @@ import anyio import anyio.lowlevel import httpx import tiktoken -from tokenizers import Tokenizer from typing_extensions import ParamSpec, TypeVar import litellm @@ -30,8 +29,10 @@ from litellm.constants import ( TOKEN_COUNTER_MAX_EXACT_CHARS, ) from litellm.litellm_core_utils.asyncify import asyncify +from litellm.litellm_core_utils.tokenizer import Encoding, HuggingFace, HuggingFaceTokenizer, OpenAIEncoding from litellm.litellm_core_utils.url_utils import safe_get from litellm.llms.custom_httpx.http_handler import _get_httpx_client +from litellm.rust_bridge.tokenizer import get_encoding from litellm.types.llms.anthropic import ( AnthropicContentParamSource, AnthropicContentParamSourceFileId, @@ -622,9 +623,11 @@ def _get_exact_count_function( if model is not None or custom_tokenizer is not None: tokenizer_json: Final = custom_tokenizer or _select_tokenizer(model) if tokenizer_json["type"] == "huggingface_tokenizer": - tokenizer: Final[Tokenizer] = tokenizer_json["tokenizer"] + tokenizer: Final[HuggingFace] = tokenizer_json["tokenizer"] def count_tokens(text: str) -> int: + if isinstance(tokenizer, HuggingFaceTokenizer): + return tokenizer.count(text) return len(tokenizer.encode_batch_fast([text])[0]) return count_tokens @@ -632,31 +635,43 @@ def _get_exact_count_function( encoding: Final = openai_tokenizer_encoding(model) def encode_length(text: str) -> int: - return len(encoding.encode(text, disallowed_special=())) + return _encoding_count(encoding, text) return _get_tiktoken_count_function(encode_length) else: raise ValueError("Unsupported tokenizer type") else: + default_encoding: Final = _get_default_encoding() def encode_length(text: str) -> int: - return len(_get_default_encoding().encode(text, disallowed_special=())) + return _encoding_count(default_encoding, text) return _get_tiktoken_count_function(encode_length) -def openai_tokenizer_encoding(model: str) -> tiktoken.Encoding: - """The tiktoken encoding `token_counter` uses for a model on the `openai_tokenizer` path.""" +def _encoding_count(encoding: Encoding, text: str) -> int: + if isinstance(encoding, OpenAIEncoding): + return encoding.count(text) + return len(encoding.encode(text, disallowed_special=())) + + +def openai_tokenizer_encoding(model: str) -> Encoding: + """The encoding `token_counter` uses for a model on the `openai_tokenizer` path.""" + return get_encoding(openai_tokenizer_encoding_name(model)) + + +def openai_tokenizer_encoding_name(model: str) -> str: + """The tiktoken encoding name for `model`, without loading the encoding.""" from litellm.utils import print_verbose model_to_use: Final = _fix_model_name(model) if "gpt-4o" in model_to_use: - return tiktoken.get_encoding("o200k_base") + return "o200k_base" try: - return tiktoken.encoding_for_model(model_to_use) + return tiktoken.encoding_name_for_model(model_to_use) except KeyError: print_verbose("Warning: model not found. Using cl100k_base encoding.") - return tiktoken.get_encoding("cl100k_base") + return "cl100k_base" def uses_legacy_message_accounting(model: str) -> bool: diff --git a/litellm/litellm_core_utils/tokenizer.py b/litellm/litellm_core_utils/tokenizer.py new file mode 100644 index 00000000000..aea187fa08e --- /dev/null +++ b/litellm/litellm_core_utils/tokenizer.py @@ -0,0 +1,402 @@ +"""Python faces of the Rust text codecs. + +``OpenAIEncoding`` mirrors ``tiktoken.Encoding`` and ``HuggingFaceTokenizer`` mirrors +``tokenizers.Tokenizer``, so a caller holding ``litellm.encoding`` or the object returned by +``litellm.create_tokenizer`` sees the same read-only surface whichever backend the Rust catalog +selected. Both wrappers are immutable: ``tokenizers`` mutators (``enable_padding``, +``enable_truncation``, ``add_tokens``) stay on the Python tokenizer. +""" + +from __future__ import annotations + +from collections.abc import Callable, Collection, Mapping, Sequence, Set +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from functools import partial +from pathlib import Path +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias, runtime_checkable + +import tiktoken +from tokenizers import AddedToken +from tokenizers import Tokenizer as PythonHuggingFaceTokenizer + +if TYPE_CHECKING: + import numpy as np + import numpy.typing as npt + + from litellm.rust_bridge._native import HuggingFaceEncoding + from litellm.rust_bridge._native import Tokenizer as NativeTokenizer + +SpecialTokens: TypeAlias = Literal["all"] | Collection[str] +AllowedSpecial: TypeAlias = Literal["all"] | Set[str] +HuggingFaceInput: TypeAlias = str | list[str] | tuple[str, ...] +HuggingFaceBatchInput: TypeAlias = HuggingFaceInput | tuple[HuggingFaceInput, HuggingFaceInput] | list[HuggingFaceInput] + + +@dataclass(frozen=True, slots=True) +class OpenAIEncoding: + """``tiktoken.Encoding`` over the Rust tiktoken codec.""" + + _native: NativeTokenizer + _special_tokens: Mapping[str, int] + + @staticmethod + def wrap(native: NativeTokenizer) -> OpenAIEncoding: + return OpenAIEncoding(native, MappingProxyType(native.special_tokens())) + + @staticmethod + def from_tiktoken(encoding: str) -> OpenAIEncoding: + from litellm.rust_bridge._native import Tokenizer as NativeTokenizer + + return OpenAIEncoding.wrap(NativeTokenizer.from_tiktoken(encoding)) + + def __repr__(self) -> str: + return f"" + + @property + def name(self) -> str: + return self._native.name + + @property + def max_token_value(self) -> int: + return self._native.max_token_value() + + @property + def n_vocab(self) -> int: + """For backwards compatibility. Prefer to use `enc.max_token_value + 1`.""" + return self.max_token_value + 1 + + @property + def eot_token(self) -> int: + return self._special_tokens["<|endoftext|>"] + + @property + def special_tokens_set(self) -> set[str]: # mutable-ok: [LIT001, LIT002] SDK return type + return set(self._special_tokens) + + def is_special_token(self, token: int) -> bool: + return self._native.is_special_token(token) + + # ---- encoding ------------------------------------------------------------------------- + + def encode_ordinary(self, text: str) -> list[int]: # mutable-ok: [LIT001, LIT002] SDK return type + return self._native.encode(text) + + def encode( + self, + text: str, + *, + allowed_special: AllowedSpecial = frozenset(), + disallowed_special: SpecialTokens = "all", + ) -> list[int]: # mutable-ok: [LIT001, LIT002] SDK return type + allowed: Final = self._allowed(text, allowed_special, disallowed_special) + if not allowed: + return self.encode_ordinary(text) + return self._native.encode_special(text, tuple(allowed)) + + def encode_to_numpy( + self, + text: str, + *, + allowed_special: AllowedSpecial = frozenset(), + disallowed_special: SpecialTokens = "all", + ) -> npt.NDArray[np.uint32]: + import numpy + + return numpy.asarray( + self.encode(text, allowed_special=allowed_special, disallowed_special=disallowed_special), + dtype=numpy.uint32, + ) + + def encode_ordinary_batch( + self, text: Sequence[str], *, num_threads: int = 8 + ) -> list[list[int]]: # mutable-ok: [LIT001, LIT002] SDK return type + with ThreadPoolExecutor(num_threads) as executor: + return list( # mutable-ok: [LIT002] SDK returns a list + executor.map(self.encode_ordinary, text) + ) + + def encode_batch( + self, + text: Sequence[str], + *, + num_threads: int = 8, + allowed_special: AllowedSpecial = frozenset(), + disallowed_special: SpecialTokens = "all", + ) -> list[list[int]]: # mutable-ok: [LIT001, LIT002] SDK return type + encode: Final = partial(self.encode, allowed_special=allowed_special, disallowed_special=disallowed_special) + with ThreadPoolExecutor(num_threads) as executor: + return list( # mutable-ok: [LIT002] SDK returns a list + executor.map(encode, text) + ) + + def encode_with_unstable( + self, + text: str, + *, + allowed_special: AllowedSpecial = frozenset(), + disallowed_special: SpecialTokens = "all", + ) -> tuple[list[int], list[list[int]]]: # mutable-ok: [LIT001, LIT002] SDK return type + """The stable tokens of `text` and every completion its unstable tail could become. + + Completions come back sorted; tiktoken returns them in hash order.""" + allowed: Final = self._allowed(text, allowed_special, disallowed_special) + return self._native.encode_with_unstable(text, tuple(allowed)) + + def encode_single_token(self, text_or_bytes: str | bytes) -> int: + """The token of one whole piece, special tokens included. Raises `KeyError` otherwise.""" + piece: Final = text_or_bytes.encode("utf-8") if isinstance(text_or_bytes, str) else text_or_bytes + return self._native.encode_single_token(piece) + + def count(self, text: str, fast: bool = False) -> int: + """Count ordinary text; `fast` accelerates supported encodings and otherwise counts normally.""" + return self._native.count(text, fast) + + # ---- decoding ------------------------------------------------------------------------- + + def decode_bytes(self, tokens: Sequence[int]) -> bytes: + return self._native.decode_bytes(tokens) + + def decode(self, tokens: Sequence[int], errors: str = "replace") -> str: + return self.decode_bytes(tokens).decode("utf-8", errors=errors) + + def decode_single_token_bytes(self, token: int) -> bytes: + return self.decode_bytes((token,)) + + def decode_tokens_bytes(self, tokens: Sequence[int]) -> list[bytes]: # mutable-ok: [LIT001, LIT002] SDK return type + return [ # mutable-ok: [LIT002] SDK returns a list + self.decode_single_token_bytes(token) for token in tokens + ] + + def decode_with_offsets( + self, tokens: Sequence[int] + ) -> tuple[str, list[int]]: # mutable-ok: [LIT001, LIT002] SDK return type + """The decoded text and, per token, the index of the first character holding its bytes. + + Like tiktoken, raises `UnicodeDecodeError` when the tokens do not decode to valid UTF-8.""" + token_bytes: Final = self.decode_tokens_bytes(tokens) + text_len = 0 + offsets: Final[list[int]] = [] # mutable-ok: [LIT001] local accumulator + for token in token_bytes: + offsets.append(max(0, text_len - (0x80 <= token[0] < 0xC0))) + text_len += sum(1 for c in token if not 0x80 <= c < 0xC0) + return b"".join(token_bytes).decode("utf-8", errors="strict"), offsets + + def decode_batch( + self, batch: Sequence[Sequence[int]], *, errors: str = "replace", num_threads: int = 8 + ) -> list[str]: # mutable-ok: [LIT001, LIT002] SDK return type + with ThreadPoolExecutor(num_threads) as executor: + return list( # mutable-ok: [LIT002] SDK returns a list + executor.map(partial(self.decode, errors=errors), batch) + ) + + def decode_bytes_batch( + self, batch: Sequence[Sequence[int]], *, num_threads: int = 8 + ) -> list[bytes]: # mutable-ok: [LIT001, LIT002] SDK return type + with ThreadPoolExecutor(num_threads) as executor: + return list( # mutable-ok: [LIT002] SDK returns a list + executor.map(self.decode_bytes, batch) + ) + + def token_byte_values(self) -> list[bytes]: # mutable-ok: [LIT001, LIT002] SDK return type + return self._native.token_byte_values() + + def __reduce__(self) -> tuple[Callable[[str], OpenAIEncoding], tuple[str]]: + return (OpenAIEncoding.from_tiktoken, (self.name,)) + + # ---- private -------------------------------------------------------------------------- + + def _allowed(self, text: str, allowed_special: AllowedSpecial, disallowed_special: SpecialTokens) -> frozenset[str]: + """tiktoken's special-token policy: which specials `text` may encode, after rejecting + any it must not contain.""" + allowed: Final = frozenset(self._special_tokens) if allowed_special == "all" else frozenset(allowed_special) + disallowed: Final = ( + frozenset(self._special_tokens) - allowed if disallowed_special == "all" else frozenset(disallowed_special) + ) + for token in disallowed: + if token in text: + raise ValueError( + f"Encountered text corresponding to disallowed special token {token!r}.\n" + "If you want this text to be encoded as a special token, " + f"pass it to `allowed_special`, e.g. `allowed_special={{{token!r}, ...}}`.\n" + "If you want this text to be encoded as normal text, disable the check for this token " + f"by passing `disallowed_special=(enc.special_tokens_set - {{{token!r}}})`.\n" + "To disable this check for all special tokens, pass `disallowed_special=()`.\n" + ) + return allowed + + +@dataclass(frozen=True, slots=True) +class HuggingFaceTokenizer: + """The read-only ``tokenizers.Tokenizer`` surface over the Rust Hugging Face codec.""" + + _native: NativeTokenizer + + @staticmethod + def from_str(json: str) -> HuggingFaceTokenizer: + from litellm.rust_bridge._native import Tokenizer as NativeTokenizer + + return HuggingFaceTokenizer(NativeTokenizer.from_json(json)) + + from_json = from_str + + @staticmethod + def from_buffer(buffer: bytes) -> HuggingFaceTokenizer: + return HuggingFaceTokenizer.from_str(buffer.decode("utf-8")) + + @staticmethod + def from_file(path: str) -> HuggingFaceTokenizer: + return HuggingFaceTokenizer.from_str(Path(path).read_text(encoding="utf-8")) + + @staticmethod + def from_pretrained(identifier: str, revision: str = "main", token: str | None = None) -> HuggingFaceTokenizer: + from litellm.rust_bridge._native import Tokenizer as NativeTokenizer + + return HuggingFaceTokenizer(NativeTokenizer.from_pretrained(identifier, revision=revision, token=token)) + + def to_str(self, pretty: bool = False) -> str: + return self._native.to_json(pretty) + + def save(self, path: str, pretty: bool = True) -> None: + Path(path).write_text(self.to_str(pretty), encoding="utf-8") + + @property + def name(self) -> str: + return self._native.name + + # ---- vocabulary ----------------------------------------------------------------------- + + def token_to_id(self, token: str) -> int | None: + return self._native.token_to_id(token) + + def id_to_token(self, id: int) -> str | None: + return self._native.id_to_token(id) + + def get_vocab( + self, with_added_tokens: bool = True + ) -> dict[str, int]: # mutable-ok: [LIT001, LIT002] SDK return type + return self._native.get_vocab(with_added_tokens) + + def get_vocab_size(self, with_added_tokens: bool = True) -> int: + return self._native.get_vocab_size(with_added_tokens) + + def get_added_tokens_decoder(self) -> dict[int, AddedToken]: # mutable-ok: [LIT001, LIT002] SDK return type + return { # mutable-ok: [LIT002] SDK returns a dict + token_id: AddedToken( + content, single_word=single_word, lstrip=lstrip, rstrip=rstrip, normalized=normalized, special=special + ) + for token_id, ( + content, + single_word, + lstrip, + rstrip, + normalized, + special, + ) in self._native.added_tokens_decoder() + } + + def num_special_tokens_to_add(self, is_pair: bool) -> int: + return self._native.num_special_tokens_to_add(is_pair) + + @property + def padding(self) -> dict[str, object] | None: # mutable-ok: [LIT001, LIT002] SDK return type + return self._native.padding() + + @property + def truncation(self) -> dict[str, object] | None: # mutable-ok: [LIT001, LIT002] SDK return type + return self._native.truncation() + + @property + def encode_special_tokens(self) -> bool: + return self._native.encode_special_tokens() + + # ---- encoding and decoding ------------------------------------------------------------ + + def encode( + self, + sequence: HuggingFaceInput, + pair: HuggingFaceInput | None = None, + is_pretokenized: bool = False, + add_special_tokens: bool = True, + ) -> HuggingFaceEncoding: + return self._native.encode_huggingface(sequence, pair, is_pretokenized, add_special_tokens) + + def encode_batch( + self, + input: Sequence[HuggingFaceBatchInput], + is_pretokenized: bool = False, + add_special_tokens: bool = True, + ) -> list[HuggingFaceEncoding]: # mutable-ok: [LIT001, LIT002] SDK return type + return self._encode_batch(input, is_pretokenized, add_special_tokens, fast=False) + + def encode_batch_fast( + self, + input: Sequence[HuggingFaceBatchInput], + is_pretokenized: bool = False, + add_special_tokens: bool = True, + ) -> list[HuggingFaceEncoding]: # mutable-ok: [LIT001, LIT002] SDK return type + return self._encode_batch(input, is_pretokenized, add_special_tokens, fast=True) + + def _encode_batch( + self, input: Sequence[HuggingFaceBatchInput], is_pretokenized: bool, add_special_tokens: bool, fast: bool + ) -> list[HuggingFaceEncoding]: # mutable-ok: [LIT001, LIT002] SDK return type + sequences: Final = tuple(_batch_input(item, is_pretokenized) for item in input) + return self._native.encode_batch_huggingface(sequences, is_pretokenized, add_special_tokens, fast) + + def count(self, text: str, fast: bool = False) -> int: + """Count with this tokenizer's configuration; `fast` uses acceleration where supported.""" + return self._native.count(text, fast) + + def decode(self, ids: Sequence[int], skip_special_tokens: bool = True) -> str: + return self._native.decode(ids, skip_special_tokens=skip_special_tokens) + + def decode_batch( + self, sequences: Sequence[Sequence[int]], skip_special_tokens: bool = True + ) -> list[str]: # mutable-ok: [LIT001, LIT002] SDK return type + return [ # mutable-ok: [LIT002] SDK returns a list + self.decode(ids, skip_special_tokens=skip_special_tokens) for ids in sequences + ] + + def __reduce__(self) -> tuple[Callable[[str], HuggingFaceTokenizer], tuple[str]]: + return (HuggingFaceTokenizer.from_str, (self.to_str(),)) + + +def _batch_input( + item: HuggingFaceBatchInput, is_pretokenized: bool +) -> tuple[HuggingFaceInput, HuggingFaceInput | None]: + if isinstance(item, str): + return (item, None) + if is_pretokenized and all(isinstance(word, str) for word in item): + return (tuple(word for word in item if isinstance(word, str)), None) + if len(item) != 2: + raise TypeError("batch input must be a sequence or a pair of sequences") + return (item[0], item[1]) + + +Encoding: TypeAlias = tiktoken.Encoding | OpenAIEncoding +HuggingFace: TypeAlias = PythonHuggingFaceTokenizer | HuggingFaceTokenizer +Tokenizer: TypeAlias = Encoding | HuggingFace + + +class _AddedToken(Protocol): + @property + def special(self) -> bool: ... + + +@runtime_checkable +class _AddedTokenDecoder(Protocol): + def get_added_tokens_decoder(self) -> Mapping[int, _AddedToken]: ... + + +def strip_special_tokens(tokenizer: object, tokens: Sequence[int]) -> Sequence[int]: + """Drop the special added tokens before a Python `tokenizers` decode; the Rust codec's + `decode(skip_special_tokens=True)` already does this itself.""" + if isinstance(tokenizer, HuggingFaceTokenizer) or not isinstance(tokenizer, _AddedTokenDecoder): + return tokens + try: + added: Final = tokenizer.get_added_tokens_decoder() + except Exception: # noqa: BLE001 # optional metadata failures historically fall back to decoding + return tokens + special_ids: Final = frozenset(token_id for token_id, token in added.items() if token.special) + return tuple(token for token in tokens if token not in special_ids) diff --git a/litellm/llms/a2a/chat/transformation.py b/litellm/llms/a2a/chat/transformation.py index 77f26b65de0..c5a71daaba5 100644 --- a/litellm/llms/a2a/chat/transformation.py +++ b/litellm/llms/a2a/chat/transformation.py @@ -23,9 +23,8 @@ from ..common_utils import ( from .streaming_iterator import A2AModelResponseIterator if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer _REGISTRY_PARAMS_KEPT_OUT_OF_OPTIONAL_PARAMS: Final = ( @@ -292,7 +291,7 @@ class A2AConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/aiml/image_generation/transformation.py b/litellm/llms/aiml/image_generation/transformation.py index 4f4cd074165..b585d35a2ac 100644 --- a/litellm/llms/aiml/image_generation/transformation.py +++ b/litellm/llms/aiml/image_generation/transformation.py @@ -14,9 +14,8 @@ from litellm.types.llms.openai import ( from litellm.types.utils import ImageObject, ImageResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -171,7 +170,7 @@ class AimlImageGenerationConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/aiohttp_openai/chat/transformation.py b/litellm/llms/aiohttp_openai/chat/transformation.py index 530896bf9b0..a06c670e3f1 100644 --- a/litellm/llms/aiohttp_openai/chat/transformation.py +++ b/litellm/llms/aiohttp_openai/chat/transformation.py @@ -16,9 +16,8 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import Choices, ModelResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -68,7 +67,7 @@ class AiohttpOpenAIChatConfig(OpenAILikeChatConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/amazon_nova/chat/transformation.py b/litellm/llms/amazon_nova/chat/transformation.py index 7551fb28c21..a93fbf1e933 100644 --- a/litellm/llms/amazon_nova/chat/transformation.py +++ b/litellm/llms/amazon_nova/chat/transformation.py @@ -17,7 +17,7 @@ from litellm.types.utils import ModelResponse from ...openai_like.chat.transformation import OpenAILikeChatConfig if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class AmazonNovaChatConfig(OpenAILikeChatConfig): @@ -86,7 +86,7 @@ class AmazonNovaChatConfig(OpenAILikeChatConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/anthropic/batches/transformation.py b/litellm/llms/anthropic/batches/transformation.py index 4f4d39f09b0..7dee7513538 100644 --- a/litellm/llms/anthropic/batches/transformation.py +++ b/litellm/llms/anthropic/batches/transformation.py @@ -14,9 +14,8 @@ from litellm.types.llms.openai import AllMessageValues, CreateBatchRequest from litellm.types.utils import LiteLLMBatch, LlmProviders, ModelResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LoggingClass = LiteLLMLoggingObj else: @@ -290,7 +289,7 @@ class AnthropicBatchesConfig(BaseBatchesConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 545e920156e..c221e9f1505 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -100,9 +100,8 @@ from ..common_utils import ( ) if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LoggingClass = LiteLLMLoggingObj else: @@ -2688,7 +2687,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/anthropic/completion/transformation.py b/litellm/llms/anthropic/completion/transformation.py index b15b0159bd9..46ab27ab0c7 100644 --- a/litellm/llms/anthropic/completion/transformation.py +++ b/litellm/llms/anthropic/completion/transformation.py @@ -33,7 +33,7 @@ from litellm.types.utils import ( ) if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class AnthropicTextError(BaseLLMException): @@ -185,7 +185,7 @@ class AnthropicTextConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/azure/chat/gpt_transformation.py b/litellm/llms/azure/chat/gpt_transformation.py index 424422612db..355714c0daf 100644 --- a/litellm/llms/azure/chat/gpt_transformation.py +++ b/litellm/llms/azure/chat/gpt_transformation.py @@ -29,9 +29,8 @@ from ...base_llm.chat.transformation import BaseConfig from ..common_utils import AzureOpenAIError if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LoggingClass = LiteLLMLoggingObj else: @@ -304,7 +303,7 @@ class AzureOpenAIConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/azure_ai/agents/transformation.py b/litellm/llms/azure_ai/agents/transformation.py index 60ce81a23c7..baba3149963 100644 --- a/litellm/llms/azure_ai/agents/transformation.py +++ b/litellm/llms/azure_ai/agents/transformation.py @@ -34,9 +34,8 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler LiteLLMLoggingObj = _LiteLLMLoggingObj @@ -297,7 +296,7 @@ class AzureAIAgentsConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/azure_ai/azure_model_router/transformation.py b/litellm/llms/azure_ai/azure_model_router/transformation.py index 9e35e396e15..0e4c8ca0d15 100644 --- a/litellm/llms/azure_ai/azure_model_router/transformation.py +++ b/litellm/llms/azure_ai/azure_model_router/transformation.py @@ -15,7 +15,7 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class AzureModelRouterConfig(AzureAIStudioConfig): @@ -59,7 +59,7 @@ class AzureModelRouterConfig(AzureAIStudioConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/azure_ai/chat/transformation.py b/litellm/llms/azure_ai/chat/transformation.py index 00e1c1e25ba..779a86629e2 100644 --- a/litellm/llms/azure_ai/chat/transformation.py +++ b/litellm/llms/azure_ai/chat/transformation.py @@ -30,7 +30,7 @@ from litellm.types.utils import ModelResponse, ProviderField from litellm.utils import _add_path_to_api_base, supports_tool_choice if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class AzureFoundryErrorStrings(str, enum.Enum): @@ -305,7 +305,7 @@ class AzureAIStudioConfig(OpenAIConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/azure_ai/image_generation/mai_transformation.py b/litellm/llms/azure_ai/image_generation/mai_transformation.py index 67b1a8bcab3..18b4b6f456a 100644 --- a/litellm/llms/azure_ai/image_generation/mai_transformation.py +++ b/litellm/llms/azure_ai/image_generation/mai_transformation.py @@ -12,9 +12,10 @@ from litellm.types.utils import ImageResponse from litellm.utils import convert_to_model_response_object if TYPE_CHECKING: - import tiktoken from litellm.litellm_core_utils.logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer + class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig): """Azure AI Foundry MAI image generation (e.g. MAI-Image-2.5).""" @@ -245,7 +246,7 @@ class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/base_llm/audio_transcription/transformation.py b/litellm/llms/base_llm/audio_transcription/transformation.py index 2296909cfe1..e4bf148abf3 100644 --- a/litellm/llms/base_llm/audio_transcription/transformation.py +++ b/litellm/llms/base_llm/audio_transcription/transformation.py @@ -12,9 +12,8 @@ from litellm.types.llms.openai import ( from litellm.types.utils import FileTypes, ModelResponse, TranscriptionResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -121,7 +120,7 @@ class BaseAudioTranscriptionConfig(BaseConfig, ABC): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/base_llm/bridges/completion_transformation.py b/litellm/llms/base_llm/bridges/completion_transformation.py index 87b55152d09..a5c03705088 100644 --- a/litellm/llms/base_llm/bridges/completion_transformation.py +++ b/litellm/llms/base_llm/bridges/completion_transformation.py @@ -7,10 +7,10 @@ from collections.abc import AsyncIterator, Iterator from typing import TYPE_CHECKING, Union if TYPE_CHECKING: - import tiktoken from pydantic import BaseModel from litellm import LiteLLMLoggingObj, ModelResponse + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.types.llms.openai import AllMessageValues @@ -39,7 +39,7 @@ class CompletionTransformationBridge(ABC): messages: list["AllMessageValues"], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> "ModelResponse": diff --git a/litellm/llms/base_llm/chat/transformation.py b/litellm/llms/base_llm/chat/transformation.py index 7bfc87a30d6..7decf1b4186 100644 --- a/litellm/llms/base_llm/chat/transformation.py +++ b/litellm/llms/base_llm/chat/transformation.py @@ -21,9 +21,8 @@ from litellm.types.llms.openai import ( ) if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer from litellm.types.utils import ModelResponse from ..base_utils import ( @@ -344,7 +343,7 @@ class BaseConfig(ABC): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> "ModelResponse": diff --git a/litellm/llms/base_llm/completion/transformation.py b/litellm/llms/base_llm/completion/transformation.py index fb472dfa63b..b8ebbfed12f 100644 --- a/litellm/llms/base_llm/completion/transformation.py +++ b/litellm/llms/base_llm/completion/transformation.py @@ -8,9 +8,8 @@ from litellm.types.llms.openai import AllMessageValues, OpenAITextCompletionUser from litellm.types.utils import ModelResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -68,7 +67,7 @@ class BaseTextCompletionConfig(BaseConfig, ABC): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/base_llm/embedding/transformation.py b/litellm/llms/base_llm/embedding/transformation.py index da87dcc7f98..46ac3ccf433 100644 --- a/litellm/llms/base_llm/embedding/transformation.py +++ b/litellm/llms/base_llm/embedding/transformation.py @@ -8,9 +8,8 @@ from litellm.types.llms.openai import AllEmbeddingInputValues, AllMessageValues from litellm.types.utils import EmbeddingResponse, ModelResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -80,7 +79,7 @@ class BaseEmbeddingConfig(BaseConfig, ABC): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/base_llm/files/transformation.py b/litellm/llms/base_llm/files/transformation.py index 254995c028f..5ecf033fb2c 100644 --- a/litellm/llms/base_llm/files/transformation.py +++ b/litellm/llms/base_llm/files/transformation.py @@ -21,9 +21,8 @@ from litellm.types.utils import LlmProviders, ModelResponse from ..chat.transformation import BaseConfig if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer from litellm.router import Router as _Router from litellm.types.llms.openai import HttpxBinaryResponseContent @@ -231,7 +230,7 @@ class BaseFilesConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/base_llm/image_generation/transformation.py b/litellm/llms/base_llm/image_generation/transformation.py index 4616441133e..7ac440c6f48 100644 --- a/litellm/llms/base_llm/image_generation/transformation.py +++ b/litellm/llms/base_llm/image_generation/transformation.py @@ -11,9 +11,8 @@ from litellm.types.llms.openai import ( from litellm.types.utils import ImageResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -93,7 +92,7 @@ class BaseImageGenerationConfig(ABC): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/base_llm/image_variations/transformation.py b/litellm/llms/base_llm/image_variations/transformation.py index d3e02139e0e..15a4e0f243c 100644 --- a/litellm/llms/base_llm/image_variations/transformation.py +++ b/litellm/llms/base_llm/image_variations/transformation.py @@ -17,9 +17,8 @@ from litellm.types.utils import ( ) if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -82,7 +81,7 @@ class BaseImageVariationConfig(BaseConfig, ABC): image: FileTypes, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, ) -> ImageResponse: pass @@ -98,7 +97,7 @@ class BaseImageVariationConfig(BaseConfig, ABC): image: FileTypes, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, ) -> ImageResponse: pass @@ -125,7 +124,7 @@ class BaseImageVariationConfig(BaseConfig, ABC): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/bedrock/chat/agentcore/transformation.py b/litellm/llms/bedrock/chat/agentcore/transformation.py index e1a9a807abc..29133bcfaf9 100644 --- a/litellm/llms/bedrock/chat/agentcore/transformation.py +++ b/litellm/llms/bedrock/chat/agentcore/transformation.py @@ -40,9 +40,8 @@ from litellm.types.utils import ( ) if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler LiteLLMLoggingObj = _LiteLLMLoggingObj @@ -990,7 +989,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 4f9b1f56a5b..21830eb0d8e 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -99,7 +99,7 @@ from ..common_utils import ( ) if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer # Computer use tool prefixes supported by Bedrock BEDROCK_COMPUTER_USE_TOOLS: Final = [ @@ -1920,7 +1920,7 @@ class AmazonConverseConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/bedrock/chat/invoke_agent/transformation.py b/litellm/llms/bedrock/chat/invoke_agent/transformation.py index d489e47c3b5..6877af74494 100644 --- a/litellm/llms/bedrock/chat/invoke_agent/transformation.py +++ b/litellm/llms/bedrock/chat/invoke_agent/transformation.py @@ -37,9 +37,8 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import Choices, Message, ModelResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -438,7 +437,7 @@ class AmazonInvokeAgentConfig(BaseConfig, BaseAWSLLM): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_deepseek_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_deepseek_transformation.py index 5a3f4f17b8b..5699f94d084 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_deepseek_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_deepseek_transformation.py @@ -25,7 +25,7 @@ from litellm.types.utils import ( from .amazon_llama_transformation import AmazonLlamaConfig if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class AmazonDeepSeekR1Config(AmazonLlamaConfig): @@ -39,7 +39,7 @@ class AmazonDeepSeekR1Config(AmazonLlamaConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_moonshot_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_moonshot_transformation.py index 5d39b68d9d5..f8b730b6cd0 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_moonshot_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_moonshot_transformation.py @@ -21,9 +21,8 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import Choices if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer from litellm.types.utils import ModelResponse LiteLLMLoggingObj = _LiteLLMLoggingObj @@ -198,7 +197,7 @@ class AmazonMoonshotConfig(AmazonInvokeConfig, MoonshotChatConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> "ModelResponse": diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_nova_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_nova_transformation.py index bc97551d57a..8d1ff1d2bd9 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_nova_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_nova_transformation.py @@ -28,7 +28,7 @@ from ..converse_transformation import AmazonConverseConfig from .base_invoke_transformation import AmazonInvokeConfig if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer _CachePointCarrier = TypeVar("_CachePointCarrier", SystemContentBlock, ContentBlock) _INJECTION_POINTS: Final = TypeAdapter(tuple[Mapping[str, object], ...]) @@ -128,7 +128,7 @@ class AmazonInvokeNovaConfig(AmazonInvokeConfig, AmazonConverseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen2_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen2_transformation.py index c78375c37bb..67364ccfda0 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen2_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen2_transformation.py @@ -21,7 +21,7 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse, Usage if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class AmazonQwen2Config(AmazonQwen3Config): @@ -44,7 +44,7 @@ class AmazonQwen2Config(AmazonQwen3Config): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen3_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen3_transformation.py index e251fb15725..2e19dbf77af 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen3_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen3_transformation.py @@ -19,7 +19,7 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse, Usage if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class AmazonQwen3Config(AmazonInvokeConfig, BaseConfig): @@ -170,7 +170,7 @@ class AmazonQwen3Config(AmazonInvokeConfig, BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py index 39cded4ed64..fe287111fdd 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py @@ -25,9 +25,8 @@ from litellm.types.utils import ModelResponse, Usage from litellm.utils import get_base64_str if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -190,7 +189,7 @@ class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py index 1326dc22ca0..a8b94fb5703 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py @@ -34,9 +34,8 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -359,7 +358,7 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py index 90a2692f68a..dcc5e249d8a 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py @@ -34,9 +34,8 @@ from litellm.types.utils import ModelResponse, Usage from litellm.utils import CustomStreamWrapper if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -288,7 +287,7 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/black_forest_labs/image_generation/transformation.py b/litellm/llms/black_forest_labs/image_generation/transformation.py index 119ffff1c34..e5c2bdf59e9 100644 --- a/litellm/llms/black_forest_labs/image_generation/transformation.py +++ b/litellm/llms/black_forest_labs/image_generation/transformation.py @@ -29,9 +29,8 @@ from ..common_utils import ( ) if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -258,7 +257,7 @@ class BlackForestLabsImageGenerationConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/brave/search/__init__.py b/litellm/llms/brave/search/__init__.py index cc1168d7ef8..de70c62e040 100644 --- a/litellm/llms/brave/search/__init__.py +++ b/litellm/llms/brave/search/__init__.py @@ -1,7 +1,7 @@ -""" -Brave Search API module. -""" - -from litellm.llms.brave.search.transformation import BraveSearchConfig - -__all__ = ["BraveSearchConfig"] +""" +Brave Search API module. +""" + +from litellm.llms.brave.search.transformation import BraveSearchConfig + +__all__ = ["BraveSearchConfig"] diff --git a/litellm/llms/bytez/chat/transformation.py b/litellm/llms/bytez/chat/transformation.py index 7977db0f056..e622761dd7f 100644 --- a/litellm/llms/bytez/chat/transformation.py +++ b/litellm/llms/bytez/chat/transformation.py @@ -23,9 +23,8 @@ from litellm.utils import CustomStreamWrapper, ModelResponse, Usage from ..common_utils import API_BASE, BytezError if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -187,7 +186,7 @@ class BytezChatConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/clarifai/chat/transformation.py b/litellm/llms/clarifai/chat/transformation.py index 76d35467497..a0946254de0 100644 --- a/litellm/llms/clarifai/chat/transformation.py +++ b/litellm/llms/clarifai/chat/transformation.py @@ -13,9 +13,8 @@ from litellm.types.utils import ModelResponse from ...openai.chat.gpt_transformation import OpenAIGPTConfig if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -87,7 +86,7 @@ class ClarifaiConfig(OpenAIGPTConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/cohere/chat/transformation.py b/litellm/llms/cohere/chat/transformation.py index fa46bd7f6cf..a26cdc81695 100644 --- a/litellm/llms/cohere/chat/transformation.py +++ b/litellm/llms/cohere/chat/transformation.py @@ -15,9 +15,8 @@ from ..common_utils import ModelResponseIterator as CohereModelResponseIterator from ..common_utils import validate_environment as cohere_validate_environment if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -227,7 +226,7 @@ class CohereChatConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/cohere/chat/v2_transformation.py b/litellm/llms/cohere/chat/v2_transformation.py index 4252e7d02e9..37c53640d18 100644 --- a/litellm/llms/cohere/chat/v2_transformation.py +++ b/litellm/llms/cohere/chat/v2_transformation.py @@ -20,9 +20,8 @@ from ..common_utils import CohereError, CohereV2ModelResponseIterator from ..common_utils import validate_environment as cohere_validate_environment if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -191,7 +190,7 @@ class CohereV2ChatConfig(OpenAIGPTConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/cohere/embed/handler.py b/litellm/llms/cohere/embed/handler.py index 3cebf6b9a90..6d9543d7c30 100644 --- a/litellm/llms/cohere/embed/handler.py +++ b/litellm/llms/cohere/embed/handler.py @@ -20,7 +20,7 @@ from litellm.types.utils import EmbeddingResponse from .v1_transformation import CohereEmbeddingConfig if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer def validate_environment(api_key, headers: dict): @@ -60,7 +60,7 @@ async def async_embedding( api_base: str, api_key: str | None, headers: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", client: AsyncHTTPHandler | None = None, ): ## LOGGING @@ -122,7 +122,7 @@ def embedding( logging_obj: LiteLLMLoggingObj, optional_params: dict, headers: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", data: dict | CohereEmbeddingRequest | None = None, complete_api_base: str | None = None, api_key: str | None = None, diff --git a/litellm/llms/cometapi/image_generation/transformation.py b/litellm/llms/cometapi/image_generation/transformation.py index 03c820de198..4432c151a64 100644 --- a/litellm/llms/cometapi/image_generation/transformation.py +++ b/litellm/llms/cometapi/image_generation/transformation.py @@ -13,9 +13,8 @@ from litellm.types.llms.openai import ( from litellm.types.utils import ImageObject, ImageResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -132,7 +131,7 @@ class CometAPIImageGenerationConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/compactifai/chat/transformation.py b/litellm/llms/compactifai/chat/transformation.py index 3c5a889ce63..a02db2338d8 100644 --- a/litellm/llms/compactifai/chat/transformation.py +++ b/litellm/llms/compactifai/chat/transformation.py @@ -17,9 +17,8 @@ from litellm.types.utils import ModelResponse from ...openai.chat.gpt_transformation import OpenAIGPTConfig if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -66,7 +65,7 @@ class CompactifAIChatConfig(OpenAIGPTConfig): messages: Sequence[AllMessageValues], optional_params: Mapping[str, object], litellm_params: Mapping[str, object], - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/custom_httpx/aiohttp_handler.py b/litellm/llms/custom_httpx/aiohttp_handler.py index 0809ef5274f..034c9514092 100644 --- a/litellm/llms/custom_httpx/aiohttp_handler.py +++ b/litellm/llms/custom_httpx/aiohttp_handler.py @@ -26,9 +26,8 @@ from litellm.types.utils import HttpHandlerRequestFields, ImageResponse, LlmProv from litellm.utils import CustomStreamWrapper, ModelResponse, ProviderConfigManager if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -268,7 +267,7 @@ class BaseLLMAIOHTTPHandler: messages: list, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, client: ClientSession | None = None, ): diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 18d4c27fec1..333ce523e34 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -192,12 +192,12 @@ def _rust_responses_websocket_enabled( from .http_handler import get_shared_realtime_ssl_context if TYPE_CHECKING: - import tiktoken from aiohttp import ClientSession from websockets.asyncio.client import ClientConnection from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer from litellm.llms.anthropic.experimental_pass_through.messages.fake_stream_iterator import ( FakeAnthropicMessagesStreamIterator, ) @@ -493,7 +493,7 @@ class BaseLLMHTTPHandler: messages: list, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, client: AsyncHTTPHandler | None = None, json_mode: bool = False, @@ -559,7 +559,7 @@ class BaseLLMHTTPHandler: api_base: str | None, custom_llm_provider: str, model_response: ModelResponse, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", logging_obj: LiteLLMLoggingObj, optional_params: dict, timeout: float | httpx.Timeout, diff --git a/litellm/llms/dashscope/image_generation/transformation.py b/litellm/llms/dashscope/image_generation/transformation.py index c0e278a96ef..ffa60a3d9bc 100644 --- a/litellm/llms/dashscope/image_generation/transformation.py +++ b/litellm/llms/dashscope/image_generation/transformation.py @@ -38,9 +38,8 @@ from litellm.types.llms.openai import ( from litellm.types.utils import ImageObject, ImageResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -165,7 +164,7 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/databricks/chat/transformation.py b/litellm/llms/databricks/chat/transformation.py index 30144d29f51..538904b34e6 100644 --- a/litellm/llms/databricks/chat/transformation.py +++ b/litellm/llms/databricks/chat/transformation.py @@ -149,9 +149,8 @@ def _split_parallel_tool_calls(messages: list[AllMessageValues]) -> list[AllMess if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -189,7 +188,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): return "databricks" @classmethod - def get_config(cls): + def get_config(cls, *, model: str | None = None): return super().get_config() def get_required_params(self) -> list[ProviderField]: @@ -651,7 +650,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/deprecated_providers/aleph_alpha.py b/litellm/llms/deprecated_providers/aleph_alpha.py index 2ad9ce4edc8..8f5c35f32f2 100644 --- a/litellm/llms/deprecated_providers/aleph_alpha.py +++ b/litellm/llms/deprecated_providers/aleph_alpha.py @@ -277,12 +277,7 @@ def completion( ## CALCULATING USAGE - baseten charges on time, not tokens - have some mapping of cost here. prompt_tokens: Final = len(encoding.encode(prompt)) - completion_tokens: Final = len( - encoding.encode( - model_response["choices"][0]["message"]["content"], - disallowed_special=(), - ) - ) + completion_tokens: Final = len(encoding.encode(model_response["choices"][0]["message"]["content"])) model_response.created = int(time.time()) model_response.model = model diff --git a/litellm/llms/edenai/chat/transformation.py b/litellm/llms/edenai/chat/transformation.py index 4fd5a9d550b..67d308d9e38 100644 --- a/litellm/llms/edenai/chat/transformation.py +++ b/litellm/llms/edenai/chat/transformation.py @@ -25,9 +25,8 @@ from litellm.types.utils import ModelResponse, ModelResponseStream, Usage from ..common_utils import EdenAIException, reported_cost, resolve_api_base, resolve_api_key if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding _OPTIONAL_MAPPING: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None) @@ -97,7 +96,7 @@ class EdenAIChatConfig(OpenAIGPTConfig): messages: list[AllMessageValues], # mutable-ok: inherited contract optional_params: dict[str, object], # mutable-ok: inherited contract litellm_params: dict[str, object], # mutable-ok: inherited contract - encoding: "tiktoken.Encoding | None", + encoding: "Encoding | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/edenai/image_generation/transformation.py b/litellm/llms/edenai/image_generation/transformation.py index 2965c4041de..7f729cd7fbb 100644 --- a/litellm/llms/edenai/image_generation/transformation.py +++ b/litellm/llms/edenai/image_generation/transformation.py @@ -19,9 +19,8 @@ from litellm.utils import convert_to_model_response_object from ..common_utils import EdenAIException, endpoint_url, json_headers, pick, reported_cost if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding _SUPPORTED_PARAMS: Final[tuple[OpenAIImageGenerationOptionalParams, ...]] = ( "background", @@ -94,7 +93,7 @@ class EdenAIImageGenerationConfig(BaseImageGenerationConfig): request_data: dict[str, object], # mutable-ok: inherited contract optional_params: dict[str, object], # mutable-ok: inherited contract litellm_params: dict[str, object], # mutable-ok: inherited contract - encoding: "tiktoken.Encoding | None", + encoding: "Encoding | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/fal_ai/image_generation/bria_transformation.py b/litellm/llms/fal_ai/image_generation/bria_transformation.py index c528550811a..53da48e62ca 100644 --- a/litellm/llms/fal_ai/image_generation/bria_transformation.py +++ b/litellm/llms/fal_ai/image_generation/bria_transformation.py @@ -8,9 +8,8 @@ from litellm.types.utils import ImageObject, ImageResponse from .transformation import FalAIBaseConfig if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -187,7 +186,7 @@ class FalAIBriaConfig(FalAIBaseConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/fal_ai/image_generation/flux_pro_v11_ultra_transformation.py b/litellm/llms/fal_ai/image_generation/flux_pro_v11_ultra_transformation.py index 6b8558b8124..7c63e1077f1 100644 --- a/litellm/llms/fal_ai/image_generation/flux_pro_v11_ultra_transformation.py +++ b/litellm/llms/fal_ai/image_generation/flux_pro_v11_ultra_transformation.py @@ -8,9 +8,8 @@ from litellm.types.utils import ImageResponse from .transformation import FalAIBaseConfig, fal_images_to_image_objects if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -194,7 +193,7 @@ class FalAIFluxProV11UltraConfig(FalAIBaseConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/fal_ai/image_generation/ideogram_v3_transformation.py b/litellm/llms/fal_ai/image_generation/ideogram_v3_transformation.py index 04b4f426878..ad1852a622b 100644 --- a/litellm/llms/fal_ai/image_generation/ideogram_v3_transformation.py +++ b/litellm/llms/fal_ai/image_generation/ideogram_v3_transformation.py @@ -8,9 +8,8 @@ from litellm.types.utils import ImageObject, ImageResponse from .transformation import FalAIBaseConfig if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -150,7 +149,7 @@ class FalAIIdeogramV3Config(FalAIBaseConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/fal_ai/image_generation/imagen4_transformation.py b/litellm/llms/fal_ai/image_generation/imagen4_transformation.py index 8a6665b2585..3624b76a4a3 100644 --- a/litellm/llms/fal_ai/image_generation/imagen4_transformation.py +++ b/litellm/llms/fal_ai/image_generation/imagen4_transformation.py @@ -8,9 +8,8 @@ from litellm.types.utils import ImageObject, ImageResponse from .transformation import FalAIBaseConfig if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -182,7 +181,7 @@ class FalAIImagen4Config(FalAIBaseConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/fal_ai/image_generation/recraft_v3_transformation.py b/litellm/llms/fal_ai/image_generation/recraft_v3_transformation.py index 4880dfec7e3..934ce420d53 100644 --- a/litellm/llms/fal_ai/image_generation/recraft_v3_transformation.py +++ b/litellm/llms/fal_ai/image_generation/recraft_v3_transformation.py @@ -8,9 +8,8 @@ from litellm.types.utils import ImageObject, ImageResponse from .transformation import FalAIBaseConfig if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -172,7 +171,7 @@ class FalAIRecraftV3Config(FalAIBaseConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/fal_ai/image_generation/stable_diffusion_transformation.py b/litellm/llms/fal_ai/image_generation/stable_diffusion_transformation.py index bc3a4d07282..79d8800773b 100644 --- a/litellm/llms/fal_ai/image_generation/stable_diffusion_transformation.py +++ b/litellm/llms/fal_ai/image_generation/stable_diffusion_transformation.py @@ -8,9 +8,8 @@ from litellm.types.utils import ImageObject, ImageResponse from .transformation import FalAIBaseConfig if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -208,7 +207,7 @@ class FalAIStableDiffusionConfig(FalAIBaseConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/fal_ai/image_generation/transformation.py b/litellm/llms/fal_ai/image_generation/transformation.py index fd8e280da1c..8f081c7228d 100644 --- a/litellm/llms/fal_ai/image_generation/transformation.py +++ b/litellm/llms/fal_ai/image_generation/transformation.py @@ -16,9 +16,8 @@ from litellm.types.llms.openai import ( from litellm.types.utils import ImageObject, ImageResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -117,7 +116,7 @@ class FalAIBaseConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/fireworks_ai/chat/transformation.py b/litellm/llms/fireworks_ai/chat/transformation.py index 28ebb39a303..196022b3558 100644 --- a/litellm/llms/fireworks_ai/chat/transformation.py +++ b/litellm/llms/fireworks_ai/chat/transformation.py @@ -46,7 +46,7 @@ from ..common_utils import ( ) if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer def _map_reasoning_effort(value: object) -> object: @@ -708,7 +708,7 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/gemini/image_generation/transformation.py b/litellm/llms/gemini/image_generation/transformation.py index d009fe4cd72..bb8d7455031 100644 --- a/litellm/llms/gemini/image_generation/transformation.py +++ b/litellm/llms/gemini/image_generation/transformation.py @@ -24,9 +24,8 @@ from litellm.types.llms.openai import ( from litellm.types.utils import ImageObject, ImageResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -173,7 +172,7 @@ class GoogleImageGenConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/gigachat/chat/transformation.py b/litellm/llms/gigachat/chat/transformation.py index d9250ea8836..c047dc0c881 100644 --- a/litellm/llms/gigachat/chat/transformation.py +++ b/litellm/llms/gigachat/chat/transformation.py @@ -26,9 +26,8 @@ from ..authenticator import get_access_token from ..file_handler import upload_file_sync if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -416,7 +415,7 @@ class GigaChatConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: tiktoken.Encoding | None, + encoding: Tokenizer | None, api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/groq/chat/transformation.py b/litellm/llms/groq/chat/transformation.py index 41a2df17c6f..1da7ad0a7b5 100644 --- a/litellm/llms/groq/chat/transformation.py +++ b/litellm/llms/groq/chat/transformation.py @@ -27,7 +27,7 @@ from litellm.types.utils import ModelResponse, ModelResponseStream, ServerToolUs from ...openai_like.chat.transformation import OpenAILikeChatConfig if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer GROQ_COMPOUND_MODELS: Final = frozenset({"compound", "compound-mini"}) @@ -286,7 +286,7 @@ class GroqChatConfig(OpenAILikeChatConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/huggingface/embedding/handler.py b/litellm/llms/huggingface/embedding/handler.py index 57d1357ee46..60917c68221 100644 --- a/litellm/llms/huggingface/embedding/handler.py +++ b/litellm/llms/huggingface/embedding/handler.py @@ -1,6 +1,5 @@ import json import os -from collections.abc import Sequence from typing import Final, Literal, Protocol, get_args import httpx @@ -32,7 +31,7 @@ hf_tasks_embeddings: Final = ( class _SupportsTokenEncode(Protocol): """Token encoder handle. Only ``encode`` is ever called on it here.""" - def encode(self, text: str, *, disallowed_special: tuple[str, ...]) -> Sequence[int]: ... + def encode(self, text: str) -> list[int]: ... def get_hf_task_embedding_for_model(model: str, task_type: str | None, api_base: str) -> str | None: @@ -214,7 +213,7 @@ class HuggingFaceEmbedding(BaseLLM): model_response.model = model input_tokens = 0 for text in input: - input_tokens += len(encoding.encode(text, disallowed_special=())) + input_tokens += len(encoding.encode_ordinary(text)) setattr( model_response, diff --git a/litellm/llms/huggingface/embedding/transformation.py b/litellm/llms/huggingface/embedding/transformation.py index 33b0e21e326..3fdd4abda73 100644 --- a/litellm/llms/huggingface/embedding/transformation.py +++ b/litellm/llms/huggingface/embedding/transformation.py @@ -25,9 +25,8 @@ from litellm.utils import token_counter from ..common_utils import HuggingFaceError, hf_task_list, hf_tasks, output_parser if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LoggingClass = LiteLLMLoggingObj else: @@ -479,7 +478,7 @@ class HuggingFaceEmbeddingConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/langflow/chat/transformation.py b/litellm/llms/langflow/chat/transformation.py index 17ae7017cf6..a53887b36af 100644 --- a/litellm/llms/langflow/chat/transformation.py +++ b/litellm/llms/langflow/chat/transformation.py @@ -14,9 +14,8 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import Choices, Message, ModelResponse, Usage if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.utils import CustomStreamWrapper @@ -225,7 +224,7 @@ class LangFlowConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/langgraph/chat/transformation.py b/litellm/llms/langgraph/chat/transformation.py index 84d79e6bd31..c9388ee472f 100644 --- a/litellm/llms/langgraph/chat/transformation.py +++ b/litellm/llms/langgraph/chat/transformation.py @@ -23,9 +23,8 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import Choices, Message, ModelResponse, Usage if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.utils import CustomStreamWrapper @@ -415,7 +414,7 @@ class LangGraphConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/lemonade/chat/transformation.py b/litellm/llms/lemonade/chat/transformation.py index c01ad2a0edd..341e8dd2e12 100644 --- a/litellm/llms/lemonade/chat/transformation.py +++ b/litellm/llms/lemonade/chat/transformation.py @@ -19,7 +19,7 @@ from litellm.types.utils import ModelResponse from ...openai_like.chat.transformation import OpenAILikeChatConfig if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class LemonadeChatConfig(OpenAILikeChatConfig): @@ -231,7 +231,7 @@ class LemonadeChatConfig(OpenAILikeChatConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/mistral/chat/transformation.py b/litellm/llms/mistral/chat/transformation.py index f77e828b59a..33b567e9710 100644 --- a/litellm/llms/mistral/chat/transformation.py +++ b/litellm/llms/mistral/chat/transformation.py @@ -32,7 +32,7 @@ from litellm.types.utils import ModelResponse, ModelResponseStream from litellm.utils import convert_to_model_response_object, supports_reasoning if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer def _accepted_reasoning_effort(model: str, requested: str, custom_llm_provider: str) -> str: @@ -580,7 +580,7 @@ class MistralConfig(OpenAIGPTConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/nlp_cloud/chat/transformation.py b/litellm/llms/nlp_cloud/chat/transformation.py index 17c547618d3..2ff48894619 100644 --- a/litellm/llms/nlp_cloud/chat/transformation.py +++ b/litellm/llms/nlp_cloud/chat/transformation.py @@ -14,9 +14,8 @@ from litellm.utils import ModelResponse, Usage from ..common_utils import NLPCloudError if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LoggingClass = LiteLLMLoggingObj else: @@ -175,7 +174,7 @@ class NLPCloudConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/oci/chat/transformation.py b/litellm/llms/oci/chat/transformation.py index 8e4e41b4ac1..ecff823a18d 100644 --- a/litellm/llms/oci/chat/transformation.py +++ b/litellm/llms/oci/chat/transformation.py @@ -65,9 +65,8 @@ from litellm.types.utils import ( from litellm.utils import supports_reasoning if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -603,7 +602,7 @@ class OCIChatConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/ollama/chat/transformation.py b/litellm/llms/ollama/chat/transformation.py index bcde8a041a6..cb3080e6534 100644 --- a/litellm/llms/ollama/chat/transformation.py +++ b/litellm/llms/ollama/chat/transformation.py @@ -31,9 +31,8 @@ from litellm.types.utils import ModelResponse, ModelResponseStream from ..common_utils import OllamaError if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -321,7 +320,7 @@ class OllamaChatConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/ollama/completion/transformation.py b/litellm/llms/ollama/completion/transformation.py index 3eb2c833094..0fc1cd926b8 100644 --- a/litellm/llms/ollama/completion/transformation.py +++ b/litellm/llms/ollama/completion/transformation.py @@ -35,9 +35,8 @@ from litellm.types.utils import ( from ..common_utils import OllamaError, OllamaModelInfo, _convert_image if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -252,7 +251,7 @@ class OllamaConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/oobabooga/chat/transformation.py b/litellm/llms/oobabooga/chat/transformation.py index 43d627102b6..05383a35389 100644 --- a/litellm/llms/oobabooga/chat/transformation.py +++ b/litellm/llms/oobabooga/chat/transformation.py @@ -11,9 +11,8 @@ from litellm.types.utils import ModelResponse, Usage from ..common_utils import OobaboogaError if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LoggingClass = LiteLLMLoggingObj else: @@ -39,7 +38,7 @@ class OobaboogaConfig(OpenAIGPTConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index 9dbcf0cc089..b63684db782 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -60,9 +60,8 @@ from litellm.utils import convert_to_model_response_object from ..common_utils import OpenAIError if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer from litellm.llms.base_llm.base_utils import BaseTokenCounter from litellm.types.llms.openai import ChatCompletionToolParam @@ -671,7 +670,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/openai/image_generation/dall_e_2_transformation.py b/litellm/llms/openai/image_generation/dall_e_2_transformation.py index 74936cf1895..ffc6f1d5fe9 100644 --- a/litellm/llms/openai/image_generation/dall_e_2_transformation.py +++ b/litellm/llms/openai/image_generation/dall_e_2_transformation.py @@ -10,9 +10,10 @@ from litellm.types.utils import ImageResponse from litellm.utils import convert_to_model_response_object if TYPE_CHECKING: - import tiktoken from litellm.litellm_core_utils.logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer + class DallE2ImageGenerationConfig(BaseImageGenerationConfig): """ @@ -52,7 +53,7 @@ class DallE2ImageGenerationConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/openai/image_generation/dall_e_3_transformation.py b/litellm/llms/openai/image_generation/dall_e_3_transformation.py index 5c561d011a9..90b7eaedf2f 100644 --- a/litellm/llms/openai/image_generation/dall_e_3_transformation.py +++ b/litellm/llms/openai/image_generation/dall_e_3_transformation.py @@ -10,9 +10,10 @@ from litellm.types.utils import ImageResponse from litellm.utils import convert_to_model_response_object if TYPE_CHECKING: - import tiktoken from litellm.litellm_core_utils.logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer + class DallE3ImageGenerationConfig(BaseImageGenerationConfig): """ @@ -52,7 +53,7 @@ class DallE3ImageGenerationConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/openai/image_generation/gpt_transformation.py b/litellm/llms/openai/image_generation/gpt_transformation.py index 8dc4d8953ea..c3a826616ed 100644 --- a/litellm/llms/openai/image_generation/gpt_transformation.py +++ b/litellm/llms/openai/image_generation/gpt_transformation.py @@ -10,9 +10,10 @@ from litellm.types.utils import ImageResponse from litellm.utils import convert_to_model_response_object if TYPE_CHECKING: - import tiktoken from litellm.litellm_core_utils.logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer + class GPTImageGenerationConfig(BaseImageGenerationConfig): """ @@ -61,7 +62,7 @@ class GPTImageGenerationConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/openai/image_variations/transformation.py b/litellm/llms/openai/image_variations/transformation.py index afd2909b697..73b44d5ea6a 100644 --- a/litellm/llms/openai/image_variations/transformation.py +++ b/litellm/llms/openai/image_variations/transformation.py @@ -12,7 +12,7 @@ from ...base_llm.image_variations.transformation import BaseImageVariationConfig from ..common_utils import OpenAIError if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class OpenAIImageVariationConfig(BaseImageVariationConfig): @@ -53,7 +53,7 @@ class OpenAIImageVariationConfig(BaseImageVariationConfig): image: FileTypes, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, ) -> ImageResponse: return model_response @@ -68,7 +68,7 @@ class OpenAIImageVariationConfig(BaseImageVariationConfig): image: FileTypes, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, ) -> ImageResponse: return model_response diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index 869ad387c5a..7ac0d988074 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -6,9 +6,10 @@ from typing import TYPE_CHECKING, Final, Literal, Optional, cast import httpx if TYPE_CHECKING: - import tiktoken from aiohttp import ClientSession + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer + import openai from openai import AsyncOpenAI, OpenAI from openai._base_client import make_request_options @@ -277,7 +278,7 @@ class OpenAIConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/openai_like/chat/transformation.py b/litellm/llms/openai_like/chat/transformation.py index 030710c8b2d..e5d6cbb7e5e 100644 --- a/litellm/llms/openai_like/chat/transformation.py +++ b/litellm/llms/openai_like/chat/transformation.py @@ -13,9 +13,8 @@ from litellm.types.utils import ModelResponse from ...openai.chat.gpt_transformation import OpenAIGPTConfig if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -131,7 +130,7 @@ class OpenAILikeChatConfig(OpenAIGPTConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/openrouter/chat/transformation.py b/litellm/llms/openrouter/chat/transformation.py index 77a902149d9..08d43c169f4 100644 --- a/litellm/llms/openrouter/chat/transformation.py +++ b/litellm/llms/openrouter/chat/transformation.py @@ -23,9 +23,8 @@ from ...openai.chat.gpt_transformation import OpenAIGPTConfig from ..common_utils import OpenRouterException if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class CacheControlSupportedModels(str, Enum): @@ -182,7 +181,7 @@ class OpenrouterConfig(OpenAIGPTConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/openrouter/image_generation/transformation.py b/litellm/llms/openrouter/image_generation/transformation.py index 6bbda324336..67d90d027ec 100644 --- a/litellm/llms/openrouter/image_generation/transformation.py +++ b/litellm/llms/openrouter/image_generation/transformation.py @@ -50,9 +50,8 @@ from litellm.types.utils import ( ) if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer else: LiteLLMLoggingObj = Any @@ -319,7 +318,7 @@ class OpenRouterImageGenerationConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/perplexity/chat/transformation.py b/litellm/llms/perplexity/chat/transformation.py index 354f7692fd5..dca2f9857b8 100644 --- a/litellm/llms/perplexity/chat/transformation.py +++ b/litellm/llms/perplexity/chat/transformation.py @@ -15,7 +15,7 @@ from litellm.types.llms.openai import AllMessageValues, ChatCompletionAnnotation from litellm.types.utils import ModelResponse, PromptTokensDetailsWrapper, Usage if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class PerplexityChatConfig(OpenAIGPTConfig): @@ -75,7 +75,7 @@ class PerplexityChatConfig(OpenAIGPTConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/petals/completion/transformation.py b/litellm/llms/petals/completion/transformation.py index 3e0de14a7b2..ee20c2b12d5 100644 --- a/litellm/llms/petals/completion/transformation.py +++ b/litellm/llms/petals/completion/transformation.py @@ -14,7 +14,7 @@ from litellm.types.utils import ModelResponse from ..common_utils import PetalsError if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class PetalsConfig(BaseConfig): @@ -112,7 +112,7 @@ class PetalsConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/predibase/chat/transformation.py b/litellm/llms/predibase/chat/transformation.py index 2a63c489395..69924396a1e 100644 --- a/litellm/llms/predibase/chat/transformation.py +++ b/litellm/llms/predibase/chat/transformation.py @@ -18,9 +18,8 @@ from litellm.types.utils import Choices, Message, ModelResponse, Usage from ..common_utils import PredibaseError if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -150,7 +149,7 @@ class PredibaseConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/recraft/image_generation/transformation.py b/litellm/llms/recraft/image_generation/transformation.py index 3a04e0a62b4..f65bf1e7292 100644 --- a/litellm/llms/recraft/image_generation/transformation.py +++ b/litellm/llms/recraft/image_generation/transformation.py @@ -14,9 +14,8 @@ from litellm.types.llms.recraft import RecraftImageGenerationRequestParams from litellm.types.utils import ImageObject, ImageResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -122,7 +121,7 @@ class RecraftImageGenerationConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/replicate/chat/transformation.py b/litellm/llms/replicate/chat/transformation.py index 769160c6ced..f7e09b7bec0 100644 --- a/litellm/llms/replicate/chat/transformation.py +++ b/litellm/llms/replicate/chat/transformation.py @@ -19,9 +19,8 @@ from litellm.utils import token_counter from ..common_utils import ReplicateError if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LoggingClass = LiteLLMLoggingObj else: @@ -237,7 +236,7 @@ class ReplicateConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/runwayml/image_generation/transformation.py b/litellm/llms/runwayml/image_generation/transformation.py index 5913709c8a0..e5e988328d8 100644 --- a/litellm/llms/runwayml/image_generation/transformation.py +++ b/litellm/llms/runwayml/image_generation/transformation.py @@ -22,9 +22,8 @@ from litellm.types.llms.openai import ( from litellm.types.utils import ImageObject, ImageResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -308,7 +307,7 @@ class RunwayMLImageGenerationConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: @@ -383,7 +382,7 @@ class RunwayMLImageGenerationConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/sagemaker/completion/transformation.py b/litellm/llms/sagemaker/completion/transformation.py index 576018f0046..e1bc496a82d 100644 --- a/litellm/llms/sagemaker/completion/transformation.py +++ b/litellm/llms/sagemaker/completion/transformation.py @@ -24,9 +24,8 @@ from litellm.utils import token_counter from ..common_utils import SagemakerError if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -198,7 +197,7 @@ class SagemakerConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/sap/chat/transformation.py b/litellm/llms/sap/chat/transformation.py index d64d7a57281..4c73ccacc16 100755 --- a/litellm/llms/sap/chat/transformation.py +++ b/litellm/llms/sap/chat/transformation.py @@ -15,9 +15,8 @@ from litellm.types.utils import ModelResponse from ...openai.chat.gpt_transformation import OpenAIGPTConfig if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -57,7 +56,7 @@ def validate_dict(data: dict, model) -> dict: return model(**data).model_dump(by_alias=True, exclude_unset=True) -def _messages_to_sap_template(messages: list[dict[str, str]]) -> list: +def _messages_to_sap_template(messages: list[AllMessageValues]) -> list: template: Final = [] for message in messages: if message["role"] == "user": @@ -311,7 +310,7 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig): def transform_request( self, model: str, - messages: list[dict[str, str]], + messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, headers: dict, @@ -383,7 +382,7 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/stability/image_generation/transformation.py b/litellm/llms/stability/image_generation/transformation.py index cf3576a9404..656ffe395c8 100644 --- a/litellm/llms/stability/image_generation/transformation.py +++ b/litellm/llms/stability/image_generation/transformation.py @@ -26,9 +26,8 @@ from litellm.types.llms.stability import ( from litellm.types.utils import ImageObject, ImageResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -207,7 +206,7 @@ class StabilityImageGenerationConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/topaz/image_variations/transformation.py b/litellm/llms/topaz/image_variations/transformation.py index f4753c8ba17..94f60d29cb9 100644 --- a/litellm/llms/topaz/image_variations/transformation.py +++ b/litellm/llms/topaz/image_variations/transformation.py @@ -23,7 +23,7 @@ from ...base_llm.image_variations.transformation import BaseImageVariationConfig from ..common_utils import TopazException, TopazModelInfo if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class TopazImageVariationConfig(TopazModelInfo, BaseImageVariationConfig): @@ -139,7 +139,7 @@ class TopazImageVariationConfig(TopazModelInfo, BaseImageVariationConfig): image: FileTypes, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, ) -> ImageResponse: image_content: Final = await raw_response.read() @@ -158,7 +158,7 @@ class TopazImageVariationConfig(TopazModelInfo, BaseImageVariationConfig): image: FileTypes, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, ) -> ImageResponse: image_content: Final = raw_response.content diff --git a/litellm/llms/triton/completion/transformation.py b/litellm/llms/triton/completion/transformation.py index 3c868b3a96f..b37bbd78f2b 100644 --- a/litellm/llms/triton/completion/transformation.py +++ b/litellm/llms/triton/completion/transformation.py @@ -29,7 +29,7 @@ from litellm.types.utils import ( from ..common_utils import TritonError if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class TritonConfig(BaseConfig): @@ -95,7 +95,7 @@ class TritonConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: @@ -215,7 +215,7 @@ class TritonGenerateConfig(TritonConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: @@ -280,7 +280,7 @@ class TritonInferConfig(TritonConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/vertex_ai/agent_engine/transformation.py b/litellm/llms/vertex_ai/agent_engine/transformation.py index c5ca9f38144..b37bf473731 100644 --- a/litellm/llms/vertex_ai/agent_engine/transformation.py +++ b/litellm/llms/vertex_ai/agent_engine/transformation.py @@ -29,9 +29,8 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import Choices, Message, ModelResponse, Usage if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.utils import CustomStreamWrapper @@ -285,7 +284,7 @@ class VertexAgentEngineConfig(BaseConfig, VertexBase): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py b/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py index b2c52c53580..17ccf16837b 100644 --- a/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py +++ b/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py @@ -24,9 +24,8 @@ from litellm.types.utils import ( ) if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -284,7 +283,7 @@ class VertexAIGeminiImageGenerationConfig(BaseImageGenerationConfig, VertexLLM): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/vertex_ai/image_generation/vertex_imagen_transformation.py b/litellm/llms/vertex_ai/image_generation/vertex_imagen_transformation.py index 8faf7b0d484..ae6e08611ae 100644 --- a/litellm/llms/vertex_ai/image_generation/vertex_imagen_transformation.py +++ b/litellm/llms/vertex_ai/image_generation/vertex_imagen_transformation.py @@ -20,9 +20,8 @@ from litellm.types.llms.openai import ( from litellm.types.utils import ImageObject, ImageResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -214,7 +213,7 @@ class VertexAIImagenImageGenerationConfig(BaseImageGenerationConfig, VertexLLM): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py index 508f68b3eca..2fab6f438f6 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py @@ -14,7 +14,7 @@ from ....anthropic.chat.transformation import AnthropicConfig from .output_params_utils import sanitize_vertex_anthropic_output_params if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class VertexAIError(Exception): @@ -197,7 +197,7 @@ class VertexAIAnthropicConfig(AnthropicConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/llama3/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/llama3/transformation.py index 89a5b8a570e..f2d2c0896d2 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/llama3/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/llama3/transformation.py @@ -21,7 +21,7 @@ from litellm.types.utils import ( from ...common_utils import VertexAIError if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class VertexAILlama3Config(OpenAIGPTConfig): @@ -112,7 +112,7 @@ class VertexAILlama3Config(OpenAIGPTConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/vertex_ai/vertex_gemma_models/transformation.py b/litellm/llms/vertex_ai/vertex_gemma_models/transformation.py index 67b01c2dc43..2ae8b4cd188 100644 --- a/litellm/llms/vertex_ai/vertex_gemma_models/transformation.py +++ b/litellm/llms/vertex_ai/vertex_gemma_models/transformation.py @@ -24,9 +24,8 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer from litellm.llms.base_llm.base_model_iterator import MockResponseIterator @@ -276,7 +275,7 @@ class VertexGemmaConfig(OpenAIGPTConfig): litellm_params: dict, client: HTTPHandler | httpx.Client | None = None, timeout: float | httpx.Timeout | None = None, - encoding: "tiktoken.Encoding | None" = None, + encoding: "Tokenizer | None" = None, ): """Synchronous completion request""" from litellm.utils import convert_to_model_response_object @@ -366,7 +365,7 @@ class VertexGemmaConfig(OpenAIGPTConfig): litellm_params: dict, client: AsyncHTTPHandler | httpx.AsyncClient | None = None, timeout: float | httpx.Timeout | None = None, - encoding: "tiktoken.Encoding | None" = None, + encoding: "Tokenizer | None" = None, ): """Asynchronous completion request""" from litellm.utils import convert_to_model_response_object diff --git a/litellm/llms/watsonx/completion/transformation.py b/litellm/llms/watsonx/completion/transformation.py index 2be007336b4..2fec8485cf9 100644 --- a/litellm/llms/watsonx/completion/transformation.py +++ b/litellm/llms/watsonx/completion/transformation.py @@ -21,9 +21,8 @@ from ..common_utils import ( ) if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -280,7 +279,7 @@ class IBMWatsonXAIConfig(IBMWatsonXMixin, BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/main.py b/litellm/main.py index 2570b93455f..7d231a1bf7a 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -37,7 +37,6 @@ if TYPE_CHECKING: import dotenv import httpx import openai -import tiktoken from pydantic import BaseModel from typing_extensions import overload @@ -100,6 +99,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( from litellm.litellm_core_utils.request_timeout_resolver import ( get_configured_request_timeout, ) +from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer from litellm.llms.azure_ai.common_utils import ( azure_ai_supports_native_responses, foundry_chat_rejects_function_tools_while_reasoning, @@ -7489,7 +7489,9 @@ def text_completion( if isinstance(prompt, list): import concurrent.futures - tokenizer: Final = tiktoken.encoding_for_model("text-davinci-003") + from litellm.rust_bridge.tokenizer import get_encoding + + tokenizer: Final = get_encoding("p50k_base") ## if it's a 2d list - each element in the list is a text_completion() request if len(prompt) > 0 and isinstance(prompt[0], list): responses: Final = [None for x in prompt] # init responses @@ -9259,7 +9261,7 @@ async def acount_tokens( except Exception as e: verbose_logger.debug("Provider token counting failed for model=%s, falling back to local: %s", model, e) - # Fallback to local tiktoken-based token counting + # Fallback to local token counting fallback_messages = messages or [] if system and fallback_messages: fallback_messages = [{"role": "system", "content": system}] + fallback_messages @@ -9278,16 +9280,16 @@ async def acount_tokens( # Cache for encoding to avoid repeated __getattr__ calls -_encoding_cache: tiktoken.Encoding | None = None +_encoding_cache: Tokenizer | None = None -def _load_module_encoding() -> tiktoken.Encoding: +def _load_module_encoding() -> Tokenizer: import sys return sys.modules[__name__].encoding -def _get_encoding() -> tiktoken.Encoding: +def _get_encoding() -> Tokenizer: """Get encoding, loading it lazily if needed.""" global _encoding_cache if _encoding_cache is None: @@ -9296,18 +9298,15 @@ def _get_encoding() -> tiktoken.Encoding: return _encoding_cache -def _load_default_encoding() -> tiktoken.Encoding: +def _load_default_encoding() -> Tokenizer: from litellm._lazy_imports import _get_default_encoding return _get_default_encoding() -def __getattr__(name: str) -> tiktoken.Encoding: +def __getattr__(name: str) -> Tokenizer: """Lazy import handler for main module""" if name == "encoding": - # Use _get_default_encoding which properly sets TIKTOKEN_CACHE_DIR - # before loading tiktoken, ensuring the local cache is used - # instead of downloading from the internet _encoding: Final = _load_default_encoding() # Cache it in the module's __dict__ for subsequent accesses import sys diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index ac9b07a55f7..e28fa2c06a4 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -36,10 +36,10 @@ from litellm.proxy.common_utils.user_api_key_cache import ( tag_cache_key, team_membership_reservation_cache_key, ) +from litellm.proxy.spend_tracking.input_tokens import count_input_tokens, count_input_tokens_for_model from litellm.proxy.spend_tracking.spend_counter_batch import PendingSpendIncrement, spend_counter_batch_scope from litellm.proxy.utils import PrismaClient, ProxyLogging from litellm.router import Router -from litellm.rust_bridge.token_counter import RustTokenizer, count_input_tokens, rust_tokenizer from litellm.types.proxy.model_access_group_budget import ModelAccessGroupBudget from litellm.types.router import DeploymentTypedDict @@ -1489,9 +1489,6 @@ def _get_request_models( return (model,) if isinstance(model, str) else tuple(model) -TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS: Final = 30_000 - - async def count_request_input_tokens( request_body: dict, route: str, @@ -1500,105 +1497,11 @@ async def count_request_input_tokens( ) -> Mapping[str, int]: """Input-token count per candidate model, counted once per request. - Tokenizing is the reservation path's dominant CPU cost and is O(prompt), so - counting a large prompt inline stalls every other request on the worker. - Models whose tokenizer the Rust bridge ports (Anthropic, tiktoken cl100k_base - and o200k_base) are counted from the raw body by the bridge when it is enabled, once per - distinct tokenizer, which parses and tokenizes with the GIL released. - Everything it declines is counted in Python, large prompts in a worker - thread. The counts are reused by both the max-cost and the input-cost - estimate. - """ + The counts are reused by both the max-cost and the input-cost estimate.""" models: Final = _get_request_models(request_body=request_body, route=route, llm_router=llm_router) if not models: return MappingProxyType({}) - tokenizers: Final[Mapping[str, RustTokenizer | None]] = MappingProxyType( - {model: rust_tokenizer(model) for model in models} - ) - distinct_tokenizers: Final[tuple[RustTokenizer, ...]] = tuple( - dict.fromkeys(tokenizer for tokenizer in tokenizers.values() if tokenizer is not None) - ) - rust_counts_by_tokenizer: Final[Mapping[RustTokenizer, int]] = MappingProxyType( - { - tokenizer: count.input_tokens - for tokenizer in distinct_tokenizers - if raw_body is not None and (count := await count_input_tokens(raw_body, tokenizer)) is not None - } - ) - rust_counts: Final = MappingProxyType( - { - model: rust_counts_by_tokenizer[tokenizer] - for model, tokenizer in tokenizers.items() - if tokenizer is not None and tokenizer in rust_counts_by_tokenizer - } - ) - python_models: Final = tuple(model for model in models if model not in rust_counts) - python_counts: Final = ( - MappingProxyType({}) - if not python_models - else _count_input_tokens_for_models(request_body=request_body, models=python_models) - if _approximate_input_size(request_body) < TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS - else await asyncio.to_thread( - _count_input_tokens_for_models, - request_body=request_body, - models=python_models, - ) - ) - verbose_proxy_logger.debug("input token counts: rust=%s python=%s", dict(rust_counts), dict(python_counts)) - return MappingProxyType({**rust_counts, **python_counts}) - - -def _count_input_tokens_for_models( - request_body: dict, - models: Sequence[str], -) -> Mapping[str, int]: - return MappingProxyType( - { - model: tokens - for model in models - if (tokens := _count_input_tokens(request_body=request_body, model=model)) is not None - } - ) - - -_INPUT_SIZE_FIELDS: Final = ("messages", "prompt", "input", "query", "documents", "tools", "tool_choice") - - -def _approximate_input_size(request_body: Mapping[str, object]) -> int: - """Length of the request's input text, a cheap stand-in for tokenizing cost. - - Every field _count_input_tokens hands the tokenizer is sized here, and - rendering rather than walking keeps mapping keys in the total, which a tool - schema's property names are.""" - return sum(len(str(request_body.get(field, ""))) for field in _INPUT_SIZE_FIELDS) - - -def _count_input_tokens(request_body: dict, model: str) -> int | None: - try: - if "messages" in request_body: - try: - return litellm.token_counter( - model=model, - messages=request_body.get("messages") or (), - tools=request_body.get("tools"), - tool_choice=request_body.get("tool_choice"), - ) - except ValueError: - return _count_text_tokens(model=model, text=request_body.get("messages")) - if "prompt" in request_body: - return _count_text_tokens(model=model, text=request_body.get("prompt")) - if "input" in request_body: - return _count_text_tokens(model=model, text=request_body.get("input")) - if "query" in request_body or "documents" in request_body: - query_tokens: Final = _count_text_tokens(model=model, text=request_body.get("query")) - document_tokens: Final = _count_text_tokens( - model=model, - text=request_body.get("documents"), - ) - return query_tokens + document_tokens - except Exception: - verbose_proxy_logger.debug("Unable to count input tokens for budget reservation", exc_info=True) - return None + return await count_input_tokens(request_body=request_body, raw_body=raw_body, models=models) def _estimate_input_tokens( @@ -1609,7 +1512,9 @@ def _estimate_input_tokens( input_tokens: int | None = None, ) -> int | None: counted: Final = ( - input_tokens if input_tokens is not None else _count_input_tokens(request_body=request_body, model=model) + input_tokens + if input_tokens is not None + else count_input_tokens_for_model(request_body=request_body, model=model) ) if counted is not None: return counted @@ -1658,26 +1563,6 @@ def _requested_output_tokens(request_body: Mapping[str, object]) -> int | None: return next((tokens for tokens in map(_to_int, candidates) if tokens is not None), None) -def _count_text_tokens(model: str, text: object) -> int: - if text is None: - return 0 - - token_count = 0 - stack: Final = [text] - while stack: - item = stack.pop() - if item is None: - continue - if isinstance(item, list): - stack.extend(item) - continue - if isinstance(item, dict): - token_count += litellm.token_counter(model=model, text=json.dumps(item)) - continue - token_count += litellm.token_counter(model=model, text=str(item)) - return token_count - - def _get_output_multiplier(request_body: dict) -> int: output_multiplier = 1 for key in ("n", "best_of"): diff --git a/litellm/proxy/spend_tracking/input_tokens.py b/litellm/proxy/spend_tracking/input_tokens.py new file mode 100644 index 00000000000..6c7083fb6db --- /dev/null +++ b/litellm/proxy/spend_tracking/input_tokens.py @@ -0,0 +1,173 @@ +"""Input-token counting for the budget reservation path. + +Tokenizing is the reservation path's dominant CPU cost and is O(prompt), so +counting a large prompt inline stalls every other request on the worker. +Models whose tokenizer the Rust bridge ports (Anthropic, tiktoken cl100k_base +and o200k_base) are counted from the raw body by the bridge, once per distinct +tokenizer, which parses and tokenizes with the GIL released. Everything it +declines, and every model with no Rust tokenizer, is counted in Python, large +prompts in a worker thread. +""" + +from __future__ import annotations + +import asyncio +import json +from collections.abc import Mapping, Sequence +from types import MappingProxyType +from typing import Final + +import litellm +from litellm._logging import verbose_proxy_logger +from litellm.rust_bridge import runtime +from litellm.rust_bridge.catalog import Route, RouteContext +from litellm.rust_bridge.token_counter import ( + TOKEN_COUNTER, + RustTokenCounterFactory, + RustTokenizer, + native_count, + rust_tokenizer, +) + +TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS: Final = 30_000 + +_INPUT_SIZE_FIELDS: Final = ("messages", "prompt", "input", "query", "documents", "tools", "tool_choice") + + +def _approximate_input_size(request_body: Mapping[str, object]) -> int: + """Length of the request's input text, a cheap stand-in for tokenizing cost. + + Every field count_input_tokens_for_model hands the tokenizer is sized here, + and rendering rather than walking keeps mapping keys in the total, which a + tool schema's property names are.""" + return sum(len(str(request_body.get(field, ""))) for field in _INPUT_SIZE_FIELDS) + + +async def count_input_tokens( + request_body: dict, + raw_body: bytes | None, + models: Sequence[str], +) -> Mapping[str, int]: + """Input-token count per model, sharing one native count across models that + select the same tokenizer.""" + tokenizers: Final[tuple[tuple[str, RustTokenizer | None], ...]] = tuple( + (model, rust_tokenizer(model)) for model in models + ) + groups: Final[tuple[RustTokenizer | None, ...]] = tuple(dict.fromkeys(tokenizer for _, tokenizer in tokenizers)) + group_counts: Final = [ + await _count_group( + request_body=request_body, + raw_body=raw_body, + tokenizer=tokenizer, + models=tuple(model for model, selected in tokenizers if selected == tokenizer), + ) + for tokenizer in groups + ] + counts: Final = MappingProxyType({model: tokens for group in group_counts for model, tokens in group.items()}) + verbose_proxy_logger.debug("input token counts: %s", dict(counts)) + return counts + + +async def _count_group( + request_body: dict, + raw_body: bytes | None, + tokenizer: RustTokenizer | None, + models: tuple[str, ...], +) -> Mapping[str, int]: + async def python() -> Mapping[str, int]: + if _approximate_input_size(request_body) < TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS: + return _count_input_tokens_for_models(request_body=request_body, models=models) + return await asyncio.to_thread( + _count_input_tokens_for_models, + request_body=request_body, + models=models, + ) + + if tokenizer is None or raw_body is None: + return await python() + try: + return await runtime.arun( + RouteContext(Route.TOKEN_COUNTER, provider=tokenizer), + binding=TOKEN_COUNTER, + native=lambda factory: _native_counts(factory, tokenizer, raw_body, models), + python=python, + ) + except (RuntimeError, ValueError) as error: + from litellm.rust_bridge.fork_guard import ForkedAfterNativeRuntimeStarted, ProcessReservedForForking + + if isinstance(error, (ForkedAfterNativeRuntimeStarted, ProcessReservedForForking)): + raise + verbose_proxy_logger.debug("Rust token counter (%s) failed, counting in Python: %s", tokenizer, error) + return await python() + + +async def _native_counts( + factory: RustTokenCounterFactory, + tokenizer: RustTokenizer, + raw_body: bytes, + models: tuple[str, ...], +) -> Mapping[str, int]: + count: Final = await native_count(factory, tokenizer, raw_body) + verbose_proxy_logger.debug("Rust token counter (%s) counted %d input tokens", tokenizer, count.input_tokens) + return MappingProxyType({model: count.input_tokens for model in models}) + + +def _count_input_tokens_for_models( + request_body: dict, + models: Sequence[str], +) -> Mapping[str, int]: + return MappingProxyType( + { + model: tokens + for model in models + if (tokens := count_input_tokens_for_model(request_body=request_body, model=model)) is not None + } + ) + + +def count_input_tokens_for_model(request_body: dict, model: str) -> int | None: + try: + if "messages" in request_body: + try: + return litellm.token_counter( + model=model, + messages=request_body.get("messages") or (), + tools=request_body.get("tools"), + tool_choice=request_body.get("tool_choice"), + ) + except ValueError: + return _count_text_tokens(model=model, text=request_body.get("messages")) + if "prompt" in request_body: + return _count_text_tokens(model=model, text=request_body.get("prompt")) + if "input" in request_body: + return _count_text_tokens(model=model, text=request_body.get("input")) + if "query" in request_body or "documents" in request_body: + query_tokens: Final = _count_text_tokens(model=model, text=request_body.get("query")) + document_tokens: Final = _count_text_tokens( + model=model, + text=request_body.get("documents"), + ) + return query_tokens + document_tokens + except Exception: + verbose_proxy_logger.debug("Unable to count input tokens for budget reservation", exc_info=True) + return None + + +def _count_text_tokens(model: str, text: object) -> int: + if text is None: + return 0 + + token_count = 0 + stack: Final = [text] + while stack: + item = stack.pop() + if item is None: + continue + if isinstance(item, list): + stack.extend(item) + continue + if isinstance(item, dict): + token_count += litellm.token_counter(model=model, text=json.dumps(item)) + continue + token_count += litellm.token_counter(model=model, text=str(item)) + return token_count diff --git a/litellm/rust_bridge/_native.pyi b/litellm/rust_bridge/_native.pyi index a033b89a6a8..61e597bf674 100644 --- a/litellm/rust_bridge/_native.pyi +++ b/litellm/rust_bridge/_native.pyi @@ -224,26 +224,121 @@ class _CacheTestResolver: @final class TokenCounter: - def __new__(cls, tokenizer_json: str) -> TokenCounter: ... @staticmethod - def from_cl100k_ranks(rank_file: str) -> TokenCounter: ... - @staticmethod - def from_o200k_ranks(rank_file: str) -> TokenCounter: ... - @staticmethod - def from_tiktoken(encoding: str) -> TokenCounter: ... + def from_tokenizer(tokenizer: Tokenizer, fast: bool = False) -> TokenCounter: ... def acount_request(self, body: bytes) -> Future[dict[str, object]]: ... +@final +class Tokenizer: + @staticmethod + def from_tiktoken(encoding: str) -> Tokenizer: ... + @staticmethod + def from_json(tokenizer_json: str) -> Tokenizer: ... + @staticmethod + def from_pretrained( + identifier: str, + revision: str = "main", + token: str | None = None, + ) -> Tokenizer: ... + def encode(self, text: str) -> list[int]: ... + def decode(self, ids: Sequence[int], skip_special_tokens: bool = True) -> str: ... + def count(self, text: str, fast: bool = False) -> int: ... + # tiktoken encodings + def encode_special(self, text: str, allowed: Sequence[str]) -> list[int]: ... + def encode_with_unstable(self, text: str, allowed: Sequence[str]) -> tuple[list[int], list[list[int]]]: ... + def encode_single_token(self, piece: bytes) -> int: ... + def special_tokens(self) -> dict[str, int]: ... + def max_token_value(self) -> int: ... + def is_special_token(self, token: int) -> bool: ... + def token_byte_values(self) -> list[bytes]: ... + def decode_bytes(self, ids: Sequence[int]) -> bytes: ... + # Hugging Face tokenizers + def to_json(self, pretty: bool = False) -> str: ... + def token_to_id(self, token: str) -> int | None: ... + def id_to_token(self, id: int) -> str | None: ... + def get_vocab(self, with_added_tokens: bool = True) -> dict[str, int]: ... + def get_vocab_size(self, with_added_tokens: bool = True) -> int: ... + def added_tokens_decoder(self) -> list[tuple[int, tuple[str, bool, bool, bool, bool, bool]]]: ... + def padding(self) -> dict[str, object] | None: ... + def truncation(self) -> dict[str, object] | None: ... + def num_special_tokens_to_add(self, is_pair: bool) -> int: ... + def encode_special_tokens(self) -> bool: ... + def encode_huggingface( + self, + sequence: str | Sequence[str], + pair: str | Sequence[str] | None = None, + is_pretokenized: bool = False, + add_special_tokens: bool = True, + fast: bool = False, + ) -> HuggingFaceEncoding: ... + def encode_batch_huggingface( + self, + inputs: Sequence[tuple[str | Sequence[str], str | Sequence[str] | None]], + is_pretokenized: bool = False, + add_special_tokens: bool = True, + fast: bool = False, + ) -> list[HuggingFaceEncoding]: ... + @property + def name(self) -> str: ... + +@final +class HuggingFaceEncoding: + def __new__(cls, json: str | None = None) -> HuggingFaceEncoding: ... + @staticmethod + def merge(encodings: Sequence[HuggingFaceEncoding], growing_offsets: bool = True) -> HuggingFaceEncoding: ... + def __len__(self) -> int: ... + def __reduce__(self) -> tuple[type[HuggingFaceEncoding], tuple[str]]: ... + def word_to_tokens(self, word_index: int, sequence_index: int = 0) -> tuple[int, int] | None: ... + def word_to_chars(self, word_index: int, sequence_index: int = 0) -> tuple[int, int] | None: ... + def token_to_sequence(self, token_index: int) -> int | None: ... + def token_to_chars(self, token_index: int) -> tuple[int, int] | None: ... + def token_to_word(self, token_index: int) -> int | None: ... + def char_to_token(self, char_pos: int, sequence_index: int = 0) -> int | None: ... + def char_to_word(self, char_pos: int, sequence_index: int = 0) -> int | None: ... + def set_sequence_id(self, sequence_id: int) -> None: ... + def pad( + self, + length: int, + direction: str = "right", + pad_id: int = 0, + pad_type_id: int = 0, + pad_token: str = "[PAD]", + ) -> None: ... + def truncate(self, max_length: int, stride: int = 0, direction: str = "right") -> None: ... + @property + def ids(self) -> list[int]: ... + @property + def tokens(self) -> list[str]: ... + @property + def offsets(self) -> list[tuple[int, int]]: ... + @property + def type_ids(self) -> list[int]: ... + @property + def attention_mask(self) -> list[int]: ... + @property + def special_tokens_mask(self) -> list[int]: ... + @property + def word_ids(self) -> list[int | None]: ... + @property + def sequence_ids(self) -> list[int | None]: ... + @property + def overflowing(self) -> list[HuggingFaceEncoding]: ... + @property + def n_sequences(self) -> int: ... + def gil_stats() -> dict[str, int]: ... def process_state_started() -> bool: ... def reserve_process_for_forking() -> None: ... __all__ = [ "ForkedAfterNativeRuntimeStarted", + "HuggingFaceEncoding", "ProcessReservedForForking", "ResponsesWebSocketConnection", "RustBridgeDeclined", "RustUpstreamError", "TokenCounter", + "Tokenizer", "achat_completions", "amessages", "aocr", diff --git a/litellm/rust_bridge/catalog.py b/litellm/rust_bridge/catalog.py index 74ceb1ba123..d7479631e04 100644 --- a/litellm/rust_bridge/catalog.py +++ b/litellm/rust_bridge/catalog.py @@ -22,6 +22,8 @@ class Route(str, Enum): RESPONSES = "responses" TRANSCRIPTION = "transcription" OCR = "ocr" + TOKEN_COUNTER = "token_counter" + TOKENIZER = "tokenizer" class Delivery(Enum): @@ -92,6 +94,8 @@ RULES: Final[Rules] = ( RouteRule(Route.OCR, Rollout.RUST_REQUIRED, providers=frozenset({"aws_textract"})), RouteRule(Route.OCR, Rollout.RUST_OPT_OUT), RouteRule(Route.MESSAGES, Rollout.RUST_OPT_IN), + RouteRule(Route.TOKEN_COUNTER, Rollout.RUST_OPT_IN), + RouteRule(Route.TOKENIZER, Rollout.RUST_OPT_IN), RouteRule(Route.TRANSCRIPTION, Rollout.RUST_REQUIRED, providers=frozenset({"bedrock"})), CacheRule(Rollout.PYTHON_ONLY, backends=frozenset({LiteLLMCacheType.LOCAL})), CacheRule(Rollout.PYTHON_ONLY, backends=frozenset({LiteLLMCacheType.REDIS})), diff --git a/litellm/rust_bridge/configuration.py b/litellm/rust_bridge/configuration.py index cd468c80655..152ba632996 100644 --- a/litellm/rust_bridge/configuration.py +++ b/litellm/rust_bridge/configuration.py @@ -2,6 +2,7 @@ from __future__ import annotations import os from enum import Enum, auto +from functools import lru_cache from typing import Final from pydantic import TypeAdapter, ValidationError @@ -32,7 +33,9 @@ class _RustConfiguration: _CONFIGURATION: Final = _RustConfiguration() +@lru_cache(maxsize=16) def _parse_env_bool(value: str | None) -> bool | None: + """`LITELLM_RUST` as a bool; cached by raw value because `decision` runs per tokenizer call.""" if value is None: return None try: diff --git a/litellm/rust_bridge/token_counter.py b/litellm/rust_bridge/token_counter.py index d36234f56c1..250ad18d44c 100644 --- a/litellm/rust_bridge/token_counter.py +++ b/litellm/rust_bridge/token_counter.py @@ -5,18 +5,19 @@ from __future__ import annotations from collections.abc import Awaitable from dataclasses import dataclass from functools import lru_cache -from typing import Final, Literal, Protocol, cast # noqa: TID251 # native extension exposes untyped callables +from typing import TYPE_CHECKING, Final, Literal, Protocol, cast # noqa: TID251 # PyO3 binding validation from pydantic import TypeAdapter +from typing_extensions import assert_never import litellm -from litellm._logging import verbose_logger -from litellm.litellm_core_utils.default_encoding import cl100k_base_rank_file, o200k_base_rank_file -from litellm.litellm_core_utils.token_counter import openai_tokenizer_encoding, uses_legacy_message_accounting +from litellm.litellm_core_utils.token_counter import openai_tokenizer_encoding_name, uses_legacy_message_accounting +from litellm.rust_bridge import tokenizer as tokenizer_dispatch from litellm.rust_bridge.bindings import NativeBinding -from litellm.rust_bridge.configuration import rust_enabled -from litellm.rust_bridge.runtime import BridgeErrorContext, RustHandled, aattempt -from litellm.utils import claude_json_str, huggingface_tokenizer_kind +from litellm.utils import huggingface_tokenizer_kind + +if TYPE_CHECKING: + from litellm.rust_bridge._native import Tokenizer as NativeTokenizer RustTokenizer = Literal["anthropic", "cl100k_base", "o200k_base"] @@ -27,13 +28,7 @@ class RustTokenCounter(Protocol): class RustTokenCounterFactory(Protocol): - def __call__(self, tokenizer_json: str) -> RustTokenCounter: - raise NotImplementedError - - def from_cl100k_ranks(self, rank_file: str) -> RustTokenCounter: - raise NotImplementedError - - def from_o200k_ranks(self, rank_file: str) -> RustTokenCounter: + def from_tokenizer(self, tokenizer: NativeTokenizer, fast: bool = False) -> RustTokenCounter: raise NotImplementedError @@ -51,7 +46,7 @@ def _as_factory(value: object) -> RustTokenCounterFactory | None: cast( # cast-ok: native extension protocol is runtime-defined RustTokenCounterFactory, value ) - if callable(value) + if callable(getattr(value, "from_tokenizer", None)) else None ) @@ -73,7 +68,7 @@ def rust_tokenizer(model: str) -> RustTokenizer | None: return "anthropic" if kind is not None or uses_legacy_message_accounting(model): return None - match openai_tokenizer_encoding(model).name: + match openai_tokenizer_encoding_name(model): case "cl100k_base": return "cl100k_base" case "o200k_base": @@ -84,31 +79,26 @@ def rust_tokenizer(model: str) -> RustTokenizer | None: @lru_cache(maxsize=4) def _counter(factory: RustTokenCounterFactory, tokenizer: RustTokenizer) -> RustTokenCounter: + return factory.from_tokenizer(_native_tokenizer(tokenizer)) + + +def _native_tokenizer(tokenizer: RustTokenizer) -> NativeTokenizer: match tokenizer: case "anthropic": - return factory(claude_json_str) - case "cl100k_base": - return factory.from_cl100k_ranks(cl100k_base_rank_file()) - case "o200k_base": - return factory.from_o200k_ranks(o200k_base_rank_file()) + native = tokenizer_dispatch.native_anthropic() + case "cl100k_base" | "o200k_base": + native = tokenizer_dispatch.native_encoding(tokenizer) + case _: + assert_never(tokenizer) + if native is None: + raise RuntimeError(f"native {tokenizer} tokenizer is unavailable") + return native -async def count_input_tokens(body: bytes, tokenizer: RustTokenizer) -> InputTokenCount | None: - if not rust_enabled(): - return None - factory: Final = TOKEN_COUNTER.load() - if factory is None: - return None - try: - attempt: Final = await aattempt( - native_call=lambda: _counter(factory, tokenizer).acount_request(body), - adapt=_INPUT_TOKEN_COUNT.validate_python, - context=BridgeErrorContext(route="token_counter", provider=tokenizer, model=""), - ) - except (RuntimeError, ValueError) as error: - verbose_logger.debug("Rust token counter (%s) failed, counting in Python: %s", tokenizer, error) - return None - if not isinstance(attempt, RustHandled): - return None - verbose_logger.debug("Rust token counter (%s) counted %d input tokens", tokenizer, attempt.value.input_tokens) - return attempt.value +async def native_count(factory: RustTokenCounterFactory, tokenizer: RustTokenizer, body: bytes) -> InputTokenCount: + """One native count, validated into the public shape. + + ``RustBridgeDeclined`` and upstream errors propagate so the caller's route + runner can map them onto its fallback policy; other failures (RuntimeError, + ValueError) propagate as-is.""" + return _INPUT_TOKEN_COUNT.validate_python(await _counter(factory, tokenizer).acount_request(body)) diff --git a/litellm/rust_bridge/tokenizer.py b/litellm/rust_bridge/tokenizer.py new file mode 100644 index 00000000000..5ed89813620 --- /dev/null +++ b/litellm/rust_bridge/tokenizer.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +from functools import lru_cache +from typing import TYPE_CHECKING, Final, cast # noqa: TID251 # native class is validated at the binding boundary + +import tiktoken +from tokenizers import Tokenizer as PythonHuggingFaceTokenizer + +from litellm.litellm_core_utils.tokenizer import Encoding, HuggingFace, HuggingFaceTokenizer, OpenAIEncoding +from litellm.rust_bridge import runtime +from litellm.rust_bridge.bindings import NativeBinding +from litellm.rust_bridge.catalog import Route, RouteContext + +if TYPE_CHECKING: + from litellm.rust_bridge._native import Tokenizer as NativeTokenizer + + +def _as_factory(value: object) -> type[NativeTokenizer] | None: + return ( + cast(type["NativeTokenizer"], value) # cast-ok: PyO3 class validated at the native boundary + if isinstance(value, type) + else None + ) + + +TOKENIZER: Final = NativeBinding("Tokenizer", validate=_as_factory) + +# The catalog contexts the tokenizer factories dispatch on. Callers that cache a tokenizer per +# backend key their cache on `decision(...)` of the same context, so key and dispatch agree. +TIKTOKEN_CONTEXT: Final = RouteContext(Route.TOKENIZER, provider="tiktoken") +HUGGINGFACE_CONTEXT: Final = RouteContext(Route.TOKENIZER, provider="huggingface") + + +@lru_cache(maxsize=8) +def _native_tiktoken(factory: type[NativeTokenizer], name: str) -> NativeTokenizer: + return factory.from_tiktoken(name) + + +@lru_cache(maxsize=1) +def _native_anthropic(factory: type[NativeTokenizer]) -> NativeTokenizer: + from litellm.utils import claude_json_str + + return factory.from_json(claude_json_str) + + +@lru_cache(maxsize=8) +def _native_encoding(factory: type[NativeTokenizer], name: str) -> OpenAIEncoding: + return OpenAIEncoding.wrap(_native_tiktoken(factory, name)) + + +def native_encoding(name: str) -> NativeTokenizer | None: + """The native tiktoken encoding behind `get_encoding(name)`, for a Rust route that counts + with the same loaded model; `None` without the extension.""" + factory: Final = TOKENIZER.load() + return None if factory is None else _native_tiktoken(factory, name) + + +def native_anthropic() -> NativeTokenizer | None: + """The native packaged Anthropic tokenizer behind `anthropic()`, parsed once per process.""" + factory: Final = TOKENIZER.load() + return None if factory is None else _native_anthropic(factory) + + +def _python_encoding(name: str) -> tiktoken.Encoding: + from litellm.litellm_core_utils.default_encoding import encoding + + return encoding if name == encoding.name else tiktoken.get_encoding(name) + + +def get_encoding(name: str) -> Encoding: + return runtime.run( + TIKTOKEN_CONTEXT, + binding=TOKENIZER, + native=lambda factory: _native_encoding(factory, name), + python=lambda: _python_encoding(name), + ) + + +def anthropic() -> HuggingFace: + """The packaged Anthropic tokenizer on the selected backend.""" + from litellm.utils import claude_json_str + + return runtime.run( + HUGGINGFACE_CONTEXT, + binding=TOKENIZER, + native=lambda factory: HuggingFaceTokenizer(_native_anthropic(factory)), + python=lambda: PythonHuggingFaceTokenizer.from_str(claude_json_str), + ) + + +def from_str(json: str) -> HuggingFace: + return runtime.run( + HUGGINGFACE_CONTEXT, + binding=TOKENIZER, + native=lambda factory: HuggingFaceTokenizer(factory.from_json(json)), + python=lambda: PythonHuggingFaceTokenizer.from_str(json), + ) + + +def from_pretrained(identifier: str, revision: str = "main", token: str | None = None) -> HuggingFace: + return runtime.run( + HUGGINGFACE_CONTEXT, + binding=TOKENIZER, + native=lambda factory: HuggingFaceTokenizer( + factory.from_pretrained(identifier, revision=revision, token=token) + ), + python=lambda: PythonHuggingFaceTokenizer.from_pretrained(identifier, revision=revision, token=token), + ) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 2e8c869b89d..65d66677b27 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -104,6 +104,8 @@ def _nested_selector( if TYPE_CHECKING: + from litellm.litellm_core_utils.tokenizer import Tokenizer + from .vector_stores import VectorStoreSearchResponse else: VectorStoreSearchResponse = Any @@ -4406,7 +4408,7 @@ class ProviderSpecificHeader(TypedDict): class SelectTokenizerResponse(TypedDict): type: Literal["openai_tokenizer", "huggingface_tokenizer"] - tokenizer: Any + tokenizer: ReadOnly["Tokenizer"] class LiteLLMFineTuningJob(FineTuningJob): diff --git a/litellm/utils.py b/litellm/utils.py index 812299560c7..e088a5988c8 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -40,14 +40,11 @@ from types import MappingProxyType import dotenv import httpx import openai -import tiktoken from httpx import Proxy from httpx._utils import get_environment_proxies from openai.lib import _parsing, _pydantic from openai.types.chat.completion_create_params import ResponseFormat from pydantic import BaseModel -from tiktoken import Encoding -from tokenizers import Tokenizer import litellm import litellm.litellm_core_utils @@ -91,6 +88,10 @@ from litellm.litellm_core_utils.fallback_generalizations import ( match_fill_missing_generalizations, ) from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload +from litellm.litellm_core_utils.tokenizer import Encoding, HuggingFace, strip_special_tokens +from litellm.rust_bridge import tokenizer as tokenizer_dispatch +from litellm.rust_bridge.catalog import decision +from litellm.rust_bridge.configuration import Decision _CachingHandlerResponse = None _LLMCachingHandler = None @@ -289,6 +290,8 @@ import importlib.metadata from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping, Sequence from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Union, cast, get_args +from typing_extensions import assert_never + from litellm import utils as litellm_utils # These are lazy loaded via __getattr__ @@ -2254,17 +2257,27 @@ def _select_tokenizer(model: str, custom_tokenizer: CustomHuggingfaceTokenizer | identifier=custom_tokenizer["identifier"], revision=custom_tokenizer["revision"], auth_token=custom_tokenizer["auth_token"], + backend=_huggingface_tokenizer_backend(), ) return _select_tokenizer_helper(model=model) +def _huggingface_tokenizer_backend() -> Decision: + """The backend `tokenizer_dispatch.from_str` / `from_pretrained` will select right now. + + Cached HuggingFace tokenizers are keyed on it, so flipping `LITELLM_RUST` or + `litellm.rust(...)` reaches a fresh object instead of the other backend's.""" + return decision(tokenizer_dispatch.HUGGINGFACE_CONTEXT) + + @lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE) -def _select_custom_tokenizer_helper(identifier: str, revision: str, auth_token: str | None) -> SelectTokenizerResponse: +def _select_custom_tokenizer_helper( + identifier: str, revision: str, auth_token: str | None, backend: Decision +) -> SelectTokenizerResponse: verbose_logger.debug("Loading custom HuggingFace tokenizer %s (revision %s)", identifier, revision) return create_pretrained_tokenizer(identifier=identifier, revision=revision, auth_token=auth_token) -@lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE) def _select_tokenizer_helper(model: str) -> SelectTokenizerResponse: if litellm.disable_hf_tokenizer_download is True: return _return_openai_tokenizer(model) @@ -2274,6 +2287,10 @@ def _select_tokenizer_helper(model: str) -> SelectTokenizerResponse: if result is not None: return result except Exception as e: + from litellm.rust_bridge.fork_guard import ForkedAfterNativeRuntimeStarted, ProcessReservedForForking + + if isinstance(e, (ForkedAfterNativeRuntimeStarted, ProcessReservedForForking)): + raise verbose_logger.debug("Error selecting tokenizer: %s", e) # default - tiktoken @@ -2308,19 +2325,26 @@ def _return_huggingface_tokenizer(model: str) -> SelectTokenizerResponse | None: kind: Final = huggingface_tokenizer_kind(model) if kind is None: return None - return {"type": "huggingface_tokenizer", "tokenizer": _load_huggingface_tokenizer(kind)} + return { + "type": "huggingface_tokenizer", + "tokenizer": _load_huggingface_tokenizer(kind, _huggingface_tokenizer_backend()), + } -def _load_huggingface_tokenizer(kind: HuggingFaceTokenizerKind) -> Tokenizer: +@lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE) +def _load_huggingface_tokenizer(kind: HuggingFaceTokenizerKind, backend: Decision) -> HuggingFace: + """One tokenizer per kind and backend; `backend` is the cache key, the dispatch re-derives it.""" match kind: case "cohere": - return Tokenizer.from_pretrained("Xenova/c4ai-command-r-v01-tokenizer") + return tokenizer_dispatch.from_pretrained("Xenova/c4ai-command-r-v01-tokenizer") case "anthropic": - return Tokenizer.from_str(claude_json_str) + return tokenizer_dispatch.anthropic() case "llama2": - return Tokenizer.from_pretrained("hf-internal-testing/llama-tokenizer") + return tokenizer_dispatch.from_pretrained("hf-internal-testing/llama-tokenizer") case "llama3": - return Tokenizer.from_pretrained("Xenova/llama-3-tokenizer") + return tokenizer_dispatch.from_pretrained("Xenova/llama-3-tokenizer") + case _: + assert_never(kind) def encode(model="", text="", custom_tokenizer: dict | None = None): @@ -2336,15 +2360,13 @@ def encode(model="", text="", custom_tokenizer: dict | None = None): enc: The encoded text. """ tokenizer_json: Final = custom_tokenizer or _select_tokenizer(model=model) - if isinstance(tokenizer_json["tokenizer"], Encoding): - enc = tokenizer_json["tokenizer"].encode(text, disallowed_special=()) - else: - enc = tokenizer_json["tokenizer"].encode(text) - # Normalize: HuggingFace Tokenizer.encode() returns an Encoding object; - # extract .ids so the return type is always List[int]. - if hasattr(enc, "ids"): - return enc.ids - return enc + if tokenizer_json["type"] == "openai_tokenizer": + openai_tokenizer: Final = cast( # cast-ok: [LIT006] caller's explicit type tag selects this interface + Encoding, tokenizer_json["tokenizer"] + ) + return openai_tokenizer.encode(text, disallowed_special=()) + encoded: Final = tokenizer_json["tokenizer"].encode(text) + return encoded.ids if hasattr(encoded, "ids") else encoded def decode( @@ -2363,26 +2385,12 @@ def decode( """ tokenizer_json: Final = custom_tokenizer or _select_tokenizer(model=model) if tokenizer_json["type"] == "huggingface_tokenizer": - if skip_special_tokens: - tokens = _strip_huggingface_special_token_ids(tokenizer_json["tokenizer"], tokens) - dec = tokenizer_json["tokenizer"].decode(tokens, skip_special_tokens=skip_special_tokens) - return dec - dec = tokenizer_json["tokenizer"].decode(tokens) - return dec - - -def _strip_huggingface_special_token_ids(tokenizer: Tokenizer, tokens: Sequence[int]) -> Sequence[int]: - try: - added_tokens_decoder: Final = tokenizer.get_added_tokens_decoder() - except Exception: - return tokens - - special_token_ids: Final = { - token_id for token_id, added_token in added_tokens_decoder.items() if getattr(added_token, "special", False) - } - if not special_token_ids: - return tokens - return [token for token in tokens if token not in special_token_ids] + ids: Final = strip_special_tokens(tokenizer_json["tokenizer"], tokens) if skip_special_tokens else tokens + hf_tokenizer: Final = cast( # cast-ok: [LIT006] caller's explicit type tag selects this interface + HuggingFace, tokenizer_json["tokenizer"] + ) + return hf_tokenizer.decode(ids, skip_special_tokens=skip_special_tokens) + return tokenizer_json["tokenizer"].decode(tokens) def create_pretrained_tokenizer(identifier: str, revision="main", auth_token: str | None = None): @@ -2398,7 +2406,7 @@ def create_pretrained_tokenizer(identifier: str, revision="main", auth_token: st dict: A dictionary with the tokenizer and its type. """ - tokenizer: Final = Tokenizer.from_pretrained(identifier, revision=revision, token=auth_token) + tokenizer: Final = tokenizer_dispatch.from_pretrained(identifier, revision=revision, token=auth_token) return {"type": "huggingface_tokenizer", "tokenizer": tokenizer} @@ -2413,7 +2421,7 @@ def create_tokenizer(json: str): dict: A dictionary with the tokenizer and its type. """ - tokenizer: Final = Tokenizer.from_str(json) + tokenizer: Final = tokenizer_dispatch.from_str(json) return {"type": "huggingface_tokenizer", "tokenizer": tokenizer} diff --git a/migrations/Dockerfile b/migrations/Dockerfile index c6d1b0cc46e..f34940c0ce0 100644 --- a/migrations/Dockerfile +++ b/migrations/Dockerfile @@ -67,6 +67,8 @@ RUN --mount=type=cache,target=/root/.cache/uv \ --extra extra_proxy \ --python python3.13 +RUN cp "$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"/litellm/rust_bridge/_native*.so litellm/rust_bridge/ + COPY migrations/run.py /app/run.py # Pre-warm the Prisma binary cache so the Job pod doesn't reach the diff --git a/pyproject.toml b/pyproject.toml index a1b276e4e8c..f447343ff33 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,11 +19,13 @@ dependencies = [ "httpx[http2]>=0.28.0,<1.0", "openai>=2.20.0,<3.0.0", "python-dotenv>=1.0.0,<2.0", + "pyyaml>=6.0.3,<7.0", + "packaging>=24.0", + "importlib-metadata>=8.0.0,<9.0", "tiktoken>=0.8.0,<1.0; python_version < '3.14'", "tiktoken>=0.12.0,<1.0; python_version >= '3.14'", - "importlib-metadata>=8.0.0,<9.0", - "packaging>=24.0", "tokenizers>=0.21.0,<1.0", + "huggingface-hub>=0.34.0,<2.0", "click>=8.0.0,<9.0", "jinja2>=3.1.6,<4.0", "aiohttp>=3.14.2,<4.0", @@ -191,6 +193,7 @@ litellm-proxy = "litellm.proxy.client.cli:litellm_proxy_cli" [dependency-groups] dev = [ + "numpy>=1.26.0,<3.0", "diff-cover==9.7.2", "hypothesis==6.165.10", "reportlab==5.0.1", @@ -290,6 +293,12 @@ healthcheck = [ "httpx==0.28.1", "pyyaml==6.0.3", ] +benchmarks = [ + "pytest==9.0.3", + "pytest-codspeed==4.3.0", + "mcp>=2.2.0,<3", + "a2a-sdk==1.1.0", +] [build-system] requires = ["maturin==1.15.0"] diff --git a/tests/benchmarks/conftest.py b/tests/benchmarks/conftest.py index c9b31cfb7d7..309ecab9991 100644 --- a/tests/benchmarks/conftest.py +++ b/tests/benchmarks/conftest.py @@ -8,8 +8,11 @@ flipping results between runs. Running the executor inline keeps each benchmark's cost self-contained and deterministic. """ +import os +import sys from collections.abc import Callable, Iterator from concurrent.futures import Future +from pathlib import Path from typing import ParamSpec, TypeVar import pytest @@ -20,6 +23,21 @@ P = ParamSpec("P") R = TypeVar("R") +def pytest_configure(config: pytest.Config) -> None: + if os.environ.get("LITELLM_REQUIRE_INSTALLED_WHEEL") != "1": + return + + import litellm + import litellm.rust_bridge._native as native + + prefix = Path(sys.prefix).resolve() + for name, module_file in (("litellm", litellm.__file__), ("litellm.rust_bridge._native", native.__file__)): + path = Path(module_file).resolve() + if not path.is_relative_to(prefix): + raise pytest.UsageError(f"{name} resolved outside the benchmark environment: {path}") + print(f"{name}: {path}") # noqa: T201 # provenance evidence must be visible in CI logs + + def _submit_inline(fn: Callable[P, R], /, *args: P.args, **kwargs: P.kwargs) -> Future[R]: future: Future[R] = Future() try: diff --git a/tests/proxy_unit_tests/test_custom_tokenizer_bug.py b/tests/proxy_unit_tests/test_custom_tokenizer_bug.py index c4b1f4f3afd..23b6b302202 100644 --- a/tests/proxy_unit_tests/test_custom_tokenizer_bug.py +++ b/tests/proxy_unit_tests/test_custom_tokenizer_bug.py @@ -22,10 +22,8 @@ from litellm.proxy.proxy_server import token_counter def _fake_hf_tokenizer(num_tokens: int) -> MagicMock: - encoding = MagicMock() - encoding.__len__.return_value = num_tokens tokenizer = MagicMock() - tokenizer.encode_batch_fast.return_value = [encoding] + tokenizer.encode_batch_fast.return_value = [[0] * num_tokens] return tokenizer @@ -58,7 +56,7 @@ async def test_custom_tokenizer_from_model_info_is_used(monkeypatch): ) monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", llm_router) - with patch.object(litellm.utils, "Tokenizer") as mock_tokenizer_cls: + with patch.object(litellm.utils, "tokenizer_dispatch") as mock_tokenizer_cls: mock_tokenizer_cls.from_pretrained.return_value = _fake_hf_tokenizer(7) response = await token_counter( @@ -92,7 +90,7 @@ async def test_model_without_custom_tokenizer_uses_default(monkeypatch): ) monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", llm_router) - with patch.object(litellm.utils, "Tokenizer") as mock_tokenizer_cls: + with patch.object(litellm.utils, "tokenizer_dispatch") as mock_tokenizer_cls: response = await token_counter( request=TokenCountRequest( model="gpt-4", diff --git a/tests/test_litellm/litellm_core_utils/test_decode_special_tokens.py b/tests/test_litellm/litellm_core_utils/test_decode_special_tokens.py index d1a5f78a859..dd2daef5484 100644 --- a/tests/test_litellm/litellm_core_utils/test_decode_special_tokens.py +++ b/tests/test_litellm/litellm_core_utils/test_decode_special_tokens.py @@ -1,23 +1,11 @@ -from tokenizers import AddedToken, Tokenizer -from tokenizers.models import WordLevel -from tokenizers.pre_tokenizers import Whitespace -from tokenizers.processors import TemplateProcessing - from litellm import decode, encode +from tokenizers import Tokenizer + +TOKENIZER_JSON = """{"version":"1.0","truncation":null,"padding":null,"added_tokens":[{"id":3,"content":"[BOS]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true}],"normalizer":null,"pre_tokenizer":{"type":"Whitespace"},"post_processor":{"type":"TemplateProcessing","single":[{"SpecialToken":{"id":"[BOS]","type_id":0}},{"Sequence":{"id":"A","type_id":0}}],"pair":[{"Sequence":{"id":"A","type_id":0}},{"Sequence":{"id":"B","type_id":1}}],"special_tokens":{"[BOS]":{"id":"[BOS]","ids":[3],"tokens":["[BOS]"]}}},"decoder":null,"model":{"type":"WordLevel","vocab":{"[UNK]":0,"Hello":1,"World":2},"unk_token":"[UNK]"}}""" def _create_custom_tokenizer(): - tokenizer = Tokenizer( - WordLevel({"[UNK]": 0, "Hello": 1, "World": 2}, unk_token="[UNK]") - ) - tokenizer.pre_tokenizer = Whitespace() - tokenizer.add_special_tokens([AddedToken("[BOS]", special=True)]) - bos_token_id = tokenizer.token_to_id("[BOS]") - assert bos_token_id is not None - tokenizer.post_processor = TemplateProcessing( - single="[BOS] $A", - special_tokens=[("[BOS]", bos_token_id)], - ) + tokenizer = Tokenizer.from_str(TOKENIZER_JSON) return {"type": "huggingface_tokenizer", "tokenizer": tokenizer} diff --git a/tests/test_litellm/litellm_core_utils/test_token_counter.py b/tests/test_litellm/litellm_core_utils/test_token_counter.py index f19a8891609..5ce6a4b1ce9 100644 --- a/tests/test_litellm/litellm_core_utils/test_token_counter.py +++ b/tests/test_litellm/litellm_core_utils/test_token_counter.py @@ -786,23 +786,23 @@ def test_token_counter(): import unittest -from litellm.utils import _select_tokenizer_helper, claude_json_str, encoding +from litellm.utils import _load_huggingface_tokenizer, _select_tokenizer_helper, claude_json_str, encoding # Clear the cache at module load to ensure clean state -_select_tokenizer_helper.cache_clear() +_load_huggingface_tokenizer.cache_clear() class TestTokenizerSelection(unittest.TestCase): def setUp(self): """Clear the LRU cache before each test method. - The _select_tokenizer_helper function is decorated with @lru_cache, - which can cause cache hits from previous tests when running with + The HuggingFace tokenizers behind _select_tokenizer_helper are cached with + @lru_cache, which can cause cache hits from previous tests when running with --dist=loadscope (tests from same file run on same worker). """ - _select_tokenizer_helper.cache_clear() + _load_huggingface_tokenizer.cache_clear() - @patch("litellm.utils.Tokenizer.from_pretrained") + @patch("litellm.utils.tokenizer_dispatch.from_pretrained") def test_llama3_tokenizer_api_failure(self, mock_from_pretrained): # Setup mock to raise an error mock_from_pretrained.side_effect = Exception("Failed to load tokenizer") @@ -817,7 +817,7 @@ class TestTokenizerSelection(unittest.TestCase): self.assertEqual(result["type"], "openai_tokenizer") self.assertEqual(result["tokenizer"], encoding) - @patch("litellm.utils.Tokenizer.from_pretrained") + @patch("litellm.utils.tokenizer_dispatch.from_pretrained") def test_cohere_tokenizer_api_failure(self, mock_from_pretrained): # Setup mock to raise an error mock_from_pretrained.side_effect = Exception("Failed to load tokenizer") @@ -837,10 +837,10 @@ class TestTokenizerSelection(unittest.TestCase): self.assertEqual(result["type"], "openai_tokenizer") self.assertEqual(result["tokenizer"], encoding) - @patch("litellm.utils.Tokenizer.from_str") - def test_claude_tokenizer_api_failure(self, mock_from_str): + @patch("litellm.utils.tokenizer_dispatch.anthropic") + def test_claude_tokenizer_api_failure(self, mock_anthropic): # Setup mock to raise an error - mock_from_str.side_effect = Exception("Failed to load tokenizer") + mock_anthropic.side_effect = Exception("Failed to load tokenizer") # Add Claude model to the list for testing litellm.anthropic_models = ["claude-2"] @@ -849,13 +849,13 @@ class TestTokenizerSelection(unittest.TestCase): result = _select_tokenizer_helper("claude-2") # Verify the attempt to load Claude tokenizer - mock_from_str.assert_called_once_with(claude_json_str) + mock_anthropic.assert_called_once_with() # Verify fallback to OpenAI tokenizer self.assertEqual(result["type"], "openai_tokenizer") self.assertEqual(result["tokenizer"], encoding) - @patch("litellm.utils.Tokenizer.from_pretrained") + @patch("litellm.utils.tokenizer_dispatch.from_pretrained") def test_llama2_tokenizer_api_failure(self, mock_from_pretrained): # Setup mock to raise an error mock_from_pretrained.side_effect = Exception("Failed to load tokenizer") diff --git a/tests/test_litellm/litellm_core_utils/test_tokenizer.py b/tests/test_litellm/litellm_core_utils/test_tokenizer.py new file mode 100644 index 00000000000..aa4a0fc6a1c --- /dev/null +++ b/tests/test_litellm/litellm_core_utils/test_tokenizer.py @@ -0,0 +1,403 @@ +import copy +import os +import pickle +import subprocess +import sys +from pathlib import Path +from typing import Final, Literal + +import pytest +import tiktoken +from tokenizers import Tokenizer as ReferenceTokenizer + +import litellm +from litellm.caching._embedding_router import truncate_embedding_input +from litellm.litellm_core_utils.tokenizer import HuggingFaceTokenizer, OpenAIEncoding +from litellm.utils import claude_json_str +from tests.test_litellm.litellm_core_utils.test_decode_special_tokens import TOKENIZER_JSON + + +@pytest.mark.parametrize( + "name", ("cl100k_base", "o200k_base", "p50k_base", "p50k_edit", "r50k_base", "gpt2", "o200k_harmony") +) +@pytest.mark.parametrize( + "text", ("hello world", "café 漢字 🙂", "", "a\ud800b", "\ud83d\ude42", "🙂\ud83d\ude42\udfff", " " * 64) +) +def test_openai_encoding_matches_python_unicode_and_batches(name: str, text: str) -> None: + reference: Final = tiktoken.get_encoding(name) + encoding: Final = OpenAIEncoding.from_tiktoken(name) + expected: Final = reference.encode(text) + + assert encoding.encode(text) == expected + assert encoding.count(text) == len(expected) + assert encoding.encode_batch([text], num_threads=2) == reference.encode_batch([text], num_threads=2) + assert encoding.encode_ordinary_batch([text]) == reference.encode_ordinary_batch([text]) + assert encoding.decode_batch([expected]) == reference.decode_batch([expected]) + assert encoding.decode_bytes_batch([expected]) == reference.decode_bytes_batch([expected]) + + +@pytest.mark.parametrize("allowed", (frozenset(), frozenset({"<|endoftext|>"}), "all")) +@pytest.mark.parametrize("disallowed", (frozenset(), frozenset({"<|fim_prefix|>"}), "all")) +def test_openai_special_token_options_match_python( + allowed: frozenset[str] | Literal["all"], disallowed: frozenset[str] | Literal["all"] +) -> None: + reference: Final = tiktoken.get_encoding("cl100k_base") + encoding: Final = OpenAIEncoding.from_tiktoken(reference.name) + text: Final = "hello<|endoftext|><|fim_prefix|>world" + allowed_set: Final = reference.special_tokens_set if allowed == "all" else allowed + disallowed_set: Final = reference.special_tokens_set - allowed_set if disallowed == "all" else disallowed + if any(token in text for token in disallowed_set): + with pytest.raises(ValueError, match="disallowed special token"): + encoding.encode(text, allowed_special=allowed, disallowed_special=disallowed) + return + assert encoding.encode(text, allowed_special=allowed, disallowed_special=disallowed) == reference.encode( + text, allowed_special=allowed, disallowed_special=disallowed + ) + assert encoding.special_tokens_set == reference.special_tokens_set + assert encoding.eot_token == reference.eot_token + + +@pytest.mark.parametrize("errors", ("replace", "ignore", "backslashreplace", "strict")) +def test_openai_partial_token_decoding_preserves_error_policy(errors: str) -> None: + reference: Final = tiktoken.get_encoding("cl100k_base") + encoding: Final = OpenAIEncoding.from_tiktoken(reference.name) + tokens: Final = reference.encode("🙂")[:1] + assert encoding.decode_bytes(tokens) == reference.decode_bytes(tokens) + if errors == "strict": + with pytest.raises(UnicodeDecodeError): + encoding.decode(tokens, errors=errors) + return + assert encoding.decode(tokens, errors=errors) == reference.decode(tokens, errors=errors) + assert encoding.decode_tokens_bytes(tokens) == reference.decode_tokens_bytes(tokens) + + +def test_public_encoding_and_semantic_cache_preserve_truncated_unicode() -> None: + reference: Final = tiktoken.get_encoding(litellm.encoding.name) + text: Final = "🙂" + tokens: Final = reference.encode(text) + + assert litellm.encoding.encode(text, disallowed_special=()) == tokens + assert litellm.encoding.encode_batch([text]) == [tokens] + assert litellm.decode(tokens=tokens[:1]) == reference.decode(tokens[:1]) + assert truncate_embedding_input(text, "", 1) == reference.decode(tokens[:1]) + + +@pytest.mark.parametrize("add_special_tokens", (True, False)) +def test_huggingface_encoding_preserves_result_fields_and_serialization(add_special_tokens: bool) -> None: + reference: Final = ReferenceTokenizer.from_str(TOKENIZER_JSON) + tokenizer: Final = HuggingFaceTokenizer.from_str(TOKENIZER_JSON) + expected: Final = reference.encode("Hello World", add_special_tokens=add_special_tokens) + actual: Final = tokenizer.encode("Hello World", add_special_tokens=add_special_tokens) + + assert (actual.ids, actual.tokens, actual.type_ids, actual.offsets, actual.word_ids, actual.sequence_ids) == ( + expected.ids, + expected.tokens, + expected.type_ids, + expected.offsets, + expected.word_ids, + expected.sequence_ids, + ) + assert (actual.attention_mask, actual.special_tokens_mask, actual.n_sequences, len(actual)) == ( + expected.attention_mask, + expected.special_tokens_mask, + expected.n_sequences, + len(expected), + ) + assert copy.deepcopy(actual).ids == expected.ids + assert pickle.loads(pickle.dumps(actual)).offsets == expected.offsets + assert tokenizer.decode(actual.ids, skip_special_tokens=False) == reference.decode( + expected.ids, skip_special_tokens=False + ) + + +def test_huggingface_character_offsets_and_pretokenized_pairs_match_python() -> None: + reference: Final = ReferenceTokenizer.from_str(claude_json_str) + tokenizer: Final = HuggingFaceTokenizer.from_str(claude_json_str) + text: Final = "café 漢字 🙂" + actual: Final = tokenizer.encode(text) + expected: Final = reference.encode(text) + + assert actual.offsets == expected.offsets + assert actual.ids == expected.ids + assert ( + tokenizer.encode(["hello", "world"], ["again"], is_pretokenized=True).ids + == reference.encode(["hello", "world"], ["again"], is_pretokenized=True).ids + ) + + +def test_huggingface_batches_apply_padding_across_inputs() -> None: + reference: Final = ReferenceTokenizer.from_str(TOKENIZER_JSON) + reference.enable_padding(pad_id=0, pad_token="[UNK]") + tokenizer: Final = HuggingFaceTokenizer.from_str(reference.to_str()) + inputs: Final = ["Hello", ("Hello World", "World")] + expected: Final = reference.encode_batch(inputs) + actual: Final = tokenizer.encode_batch(inputs) + fast: Final = tokenizer.encode_batch_fast(inputs) + + assert [(item.ids, item.attention_mask, item.offsets) for item in actual] == [ + (item.ids, item.attention_mask, item.offsets) for item in expected + ] + assert [item.ids for item in fast] == [item.ids for item in expected] + assert tokenizer.decode_batch([item.ids for item in actual]) == reference.decode_batch( + [item.ids for item in expected] + ) + + +def test_caller_supplied_huggingface_tokenizer_preserves_public_encode_and_count() -> None: + tokenizer: Final = ReferenceTokenizer.from_str(TOKENIZER_JSON) + custom: Final = {"type": "huggingface_tokenizer", "tokenizer": tokenizer} + expected: Final = tokenizer.encode("Hello World").ids + + assert litellm.encode(text="Hello World", custom_tokenizer=custom) == expected + assert litellm.token_counter(text="Hello World", custom_tokenizer=custom) == len(expected) + assert litellm.decode(tokens=expected, custom_tokenizer=custom) == "Hello World" + + +def test_caller_supplied_tiktoken_treats_special_spellings_as_text() -> None: + tokenizer: Final = tiktoken.get_encoding("cl100k_base") + custom: Final = {"type": "openai_tokenizer", "tokenizer": tokenizer} + text: Final = "<|endoftext|>" + + assert litellm.encode(text=text, custom_tokenizer=custom) == tokenizer.encode(text, disallowed_special=()) + + +def test_public_tokenizer_objects_survive_pickle_and_deepcopy(tmp_path: Path) -> None: + custom: Final = litellm.create_tokenizer(TOKENIZER_JSON) + tokenizer: Final = custom["tokenizer"] + path: Final = tmp_path / "tokenizer.json" + tokenizer.save(str(path)) + + assert copy.deepcopy(custom)["tokenizer"].encode("Hello World").ids == tokenizer.encode("Hello World").ids + assert ( + pickle.loads(pickle.dumps(custom))["tokenizer"].encode("Hello World").ids == tokenizer.encode("Hello World").ids + ) + assert HuggingFaceTokenizer.from_file(str(path)).encode("Hello World").ids == tokenizer.encode("Hello World").ids + assert copy.deepcopy(litellm.encoding).encode("hello") == litellm.encoding.encode("hello") + assert pickle.loads(pickle.dumps(litellm.encoding)).encode("hello") == litellm.encoding.encode("hello") + + +@pytest.mark.parametrize("offline", ("0", "1")) +def test_hub_loader_preserves_environment_auth_cache_and_offline(tmp_path: Path, offline: str) -> None: + script: Final = """ +import json +import sys +from pathlib import Path +sys.path.insert(0, sys.argv[1]) +import httpx +import huggingface_hub +from huggingface_hub.errors import LocalEntryNotFoundError +import litellm +payload = sys.argv[2].encode() +offline = sys.argv[3] == "1" +observed = [] +def handle(request): + assert not offline, "offline loading issued a request" + if request.url.path.endswith("/tokenizer.json"): + observed.append(request.headers.get("authorization")) + if request.headers.get("authorization") != "Bearer audit-fixture-token": + return httpx.Response(401) + return httpx.Response(200, headers={"content-length": str(len(payload)), "etag": '"fixture"', "x-repo-commit": "a" * 40}, content=payload if request.method == "GET" else b"") +if not offline: + huggingface_hub.set_client_factory(lambda: httpx.Client(transport=httpx.MockTransport(handle))) +try: + tokenizer = litellm.create_pretrained_tokenizer("test-fixture/tokenizer")["tokenizer"] +except LocalEntryNotFoundError: + assert offline + assert observed == [] +else: + assert not offline + assert "Bearer audit-fixture-token" in observed + assert tokenizer.decode(tokenizer.encode("Hello World").ids) == "Hello World" + assert tuple(Path(sys.argv[4]).rglob("tokenizer.json")) +print("compatible") +""" + result: Final = subprocess.run( + [ + sys.executable, + "-I", + "-c", + script, + str(Path(litellm.__file__).parent.parent), + TOKENIZER_JSON, + offline, + str(tmp_path / "cache"), + ], + capture_output=True, + text=True, + timeout=30, + env={ + **os.environ, + "HF_HOME": str(tmp_path / "home"), + "HF_HUB_CACHE": str(tmp_path / "cache"), + "HF_ENDPOINT": "http://127.0.0.1:9", + "HF_TOKEN": "audit-fixture-token", + "HF_HUB_OFFLINE": offline, + "HF_HUB_DISABLE_IMPLICIT_TOKEN": "0", + "LITELLM_LOCAL_MODEL_COST_MAP": "True", + }, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert result.stdout.strip() == "compatible" + + +@pytest.mark.parametrize("rust", (None, "0", "1")) +def test_tokenization_without_native_extension_stays_offline(tmp_path: Path, rust: str | None) -> None: + script: Final = """ +import importlib.abc +import sys +sys.path.insert(0, sys.argv[1]) +def reject_network(event, args): + if event == "socket.connect": + raise AssertionError("tokenizer attempted a network connection") +sys.addaudithook(reject_network) +class Block(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname == "litellm.rust_bridge._native": + raise ImportError("native extension is unavailable") +sys.meta_path.insert(0, Block()) +import litellm +from litellm.rust_bridge.tokenizer import get_encoding +import tiktoken +from tokenizers import Tokenizer +assert isinstance(litellm.encoding, tiktoken.Encoding) +for name in ("cl100k_base", "o200k_base", "o200k_harmony", "p50k_base", "p50k_edit"): + encoding = get_encoding(name) + text = "offline café 漢字 🙂" + " " * 64 + assert encoding.decode(encoding.encode(text)) == text +ids = litellm.encode(text="hello world") +assert litellm.decode(tokens=ids) == "hello world" +assert litellm.token_counter(model=None, text="hello world") == len(ids) +custom = litellm.create_tokenizer(sys.argv[2]) +assert isinstance(custom["tokenizer"], Tokenizer) +custom["tokenizer"].enable_padding(pad_id=0, pad_token="[UNK]") +assert litellm.decode(tokens=litellm.encode(text="Hello World", custom_tokenizer=custom), custom_tokenizer=custom) == "Hello World" +print("compatible") +""" + result: Final = subprocess.run( + [sys.executable, "-I", "-c", script, str(Path(litellm.__file__).parent.parent), TOKENIZER_JSON], + capture_output=True, + text=True, + timeout=30, + cwd=tmp_path, + env={ + **{key: value for key, value in os.environ.items() if key != "LITELLM_RUST"}, + **({"LITELLM_RUST": rust} if rust is not None else {}), + "LITELLM_LOCAL_MODEL_COST_MAP": "True", + "TIKTOKEN_CACHE_DIR": str(tmp_path / "unused-tokenizer-cache"), + }, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert result.stdout.strip() == "compatible" + assert not (tmp_path / "unused-tokenizer-cache").exists() + + +@pytest.mark.parametrize("is_pretokenized", (False, True)) +def test_huggingface_batch_sequence_containers_match_python(is_pretokenized: bool) -> None: + reference: Final = ReferenceTokenizer.from_str(TOKENIZER_JSON) + tokenizer: Final = HuggingFaceTokenizer.from_str(TOKENIZER_JSON) + inputs: Final = [["Hello", "World"], ("Hello", "World")] + actual: Final = tokenizer.encode_batch(inputs, is_pretokenized=is_pretokenized) + expected: Final = reference.encode_batch(inputs, is_pretokenized=is_pretokenized) + assert [(item.ids, item.type_ids, item.sequence_ids) for item in actual] == [ + (item.ids, item.type_ids, item.sequence_ids) for item in expected + ] + + +@pytest.mark.parametrize("name", ("cl100k_base", "o200k_base", "p50k_edit", "gpt2")) +def test_openai_encoding_exposes_the_tiktoken_vocabulary_surface(name: str) -> None: + reference: Final = tiktoken.get_encoding(name) + encoding: Final = OpenAIEncoding.from_tiktoken(name) + text: Final = "hello fanta" + + assert repr(encoding) == repr(reference) == f"" + assert (encoding.name, encoding.n_vocab, encoding.max_token_value) == ( + reference.name, + reference.n_vocab, + reference.max_token_value, + ) + assert encoding.token_byte_values() == reference.token_byte_values() + assert encoding.encode_single_token("hello") == reference.encode_single_token("hello") + assert encoding.encode_single_token(b"<|endoftext|>") == reference.eot_token + assert [encoding.is_special_token(token) for token in (0, reference.eot_token)] == [False, True] + assert encoding.decode_with_offsets(reference.encode(text)) == reference.decode_with_offsets(reference.encode(text)) + assert encoding.encode_to_numpy(text).tolist() == reference.encode_to_numpy(text).tolist() + stable, completions = encoding.encode_with_unstable(text) + expected_stable, expected_completions = reference.encode_with_unstable(text) + assert (stable, sorted(completions)) == (expected_stable, sorted(expected_completions)) + with pytest.raises(KeyError): + encoding.encode_single_token("<|not-a-token|>") + + +def test_huggingface_tokenizer_exposes_the_tokenizers_vocabulary_surface() -> None: + reference: Final = ReferenceTokenizer.from_str(TOKENIZER_JSON) + reference.enable_padding(pad_id=0, pad_token="[UNK]", length=4) + reference.enable_truncation(max_length=3, stride=1, strategy="only_first", direction="left") + tokenizer: Final = HuggingFaceTokenizer.from_str(reference.to_str()) + + assert tokenizer.token_to_id("Hello") == reference.token_to_id("Hello") == 1 + assert tokenizer.id_to_token(3) == reference.id_to_token(3) == "[BOS]" + assert tokenizer.id_to_token(99) is None + assert tokenizer.get_vocab() == reference.get_vocab() + assert tokenizer.get_vocab(with_added_tokens=False) == reference.get_vocab(with_added_tokens=False) + assert tokenizer.get_vocab_size() == reference.get_vocab_size() == 4 + assert tokenizer.get_vocab_size(with_added_tokens=False) == reference.get_vocab_size(with_added_tokens=False) + added: Final = tokenizer.get_added_tokens_decoder() + expected_added: Final = reference.get_added_tokens_decoder() + assert {token_id: str(token) for token_id, token in added.items()} == { + token_id: str(token) for token_id, token in expected_added.items() + } + assert added[3].special == expected_added[3].special + assert tokenizer.num_special_tokens_to_add(False) == reference.num_special_tokens_to_add(False) == 1 + assert tokenizer.num_special_tokens_to_add(True) == reference.num_special_tokens_to_add(True) == 0 + assert tokenizer.padding == reference.padding + assert tokenizer.truncation == reference.truncation + assert tokenizer.encode_special_tokens == reference.encode_special_tokens is False + assert HuggingFaceTokenizer.from_buffer(TOKENIZER_JSON.encode()).encode("Hello").ids == [3, 1] + assert HuggingFaceTokenizer.from_str(TOKENIZER_JSON).padding is None + assert HuggingFaceTokenizer.from_str(TOKENIZER_JSON).truncation is None + + +def test_huggingface_encoding_exposes_the_tokenizers_lookup_and_mutation_surface() -> None: + reference: Final = ReferenceTokenizer.from_str(claude_json_str) + tokenizer: Final = HuggingFaceTokenizer.from_str(claude_json_str) + text: Final = "hello wide world" + actual: Final = tokenizer.encode(text, "again") + expected: Final = reference.encode(text, "again") + + lookups: Final = ( + lambda encoding: [encoding.token_to_chars(index) for index in range(len(encoding))], + lambda encoding: [encoding.token_to_word(index) for index in range(len(encoding))], + lambda encoding: [encoding.token_to_sequence(index) for index in range(len(encoding))], + lambda encoding: [encoding.char_to_token(position) for position in range(len(text))], + lambda encoding: [encoding.char_to_word(position) for position in range(len(text))], + lambda encoding: [encoding.char_to_token(position, 1) for position in range(5)], + lambda encoding: [encoding.word_to_tokens(word) for word in range(3)], + lambda encoding: [encoding.word_to_chars(word) for word in range(3)], + lambda encoding: [encoding.word_to_tokens(0, 1), encoding.word_to_chars(0, 1)], + ) + for lookup in lookups: + assert lookup(actual) == lookup(expected) + assert repr(actual) == repr(expected) + + actual.truncate(4, stride=1, direction="left") + expected.truncate(4, stride=1, direction="left") + assert (actual.ids, [item.ids for item in actual.overflowing]) == ( + expected.ids, + [item.ids for item in expected.overflowing], + ) + actual.pad(6, direction="left", pad_id=7, pad_type_id=1, pad_token="") + expected.pad(6, direction="left", pad_id=7, pad_type_id=1, pad_token="") + assert (actual.ids, actual.attention_mask, actual.type_ids, actual.tokens) == ( + expected.ids, + expected.attention_mask, + expected.type_ids, + expected.tokens, + ) + actual.set_sequence_id(3) + expected.set_sequence_id(3) + assert actual.sequence_ids == expected.sequence_ids + merged: Final = type(actual).merge([actual, tokenizer.encode("more")]) + assert merged.ids == type(expected).merge([expected, reference.encode("more")]).ids + assert merged.offsets == type(expected).merge([expected, reference.encode("more")]).offsets + with pytest.raises(ValueError, match="direction"): + actual.pad(8, direction="sideways") diff --git a/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py b/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py index 214b5cde7da..13488106df4 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py +++ b/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py @@ -4,7 +4,7 @@ import json import math from datetime import datetime, timedelta, timezone from types import MappingProxyType -from typing import Final +from typing import Final, cast import pytest @@ -33,6 +33,7 @@ from litellm.proxy.utils import ProxyLogging from litellm.router import Router from litellm.rust_bridge import bindings, configuration from litellm.rust_bridge import token_counter as rust_token_counter +from litellm.rust_bridge import tokenizer as tokenizer_dispatch from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo TOKEN_COUNTING_ROUTES: Final = ( @@ -239,6 +240,22 @@ class _FakeUpstream(Exception): pass +class _FakeTokenizer: + """Stands in for one shared native `Tokenizer`; only its name identifies it.""" + + def __init__(self, name: str, json: str | None = None) -> None: + self.name = name + self.json = json + + +def _fake_native_tokenizers(monkeypatch: pytest.MonkeyPatch, anthropic_json: str | None = None) -> None: + """Point the counter's tokenizer lookups at fakes; the codec path keeps falling back to Python.""" + fakes: Final = {name: _FakeTokenizer(name) for name in ("cl100k_base", "o200k_base")} + anthropic: Final = _FakeTokenizer("anthropic", anthropic_json) + monkeypatch.setattr(tokenizer_dispatch, "native_encoding", fakes.__getitem__) + monkeypatch.setattr(tokenizer_dispatch, "native_anthropic", lambda: anthropic) + + class _FakeNative: RustBridgeDeclined = _FakeDeclined RustUpstreamError = _FakeUpstream @@ -257,19 +274,13 @@ class _RecordingCounter: class _RecordingFactory: - """Stands in for the native `TokenCounter` class: called with tokenizer JSON, or `from_*_ranks`.""" + """Stands in for the native `TokenCounter` class, built over a loaded `Tokenizer`.""" def __init__(self) -> None: self.calls: list[tuple[rust_token_counter.RustTokenizer, bytes]] = [] - def __call__(self, tokenizer_json: str) -> _RecordingCounter: - return _RecordingCounter(self, "anthropic") - - def from_cl100k_ranks(self, rank_file: str) -> _RecordingCounter: - return _RecordingCounter(self, "cl100k_base") - - def from_o200k_ranks(self, rank_file: str) -> _RecordingCounter: - return _RecordingCounter(self, "o200k_base") + def from_tokenizer(self, tokenizer: _FakeTokenizer, fast: bool = False) -> _RecordingCounter: + return _RecordingCounter(self, cast(rust_token_counter.RustTokenizer, tokenizer.name)) class _DecliningCounter: @@ -278,19 +289,14 @@ class _DecliningCounter: class _DecliningFactory: - def __call__(self, tokenizer_json: str) -> _DecliningCounter: - return _DecliningCounter() - - def from_cl100k_ranks(self, rank_file: str) -> _DecliningCounter: - return _DecliningCounter() - - def from_o200k_ranks(self, rank_file: str) -> _DecliningCounter: + def from_tokenizer(self, tokenizer: _FakeTokenizer, fast: bool = False) -> _DecliningCounter: return _DecliningCounter() @pytest.fixture def rust_counter(monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr(bindings, "get_native_bridge", lambda: _FakeNative()) + _fake_native_tokenizers(monkeypatch) rust_token_counter._counter.cache_clear() configuration.reset_rust_configuration() yield diff --git a/tests/test_litellm/proxy/spend_tracking/test_input_tokens.py b/tests/test_litellm/proxy/spend_tracking/test_input_tokens.py new file mode 100644 index 00000000000..49bbe148386 --- /dev/null +++ b/tests/test_litellm/proxy/spend_tracking/test_input_tokens.py @@ -0,0 +1,191 @@ +"""Tests for input-token counting shared across the reservation path's models.""" + +from __future__ import annotations + +import json +from types import MappingProxyType +from typing import Final, cast + +import pytest + +import litellm +from litellm.proxy.spend_tracking.input_tokens import ( + TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS, + count_input_tokens, + count_input_tokens_for_model, +) +from litellm.rust_bridge import bindings, configuration, token_counter +from litellm.rust_bridge import tokenizer as tokenizer_dispatch +from litellm.rust_bridge.token_counter import RustTokenizer + +ANTHROPIC_MODEL: Final = "claude-sonnet-4-5-20250929" +CL100K_MODEL: Final = "gpt-4" +O200K_MODEL: Final = "gpt-4o" +PYTHON_ONLY_MODEL: Final = "replicate/meta/llama-2-70b-chat" +MESSAGES: Final = [{"role": "user", "content": "hello"}] +RUST_TOKENS: Final = 777 + + +class _FakeDeclined(Exception): + pass + + +class _FakeUpstream(Exception): + pass + + +class _FakeTokenizer: + """Stands in for one shared native `Tokenizer`; only its name identifies it.""" + + def __init__(self, name: str, json: str | None = None) -> None: + self.name = name + self.json = json + + +def _fake_native_tokenizers(monkeypatch: pytest.MonkeyPatch, anthropic_json: str | None = None) -> None: + """Point the counter's tokenizer lookups at fakes; the codec path keeps falling back to Python.""" + fakes: Final = {name: _FakeTokenizer(name) for name in ("cl100k_base", "o200k_base")} + anthropic: Final = _FakeTokenizer("anthropic", anthropic_json) + monkeypatch.setattr(tokenizer_dispatch, "native_encoding", fakes.__getitem__) + monkeypatch.setattr(tokenizer_dispatch, "native_anthropic", lambda: anthropic) + + +class _FakeNative: + RustBridgeDeclined = _FakeDeclined + RustUpstreamError = _FakeUpstream + + +class _RecordingCounter: + def __init__(self, factory: _RecordingFactory, tokenizer: RustTokenizer) -> None: + self.factory = factory + self.tokenizer = tokenizer + + async def acount_request(self, body: bytes) -> object: + self.factory.calls.append((self.tokenizer, body)) + return {"model": "", "input_tokens": RUST_TOKENS} + + +class _RecordingFactory: + def __init__(self) -> None: + self.calls: list[tuple[RustTokenizer, bytes]] = [] + + def from_tokenizer(self, tokenizer: _FakeTokenizer, fast: bool = False) -> _RecordingCounter: + return _RecordingCounter(self, cast(RustTokenizer, tokenizer.name)) + + +class _DecliningCounter: + async def acount_request(self, body: bytes) -> object: + raise _FakeDeclined("unsupported request shape") + + +class _DecliningFactory: + def from_tokenizer(self, tokenizer: _FakeTokenizer, fast: bool = False) -> _DecliningCounter: + return _DecliningCounter() + + +@pytest.fixture(autouse=True) +def _reset_bridge(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(bindings, "get_native_bridge", lambda: _FakeNative()) + _fake_native_tokenizers(monkeypatch) + token_counter.TOKEN_COUNTER.reset() + token_counter._counter.cache_clear() + configuration.reset_rust_configuration() + yield + token_counter.TOKEN_COUNTER.reset() + token_counter._counter.cache_clear() + configuration.reset_rust_configuration() + + +def _body(model: object) -> tuple[dict[str, object], bytes]: + body: Final = {"model": model, "messages": MESSAGES} + return body, json.dumps(body).encode() + + +@pytest.mark.asyncio +async def test_models_sharing_a_tokenizer_are_counted_once_and_merged() -> None: + factory: Final = _RecordingFactory() + litellm.rust(True) + token_counter.TOKEN_COUNTER.override(factory) + request_body, raw_body = _body([ANTHROPIC_MODEL, CL100K_MODEL, O200K_MODEL, "gpt-5", PYTHON_ONLY_MODEL]) + + counts: Final = await count_input_tokens( + request_body=request_body, + raw_body=raw_body, + models=(ANTHROPIC_MODEL, CL100K_MODEL, O200K_MODEL, "gpt-5", PYTHON_ONLY_MODEL), + ) + + assert factory.calls == [("anthropic", raw_body), ("cl100k_base", raw_body), ("o200k_base", raw_body)] + assert dict(counts) == { + ANTHROPIC_MODEL: RUST_TOKENS, + CL100K_MODEL: RUST_TOKENS, + O200K_MODEL: RUST_TOKENS, + "gpt-5": RUST_TOKENS, + PYTHON_ONLY_MODEL: count_input_tokens_for_model(request_body=request_body, model=PYTHON_ONLY_MODEL), + } + + +@pytest.mark.asyncio +async def test_rust_disabled_counts_everything_in_python() -> None: + factory: Final = _RecordingFactory() + litellm.rust(False) + token_counter.TOKEN_COUNTER.override(factory) + request_body, raw_body = _body([ANTHROPIC_MODEL, CL100K_MODEL]) + + counts: Final = await count_input_tokens( + request_body=request_body, raw_body=raw_body, models=(ANTHROPIC_MODEL, CL100K_MODEL) + ) + + assert factory.calls == [] + assert dict(counts) == { + model: count_input_tokens_for_model(request_body=request_body, model=model) + for model in (ANTHROPIC_MODEL, CL100K_MODEL) + } + + +@pytest.mark.asyncio +async def test_missing_raw_body_counts_in_python() -> None: + factory: Final = _RecordingFactory() + litellm.rust(True) + token_counter.TOKEN_COUNTER.override(factory) + request_body, _ = _body(ANTHROPIC_MODEL) + + counts: Final = await count_input_tokens(request_body=request_body, raw_body=None, models=(ANTHROPIC_MODEL,)) + + assert factory.calls == [] + assert counts[ANTHROPIC_MODEL] == count_input_tokens_for_model(request_body=request_body, model=ANTHROPIC_MODEL) + + +@pytest.mark.asyncio +async def test_missing_binding_counts_in_python() -> None: + litellm.rust(True) + token_counter.TOKEN_COUNTER.override(None) + request_body, raw_body = _body(ANTHROPIC_MODEL) + + counts: Final = await count_input_tokens(request_body=request_body, raw_body=raw_body, models=(ANTHROPIC_MODEL,)) + + assert counts[ANTHROPIC_MODEL] == count_input_tokens_for_model(request_body=request_body, model=ANTHROPIC_MODEL) + + +@pytest.mark.asyncio +async def test_declined_request_counts_in_python() -> None: + litellm.rust(True) + token_counter.TOKEN_COUNTER.override(_DecliningFactory()) + request_body, raw_body = _body(ANTHROPIC_MODEL) + + counts: Final = await count_input_tokens(request_body=request_body, raw_body=raw_body, models=(ANTHROPIC_MODEL,)) + + assert counts[ANTHROPIC_MODEL] == count_input_tokens_for_model(request_body=request_body, model=ANTHROPIC_MODEL) + assert counts[ANTHROPIC_MODEL] != RUST_TOKENS + + +@pytest.mark.asyncio +async def test_large_input_is_still_counted() -> None: + request_body: Final = { + "model": CL100K_MODEL, + "messages": [{"role": "user", "content": "x" * (TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS + 1)}], + } + + counts: Final = await count_input_tokens(request_body=request_body, raw_body=None, models=(CL100K_MODEL,)) + + assert counts[CL100K_MODEL] == count_input_tokens_for_model(request_body=request_body, model=CL100K_MODEL) + assert isinstance(counts, MappingProxyType) diff --git a/tests/test_litellm/proxy/test_budget_reservation.py b/tests/test_litellm/proxy/test_budget_reservation.py index 86bf896188f..b3913079bb2 100644 --- a/tests/test_litellm/proxy/test_budget_reservation.py +++ b/tests/test_litellm/proxy/test_budget_reservation.py @@ -39,8 +39,6 @@ from litellm.proxy.common_utils.user_api_key_cache import ( model_access_group_spend_counter_key, ) from litellm.proxy.spend_tracking.budget_reservation import ( - TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS, - _approximate_input_size, _get_model_access_group_budget_counters, estimate_request_max_cost, get_budget_window_start, @@ -49,6 +47,10 @@ from litellm.proxy.spend_tracking.budget_reservation import ( release_budget_reservation_on_cancel, reserve_budget_for_request, ) +from litellm.proxy.spend_tracking.input_tokens import ( + TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS, + _approximate_input_size, +) from litellm.proxy.utils import ProxyLogging from litellm.router import Router from litellm.types.proxy.model_access_group_budget import ModelAccessGroupBudget diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 2792e176e0e..26bfd5c52bd 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -14877,7 +14877,7 @@ async def test_token_counter_keeps_the_event_loop_free_during_a_huggingface_coun async def test_token_counter_loads_a_custom_tokenizer_off_the_event_loop(monkeypatch): - from tokenizers import Tokenizer + from litellm.rust_bridge._native import Tokenizer from litellm import Router from tests.test_litellm.litellm_core_utils.event_loop_lag import assert_loop_stayed_free, timed_with_loop_lags @@ -14890,7 +14890,7 @@ async def test_token_counter_loads_a_custom_tokenizer_off_the_event_loop(monkeyp time.sleep(0.3) return claude_tokenizer - monkeypatch.setattr(litellm.utils, "Tokenizer", SlowHubTokenizer) + monkeypatch.setattr("litellm.rust_bridge.tokenizer.from_pretrained", SlowHubTokenizer.from_pretrained) monkeypatch.setattr( "litellm.proxy.proxy_server.llm_router", Router( @@ -14914,7 +14914,7 @@ async def test_token_counter_loads_a_custom_tokenizer_off_the_event_loop(monkeyp async def test_token_counter_loads_a_custom_tokenizer_once_per_identifier_revision_and_token(monkeypatch): - from tokenizers import Tokenizer + from litellm.rust_bridge._native import Tokenizer from litellm import Router from litellm.types.router import DeploymentTypedDict @@ -14931,7 +14931,7 @@ async def test_token_counter_loads_a_custom_tokenizer_once_per_identifier_revisi }, } - monkeypatch.setattr(litellm.utils, "Tokenizer", MagicMock(from_pretrained=from_pretrained)) + monkeypatch.setattr("litellm.rust_bridge.tokenizer.from_pretrained", from_pretrained) monkeypatch.setattr( "litellm.proxy.proxy_server.llm_router", Router( diff --git a/tests/test_litellm/rust_bridge/test_catalog.py b/tests/test_litellm/rust_bridge/test_catalog.py index 3fa5f3de7ab..82e3766e8a2 100644 --- a/tests/test_litellm/rust_bridge/test_catalog.py +++ b/tests/test_litellm/rust_bridge/test_catalog.py @@ -53,7 +53,7 @@ def test_shipped_decisions( enabled: Final = environment == "1" if environment is not None else process is not False assert catalog.rollout(context) is Rollout.RUST_OPT_OUT assert catalog.decision(context) is (Decision.RUST_WITH_FALLBACK if enabled else Decision.PYTHON) - elif route is Route.MESSAGES: + elif route in (Route.MESSAGES, Route.TOKEN_COUNTER, Route.TOKENIZER): enabled: Final = environment == "1" if environment is not None else process is True assert catalog.rollout(context) is Rollout.RUST_OPT_IN assert catalog.decision(context) is (Decision.RUST_WITH_FALLBACK if enabled else Decision.PYTHON) diff --git a/tests/test_litellm/rust_bridge/test_token_counter.py b/tests/test_litellm/rust_bridge/test_token_counter.py index 71aa79cc4bb..3da291c898d 100644 --- a/tests/test_litellm/rust_bridge/test_token_counter.py +++ b/tests/test_litellm/rust_bridge/test_token_counter.py @@ -12,25 +12,32 @@ from types import MappingProxyType from typing import Final import pytest -import tiktoken -from tokenizers import Tokenizer import litellm from litellm.constants import TIKTOKEN_ENCODE_CHUNK_SIZE_CHARS from litellm.litellm_core_utils.token_counter import openai_tokenizer_encoding -from litellm.proxy.spend_tracking.budget_reservation import _count_input_tokens +from litellm.proxy.spend_tracking.input_tokens import count_input_tokens, count_input_tokens_for_model from litellm.rust_bridge import bindings, configuration from litellm.rust_bridge import token_counter as bridge +from litellm.rust_bridge import tokenizer as tokenizer_dispatch +from litellm.rust_bridge._native import Tokenizer from litellm.utils import claude_json_str MODEL: Final = "claude-sonnet-4-5-20250929" CL100K_MODEL: Final = "gpt-4" O200K_MODEL: Final = "gpt-4o" +MODEL_BY_TOKENIZER: Final[MappingProxyType[bridge.RustTokenizer, str]] = MappingProxyType( + {"anthropic": MODEL, "cl100k_base": CL100K_MODEL, "o200k_base": O200K_MODEL} +) TOKENIZERS: Final[tuple[bridge.RustTokenizer, ...]] = ("anthropic", "cl100k_base", "o200k_base") -RANK_FILE_LINES: Final = MappingProxyType({"cl100k_base": 100_256, "o200k_base": 199_998}) BODY: Final = json.dumps({"model": MODEL, "messages": [{"role": "user", "content": "hello"}]}).encode() +def _counted(body: dict[str, object], model: str) -> tuple[bytes, dict[str, object]]: + raw: Final = json.dumps({**body, "model": model}).encode() + return raw, json.loads(raw) + + class _FakeDeclined(Exception): pass @@ -39,14 +46,41 @@ class _FakeUpstream(Exception): pass +class _FakeTokenizer: + """Stands in for one shared native `Tokenizer`; only its name identifies it.""" + + def __init__(self, name: str, json: str | None = None) -> None: + self.name = name + self.json = json + + +def _fake_native_tokenizers(monkeypatch: pytest.MonkeyPatch, anthropic_json: str | None = None) -> None: + """Point the counter's tokenizer lookups at fakes while the bridge is faked; the codec path + keeps falling back to Python. Parity tests that restore the real extension get the real + lookups back.""" + fakes: Final = {name: _FakeTokenizer(name) for name in ("cl100k_base", "o200k_base")} + anthropic: Final = _FakeTokenizer("anthropic", anthropic_json) + real_encoding: Final = tokenizer_dispatch.native_encoding + real_anthropic: Final = tokenizer_dispatch.native_anthropic + + def faked() -> bool: + return isinstance(bindings.get_native_bridge(), _FakeNative) + + monkeypatch.setattr( + tokenizer_dispatch, "native_encoding", lambda name: fakes[name] if faked() else real_encoding(name) + ) + monkeypatch.setattr(tokenizer_dispatch, "native_anthropic", lambda: anthropic if faked() else real_anthropic()) + + class _FakeNative: RustBridgeDeclined = _FakeDeclined RustUpstreamError = _FakeUpstream class _RecordingCounter: - def __init__(self, tokenizer_json: str) -> None: - self.tokenizer_json = tokenizer_json + def __init__(self, tokenizer: _FakeTokenizer, fast: bool) -> None: + self.tokenizer = tokenizer + self.fast = fast self.bodies: list[bytes] = [] async def acount_request(self, body: bytes) -> object: @@ -55,25 +89,16 @@ class _RecordingCounter: class _RecordingFactory: - """Stands in for the native `TokenCounter` class: callable for tokenizer JSON, `from_*_ranks` for rank files.""" + """Stands in for the native `TokenCounter` class, built over a loaded `Tokenizer`.""" def __init__(self) -> None: self.counters: list[_RecordingCounter] = [] - self.rank_files: list[str] = [] - def __call__(self, tokenizer_json: str) -> _RecordingCounter: - counter = _RecordingCounter(tokenizer_json) + def from_tokenizer(self, tokenizer: _FakeTokenizer, fast: bool = False) -> _RecordingCounter: + counter = _RecordingCounter(tokenizer, fast) self.counters.append(counter) return counter - def from_cl100k_ranks(self, rank_file: str) -> _RecordingCounter: - self.rank_files.append(rank_file) - return self("cl100k_base") - - def from_o200k_ranks(self, rank_file: str) -> _RecordingCounter: - self.rank_files.append(rank_file) - return self("o200k_base") - class _RaisingCounter: def __init__(self, error: Exception) -> None: @@ -89,13 +114,7 @@ class _RaisingFactory: def __init__(self, error: Exception) -> None: self.error = error - def __call__(self, tokenizer_json: str) -> _RaisingCounter: - return _RaisingCounter(self.error) - - def from_cl100k_ranks(self, rank_file: str) -> _RaisingCounter: - return _RaisingCounter(self.error) - - def from_o200k_ranks(self, rank_file: str) -> _RaisingCounter: + def from_tokenizer(self, tokenizer: _FakeTokenizer, fast: bool = False) -> _RaisingCounter: return _RaisingCounter(self.error) @@ -105,6 +124,7 @@ def _reset_bridge(monkeypatch: pytest.MonkeyPatch): bridge._counter.cache_clear() configuration.reset_rust_configuration() monkeypatch.setattr(bindings, "get_native_bridge", lambda: _FakeNative()) + _fake_native_tokenizers(monkeypatch, anthropic_json=claude_json_str) yield bridge.TOKEN_COUNTER.reset() bridge._counter.cache_clear() @@ -117,8 +137,12 @@ async def test_disabled_bridge_never_constructs_a_counter(tokenizer: bridge.Rust factory: Final = _RecordingFactory() litellm.rust(False) bridge.TOKEN_COUNTER.override(factory) + model: Final = MODEL_BY_TOKENIZER[tokenizer] + raw, request_body = _counted({"messages": [{"role": "user", "content": "hello"}]}, model) - assert await bridge.count_input_tokens(BODY, tokenizer) is None + counts: Final = await count_input_tokens(request_body=request_body, raw_body=raw, models=(model,)) + + assert counts[model] == count_input_tokens_for_model(request_body=request_body, model=model) assert factory.counters == [] @@ -128,31 +152,33 @@ async def test_enabled_bridge_returns_typed_count_and_reuses_one_counter() -> No litellm.rust(True) bridge.TOKEN_COUNTER.override(factory) - first: Final = await bridge.count_input_tokens(BODY, "anthropic") - second: Final = await bridge.count_input_tokens(BODY, "anthropic") + first: Final = await bridge.native_count(factory, "anthropic", BODY) + second: Final = await bridge.native_count(factory, "anthropic", BODY) assert first == bridge.InputTokenCount(model=MODEL, input_tokens=42) assert second == first assert len(factory.counters) == 1 assert factory.counters[0].bodies == [BODY, BODY] - assert json.loads(factory.counters[0].tokenizer_json)["model"]["type"] == "BPE" + assert factory.counters[0].fast is False + assert factory.counters[0].tokenizer is tokenizer_dispatch.native_anthropic() + assert json.loads(factory.counters[0].tokenizer.json or "")["model"]["type"] == "BPE" @pytest.mark.asyncio @pytest.mark.parametrize("tokenizer", ("cl100k_base", "o200k_base")) -async def test_tiktoken_counter_is_built_from_the_vendored_rank_file_once(tokenizer: bridge.RustTokenizer) -> None: +async def test_tiktoken_counter_is_built_over_the_shared_encoding_once(tokenizer: bridge.RustTokenizer) -> None: factory: Final = _RecordingFactory() litellm.rust(True) bridge.TOKEN_COUNTER.override(factory) - first: Final = await bridge.count_input_tokens(BODY, tokenizer) - second: Final = await bridge.count_input_tokens(BODY, tokenizer) + first: Final = await bridge.native_count(factory, tokenizer, BODY) + second: Final = await bridge.native_count(factory, tokenizer, BODY) assert first == second == bridge.InputTokenCount(model=MODEL, input_tokens=42) - assert len(factory.rank_files) == 1 - assert factory.rank_files[0].startswith("IQ== 0\n") - assert factory.rank_files[0].count("\n") == RANK_FILE_LINES[tokenizer] - assert factory.counters[0].tokenizer_json == tokenizer + assert len(factory.counters) == 1 + assert factory.counters[0].tokenizer.name == tokenizer + assert factory.counters[0].tokenizer is tokenizer_dispatch.native_encoding(tokenizer) + assert factory.counters[0].fast is False assert factory.counters[0].bodies == [BODY, BODY] @@ -162,13 +188,13 @@ async def test_each_tokenizer_gets_its_own_cached_counter() -> None: litellm.rust(True) bridge.TOKEN_COUNTER.override(factory) - await bridge.count_input_tokens(BODY, "anthropic") - await bridge.count_input_tokens(BODY, "cl100k_base") - await bridge.count_input_tokens(BODY, "o200k_base") - await bridge.count_input_tokens(BODY, "anthropic") - await bridge.count_input_tokens(BODY, "o200k_base") + await bridge.native_count(factory, "anthropic", BODY) + await bridge.native_count(factory, "cl100k_base", BODY) + await bridge.native_count(factory, "o200k_base", BODY) + await bridge.native_count(factory, "anthropic", BODY) + await bridge.native_count(factory, "o200k_base", BODY) - assert [counter.tokenizer_json for counter in factory.counters][1:] == ["cl100k_base", "o200k_base"] + assert [counter.tokenizer.name for counter in factory.counters] == ["anthropic", "cl100k_base", "o200k_base"] assert [len(counter.bodies) for counter in factory.counters] == [2, 1, 2] @@ -176,8 +202,11 @@ async def test_each_tokenizer_gets_its_own_cached_counter() -> None: async def test_missing_native_module_falls_back(monkeypatch: pytest.MonkeyPatch) -> None: litellm.rust(True) monkeypatch.setattr(bindings, "get_native_bridge", lambda: None) + raw, request_body = _counted({"messages": [{"role": "user", "content": "hello"}]}, MODEL) - assert [await bridge.count_input_tokens(BODY, tokenizer) for tokenizer in TOKENIZERS] == [None, None, None] + counts: Final = await count_input_tokens(request_body=request_body, raw_body=raw, models=(MODEL,)) + + assert counts[MODEL] == count_input_tokens_for_model(request_body=request_body, model=MODEL) @pytest.mark.asyncio @@ -185,8 +214,12 @@ async def test_missing_native_module_falls_back(monkeypatch: pytest.MonkeyPatch) async def test_declined_request_falls_back(tokenizer: bridge.RustTokenizer) -> None: litellm.rust(True) bridge.TOKEN_COUNTER.override(_RaisingFactory(_FakeDeclined("request has no messages"))) + model: Final = MODEL_BY_TOKENIZER[tokenizer] + raw, request_body = _counted({"messages": [{"role": "user", "content": "hello"}]}, model) - assert await bridge.count_input_tokens(BODY, tokenizer) is None + counts: Final = await count_input_tokens(request_body=request_body, raw_body=raw, models=(model,)) + + assert counts[model] == count_input_tokens_for_model(request_body=request_body, model=model) @pytest.mark.asyncio @@ -194,8 +227,12 @@ async def test_declined_request_falls_back(tokenizer: bridge.RustTokenizer) -> N async def test_runtime_failure_falls_back(tokenizer: bridge.RustTokenizer) -> None: litellm.rust(True) bridge.TOKEN_COUNTER.override(_RaisingFactory(RuntimeError("encode failed"))) + model: Final = MODEL_BY_TOKENIZER[tokenizer] + raw, request_body = _counted({"messages": [{"role": "user", "content": "hello"}]}, model) - assert await bridge.count_input_tokens(BODY, tokenizer) is None + counts: Final = await count_input_tokens(request_body=request_body, raw_body=raw, models=(model,)) + + assert counts[model] == count_input_tokens_for_model(request_body=request_body, model=model) @pytest.mark.parametrize( @@ -273,8 +310,8 @@ def test_rust_tokenizer_names_the_encoding_python_actually_counts_with(model: st "Hello, world! camelCase ABCdef \u00e9\u00e8 12345 \u3053\u3093\u306b\u3061\u306f <|endoftext|>\r\n" * 9 ) python_count: Final = litellm.token_counter(model=model, text=text) - cl100k_count: Final = len(tiktoken.get_encoding("cl100k_base").encode(text, disallowed_special=())) - o200k_count: Final = len(tiktoken.get_encoding("o200k_base").encode(text, disallowed_special=())) + cl100k_count: Final = Tokenizer.from_tiktoken("cl100k_base").count(text) + o200k_count: Final = Tokenizer.from_tiktoken("o200k_base").count(text) assert cl100k_count != o200k_count match bridge.rust_tokenizer(model): case "cl100k_base": @@ -282,7 +319,7 @@ def test_rust_tokenizer_names_the_encoding_python_actually_counts_with(model: st case "o200k_base": assert python_count == o200k_count case "anthropic": - assert python_count == len(Tokenizer.from_str(claude_json_str).encode(text).ids) + assert python_count == Tokenizer.from_json(claude_json_str).count(text) assert python_count not in {cl100k_count, o200k_count} case None: pytest.fail(f"{model} must have a Rust tokenizer") @@ -386,12 +423,11 @@ async def test_native_count_matches_python_budget_counter( litellm.rust(True) body: Final = json.dumps(request_body).replace(MODEL, model) - rust_count: Final = await bridge.count_input_tokens(body.encode(), tokenizer) - python_count: Final = _count_input_tokens(request_body=json.loads(body), model=model) + request_body_parsed: Final = json.loads(body) + counts: Final = await count_input_tokens(request_body=request_body_parsed, raw_body=body.encode(), models=(model,)) + python_count: Final = count_input_tokens_for_model(request_body=request_body_parsed, model=model) - assert rust_count is not None - assert rust_count.model == json.loads(body).get("model") - assert rust_count.input_tokens == python_count + assert counts[model] == python_count @pytest.mark.asyncio @@ -405,15 +441,14 @@ async def test_tiktoken_counts_long_text_exactly_where_python_chunks( litellm.rust(True) text: Final = "x " * 20_000 body: Final = {"model": model, "messages": [{"role": "user", "content": text}]} - encoding: Final = tiktoken.get_encoding(tokenizer) - exact: Final = 3 + len(encoding.encode("user")) + len(encoding.encode(text)) + 3 + encoding: Final = Tokenizer.from_tiktoken(tokenizer) + exact: Final = 3 + encoding.count("user") + encoding.count(text) + 3 chunks: Final = -(-len(text) // TIKTOKEN_ENCODE_CHUNK_SIZE_CHARS) - rust_count: Final = await bridge.count_input_tokens(json.dumps(body).encode(), tokenizer) - python_count: Final = _count_input_tokens(request_body=body, model=model) + counts: Final = await count_input_tokens(request_body=body, raw_body=json.dumps(body).encode(), models=(model,)) + python_count: Final = count_input_tokens_for_model(request_body=body, model=model) - assert rust_count is not None - assert rust_count.input_tokens == exact + assert counts[model] == exact assert python_count is not None assert exact < python_count <= exact + chunks @@ -440,5 +475,9 @@ async def test_native_declines_shapes_python_prices_differently( native: Final = pytest.importorskip("litellm.rust_bridge._native") monkeypatch.setattr(bindings, "get_native_bridge", lambda: native) litellm.rust(True) + model: Final = MODEL_BY_TOKENIZER[tokenizer] + raw, parsed = _counted(request_body, model) - assert await bridge.count_input_tokens(json.dumps(request_body).encode(), tokenizer) is None + counts: Final = await count_input_tokens(request_body=parsed, raw_body=raw, models=(model,)) + + assert counts.get(model) == count_input_tokens_for_model(request_body=parsed, model=model) diff --git a/tests/test_litellm/rust_bridge/test_tokenizer.py b/tests/test_litellm/rust_bridge/test_tokenizer.py new file mode 100644 index 00000000000..0de7ad50b1e --- /dev/null +++ b/tests/test_litellm/rust_bridge/test_tokenizer.py @@ -0,0 +1,134 @@ +from collections.abc import Generator +from typing import Final + +import pytest +import tiktoken +from tokenizers import Tokenizer + +import litellm +from litellm.litellm_core_utils.tokenizer import HuggingFaceTokenizer, OpenAIEncoding +from litellm.rust_bridge import configuration, tokenizer +from litellm.utils import _select_tokenizer +from tests.test_litellm.litellm_core_utils.test_decode_special_tokens import TOKENIZER_JSON + + +@pytest.fixture(autouse=True) +def isolated_configuration(monkeypatch: pytest.MonkeyPatch) -> Generator[None]: + monkeypatch.delenv("LITELLM_RUST", raising=False) + configuration.reset_rust_configuration() + yield + tokenizer.TOKENIZER.reset() + configuration.reset_rust_configuration() + + +@pytest.mark.parametrize("environment", (None, "0", "1")) +@pytest.mark.parametrize("process", (None, False, True)) +def test_tokenizer_factories_follow_rollout( + monkeypatch: pytest.MonkeyPatch, environment: str | None, process: bool | None +) -> None: + configuration.rust(process) + if environment is not None: + monkeypatch.setenv("LITELLM_RUST", environment) + enabled: Final = environment == "1" if environment is not None else process is True + encoding: Final = tokenizer.get_encoding("cl100k_base") + custom: Final = litellm.create_tokenizer(TOKENIZER_JSON) + reference: Final = Tokenizer.from_str(TOKENIZER_JSON) + + assert isinstance(encoding, OpenAIEncoding if enabled else tiktoken.Encoding) + assert isinstance(custom["tokenizer"], HuggingFaceTokenizer if enabled else Tokenizer) + assert encoding.encode("café 漢字 🙂") == tiktoken.get_encoding(encoding.name).encode("café 漢字 🙂") + assert litellm.encode(text="Hello World", custom_tokenizer=custom) == reference.encode("Hello World").ids + assert litellm.token_counter(text="Hello World", custom_tokenizer=custom) == len(reference.encode("Hello World")) + + +def test_missing_native_binding_keeps_python_tokenizer_api() -> None: + configuration.rust(True) + tokenizer.TOKENIZER.override(None) + encoding: Final = tokenizer.get_encoding("cl100k_base") + custom: Final = litellm.create_tokenizer(TOKENIZER_JSON)["tokenizer"] + + assert isinstance(encoding, tiktoken.Encoding) + assert isinstance(custom, Tokenizer) + custom.enable_padding(pad_id=0, pad_token="[UNK]") + assert [item.ids for item in custom.encode_batch(["Hello", "Hello World"])] == [[3, 1, 0], [3, 1, 2]] + + +def test_cached_selection_follows_backend_changes(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "disable_hf_tokenizer_download", True) + configuration.rust(True) + native: Final = _select_tokenizer("dispatch-fixture")["tokenizer"] + configuration.rust(False) + python: Final = _select_tokenizer("dispatch-fixture")["tokenizer"] + + assert isinstance(native, OpenAIEncoding) + assert isinstance(python, tiktoken.Encoding) + assert native.encode("hello") == python.encode("hello") + + +def test_declined_native_factory_falls_back_before_tokenizing() -> None: + from litellm.rust_bridge._native import RustBridgeDeclined + + class UnavailableTokenizer: + @staticmethod + def from_json(json: str) -> None: + raise RustBridgeDeclined("huggingface feature is disabled") + + configuration.rust(True) + binding: Final = tokenizer._as_factory(UnavailableTokenizer) + tokenizer.TOKENIZER.override(binding) + custom: Final = litellm.create_tokenizer(TOKENIZER_JSON) + + assert isinstance(custom["tokenizer"], Tokenizer) + assert ( + litellm.decode(tokens=litellm.encode(text="Hello World", custom_tokenizer=custom), custom_tokenizer=custom) + == "Hello World" + ) + + +@pytest.mark.parametrize( + ("model", "text"), + ( + ("gpt-4o", "hello <|endoftext|> world"), + ("gpt-3.5-turbo", "café 漢字 🙂"), + ("text-davinci-003", " def f():\n return 1\n"), + ("tokenizer-parity-fixture", "hello again"), + ), +) +def test_public_token_api_is_identical_across_backends(monkeypatch: pytest.MonkeyPatch, model: str, text: str) -> None: + """`litellm.token_counter`, `encode` and `decode` return the same values whichever backend + the catalog picks; only the object types differ.""" + monkeypatch.setattr(litellm, "anthropic_models", {*litellm.anthropic_models, "tokenizer-parity-fixture"}) + messages: Final = [{"role": "user", "content": text}, {"role": "assistant", "content": "ok"}] + + def observe() -> tuple[int, int, list[int], str]: + ids: Final = litellm.encode(model=model, text=text) + return ( + litellm.token_counter(model=model, text=text), + litellm.token_counter(model=model, messages=messages), + ids, + litellm.decode(model=model, tokens=ids), + ) + + configuration.rust(False) + python: Final = observe() + configuration.rust(True) + rust: Final = observe() + + assert rust == python + + +def test_cached_huggingface_tokenizers_follow_backend_changes(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.litellm_core_utils.tokenizer import HuggingFaceTokenizer as RustHuggingFaceTokenizer + from litellm.utils import _load_huggingface_tokenizer + + monkeypatch.setattr(litellm, "anthropic_models", {*litellm.anthropic_models, "tokenizer-cache-fixture"}) + _load_huggingface_tokenizer.cache_clear() + configuration.rust(True) + native: Final = _select_tokenizer("tokenizer-cache-fixture")["tokenizer"] + configuration.rust(False) + python: Final = _select_tokenizer("tokenizer-cache-fixture")["tokenizer"] + configuration.rust(True) + + assert isinstance(native, RustHuggingFaceTokenizer) + assert isinstance(python, Tokenizer) + assert _select_tokenizer("tokenizer-cache-fixture")["tokenizer"] is native diff --git a/tests/test_litellm/rust_bridge/test_verify_linux_native_wheel.py b/tests/test_litellm/rust_bridge/test_verify_linux_native_wheel.py index e449d4392d8..0eae041f535 100644 --- a/tests/test_litellm/rust_bridge/test_verify_linux_native_wheel.py +++ b/tests/test_litellm/rust_bridge/test_verify_linux_native_wheel.py @@ -56,10 +56,11 @@ def _write_wheel( metadata_tags: tuple[str, ...] | None = (_EXPECTED_TAG,), dist_info: str = _DIST_INFO, duplicate_wheel: bool = False, + native_bytes: bytes = b"synthetic native extension", ) -> Path: wheel: Final = tmp_path / f"litellm-1.100.0-{filename_tag}.whl" with zipfile.ZipFile(wheel, "w", compression=zipfile.ZIP_DEFLATED) as archive: - archive.writestr(_NATIVE_MEMBER, b"synthetic native extension") + archive.writestr(_NATIVE_MEMBER, native_bytes) archive.writestr( f"{dist_info}/METADATA", "Metadata-Version: 2.1\nName: litellm\nVersion: 1.100.0\n", @@ -195,3 +196,17 @@ def test_rejects_production_module_exposing_panic_hook(tmp_path: Path) -> None: wheel: Final = _write_wheel(tmp_path, filename_tag=_EXPECTED_TAG) assert _run_verifier(wheel, exposes_panic=True) == 1 + + +@pytest.mark.parametrize("embedded", (False, True)) +def test_vocabulary_is_packaged_once(tmp_path: Path, embedded: bool) -> None: + ranks: Final = b"AA== 0\nAQ== 1\nAg== 2\n" + wheel: Final = _write_wheel( + tmp_path, + filename_tag=_EXPECTED_TAG, + native_bytes=b"native engine" + (ranks if embedded else b""), + ) + with zipfile.ZipFile(wheel, "a") as archive: + archive.writestr("litellm/litellm_core_utils/tokenizers/" + "a" * 40, ranks) + + assert _run_verifier(wheel) == (1 if embedded else 0) diff --git a/tests/test_litellm_rust/test_fork_guard.py b/tests/test_litellm_rust/test_fork_guard.py index 086397bab5c..2a8fb6f9fca 100644 --- a/tests/test_litellm_rust/test_fork_guard.py +++ b/tests/test_litellm_rust/test_fork_guard.py @@ -1,5 +1,6 @@ import os import textwrap +from typing import Final import pytest @@ -144,3 +145,76 @@ def test_sdk_call_in_a_child_forked_after_native_use_raises_instead_of_hanging() result = run_child_interpreter(_SDK_CONTRACT, env=env, timeout=120) assert result.returncode == 0, result.stderr + + +@pytest.mark.skipif(not hasattr(os, "fork"), reason="fork only") +@pytest.mark.parametrize("warm_fast_counter", (False, True)) +def test_tokenizers_share_the_native_process_guard(warm_fast_counter: bool) -> None: + script: Final = """ +import asyncio +import os +import litellm +from litellm.proxy.spend_tracking.input_tokens import count_input_tokens +from litellm.rust_bridge import _native +from litellm.litellm_core_utils.tokenizer import HuggingFaceTokenizer +from litellm.utils import claude_json_str + +litellm.anthropic_models = {*litellm.anthropic_models, "tokenizer-fork-fixture"} +_native.reserve_process_for_forking() +for create in ( + lambda: _native.Tokenizer.from_tiktoken("cl100k_base"), + lambda: _native.Tokenizer.from_json(claude_json_str), + lambda: litellm.token_counter(model="tokenizer-fork-fixture", text="hello"), +): + try: + create() + except _native.ProcessReservedForForking: + pass + else: + raise AssertionError("reserved parent ran a native tokenizer") +assert not _native.process_state_started() + +pid = os.fork() +if pid == 0: + tokenizer = HuggingFaceTokenizer.from_str(claude_json_str) + encoding = _native.Tokenizer.from_tiktoken("cl100k_base") + if os.environ["WARM_FAST_COUNTER"] == "True": + _native.TokenCounter.from_tokenizer(encoding, fast=True) + expected = [item.ids for item in tokenizer.encode_batch(["hello", "world"])] + assert _native.process_state_started() + grandchild = os.fork() + if grandchild == 0: + for call in ( + lambda: tokenizer.encode_batch(["hello", "world"]), + lambda: tokenizer.encode("hello"), + lambda: encoding.count("hello"), + lambda: encoding.count("hello", fast=True), + lambda: _native.TokenCounter.from_tokenizer(encoding), + lambda: _native.TokenCounter.from_tokenizer(encoding, fast=True), + lambda: _native.Tokenizer.from_tiktoken("cl100k_base"), + lambda: asyncio.run(count_input_tokens({"prompt": "hello"}, b'{"prompt": "hello"}', ("counter-fork-fixture",))), + ): + try: + call() + except _native.ForkedAfterNativeRuntimeStarted: + pass + else: + os._exit(1) + os._exit(0) + assert os.waitpid(grandchild, 0)[1] == 0 + assert [item.ids for item in tokenizer.encode_batch(["hello", "world"])] == expected + os._exit(0) +assert os.waitpid(pid, 0)[1] == 0 +""" + result: Final = run_child_interpreter( + script, + env={ + **os.environ, + "OBJC_DISABLE_INITIALIZE_FORK_SAFETY": "YES", + "LITELLM_LOCAL_MODEL_COST_MAP": "True", + "WARM_FAST_COUNTER": str(warm_fast_counter), + }, + timeout=30, + ) + + assert result.returncode == 0, result.stderr diff --git a/tests/test_litellm_rust/test_tokenizer.py b/tests/test_litellm_rust/test_tokenizer.py new file mode 100644 index 00000000000..98d5259b652 --- /dev/null +++ b/tests/test_litellm_rust/test_tokenizer.py @@ -0,0 +1,130 @@ +import json +from typing import Final + +import pytest +import tiktoken +from tokenizers import Tokenizer as ReferenceTokenizer + +from litellm.rust_bridge import _native +from litellm.utils import claude_json_str +from tests.test_litellm.litellm_core_utils.test_decode_special_tokens import TOKENIZER_JSON + +pytestmark = pytest.mark.requires_rust_extension + + +def test_tiktoken_codec_round_trips_and_counts() -> None: + tokenizer: Final = _native.Tokenizer.from_tiktoken("cl100k_base") + encoded: Final = tokenizer.encode("hello world") + + assert tokenizer.name == "cl100k_base" + assert tokenizer.count("hello world") == len(encoded) + assert tokenizer.decode(encoded) == "hello world" + + +def test_huggingface_codec_skips_special_tokens() -> None: + tokenizer: Final = _native.Tokenizer.from_json(claude_json_str) + encoded: Final = tokenizer.encode("hello") + + assert "" in tokenizer.decode(encoded, skip_special_tokens=False) + assert tokenizer.decode(encoded, skip_special_tokens=True) == "hello" + + +def test_tiktoken_codec_keeps_the_requested_encoding_name() -> None: + assert _native.Tokenizer.from_tiktoken("gpt2").name == "gpt2" + assert _native.Tokenizer.from_tiktoken("r50k_base").name == "r50k_base" + assert _native.Tokenizer.from_tiktoken("gpt2").encode("hi") == _native.Tokenizer.from_tiktoken("r50k_base").encode( + "hi" + ) + + +def test_tiktoken_codec_exposes_its_vocabulary() -> None: + reference: Final = tiktoken.get_encoding("cl100k_base") + tokenizer: Final = _native.Tokenizer.from_tiktoken("cl100k_base") + + assert tokenizer.special_tokens() == reference._special_tokens + assert tokenizer.max_token_value() == reference.max_token_value + assert tokenizer.token_byte_values() == reference.token_byte_values() + assert tokenizer.encode_single_token(b"hello") == reference.encode_single_token("hello") + assert tokenizer.is_special_token(reference.eot_token) and not tokenizer.is_special_token(0) + with pytest.raises(KeyError): + tokenizer.encode_single_token(b"<|not-a-token|>") + + +def test_huggingface_codec_rejects_tiktoken_only_calls() -> None: + tokenizer: Final = _native.Tokenizer.from_json(claude_json_str) + with pytest.raises(ValueError, match="requires a tiktoken encoding"): + tokenizer.token_byte_values() + with pytest.raises(ValueError, match="requires a Hugging Face tokenizer"): + _native.Tokenizer.from_tiktoken("cl100k_base").get_vocab() + + +def test_unknown_tiktoken_encoding_raises_value_error() -> None: + with pytest.raises(ValueError, match="unsupported tokenizer"): + _native.Tokenizer.from_tiktoken("unknown-encoding") + + +def test_tiktoken_codec_decodes_truncated_unicode_like_python() -> None: + reference: Final = tiktoken.get_encoding("cl100k_base") + tokenizer: Final = _native.Tokenizer.from_tiktoken(reference.name) + encoded: Final = reference.encode("🙂漢字") + + assert tuple(tokenizer.decode(encoded[:end]) for end in range(1, len(encoded) + 1)) == tuple( + reference.decode(encoded[:end]) for end in range(1, len(encoded) + 1) + ) + + +FAST_TEXTS: Final = ( + "", + "hello world <|endoftext|>", + "café 漢字 ع 🙂 line\r\n indented 123456789", + "x a\u0301 fi", +) + + +def test_fast_counting_is_an_opt_in_over_the_same_loaded_tokenizer() -> None: + for tokenizer in ( + _native.Tokenizer.from_tiktoken("cl100k_base"), + _native.Tokenizer.from_tiktoken("o200k_base"), + _native.Tokenizer.from_json(claude_json_str), + ): + assert [tokenizer.count(text, fast=True) for text in FAST_TEXTS] == [ + tokenizer.count(text) for text in FAST_TEXTS + ] + + +@pytest.mark.parametrize( + "name", ("cl100k_base", "o200k_base", "o200k_harmony", "p50k_base", "p50k_edit", "r50k_base", "gpt2") +) +@pytest.mark.asyncio +async def test_token_counter_counts_over_a_shared_tokenizer(name: str) -> None: + messages: Final = [{"role": "user", "content": "hello wide world"}, {"role": "assistant", "content": "ok"}] + body: Final = json.dumps({"model": "gpt-4", "messages": messages}).encode() + tokenizer: Final = _native.Tokenizer.from_tiktoken(name) + reference: Final = tiktoken.get_encoding(name) + for text in FAST_TEXTS: + assert tokenizer.count(text, fast=True) == tokenizer.count(text) == len(reference.encode_ordinary(text)) + + exact: Final = await _native.TokenCounter.from_tokenizer(tokenizer).acount_request(body) + fast: Final = await _native.TokenCounter.from_tokenizer(tokenizer, fast=True).acount_request(body) + + assert exact == fast + assert exact["input_tokens"] == 3 + sum( + 3 + len(reference.encode_ordinary(message["role"])) + len(reference.encode_ordinary(message["content"])) + for message in messages + ) + + +@pytest.mark.parametrize("configured", (False, True)) +@pytest.mark.asyncio +async def test_fast_count_preserves_huggingface_configuration(configured: bool) -> None: + reference: Final = ReferenceTokenizer.from_str(TOKENIZER_JSON) + if configured: + reference.enable_truncation(max_length=3) + reference.enable_padding(pad_id=0, pad_token="[UNK]", length=5) + tokenizer: Final = _native.Tokenizer.from_json(reference.to_str()) + counter: Final = _native.TokenCounter.from_tokenizer(tokenizer, fast=True) + for text in ("", "Hello", "Hello World Hello World", "[BOS] Hello"): + expected: Final = len(reference.encode(text)) + assert tokenizer.count(text, fast=True) == tokenizer.count(text) == expected + result: Final = await counter.acount_request(json.dumps({"prompt": text}).encode()) + assert result["input_tokens"] == expected diff --git a/uv.lock b/uv.lock index 18d57aa991b..32d7580f61b 100644 --- a/uv.lock +++ b/uv.lock @@ -1162,14 +1162,11 @@ wheels = [ [[package]] name = "click" -version = "8.3.3" +version = "8.5.0" source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/bb/63/f9e1ea081ce35720d8b92acde70daaedace594dc93b693c869e0d5910718/click-8.3.3.tar.gz", hash = "sha256:398329ad4837b2ff7cbe1dd166a4c0f8900c3ca3a218de04466f38f6497f18a2", size = 328061, upload-time = "2026-04-22T15:11:27.506Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c7/0e/7fa0ef50764b67090eca4114772a2abf8b6148198475e54c660b97caeee6/click-8.5.0.tar.gz", hash = "sha256:ba0d2089de75ea0310e2dde03160e6ca10009947fb95a182f9b54021bb272e34", size = 382235, upload-time = "2026-08-26T13:33:14.56Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ae/44/c1221527f6a71a01ec6fbad7fa78f1d50dfa02217385cf0fa3eec7087d59/click-8.3.3-py3-none-any.whl", hash = "sha256:a2bf429bb3033c89fa4936ffb35d5cb471e3719e1f3c8a7c3fff0b8314305613", size = 110502, upload-time = "2026-04-22T15:11:25.044Z" }, + { url = "https://files.pythonhosted.org/packages/58/50/6c0d534c5f134586a8e1ba4e330569e32f057e33372ae556463212fb4cd3/click-8.5.0-py3-none-any.whl", hash = "sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360", size = 125251, upload-time = "2026-08-26T13:33:12.928Z" }, ] [[package]] @@ -1989,11 +1986,11 @@ wheels = [ [[package]] name = "filelock" -version = "3.29.0" +version = "3.32.7" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b5/fe/997687a931ab51049acce6fa1f23e8f01216374ea81374ddee763c493db5/filelock-3.29.0.tar.gz", hash = "sha256:69974355e960702e789734cb4871f884ea6fe50bd8404051a3530bc07809cf90", size = 57571, upload-time = "2026-04-19T15:39:10.068Z" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/59/e19834834cb01a32febfbb0f8a23a9088088f5d45991824ff2bc3b5e8acb/filelock-3.32.7.tar.gz", hash = "sha256:37b8a3d9811b0f9aef7e5ec5c71bb320de52df51e6ca9bcd6f5ad81187660da7", size = 225154, upload-time = "2026-09-16T00:24:20.907Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/81/47/dd9a212ef6e343a6857485ffe25bba537304f1913bdbed446a23f7f592e1/filelock-3.29.0-py3-none-any.whl", hash = "sha256:96f5f6344709aa1572bbf631c640e4ebeeb519e08da902c39a001882f30ac258", size = 39812, upload-time = "2026-04-19T15:39:08.752Z" }, + { url = "https://files.pythonhosted.org/packages/15/df/31098c5aeb4d966b553641472bd55fcf5fdfac953549894b8a765ba44e91/filelock-3.32.7-py3-none-any.whl", hash = "sha256:65ff0d0190ea42038b32bda4b77834fb05be2cad4c5b9b01aa4dfb3614536e52", size = 100157, upload-time = "2026-09-16T00:24:19.543Z" }, ] [[package]] @@ -2225,11 +2222,11 @@ wheels = [ [[package]] name = "fsspec" -version = "2026.4.0" +version = "2026.7.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d5/8d/1c51c094345df128ca4a990d633fe1a0ff28726c9e6b3c41ba65087bba1d/fsspec-2026.4.0.tar.gz", hash = "sha256:301d8ac70ae90ef3ad05dcf94d6c3754a097f9b5fe4667d2787aa359ec7df7e4", size = 312760, upload-time = "2026-04-29T20:42:38.635Z" } +sdist = { url = "https://files.pythonhosted.org/packages/00/78/f34251dadb8f3921264a1d9b8946f5e542014ee2614b285261b4e40e6775/fsspec-2026.7.0.tar.gz", hash = "sha256:c803c40f4cf860b49dea58ee3e1c33cb9c790520e233537e1340049f89b82a88", size = 317040, upload-time = "2026-07-28T16:34:51.052Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d5/0c/043d5e551459da400957a1395e0febbf771446ff34291afcbe3d8be2a279/fsspec-2026.4.0-py3-none-any.whl", hash = "sha256:11ef7bb35dab8a394fde6e608221d5cf3e8499401c249bebaeaad760a1a8dec2", size = 203402, upload-time = "2026-04-29T20:42:36.842Z" }, + { url = "https://files.pythonhosted.org/packages/fd/3c/6a2bf344106328fd04963664a60b9bb6496fc25df8e962fcdc1367285fb9/fsspec-2026.7.0-py3-none-any.whl", hash = "sha256:b57ddbafedfaef7018c1ecab32aa200a9d7ca26b77965f64e48b70061249d279", size = 206583, upload-time = "2026-07-28T16:34:49.538Z" }, ] [[package]] @@ -3146,34 +3143,26 @@ wheels = [ [[package]] name = "hf-xet" -version = "1.5.0" +version = "1.6.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/74/d8/5c06fc76461418326a7decf8367480c35be11a41fd938633929c60a9ec6b/hf_xet-1.5.0.tar.gz", hash = "sha256:e0fb0a34d9f406eed88233e829a67ec016bec5af19e480eac65a233ea289a948", size = 837196, upload-time = "2026-05-06T06:18:15.583Z" } +sdist = { url = "https://files.pythonhosted.org/packages/1b/ab/522a2ab67f27971a9d48ca666d4fca85ef7d5282d142e31fd087e27b1bbe/hf_xet-1.6.0.tar.gz", hash = "sha256:2e58454a340b3556dfa4972d5451aff4fba8dd42a236600ba1a1d2b1514f0fef", size = 920527, upload-time = "2026-08-03T22:33:13.243Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/68/9b/6912c99070915a4f28119e3c5b52a9abd1eec0ad5cb293b8c967a0c6f5a2/hf_xet-1.5.0-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:7d70fe2ce97b9db73b9c9b9c81fe3693640aec83416a966c446afea54acfae3c", size = 4023383, upload-time = "2026-05-06T06:17:53.947Z" }, - { url = "https://files.pythonhosted.org/packages/0f/6d/9563cfde59b5d8128a9c7ec972a087f4c782e4f7bac5a85234edfd5d5e49/hf_xet-1.5.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:73a0dae8c71de3b0633a45c73f4a4a5ed09e94b43441d82981a781d4f12baa42", size = 3792751, upload-time = "2026-05-06T06:17:51.791Z" }, - { url = "https://files.pythonhosted.org/packages/07/a5/ed5a0cf35b49a0571af5a8f53416dad1877a718c021c9937c3a53cb45781/hf_xet-1.5.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a60290ec57e9b71767fba7c3645ddafdd0759974b540441510c629c6db6db24a", size = 4456058, upload-time = "2026-05-06T06:17:40.735Z" }, - { url = "https://files.pythonhosted.org/packages/60/fb/3ae8bf2a7a37a4197d0195d7247fd25b3952e15cb8a599e285dfaa6f52b3/hf_xet-1.5.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:e5de0f6deada0dada870bb376a11bcd1f08abf3a968a6d118f33e72d1b1eb480", size = 4250783, upload-time = "2026-05-06T06:17:38.412Z" }, - { url = "https://files.pythonhosted.org/packages/a2/9b/8bae40d4d91525085137196e84eb0ed49cf65b5e96e5c3ecdadd8bd0fac2/hf_xet-1.5.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:c799d49f1a5544a0ef7591c0ee75e0d6b93d6f56dc7a4979f59f7518d2872216", size = 4445594, upload-time = "2026-05-06T06:18:04.219Z" }, - { url = "https://files.pythonhosted.org/packages/13/59/c74efbbd4e8728172b2cc72a2bc014d2947a4b7bdced932fbd3f5da1a4e5/hf_xet-1.5.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2baea1b0b989e5c152fe81425f7745ddc8901280ba3d97c98d8cdece7b706c60", size = 4663995, upload-time = "2026-05-06T06:18:06.1Z" }, - { url = "https://files.pythonhosted.org/packages/73/32/8e1e0410af64cda9b139d1dcebdc993a8ff9c8c7c0e2696ae356d75ccc0d/hf_xet-1.5.0-cp313-cp313t-win_amd64.whl", hash = "sha256:526345b3ed45f374f6317349df489167606736c876241ba984105afe7fd4839d", size = 3966608, upload-time = "2026-05-06T06:18:19.74Z" }, - { url = "https://files.pythonhosted.org/packages/fc/34/a8febc8f4edbea8b3e21b02ebc8b628679b84ba7e45cde624a7736b51500/hf_xet-1.5.0-cp313-cp313t-win_arm64.whl", hash = "sha256:786d28e2eb8315d5035544b9d137b4a842d600c434bb91bf7d0d953cce906ad4", size = 3796946, upload-time = "2026-05-06T06:18:17.568Z" }, - { url = "https://files.pythonhosted.org/packages/2a/20/8fc8996afe5815fa1a6be8e9e5c02f24500f409d599e905800d498a4e14d/hf_xet-1.5.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:872d5601e6deea30d15865ede55d29eac6daf5a534ab417b99b6ef6b076dd96c", size = 4023495, upload-time = "2026-05-06T06:18:01.94Z" }, - { url = "https://files.pythonhosted.org/packages/32/6a/93d84463c00cecb561a7508aa6303e35ee2894294eac14245526924415fe/hf_xet-1.5.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9929561f5abf4581c8ea79587881dfef6b8abb2a0d8a51915936fc2a614f4e73", size = 3792731, upload-time = "2026-05-06T06:18:00.021Z" }, - { url = "https://files.pythonhosted.org/packages/9d/5a/8ec8e0c863b382d00b3c2e2af6ded6b06371be617144a625903a6d562f4b/hf_xet-1.5.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f7b7bbae318e583a86fb21e5a4a175d6721d628a2874f4bd022d0e660c32a682", size = 4456738, upload-time = "2026-05-06T06:17:49.574Z" }, - { url = "https://files.pythonhosted.org/packages/c5/ca/f7effa1a67717da2bcc6b6c28f71c6ca648c77acaec4e2c32f40cbe16d85/hf_xet-1.5.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:cf7b2dc6f31a4ea754bb50f74cde482dcf5d366d184076d8530b9872787f3761", size = 4251622, upload-time = "2026-05-06T06:17:47.096Z" }, - { url = "https://files.pythonhosted.org/packages/65/f2/19247dba3e231cf77dec59ddfb878f00057635ff773d099c9b59d37812c3/hf_xet-1.5.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8dbcbab554c9ef158ef2c991545c3e970ddd8cc7acdcd0a78c5a41095dab4ded", size = 4445667, upload-time = "2026-05-06T06:18:11.983Z" }, - { url = "https://files.pythonhosted.org/packages/7f/64/6f116801a3bcfb6f59f5c251f48cadc47ea54026441c4a385079286a94fa/hf_xet-1.5.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5906bf7718d3636dc13402914736abe723492cb730f744834f5f5b67d3a12702", size = 4664619, upload-time = "2026-05-06T06:18:13.771Z" }, - { url = "https://files.pythonhosted.org/packages/5c/e8/069542d37946ed08669b127e1496fa99e78196d71de8d41eda5e9f1b7a58/hf_xet-1.5.0-cp314-cp314t-win_amd64.whl", hash = "sha256:5f3dc2248fc01cc0a00cd392ab497f1ca373fcbc7e3f2da1f452480b384e839e", size = 3966802, upload-time = "2026-05-06T06:18:28.162Z" }, - { url = "https://files.pythonhosted.org/packages/f9/91/fc6fdec27b14d04e88c386ac0a0129732b53fa23f7c4a78f4b83a039c567/hf_xet-1.5.0-cp314-cp314t-win_arm64.whl", hash = "sha256:b285cea1b5bab46b758772716ba8d6854a1a0310fed1c249d678a8b38601e5a0", size = 3797168, upload-time = "2026-05-06T06:18:26.287Z" }, - { url = "https://files.pythonhosted.org/packages/3d/fb/69ff198a82cae7eb1a69fb84d93b3a3e4816564d76817fe541ddc96874eb/hf_xet-1.5.0-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:dad0dc84e941b8ba3c860659fe1fdc35c049d47cce293f003287757e971a8f56", size = 4030814, upload-time = "2026-05-06T06:17:57.933Z" }, - { url = "https://files.pythonhosted.org/packages/9b/ff/edcc2b40162bef3ff78e14ab637e5f3b89243d6aee72f5949d3bb6a5af83/hf_xet-1.5.0-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:fd6e5a9b0fdac4ed03ed45ef79254a655b1aaab514a02202617fbf643f5fdf7a", size = 3798444, upload-time = "2026-05-06T06:17:55.79Z" }, - { url = "https://files.pythonhosted.org/packages/49/4d/103f76b04310e5e57656696cc184690d20c466af0bca3ca88f8c8ea5d4f3/hf_xet-1.5.0-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3531b1823a0e6d77d80f9ed15ca0e00f0d115094f8ac033d5cae88f4564cc949", size = 4465986, upload-time = "2026-05-06T06:17:44.886Z" }, - { url = "https://files.pythonhosted.org/packages/c4/a2/546f47f464737b3edbab6f8ddb57f2599b93d2cbb66f06abb475ccb48651/hf_xet-1.5.0-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:9a0ee58cd18d5ea799f7ed11290bbccbe56bdd8b1d97ca74b9cc49a3945d7a3b", size = 4259865, upload-time = "2026-05-06T06:17:42.639Z" }, - { url = "https://files.pythonhosted.org/packages/95/7f/1be593c1f28613be2e196473481cd81bfc5910795e30a34e8f744f6cac4f/hf_xet-1.5.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1e60df5a42e9bed8628b6416af2cba4cba57ae9f02de226a06b020d98e1aab18", size = 4459835, upload-time = "2026-05-06T06:18:08.026Z" }, - { url = "https://files.pythonhosted.org/packages/aa/b2/703569fc881f3284487e68cda7b42179978480da3c438042a6bbbb4a671c/hf_xet-1.5.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:4b35549ce62601b84da4ff9b24d970032ace3d4430f52d91bcbb26c901d6c690", size = 4672414, upload-time = "2026-05-06T06:18:09.864Z" }, - { url = "https://files.pythonhosted.org/packages/af/37/1b6def445c567286b50aa3b33828158e135b1be44938dde59f11382a500c/hf_xet-1.5.0-cp37-abi3-win_amd64.whl", hash = "sha256:2806c7c17b4d23f8d88f7c4814f838c3b6150773fe339c20af23e1cfaf2797e4", size = 3977238, upload-time = "2026-05-06T06:18:23.621Z" }, - { url = "https://files.pythonhosted.org/packages/62/94/3b66b148778ee100dcfd69c2ca22b57b41b44d3063ceec934f209e9184ce/hf_xet-1.5.0-cp37-abi3-win_arm64.whl", hash = "sha256:b6c9df403040248c76d808d3e047d64db2d923bae593eb244c41e425cf6cd7be", size = 3806916, upload-time = "2026-05-06T06:18:21.7Z" }, + { url = "https://files.pythonhosted.org/packages/41/62/3c062f593bd92ef4e77a0ef39541e3d82a0a1d3947c8a777a02a13a27828/hf_xet-1.6.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:70cbb9c896901600128cb9b6f06e132954fbede1db30f31f7c6c63f84cb7c31d", size = 4074584, upload-time = "2026-08-03T22:32:47.364Z" }, + { url = "https://files.pythonhosted.org/packages/bb/1e/c0ad437dd267a8e435bef594acf781bbc3874ff0b6435b4962d03ecf7cc4/hf_xet-1.6.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:23379c2f9ec8696d952b16414a2bae72cad86a52df869b050698ba60f538c675", size = 3867381, upload-time = "2026-08-03T22:32:49.049Z" }, + { url = "https://files.pythonhosted.org/packages/d5/ee/7c0d7b6ab336167531b1c30af2af003f054af4c749becbd7209ae33a77c3/hf_xet-1.6.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f2f7278c05c22fd60cb436cda1269649b3e81db65ecdc8496e5e164aa4143e7b", size = 4453982, upload-time = "2026-08-03T22:32:50.568Z" }, + { url = "https://files.pythonhosted.org/packages/63/06/ad8eab1c9525246650cbaa821caa3cdbaca734ab1a5b8c91bea09cbd8d69/hf_xet-1.6.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:948f15d3a9545cfe5932f6bd8b440f6ae630aee108f14b7bd6c561f7c2dcc522", size = 4249445, upload-time = "2026-08-03T22:32:52.391Z" }, + { url = "https://files.pythonhosted.org/packages/d8/26/1eee8aedb0dafc1ab9717dc9ac602cde33361b232dc06803f1f6ed18b58c/hf_xet-1.6.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5153e6bb103ad49d6ea9f1b2e230db5a2ea32551ad09a706d2f61d7c7c80d80e", size = 4451099, upload-time = "2026-08-03T22:32:54.114Z" }, + { url = "https://files.pythonhosted.org/packages/67/57/0b88af1f194ab6c9c650547d9cc06bfeaab836ae4dcdb331676bfb8be95a/hf_xet-1.6.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:35cec30d75c6f9eb9c16a77cef68e85a103b72e24d4b473714ec9ff06428bab9", size = 4664712, upload-time = "2026-08-03T22:32:55.547Z" }, + { url = "https://files.pythonhosted.org/packages/53/a0/26b717a9d1840e8abf48dcec64b5ed8fbe472671d38ad28d30e147132b33/hf_xet-1.6.0-cp314-cp314t-win_amd64.whl", hash = "sha256:5789835d7c6bc9436962853192082374297fb72d7eff7e7762ec25ceb7e25338", size = 4025906, upload-time = "2026-08-03T22:32:57.391Z" }, + { url = "https://files.pythonhosted.org/packages/49/f6/4a9966633c6fef83af997e2cff68ec1963676d412bdfd096df2a93b8e185/hf_xet-1.6.0-cp314-cp314t-win_arm64.whl", hash = "sha256:75765820ce4700db3750c94acc8fe27c5fae4c9ec000a0dbac3ca082acf97765", size = 3849221, upload-time = "2026-08-03T22:32:59.123Z" }, + { url = "https://files.pythonhosted.org/packages/a2/50/7afa2c9c787405864fc47a0d1bbc02c62e9101947ed43c1f43899fc7d91d/hf_xet-1.6.0-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:633dc0cd71d32da58ab8c03ad38e2fac452c15c2b0a2866ebf6ededfe0a5061d", size = 4071729, upload-time = "2026-08-03T22:33:00.721Z" }, + { url = "https://files.pythonhosted.org/packages/4b/69/55b8dcf636142ae660fec1869fcac14c4da2e8412e14d6eee1523be77e9f/hf_xet-1.6.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:f0906082d9932ae0c0057fa194041c22b4e2cdb46b2592ef3b91f020d62a081a", size = 3876287, upload-time = "2026-08-03T22:33:02.251Z" }, + { url = "https://files.pythonhosted.org/packages/67/4e/a28359bf1c1ecf11eba22123168c138698f7cb576ac678f5a2e16cd5da08/hf_xet-1.6.0-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d62671bb130879cef0ee4c9ebe47a14af6c66ec53e6d84dc15936e5ffdfac82f", size = 4464663, upload-time = "2026-08-03T22:33:03.802Z" }, + { url = "https://files.pythonhosted.org/packages/9a/69/1f0cbc2fb22ae6082d094f743d1b8945a3f36f6089cb95f42b7ee348cda7/hf_xet-1.6.0-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:0e6e21fa3cdfcdcd76748564bf593870a5e013f47d97cf10aed63aa222cff5b7", size = 4262538, upload-time = "2026-08-03T22:33:05.287Z" }, + { url = "https://files.pythonhosted.org/packages/d1/3a/4f4f2301ade26e404462d3336fa11f7958d914cabbabdd6e03c3c5d5658c/hf_xet-1.6.0-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:4fc74352a17015bd0ee90038bc9efe38db894cde45f268b6712b04fce8cd0acb", size = 4460520, upload-time = "2026-08-03T22:33:06.81Z" }, + { url = "https://files.pythonhosted.org/packages/ab/5f/311725e2a905534dfee2dcb5b08414f249147f1f12252bfc2bd24caa075c/hf_xet-1.6.0-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8fb4f71cba6129110c3374a33f919001ff130488fc23553698e34cc1c2a1198c", size = 4675937, upload-time = "2026-08-03T22:33:08.616Z" }, + { url = "https://files.pythonhosted.org/packages/98/b7/8c59a66d15205024662f1d66968136f13893f96df1ddc5087e2e281fc95f/hf_xet-1.6.0-cp38-abi3-win_amd64.whl", hash = "sha256:fb4fadde1b2b70bf4c0c14a6dccbe7194b1c28947fefd5bbe3fed9d940676c3b", size = 4033128, upload-time = "2026-08-03T22:33:10.171Z" }, + { url = "https://files.pythonhosted.org/packages/73/63/ca511b6f802f28cf3489b280fe77475bcca8de85e81a6299d7916b5b5555/hf_xet-1.6.0-cp38-abi3-win_arm64.whl", hash = "sha256:3dc3e35441ba395006af5aaacc40ef2e603c51ef46c3530b9156185f00935ea3", size = 3859359, upload-time = "2026-08-03T22:33:11.725Z" }, ] [[package]] @@ -3381,22 +3370,23 @@ wheels = [ [[package]] name = "huggingface-hub" -version = "1.14.0" +version = "1.32.0" source = { registry = "https://pypi.org/simple" } dependencies = [ + { name = "click" }, { name = "filelock" }, { name = "fsspec" }, { name = "hf-xet", marker = "platform_machine == 'AMD64' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" }, { name = "httpx" }, { name = "packaging" }, { name = "pyyaml" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, { name = "tqdm" }, - { name = "typer" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/39/40/43109e943fd718b0ccd0cd61eb4f1c347df22bf81f5874c6f22adf44bcff/huggingface_hub-1.14.0.tar.gz", hash = "sha256:d6d2c9cd6be1d02ae9ec6672d5587d10a427f377db688e82528f426a041622c2", size = 782365, upload-time = "2026-05-06T14:14:34.278Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fe/0f/e83fdd856da8fca26bf78d71709ebd120432a0ce535e72b9597cab1eb5bf/huggingface_hub-1.32.0.tar.gz", hash = "sha256:ed70a45498abe86039df7c2f4e5f7575de524be908d3840e8f828d5525eafd6a", size = 1038662, upload-time = "2026-09-17T10:27:48.049Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/89/a5/33b49ba7bea7c41bb37f74ec0f8beea0831e052330196633fe2c77516ea6/huggingface_hub-1.14.0-py3-none-any.whl", hash = "sha256:efe075535c62e130b30e836b138e13785f6f043d1f0539e0a39aa411a99e90b8", size = 661479, upload-time = "2026-05-06T14:14:32.029Z" }, + { url = "https://files.pythonhosted.org/packages/1b/cf/d98dd561d6d0d7b7d7a64d1563f8aaaa7c235daee41c1c9bcc3da62420ed/huggingface_hub-1.32.0-py3-none-any.whl", hash = "sha256:b0c7c80561969d9cdacdd55fce67ba9584cca0b9d4ea80957a3a5c1445fac5c8", size = 842906, upload-time = "2026-09-17T10:27:46.102Z" }, ] [[package]] @@ -4518,6 +4508,7 @@ dependencies = [ { name = "fastuuid" }, { name = "filelock" }, { name = "httpx", extra = ["http2"] }, + { name = "huggingface-hub" }, { name = "importlib-metadata" }, { name = "jinja2" }, { name = "jsonschema" }, @@ -4526,6 +4517,7 @@ dependencies = [ { name = "pydantic" }, { name = "pydantic-settings" }, { name = "python-dotenv" }, + { name = "pyyaml" }, { name = "tiktoken" }, { name = "tokenizers" }, ] @@ -4650,6 +4642,12 @@ utils = [ ] [package.dev-dependencies] +benchmarks = [ + { name = "a2a-sdk" }, + { name = "mcp" }, + { name = "pytest" }, + { name = "pytest-codspeed" }, +] ci = [ { name = "aiodynamo" }, { name = "anthropic" }, @@ -4689,6 +4687,8 @@ dev = [ { name = "keyring" }, { name = "langfuse" }, { name = "mypy" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "numpy", version = "2.4.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "openapi-core" }, { name = "opentelemetry-api" }, { name = "opentelemetry-exporter-otlp" }, @@ -4787,6 +4787,7 @@ requires-dist = [ { name = "httpx", extras = ["http2"], specifier = ">=0.28.0,<1.0" }, { name = "httpx2", marker = "extra == 'mcp'", specifier = ">=2.5.0,<3" }, { name = "httpx2", marker = "extra == 'proxy'", specifier = ">=2.5.0,<3" }, + { name = "huggingface-hub", specifier = ">=0.34.0,<2.0" }, { name = "importlib-metadata", specifier = ">=8.0.0,<9.0" }, { name = "inquirerpy", marker = "extra == 'cli'", specifier = ">=0.3.4,<1.0" }, { name = "inquirerpy", marker = "extra == 'proxy'", specifier = ">=0.3.4,<1.0" }, @@ -4805,12 +4806,12 @@ requires-dist = [ { name = "numpydoc", marker = "extra == 'utils'", specifier = ">=1.8.0,<2.0" }, { name = "nvidia-riva-client", marker = "extra == 'stt-nvidia-riva'", specifier = ">=2.15.0" }, { name = "openai", specifier = ">=2.20.0,<3.0.0" }, - { name = "packaging", specifier = ">=24.0" }, { name = "opentelemetry-api", marker = "extra == 'proxy-runtime'", specifier = "==1.28.0" }, { name = "opentelemetry-exporter-otlp", marker = "extra == 'proxy-runtime'", specifier = "==1.28.0" }, { name = "opentelemetry-instrumentation-fastapi", marker = "extra == 'proxy-runtime'", specifier = "==0.49b0" }, { name = "opentelemetry-sdk", marker = "extra == 'proxy-runtime'", specifier = "==1.28.0" }, { name = "orjson", marker = "extra == 'proxy'", specifier = ">=3.11.6,<4.0" }, + { name = "packaging", specifier = ">=24.0" }, { name = "polars", marker = "extra == 'proxy'", specifier = ">=1.38.1,<2.0" }, { name = "prisma", marker = "extra == 'extra-proxy'", specifier = ">=0.11.0,<1.0" }, { name = "prometheus-client", marker = "extra == 'proxy-runtime'", specifier = ">=0.20.0,<1.0" }, @@ -4828,6 +4829,7 @@ requires-dist = [ { name = "python-dotenv", specifier = ">=1.0.0,<2.0" }, { name = "python-multipart", marker = "extra == 'proxy'", specifier = ">=0.0.27,<1.0" }, { name = "python3-saml", marker = "extra == 'saml'", specifier = ">=1.16.0,<2.0" }, + { name = "pyyaml", specifier = ">=6.0.3,<7.0" }, { name = "pyyaml", marker = "extra == 'cli'", specifier = ">=6.0.3,<7.0" }, { name = "pyyaml", marker = "extra == 'proxy'", specifier = ">=6.0.3,<7.0" }, { name = "redisvl", marker = "extra == 'extra-proxy'", specifier = ">=0.4.1,<1.0" }, @@ -4854,6 +4856,12 @@ requires-dist = [ provides-extras = ["proxy", "cli", "extra-proxy", "utils", "caching", "mcp", "saml", "semantic-router", "mlflow", "grpc", "stt-vertex-chirp", "stt-nvidia-riva", "google", "bedrock-realtime", "proxy-runtime"] [package.metadata.requires-dev] +benchmarks = [ + { name = "a2a-sdk", specifier = "==1.1.0" }, + { name = "mcp", specifier = ">=2.2.0,<3" }, + { name = "pytest", specifier = "==9.0.3" }, + { name = "pytest-codspeed", specifier = "==4.3.0" }, +] ci = [ { name = "aiodynamo", specifier = "==24.7" }, { name = "anthropic", specifier = "==0.84.0" }, @@ -4893,6 +4901,7 @@ dev = [ { name = "keyring", specifier = "==25.7.0" }, { name = "langfuse", specifier = "==2.59.7" }, { name = "mypy", specifier = "==1.20.1" }, + { name = "numpy", specifier = ">=1.26.0,<3.0" }, { name = "openapi-core", specifier = "==0.22.0" }, { name = "opentelemetry-api", specifier = "==1.28.0" }, { name = "opentelemetry-exporter-otlp", specifier = "==1.28.0" }, @@ -9171,15 +9180,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5d/40/e1e72872c6354b306daef1703549e8e83b4d43cfea356311bf722a043752/setuptools-83.0.0-py3-none-any.whl", hash = "sha256:29b23c360f22f414dc7336bb39178cc7bcbf6021ed2733cde173f09dba19abb3", size = 1008090, upload-time = "2026-07-04T15:31:20.885Z" }, ] -[[package]] -name = "shellingham" -version = "1.5.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/58/15/8b3609fd3830ef7b27b655beb4b4e9c62313a4e8da8c676e142cc210d58e/shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de", size = 10310, upload-time = "2023-10-24T04:13:40.426Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, -] - [[package]] name = "simple-websocket" version = "1.1.0" @@ -9920,21 +9920,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/19/97/56608b2249fe206a67cd573bc93cd9896e1efb9e98bce9c163bcdc704b88/truststore-0.10.4-py3-none-any.whl", hash = "sha256:adaeaecf1cbb5f4de3b1959b42d41f6fab57b2b1666adb59e89cb0b53361d981", size = 18660, upload-time = "2025-08-12T18:49:01.46Z" }, ] -[[package]] -name = "typer" -version = "0.25.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "annotated-doc" }, - { name = "click" }, - { name = "rich" }, - { name = "shellingham" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e4/51/9aed62104cea109b820bbd6c14245af756112017d309da813ef107d42e7e/typer-0.25.1.tar.gz", hash = "sha256:9616eb8853a09ffeabab1698952f33c6f29ffdbceb4eaeecf571880e8d7664cc", size = 122276, upload-time = "2026-04-30T19:32:16.964Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3f/f9/2b3ff4e56e5fa7debfaf9eb135d0da96f3e9a1d5b27222223c7296336e5f/typer-0.25.1-py3-none-any.whl", hash = "sha256:75caa44ed46a03fb2dab8808753ffacdbfea88495e74c85a28c5eefcf5f39c89", size = 58409, upload-time = "2026-04-30T19:32:18.271Z" }, -] - [[package]] name = "types-awscrt" version = "0.34.1" From 55e95c0279928c30376b6ecb4421893ad6d02593 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 21:42:52 -0700 Subject: [PATCH 13/15] fix(presidio): mask PII in streaming /v1/messages output (#42351) * fix(presidio): mask PII in streaming /v1/messages output Raw Anthropic SSE frames were passed through the post_call output masking callback untouched, and ProxyLogging rerouted the callback to the unified apply_guardrail path on /v1/messages because mask_response_content was false. Buffer the raw frames, assemble them with the shared Anthropic SSE helpers, mask through Presidio, and re-emit the masked frames. Resolves LIT-8288 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(presidio): replay raw SSE frames when masking fails mid-stream Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(presidio): propagate upstream stream errors instead of returning an empty stream Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(presidio): extract buffered stream masking to satisfy complexity budget Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(presidio): let BLOCK on generated PII refuse the streaming /v1/messages response Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(presidio): fold the BLOCK re-raise into the existing except to stay within the complexity budget Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(presidio): move the blocked stream consumption into a helper so pytest.raises holds one statement Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(presidio): fail closed when output masking of a raw SSE stream errors A Presidio outage on streaming /v1/messages replayed the unscanned frames to the caller. Propagate the error instead, matching the non streaming path and the merge base Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(presidio): cover structured chat stream output masking and trailing bytes passthrough Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yucheng Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../guardrails/guardrail_hooks/presidio.py | 85 +++-- .../guardrails/guardrail_initializers.py | 1 + tests/e2e/coverage_registry/guardrail.yaml | 1 + tests/e2e/guardrails/guardrails_client.py | 22 +- .../guardrails/test_presidio_masking_e2e.py | 195 ++++++++++- .../guardrail_hooks/test_presidio.py | 310 +++++++++++++++++- .../test_proxy_logging_hook_detection.py | 62 ++++ 7 files changed, 625 insertions(+), 51 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/presidio.py b/litellm/proxy/guardrails/guardrail_hooks/presidio.py index 70ea21320ee..fe91d6d7a28 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/presidio.py +++ b/litellm/proxy/guardrails/guardrail_hooks/presidio.py @@ -11,7 +11,7 @@ import asyncio import json import threading -from collections.abc import AsyncGenerator, AsyncIterable, Awaitable, Sequence +from collections.abc import AsyncGenerator, AsyncIterable, AsyncIterator, Awaitable, Sequence from contextlib import asynccontextmanager from datetime import datetime from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, TypedDict, cast @@ -39,6 +39,11 @@ from litellm.integrations.custom_guardrail import ( log_guardrail_information, ) from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.guardrails.anthropic_sse import ( + anthropic_sse_chunks_from_response, + assemble_anthropic_sse_stream, + model_response_text, +) from litellm.types.guardrails import ( GuardrailEventHooks, LitellmParams, @@ -1327,30 +1332,44 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): ) return response - async def _stream_apply_output_masking( - self, - response: AsyncIterable[object], - request_data: dict, - ) -> AsyncGenerator[ModelResponseStream | bytes, None]: - """Apply Presidio masking to streaming output (apply_to_output=True path).""" + async def _mask_buffered_model_response_stream( + self, all_chunks: Sequence[ModelResponseStream], request_data: dict + ) -> tuple[object, ...]: from litellm.llms.base_llm.base_model_iterator import ( convert_model_response_to_streaming, ) from litellm.main import stream_chunk_builder from litellm.types.utils import ModelResponse + assembled: Final = stream_chunk_builder(chunks=list(all_chunks), messages=request_data.get("messages")) + if not isinstance(assembled, ModelResponse): + return tuple(all_chunks) + await self._process_response_for_pii(response=assembled, request_data=request_data, mode="mask") + return (convert_model_response_to_streaming(assembled),) + + async def _stream_apply_output_masking( + self, + response: AsyncIterable[object], + request_data: dict, + ) -> AsyncGenerator[object, None]: + """Apply Presidio masking to streaming output (apply_to_output=True path).""" all_chunks: list[ModelResponseStream] = [] passthrough_due_to_unknown_stream_shape = False try: - async for chunk in response: + stream: Final = response.__aiter__() + async for chunk in stream: if isinstance(chunk, ModelResponseStream): if passthrough_due_to_unknown_stream_shape: yield chunk else: all_chunks.append(chunk) elif isinstance(chunk, bytes): - yield chunk - continue + if passthrough_due_to_unknown_stream_shape or all_chunks: + yield chunk + continue + for masked_chunk in await self._mask_anthropic_sse_stream(chunk, stream, request_data): + yield masked_chunk + return else: if all_chunks: # Flush buffered chunks and switch to transparent passthrough for this stream shape. @@ -1375,33 +1394,39 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): if not all_chunks: verbose_proxy_logger.warning( "Presidio apply_to_output: streaming response contained no " - "ModelResponseStream chunks (e.g. raw SSE bytes or an empty " - "upstream stream). Output PII masking was skipped for this " - "response." + "ModelResponseStream chunks (an empty upstream stream). " + "Output PII masking was skipped for this response." ) return - assembled_model_response = stream_chunk_builder(chunks=all_chunks, messages=request_data.get("messages")) - - if not isinstance(assembled_model_response, ModelResponse): - for chunk in all_chunks: - yield chunk - return - - await self._process_response_for_pii( - response=assembled_model_response, - request_data=request_data, - mode="mask", - ) - - mock_response_stream: Final = convert_model_response_to_streaming(assembled_model_response) - yield mock_response_stream + for masked_chunk in await self._mask_buffered_model_response_stream(all_chunks, request_data): + yield masked_chunk except Exception as e: + if not all_chunks or isinstance(e, BlockedPiiEntityError): + raise verbose_proxy_logger.error("Error masking streaming PII output: %s", e) for chunk in all_chunks: yield chunk + async def _mask_anthropic_sse_stream( + self, first_chunk: bytes, rest: AsyncIterator[object], request_data: dict + ) -> tuple[object, ...]: + rest_chunks: Final = [chunk async for chunk in rest] # mutable-ok: tuple() cannot consume an async iterator + chunks: Final = (first_chunk, *rest_chunks) + assembled: Final = assemble_anthropic_sse_stream(chunks, restore_identity=True) + if assembled is None: + verbose_proxy_logger.warning( + "Presidio apply_to_output: raw SSE stream could not be assembled into a response. " + "Output PII masking was skipped for this response." + ) + return chunks + original_text: Final = model_response_text(assembled) + await self._process_response_for_pii(response=assembled, request_data=request_data, mode="mask") + if model_response_text(assembled) == original_text: + return chunks + return anthropic_sse_chunks_from_response(assembled) + @staticmethod def _unmask_sse_bytes_chunk(chunk: bytes, pii_tokens: dict[str, str]) -> bytes: try: @@ -1460,7 +1485,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): self, response: AsyncIterable[object], request_data: dict, - ) -> AsyncGenerator[ModelResponseStream | bytes, None]: + ) -> AsyncGenerator[object, None]: """Apply PII unmasking to streaming output (output_parse_pii=True path).""" from litellm.llms.base_llm.base_model_iterator import ( convert_model_response_to_streaming, @@ -1536,7 +1561,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): user_api_key_dict: UserAPIKeyAuth, response: AsyncIterable[object], request_data: dict, - ) -> AsyncGenerator[ModelResponseStream | bytes, None]: + ) -> AsyncGenerator[object, None]: """ Process streaming response chunks to unmask PII tokens when needed. diff --git a/litellm/proxy/guardrails/guardrail_initializers.py b/litellm/proxy/guardrails/guardrail_initializers.py index 356eb7c96c6..7bafad26569 100644 --- a/litellm/proxy/guardrails/guardrail_initializers.py +++ b/litellm/proxy/guardrails/guardrail_initializers.py @@ -165,6 +165,7 @@ def initialize_presidio(litellm_params: LitellmParams, guardrail: Guardrail) -> apply_to_output=True, event_hook=GuardrailEventHooks.post_call.value, output_parse_pii=False, + mask_response_content=True, ) if run_output else None diff --git a/tests/e2e/coverage_registry/guardrail.yaml b/tests/e2e/coverage_registry/guardrail.yaml index 86eb44f6cb1..4fe9f949fae 100644 --- a/tests/e2e/coverage_registry/guardrail.yaml +++ b/tests/e2e/coverage_registry/guardrail.yaml @@ -2,6 +2,7 @@ # Rolls up into the "Logging & Guardrails" dashboard module together with logging.* - {id: guardrail.presidio.pre_call.masks, module: guardrail, tier: P0, hook_point: pre_call, assertions: [masks], exercised_on: [chat_completions, messages], source: "guardrail_hooks/presidio.py", rationale: "PII masking pre-call; data-leak blast radius"} - {id: guardrail.presidio.post_call.masks, module: guardrail, tier: P0, hook_point: post_call, assertions: [masks], exercised_on: [chat_completions, messages], source: "guardrail_hooks/presidio.py", rationale: "Mask PII in model output"} +- {id: guardrail.presidio.post_call.masks_generated_output, module: guardrail, tier: P0, hook_point: post_call, assertions: [masks], exercised_on: [chat_completions, chat_completions_stream, anthropic_messages_stream], source: "guardrail_hooks/presidio.py", rationale: "Mask model-generated credit-card output with the UI default scope"} - {id: guardrail.presidio.logging_only.masks, module: guardrail, tier: P0, hook_point: logging_only, assertions: [masks], exercised_on: [chat_completions, messages], source: "guardrail_hooks/presidio.py", rationale: "Redact in logs without blocking"} - {id: guardrail.presidio.pre_call.logs_masked_entities, module: guardrail, tier: P0, hook_point: pre_call, assertions: [logs_masked_entities], exercised_on: [chat_completions], source: "guardrail_hooks/presidio.py", rationale: "A masking run must record itself on the spend log: the dashboard's guardrail panel renders the masked-entity counts and per-entity scores straight off metadata.guardrail_information, so a run that masks but records nothing leaves an operator unable to audit it"} - {id: guardrail.bedrock.pre_call.blocks, module: guardrail, tier: P0, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/bedrock_guardrails.py", rationale: "AWS content guardrail blocks harmful input"} diff --git a/tests/e2e/guardrails/guardrails_client.py b/tests/e2e/guardrails/guardrails_client.py index 3ceac737399..97ecac0290f 100644 --- a/tests/e2e/guardrails/guardrails_client.py +++ b/tests/e2e/guardrails/guardrails_client.py @@ -46,7 +46,7 @@ class BlockedWordBody(BaseModel): class GuardrailParamsBase(BaseModel): - mode: GuardrailMode + mode: GuardrailMode | list[GuardrailMode] default_on: bool @@ -381,6 +381,26 @@ class GuardrailsClient: ), ) + def messages_stream_raw( + self, + key: str, + model: str, + text: str, + *, + guardrails: list[str] | None = None, + max_tokens: int = 64, + ) -> StreamingResponse: + return self.proxy.messages_stream( + key, + AnthropicMessagesBody( + model=model, + messages=[ChatMessage(role="user", content=text)], + max_tokens=max_tokens, + stream=True, + guardrails=guardrails, + ), + ) + def responses( self, key: str, diff --git a/tests/e2e/guardrails/test_presidio_masking_e2e.py b/tests/e2e/guardrails/test_presidio_masking_e2e.py index 49d698938ce..d47d64be9e3 100644 --- a/tests/e2e/guardrails/test_presidio_masking_e2e.py +++ b/tests/e2e/guardrails/test_presidio_masking_e2e.py @@ -30,6 +30,7 @@ this suite deliberately requires the detected-entity details to remain visible. from __future__ import annotations import os +import re import time from collections.abc import Callable from typing import Final, Literal @@ -65,9 +66,13 @@ GUARDRAIL_PROPAGATION_POLL_INTERVAL_SECONDS = 5.0 # angle brackets; the logged payload keeps the placeholder verbatim. MASKED_EMAIL_TOKEN = "EMAIL_ADDRESS" MASKED_PHONE_TOKEN = "PHONE_NUMBER" +MASKED_CREDIT_CARD_TOKEN = "CREDIT_CARD" # Fictional NANP 555 number; a standard format Presidio's phone recognizer detects. FAKE_PHONE = "+1 415-555-0134" +FAKE_VISA_TEST_CARD = "4111 1111 1111 1111" + +_CARD_DIGIT_RUN: Final = re.compile(r"(?:\d[ -]?){13,19}") def _presidio_bases() -> tuple[str, str]: @@ -86,8 +91,8 @@ def _register_presidio( resources: ResourceManager, *, name: str, - mode: GuardrailMode = "pre_call", - filter_scope: Literal["input", "output", "both"] = "input", + mode: GuardrailMode | list[GuardrailMode] = "pre_call", + filter_scope: Literal["input", "output", "both"] | None = "input", entities: dict[PiiEntity, PiiAction] | None = None, ) -> None: analyzer, anonymizer = _presidio_bases() @@ -123,6 +128,74 @@ def _first_content(response: ChatResponse) -> str: return (message.content if message else None) or "" +class _StreamDelta(BaseModel): + content: str | None = None + + +class _StreamChoice(BaseModel): + delta: _StreamDelta + + +class _StreamChunk(BaseModel): + choices: tuple[_StreamChoice, ...] = () + + +class _AnthropicStreamDelta(BaseModel): + type: str | None = None + text: str | None = None + + +class _AnthropicStreamEvent(BaseModel): + type: str + delta: _AnthropicStreamDelta | None = None + + +def _credit_card_prompt(marker: str) -> str: + return ( + f"{marker} Reply with only the well known Visa sandbox test card number that starts with 4111, " + "the 16 digits grouped in fours separated by spaces, and nothing else." + ) + + +def _passes_luhn(digits: str) -> bool: + checksum = sum( + digit if position % 2 == 0 else (digit * 2 - 9 if digit * 2 > 9 else digit * 2) + for position, digit in enumerate(int(char) for char in reversed(digits)) + ) + return checksum % 10 == 0 + + +def _contains_card_number(text: str) -> bool: + """Presidio's CREDIT_CARD recognizer only reports Luhn-valid digit runs, so a + Luhn-invalid number the model hallucinates is not something masking can catch.""" + return any( + 13 <= len(digits) <= 19 and _passes_luhn(digits) + for digits in (re.sub(r"[ -]", "", match.group()) for match in _CARD_DIGIT_RUN.finditer(text)) + ) + + +def _stream_content(result: StreamingResponse) -> str: + return "".join( + choice.delta.content + for event in result.stream_events + if event != "[DONE]" + for choice in _StreamChunk.model_validate_json(event).choices[:1] + if choice.delta.content + ) + + +def _anthropic_stream_content(result: StreamingResponse) -> str: + return "".join( + event.delta.text + for payload in result.stream_events + for event in [_AnthropicStreamEvent.model_validate_json(payload)] + if event.type == "content_block_delta" + and event.delta is not None + and event.delta.type == "text_delta" + and event.delta.text + ) + + def _messages_text(response: AnthropicMessagesResponse) -> str: """The text of a /v1/messages answer, whichever shape the proxy produced (Anthropic-native content blocks or OpenAI-normalized choices).""" @@ -290,6 +363,124 @@ class TestPresidioPostCallMasking: time.sleep(GUARDRAIL_PROPAGATION_POLL_INTERVAL_SECONDS) +def _assert_eventually_masks_generated_card(fetch: Callable[[], str | None]) -> None: + deadline = time.monotonic() + GUARDRAIL_PROPAGATION_DEADLINE_SECONDS + last: str = "" + while True: + content = fetch() + if content is not None: + last = content + if _contains_card_number(content): + pytest.fail( + "the post_call output masking let a card number through: " + f"{content[:300]!r}" + ) + if MASKED_CREDIT_CARD_TOKEN in content: + return + if time.monotonic() >= deadline: + pytest.fail( + "presidio post_call output masking never masked the generated card within " + f"{GUARDRAIL_PROPAGATION_DEADLINE_SECONDS}s; last observation: {last[:300]!r}" + ) + time.sleep(GUARDRAIL_PROPAGATION_POLL_INTERVAL_SECONDS) + + +class TestPresidioCreditCardOutputMasking: + """Proves the UI-default Presidio scope masks model-generated card output.""" + + @pytest.mark.covers( + "guardrail.presidio.post_call.masks_generated_output", + exercised_on=["chat_completions"], + ) + def test_ui_default_scope_masks_a_card_number_the_model_generates_on_chat_completions( + self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str + ) -> None: + name = f"e2e-presidio-card-chat-{unique_marker()}" + _register_presidio( + client, + resources, + name=name, + mode=["pre_call", "post_call"], + filter_scope=None, + entities={"CREDIT_CARD": "MASK"}, + ) + prompt: Final = _credit_card_prompt(unique_marker()) + + def fetch() -> str | None: + result: Final = client.chat(scoped_key, MODEL, prompt, guardrails=[name], max_tokens=512) + match result: + case Success(data=data): + return _first_content(data) + case _: + return None + + _assert_eventually_masks_generated_card(fetch) + + @pytest.mark.covers( + "guardrail.presidio.post_call.masks_generated_output", + exercised_on=["chat_completions_stream"], + ) + def test_ui_default_scope_masks_a_card_number_the_model_generates_on_streaming_chat_completions( + self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str + ) -> None: + name = f"e2e-presidio-card-stream-{unique_marker()}" + _register_presidio( + client, + resources, + name=name, + mode=["pre_call", "post_call"], + filter_scope=None, + entities={"CREDIT_CARD": "MASK"}, + ) + prompt: Final = _credit_card_prompt(unique_marker()) + + def fetch() -> str | None: + result: Final = client.chat_stream_raw( + scoped_key, + MODEL, + prompt, + guardrails=[name], + max_tokens=512, + ) + if not result.ok or result.stream_error: + return None + return _stream_content(result) + + _assert_eventually_masks_generated_card(fetch) + + @pytest.mark.covers( + "guardrail.presidio.post_call.masks_generated_output", + exercised_on=["anthropic_messages_stream"], + ) + def test_ui_default_scope_masks_a_card_number_the_model_generates_on_streaming_anthropic_messages( + self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str + ) -> None: + name = f"e2e-presidio-card-messages-stream-{unique_marker()}" + _register_presidio( + client, + resources, + name=name, + mode=["pre_call", "post_call"], + filter_scope=None, + entities={"CREDIT_CARD": "MASK"}, + ) + prompt: Final = _credit_card_prompt(unique_marker()) + + def fetch() -> str | None: + result: Final = client.messages_stream_raw( + scoped_key, + MODEL, + prompt, + guardrails=[name], + max_tokens=512, + ) + if not result.ok or result.stream_error: + return None + return _anthropic_stream_content(result) + + _assert_eventually_masks_generated_card(fetch) + + _LOGGED_ENTITIES: dict[PiiEntity, PiiAction] = {"EMAIL_ADDRESS": "MASK", "PHONE_NUMBER": "MASK"} _ENTITY_LIST_ADAPTER: Final = TypeAdapter(list[GuardrailEntityMatch]) diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py index 33614d2eeca..89e72debbf5 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py @@ -4,6 +4,7 @@ Tests PII detection and masking for different message formats """ import asyncio +import json from contextlib import asynccontextmanager from unittest.mock import MagicMock, patch @@ -18,7 +19,7 @@ from litellm.proxy.guardrails.guardrail_hooks.presidio import ( ) from litellm.exceptions import GuardrailRaisedException from litellm.types.guardrails import LitellmParams, PiiAction, PiiEntityType -from litellm.types.utils import Choices, Message, ModelResponse +from litellm.types.utils import Choices, Delta, Message, ModelResponse, StreamingChoices from litellm.exceptions import BlockedPiiEntityError @@ -2331,47 +2332,320 @@ async def test_apply_guardrail_masks_on_request(): assert "John Smith" not in result["texts"][0] +def _anthropic_sse(event_type: str, payload: dict) -> bytes: + return f"event: {event_type}\ndata: {json.dumps(payload)}\n\n".encode() + + +def _anthropic_text_deltas(chunks: list[bytes]) -> list[tuple[int, str]]: + deltas = [] + for line in b"".join(chunks).decode().split("\n"): + if not line.startswith("data: "): + continue + event = json.loads(line[6:]) + if event.get("type") == "content_block_delta" and event["delta"].get("type") == "text_delta": + deltas.append((event["index"], event["delta"]["text"])) + return deltas + + +def _chat_delta_chunk(text: str, finish_reason: str | None = None) -> ModelResponseStream: + return ModelResponseStream( + id="chatcmpl-out-mask", + choices=[StreamingChoices(index=0, delta=Delta(content=text, role="assistant"), finish_reason=finish_reason)], + created=1, + model="gpt-4", + object="chat.completion.chunk", + ) + + @pytest.mark.asyncio -async def test_apply_to_output_streaming_bytes_only_logs_warning(): +async def test_apply_to_output_streaming_chat_chunks_are_masked_as_one_response(): """ - Regression test: when apply_to_output=True and the stream contains only - bytes chunks (Anthropic native SSE), output masking is skipped. - A warning must be logged so operators are aware. + Structured chat completion chunks are buffered, assembled and masked as a + whole, so a card number split across deltas cannot reach the caller. """ guardrail = _OPTIONAL_PresidioPIIMasking( mock_testing=True, apply_to_output=True, + mock_redacted_text={"text": "my card is "}, + ) + + async def mock_stream(): + yield _chat_delta_chunk("my card is 4111") + yield _chat_delta_chunk(" 1111 1111 1111") + yield _chat_delta_chunk("", finish_reason="stop") + + collected = [] + async for chunk in guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), + response=mock_stream(), + request_data={"messages": [{"role": "user", "content": "what is my card"}]}, + ): + collected.append(chunk) + + assert all(isinstance(chunk, ModelResponseStream) for chunk in collected) + joined = "".join(chunk.choices[0].delta.content or "" for chunk in collected) + assert joined == "my card is " + assert collected[-1].choices[0].finish_reason == "stop" + + +@pytest.mark.asyncio +async def test_apply_to_output_streaming_bytes_after_chat_chunks_are_passed_through_in_order(): + """ + Once structured chunks have been buffered, a trailing bytes frame belongs to + the same stream and must be forwarded rather than treated as a new SSE stream. + """ + guardrail = _OPTIONAL_PresidioPIIMasking( + mock_testing=True, + apply_to_output=True, + mock_redacted_text={"text": "hello"}, + ) + trailer = b"data: [DONE]\n\n" + + async def mock_stream(): + yield _chat_delta_chunk("hello", finish_reason="stop") + yield trailer + + collected = [] + async for chunk in guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), + response=mock_stream(), + request_data={}, + ): + collected.append(chunk) + + assert collected[0] == trailer + assert len(collected) == 2 + assert isinstance(collected[1], ModelResponseStream) + assert collected[1].choices[0].delta.content == "hello" + + +@pytest.mark.asyncio +async def test_apply_to_output_streaming_anthropic_sse_bytes_masks_text_split_across_deltas(): + """ + Anthropic native /v1/messages streams reach the post_call hook as raw SSE + bytes. Output masking must run over the whole content block so a card + number split across text_delta events cannot reach the caller. + """ + guardrail = _OPTIONAL_PresidioPIIMasking( + mock_testing=True, + apply_to_output=True, + mock_redacted_text={"text": ""}, ) byte_chunks = [ - b'data: {"type":"content_block_delta","delta":{"text":"Hello"}}\n\n', - b'data: {"type":"content_block_delta","delta":{"text":" world"}}\n\n', + _anthropic_sse( + "message_start", + {"type": "message_start", "message": {"id": "msg_1", "model": "claude", "content": [], "usage": {}}}, + ), + _anthropic_sse( + "content_block_start", + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + ), + _anthropic_sse( + "content_block_delta", + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "4111"}}, + ), + _anthropic_sse( + "content_block_delta", + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": " 1111 1111 1111"}}, + ), + _anthropic_sse("content_block_stop", {"type": "content_block_stop", "index": 0}), + _anthropic_sse("message_delta", {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {}}), + _anthropic_sse("message_stop", {"type": "message_stop"}), ] async def mock_stream(): for b in byte_chunks: yield b - mock_user_api_key = UserAPIKeyAuth(api_key="test-key") + collected = [] + async for chunk in guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), + response=mock_stream(), + request_data={}, + ): + collected.append(chunk) + + assert all(isinstance(chunk, bytes) for chunk in collected) + joined = b"".join(collected).decode() + assert "4111" not in joined + assert "".join(text for _, text in _anthropic_text_deltas(collected)) == "" + assert joined.count("event: message_start") == 1 + assert joined.count("event: message_stop") == 1 + + +@pytest.mark.asyncio +async def test_apply_to_output_streaming_anthropic_sse_bytes_without_pii_are_forwarded_unchanged(): + guardrail = _OPTIONAL_PresidioPIIMasking( + mock_testing=True, + apply_to_output=True, + mock_redacted_text={"text": "Hello world"}, + ) + + byte_chunks = [ + _anthropic_sse( + "message_start", + {"type": "message_start", "message": {"id": "msg_1", "model": "claude", "content": [], "usage": {}}}, + ), + _anthropic_sse( + "content_block_start", + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + ), + _anthropic_sse( + "content_block_delta", + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "Hello"}}, + ), + _anthropic_sse( + "content_block_delta", + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": " world"}}, + ), + _anthropic_sse("content_block_stop", {"type": "content_block_stop", "index": 0}), + _anthropic_sse("message_delta", {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {}}), + _anthropic_sse("message_stop", {"type": "message_stop"}), + ] + + async def mock_stream(): + for b in byte_chunks: + yield b collected = [] - with patch("litellm.proxy.guardrails.guardrail_hooks.presidio.verbose_proxy_logger") as mock_logger: + async for chunk in guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), + response=mock_stream(), + request_data={}, + ): + collected.append(chunk) + + assert collected == byte_chunks + + +@pytest.mark.asyncio +async def test_apply_to_output_streaming_anthropic_sse_bytes_fail_closed_when_presidio_is_unreachable(): + """ + The raw SSE stream is fully drained before masking, so a Presidio outage + must surface as an error to the caller: replaying the unscanned frames + would hand over whatever PII the model generated. + """ + guardrail = _OPTIONAL_PresidioPIIMasking( + mock_testing=True, + apply_to_output=True, + presidio_analyzer_api_base="http://127.0.0.1:9", + presidio_anonymizer_api_base="http://127.0.0.1:9", + ) + + byte_chunks = [ + _anthropic_sse( + "message_start", + {"type": "message_start", "message": {"id": "msg_1", "model": "claude", "content": [], "usage": {}}}, + ), + _anthropic_sse( + "content_block_start", + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + ), + _anthropic_sse( + "content_block_delta", + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "Hello world"}}, + ), + _anthropic_sse("content_block_stop", {"type": "content_block_stop", "index": 0}), + _anthropic_sse("message_stop", {"type": "message_stop"}), + ] + + async def mock_stream(): + for b in byte_chunks: + yield b + + collected = [] + + async def collect_masked_stream(): async for chunk in guardrail.async_post_call_streaming_iterator_hook( - user_api_key_dict=mock_user_api_key, + user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), response=mock_stream(), request_data={}, ): collected.append(chunk) - # All bytes should be yielded through - assert len(collected) == len(byte_chunks) - for original, received in zip(byte_chunks, collected): - assert original == received + with pytest.raises(Exception, match="Presidio PII analysis failed"): + await collect_masked_stream() - # Warning must be logged about skipped masking - mock_logger.warning.assert_called_once() - warning_msg = mock_logger.warning.call_args[0][0] - assert "Output PII masking was skipped" in warning_msg + assert collected == [] + + +@pytest.mark.asyncio +async def test_apply_to_output_streaming_anthropic_sse_bytes_block_action_raises_instead_of_replaying(): + """ + A BLOCK on generated PII must refuse the streaming /v1/messages response the + same way it refuses the non streaming one, not replay the raw frames. + """ + guardrail = _OPTIONAL_PresidioPIIMasking( + apply_to_output=True, + mock_testing=False, + presidio_analyzer_api_base="http://test-analyzer/", + presidio_anonymizer_api_base="http://test-anonymizer/", + pii_entities_config={PiiEntityType.CREDIT_CARD: PiiAction.BLOCK}, + ) + + byte_chunks = [ + _anthropic_sse( + "message_start", + {"type": "message_start", "message": {"id": "msg_1", "model": "claude", "content": [], "usage": {}}}, + ), + _anthropic_sse( + "content_block_start", + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + ), + _anthropic_sse( + "content_block_delta", + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "4111 1111 1111 1111"}}, + ), + _anthropic_sse("content_block_stop", {"type": "content_block_stop", "index": 0}), + _anthropic_sse("message_stop", {"type": "message_stop"}), + ] + + async def mock_stream(): + for b in byte_chunks: + yield b + + analyzer_hit = [{"entity_type": "CREDIT_CARD", "score": 0.99, "start": 0, "end": 19}] + collected = [] + + async def collect_masked_stream(): + async for chunk in guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), + response=mock_stream(), + request_data={}, + ): + collected.append(chunk) + + with patch.object(guardrail, "_get_session_iterator", _make_mock_session_iterator(analyzer_hit)): + with pytest.raises(BlockedPiiEntityError): + await collect_masked_stream() + + assert collected == [] + + +@pytest.mark.asyncio +async def test_apply_to_output_streaming_propagates_upstream_error_when_nothing_was_buffered(): + """ + An upstream guardrail that rejects the stream before the first chunk must + surface as an error to the caller, not as an empty 200 stream. + """ + guardrail = _OPTIONAL_PresidioPIIMasking( + mock_testing=True, + apply_to_output=True, + mock_redacted_text={"text": ""}, + ) + + async def failing_stream(): + raise RuntimeError("upstream guardrail rejected the stream") + yield b"" + + with pytest.raises(RuntimeError, match="upstream guardrail rejected the stream"): + async for _ in guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), + response=failing_stream(), + request_data={}, + ): + pass @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py b/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py index dd330d32ce6..f45715953f9 100644 --- a/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py +++ b/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py @@ -489,6 +489,68 @@ async def test_post_call_stream_masking_guardrail_keeps_own_iterator_on_anthropi assert delivered == chunks +@pytest.mark.asyncio +async def test_post_call_stream_presidio_output_masking_masks_anthropic_messages_stream(monkeypatch): + """Regression: the presidio output-masking callback built by initialize_presidio + was rerouted onto the unified scan-only path on /v1/messages, so a card number + the analyzer flagged still streamed to the caller unmasked.""" + import json + + from litellm.caching.caching import DualCache + from litellm.proxy.guardrails.guardrail_registry import InMemoryGuardrailHandler + from litellm.types.guardrails import SupportedGuardrailIntegrations + + handler = InMemoryGuardrailHandler() + result = handler.initialize_guardrail( + guardrail={ + "guardrail_name": "presidio-card-mask", + "litellm_params": { + "guardrail": SupportedGuardrailIntegrations.PRESIDIO.value, + "mode": ["pre_call", "post_call"], + "default_on": True, + "presidio_analyzer_api_base": "https://fakelink.com/v1/presidio/analyze", + "presidio_anonymizer_api_base": "https://fakelink.com/v1/presidio/anonymize", + "pii_entities_config": {"CREDIT_CARD": "MASK"}, + "mock_redacted_text": {"text": "", "items": []}, + }, + } + ) + guardrail_id = result["guardrail_id"] + callbacks = [ + handler.guardrail_id_to_custom_guardrail[guardrail_id], + *handler.guardrail_id_to_sibling_callbacks[guardrail_id], + ] + monkeypatch.setattr(litellm, "callbacks", callbacks) + + chunks = _anthropic_stream_chunks(["4111", " 1111 1111 1111"]) + + async def fake_stream(): + for chunk in chunks: + yield chunk + + delivered = [] + async for chunk in ProxyLogging(user_api_key_cache=DualCache()).async_post_call_streaming_iterator_hook( + response=fake_stream(), + user_api_key_dict=UserAPIKeyAuth(api_key="sk-1234", request_route="/v1/messages"), + request_data={ + "model": "claude-sonnet-5", + "litellm_logging_obj": _streaming_logging_obj(), + "metadata": {}, + }, + ): + delivered.append(chunk) + + wire = b"".join(delivered).decode() + text_deltas = [ + json.loads(line[6:])["delta"]["text"] + for line in wire.split("\n") + if line.startswith("data: ") and json.loads(line[6:]).get("delta", {}).get("type") == "text_delta" + ] + assert "4111" not in wire, wire + assert "".join(text_deltas) == "", wire + assert wire.count("event: message_stop") == 1, wire + + class _AppliesGuardrail(CustomGuardrail): """Implements the unified interface only, so the proxy routes it to unified_guardrail.""" From 13b374873d7b75f400fc1702c759d68195620f62 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 21:57:44 -0700 Subject: [PATCH 14/15] fix(otel v2): map completions, images, speech, transcription and moderation output onto the Langfuse generation output (#42394) * fix(otel v2): map completions, images, speech, transcription and moderation output onto the generation output Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(redaction): redact text completion choices in the standard logging payload Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(e2e): compare decoded generation output text and follow the live moderation verdict Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(otel v2): compare logged byte counts with the received media and move e2e schemas into models.py Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * ci(e2e): keep the otel_v2 Langfuse output e2e file out of the stage-mirror gate it cannot run in Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yucheng Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/e2e-stack/select_tests.py | 1 + litellm/integrations/otel/model/payloads.py | 105 ++++++- litellm/litellm_core_utils/litellm_logging.py | 2 + litellm/litellm_core_utils/redact_messages.py | 2 + litellm/types/llms/openai.py | 21 ++ .../test_e2e_changed_gate.py | 5 + tests/e2e/CONTRIBUTING.md | 2 +- tests/e2e/conftest.py | 6 + tests/e2e/e2e_config.py | 1 + tests/e2e/logging/logging_client.py | 35 +++ ..._otel_v2_langfuse_generation_output_e2e.py | 210 +++++++++++++ tests/e2e/models.py | 71 +++++ tests/e2e/pytest.ini | 1 + .../otel/test_otel_v2_sources_of_truth.py | 287 +++++++++++++++--- .../otel/test_otel_v2_vendor_mappers.py | 30 ++ .../test_redact_messages.py | 20 ++ 16 files changed, 738 insertions(+), 61 deletions(-) create mode 100644 tests/e2e/logging/test_otel_v2_langfuse_generation_output_e2e.py diff --git a/.github/e2e-stack/select_tests.py b/.github/e2e-stack/select_tests.py index dbc4ae8f5c2..2386b184e54 100644 --- a/.github/e2e-stack/select_tests.py +++ b/.github/e2e-stack/select_tests.py @@ -9,6 +9,7 @@ UNSUPPORTED: Final = re.compile( r"|^tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e\.py$" r"|^tests/e2e/batches/test_managed_files_enforcement_e2e\.py$" r"|^tests/e2e/guardrails/test_presidio_masking_e2e\.py$" + r"|^tests/e2e/logging/test_otel_v2_langfuse_generation_output_e2e\.py$" ) HARNESS: Final = re.compile( r"^tests/e2e/[A-Za-z0-9_.-]+\.(py|ini)$" diff --git a/litellm/integrations/otel/model/payloads.py b/litellm/integrations/otel/model/payloads.py index d3ad7234d93..7ecbeea255c 100644 --- a/litellm/integrations/otel/model/payloads.py +++ b/litellm/integrations/otel/model/payloads.py @@ -427,7 +427,7 @@ class LLMCallSpanData: # plain ``.get`` — no repeated ``isinstance`` guards. raw_response: Final = payload.get("response") response: Final = cast(Mapping[str, object], raw_response if isinstance(raw_response, dict) else {}) - choices_out: Final = _dicts(response.get("choices")) or _responses_choices(response) or _ocr_choices(response) + choices_out: Final = _output_choices(response) # ``finish_reasons`` is metadata, not content, so derive it from # ``choices_out`` before gating. The raw message/choice bodies are only # retained when content capture is enabled (see ``capture_span_content``); @@ -752,20 +752,99 @@ def _responses_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]: return (choice,) -def _ocr_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]: - markdowns: Final = tuple( - text for page in _dicts(response.get("pages")) if (text := as_str(page.get("markdown"))) is not None +def _output_choices(response: Mapping[str, object]) -> tuple[Mapping[str, object], ...]: + """The response output as chat-shaped choices; images and binary bodies become size summaries, never bytes.""" + return ( + _completion_choices(response) + or _responses_choices(response) + or _ocr_choices(response) + or _transcription_choices(response) + or _moderation_choices(response) + or _image_choices(response) + or _binary_choices(response) ) - if not markdowns: + + +def _text_choice(content: str, finish_reason: str | None = None) -> _Choice: + message: Final[_AssistantMessage] = {"role": "assistant", "content": content, "refusal": None, "tool_calls": None} + return {"message": message, "finish_reason": finish_reason} + + +def _joined_choice(parts: tuple[str, ...]) -> tuple[_Choice, ...]: + return (_text_choice("\n\n".join(parts)),) if parts else () + + +def _completion_choices(response: Mapping[str, object]) -> tuple[Mapping[str, object], ...]: + return tuple( + _text_choice(text, as_str(choice.get("finish_reason"))) + if "message" not in choice and isinstance(text := choice.get("text"), str) + else choice + for choice in _dicts(response.get("choices")) + ) + + +def _ocr_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]: + return _joined_choice( + tuple(text for page in _dicts(response.get("pages")) if (text := as_str(page.get("markdown"))) is not None) + ) + + +def _transcription_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]: + text: Final = response.get("text") + return (_text_choice(text),) if isinstance(text, str) and text else () + + +def _moderation_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]: + return _joined_choice( + tuple( + _moderation_verdict(flagged, result.get("categories")) + for result in _dicts(response.get("results")) + if isinstance(flagged := result.get("flagged"), bool) + ) + ) + + +def _moderation_verdict(flagged: bool, categories: object) -> str: + if not flagged: + return "not flagged" + hits: Final = ( + tuple(name for name, hit in cast(Mapping[str, object], categories).items() if hit is True) + if isinstance(categories, dict) + else () + ) + return f"flagged: {', '.join(hits)}" if hits else "flagged" + + +def _image_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]: + return _joined_choice( + tuple(summary for item in _dicts(response.get("data")) if (summary := _image_summary(item)) is not None) + ) + + +def _image_summary(item: Mapping[str, object]) -> str | None: + location: Final = _image_location(item) + if location is None: + return None + revised: Final = as_str(item.get("revised_prompt")) + return f"{revised}\n{location}" if revised else location + + +def _image_location(item: Mapping[str, object]) -> str | None: + url: Final = as_str(item.get("url")) + if url is not None: + return url + encoded: Final = item.get("b64_json") + if not isinstance(encoded, str): + return None + return f"b64_json image ({len(encoded) * 3 // 4 - encoded[-2:].count('=')} bytes)" + + +def _binary_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]: + size: Final = as_int(response.get("num_bytes")) + if size is None: return () - message: Final[_AssistantMessage] = { - "role": "assistant", - "content": "\n\n".join(markdowns), - "refusal": None, - "tool_calls": None, - } - choice: Final[_Choice] = {"message": message, "finish_reason": None} - return (choice,) + content_type: Final = as_str(response.get("content_type")) + return (_text_choice(f"{content_type} ({size} bytes)" if content_type else f"{size} bytes"),) def _responses_parts_text(parts: tuple[Mapping[str, object], ...], part_type: str, field: str) -> str | None: diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index f5967185af0..0f14b46c8ba 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -6277,6 +6277,8 @@ def _extract_response_obj_and_hidden_params( hidden_params = getattr(init_response_obj, "_hidden_params", None) elif isinstance(init_response_obj, dict): response_obj = init_response_obj + elif isinstance(init_response_obj, HttpxBinaryResponseContent): + response_obj = dict(init_response_obj.logging_summary()) else: response_obj = {} diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index b409b181a79..15bccf0301e 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -245,6 +245,8 @@ def _redact_model_response_dict_choices(choices, redacted_str: str): if "audio" in choice["delta"]: choice["delta"]["audio"] = None _redact_tool_calls_dict(choice["delta"]) + elif choice.get("text") is not None: + choice["text"] = redacted_str else: _redact_choice_content(choice) diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index bc44eb5b5b7..599b1a76249 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -110,6 +110,14 @@ FileTypes = ( EmbeddingInput = str | list[str] +class BinaryResponseSummary(TypedDict): + """What logging keeps of a binary response (speech audio, file content): size and media type, never the bytes.""" + + object: ReadOnly[Literal["binary"]] + content_type: ReadOnly[str | None] + num_bytes: ReadOnly[int] + + class HttpxBinaryResponseContent(_HttpxBinaryResponseContent): _hidden_params: dict @@ -117,6 +125,19 @@ class HttpxBinaryResponseContent(_HttpxBinaryResponseContent): super().__init__(response) self._hidden_params = {} # mutable-ok: mutable-dict contract shared with ModelResponse logging consumers + def logging_summary(self) -> BinaryResponseSummary: + return { + "object": "binary", + "content_type": self.response.headers.get("content-type"), + "num_bytes": self._num_bytes(), + } + + def _num_bytes(self) -> int: + try: + return len(self.response.content) + except httpx.ResponseNotRead: + return self.response.num_bytes_downloaded + def set_response_cost(self, response_cost: float | None) -> None: if response_cost is None: self._hidden_params.pop("response_cost", None) diff --git a/tests/code_coverage_tests/test_e2e_changed_gate.py b/tests/code_coverage_tests/test_e2e_changed_gate.py index 78e6562a4a8..2b37ab14e7f 100644 --- a/tests/code_coverage_tests/test_e2e_changed_gate.py +++ b/tests/code_coverage_tests/test_e2e_changed_gate.py @@ -212,6 +212,11 @@ def select_tests(changed: tuple[str, ...]) -> tuple[str, ...]: (("tests/e2e/batches/test_managed_files_enforcement_e2e.py",), ()), (("tests/e2e/guardrails/test_presidio_masking_e2e.py",), ()), (("tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e.py",), ()), + (("tests/e2e/logging/test_otel_v2_langfuse_generation_output_e2e.py",), ()), + ( + ("tests/e2e/logging/test_team_langfuse_callback_e2e.py",), + ("tests/e2e/logging/test_team_langfuse_callback_e2e.py",), + ), ( ("tests/e2e/llm_translation/realtime/test_realtime_e2e.py",), ("tests/e2e/llm_translation/realtime/test_realtime_e2e.py",), diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md index 2b59e8770b5..15c2d6763d6 100644 --- a/tests/e2e/CONTRIBUTING.md +++ b/tests/e2e/CONTRIBUTING.md @@ -105,7 +105,7 @@ A couple of logging destinations are configured on the proxy rather than by the ### The pull request check -Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite and both JWT suites as canaries, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Keycloak, Jaeger, and TLS cluster-mode Valkey. Realm-only edits also trigger these canaries. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories, `batches/test_managed_files_enforcement_e2e.py`, `llm_translation/realtime/test_realtime_pipecat_audio_e2e.py`, and `guardrails/test_presidio_masking_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack: the pipecat audio suite skips itself at import time unless the NLTK `punkt_tab` data is installed, and the presidio suite fails without the analyzer and anonymizer services this stack does not start. The Redis chaos test under `load/` needs a proxy it can pause the Redis of on the same host (`gateway/redis_chaos_ci_config.yml`), which `.github/workflows/test-e2e-redis-chaos.yml` boots, and which the Buildkite `e2e-redis-chaos` step in project-releaser runs co-located with Postgres and Valkey in one pod; it is deselected unless `E2E_REDIS_CHAOS` is set +Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite and both JWT suites as canaries, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Keycloak, Jaeger, and TLS cluster-mode Valkey. Realm-only edits also trigger these canaries. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories, `batches/test_managed_files_enforcement_e2e.py`, `llm_translation/realtime/test_realtime_pipecat_audio_e2e.py`, and `guardrails/test_presidio_masking_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack: the pipecat audio suite skips itself at import time unless the NLTK `punkt_tab` data is installed, and the presidio suite fails without the analyzer and anonymizer services this stack does not start. `logging/test_otel_v2_langfuse_generation_output_e2e.py` is marked `otel_v2` and deselects itself unless `E2E_OTEL_V2` is set, because it needs a gateway booted with `LITELLM_OTEL_V2=true` and Langfuse credentials, neither of which this stack provides, so run it with `E2E_OTEL_V2=1` against a local OTel v2 proxy. The Redis chaos test under `load/` needs a proxy it can pause the Redis of on the same host (`gateway/redis_chaos_ci_config.yml`), which `.github/workflows/test-e2e-redis-chaos.yml` boots, and which the Buildkite `e2e-redis-chaos` step in project-releaser runs co-located with Postgres and Valkey in one pod; it is deselected unless `E2E_REDIS_CHAOS` is set Every selected file must execute at least one passing test in each pass, and any test failure, collection error, or entirely skipped or deselected file fails the check. A file whose tests are all marked skip therefore cannot pass this check, so unskip at least one of them, or add the file to `UNSUPPORTED` in `select_tests.py` with the reason, before changing one. A failed pass stops the run. The public log prints pytest's one-line summary for each pass, including the rerun count, and names each failed or errored test as `classname::name`, so a retried network error or a failing test is visible without the raw output. The final `e2e-changed-tests` job succeeds only when no supported test files changed or the approved run completed all three passes. Fork PRs with selected tests fail this gate until a maintainer brings the reviewed change onto a same-repository branch diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 8776d00d502..ca0fbd84c35 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -30,6 +30,7 @@ from e2e_config import ( FIXTURE_MODE_RAW, MANAGED_FILES_OPT_IN_ENV, MCP_OAUTH_LIVE_OPT_IN_ENV, + OTEL_V2_OPT_IN_ENV, PROMPT_CACHING_OPT_IN_ENV, PROVIDER_EDGE_HOST_OPT_IN_ENV, PROXY_BASE_URL, @@ -61,6 +62,7 @@ OPT_IN_MARKERS: Final = MappingProxyType( "cli_determinism": CLI_DETERMINISM_OPT_IN_ENV, "mcp_oauth_live": MCP_OAUTH_LIVE_OPT_IN_ENV, "provider_edge_host": PROVIDER_EDGE_HOST_OPT_IN_ENV, + "otel_v2": OTEL_V2_OPT_IN_ENV, } ) @@ -150,6 +152,10 @@ def pytest_configure(config: pytest.Config) -> None: "provider_edge_host: routes provider traffic through the pytest host's edge in every fixture mode, so the " "gateway must reach the pytest host; deselected unless E2E_PROVIDER_EDGE_HOST_REACHABLE is set", ) + config.addinivalue_line( + "markers", + "otel_v2: needs a proxy running with LITELLM_OTEL_V2=true; deselected unless E2E_OTEL_V2 is set", + ) def pytest_sessionstart(session: pytest.Session) -> None: diff --git a/tests/e2e/e2e_config.py b/tests/e2e/e2e_config.py index 14de4619664..311c944eeb4 100644 --- a/tests/e2e/e2e_config.py +++ b/tests/e2e/e2e_config.py @@ -147,6 +147,7 @@ REDIS_CHAOS_OPT_IN_ENV = "E2E_REDIS_CHAOS" CLI_DETERMINISM_OPT_IN_ENV = "E2E_CLI_DETERMINISM" MCP_OAUTH_LIVE_OPT_IN_ENV: Final = "E2E_MCP_OAUTH_LIVE" PROVIDER_EDGE_HOST_OPT_IN_ENV: Final = "E2E_PROVIDER_EDGE_HOST_REACHABLE" +OTEL_V2_OPT_IN_ENV: Final = "E2E_OTEL_V2" ANOMALY_SESSIONS = int(os.environ.get("E2E_ANOMALY_SESSIONS", "6")) ANOMALY_TURNS_PER_SESSION = int(os.environ.get("E2E_ANOMALY_TURNS_PER_SESSION", "6")) ANOMALY_TURN_ATTEMPTS = int(os.environ.get("E2E_ANOMALY_TURN_ATTEMPTS", "3")) diff --git a/tests/e2e/logging/logging_client.py b/tests/e2e/logging/logging_client.py index 66dfa233ec4..c2f987ea33d 100644 --- a/tests/e2e/logging/logging_client.py +++ b/tests/e2e/logging/logging_client.py @@ -146,6 +146,29 @@ class LangfuseObservationList(BaseModel): data: list[LangfuseObservation] = [] +class LangfuseOtelMetadata(BaseModel): + """Langfuse stores every OTel span attribute under metadata.attributes.""" + + model_config = ConfigDict(extra="ignore") + + attributes: dict[str, str] = {} + + +def otel_attributes(obs: LangfuseObservation) -> dict[str, str]: + try: + return LangfuseOtelMetadata.model_validate(obs.metadata).attributes + except ValidationError: + return {} + + +def is_otel_v2_generation(obs: LangfuseObservation, *, key_alias: str) -> bool: + attributes = otel_attributes(obs) + return ( + attributes.get("langfuse.observation.type") == "generation" + and attributes.get("litellm.metadata.user_api_key_alias") == key_alias + ) + + class LangfuseListParams(BaseModel): model_config = ConfigDict(populate_by_name=True) @@ -630,6 +653,18 @@ class LoggingClient: time.sleep(POLL_INTERVAL) return last + def poll_langfuse_generation( + self, creds: LangfuseCreds, *, key_alias: str, from_start_time: str + ) -> LangfuseObservation | None: + """The OTel v2 generation the proxy exported for one key alias since from_start_time.""" + deadline = time.monotonic() + POLL_TIMEOUT + while time.monotonic() < deadline: + for obs in self.list_langfuse_observations(creds, from_start_time=from_start_time): + if is_otel_v2_generation(obs, key_alias=key_alias): + return obs + time.sleep(POLL_INTERVAL) + return None + def poll_langfuse_trace_observations( self, creds: LangfuseCreds, diff --git a/tests/e2e/logging/test_otel_v2_langfuse_generation_output_e2e.py b/tests/e2e/logging/test_otel_v2_langfuse_generation_output_e2e.py new file mode 100644 index 00000000000..71008d94557 --- /dev/null +++ b/tests/e2e/logging/test_otel_v2_langfuse_generation_output_e2e.py @@ -0,0 +1,210 @@ +"""Live e2e: the OTel v2 Langfuse generation carries output for every non-chat endpoint (LIT-8309). + +With LITELLM_OTEL_V2=true the proxy exports one generation per request to the +team's Langfuse destination. Chat, Responses, embeddings and OCR already fill +its output; this file pins the remaining five families. Each test registers a +real OpenAI deployment, drives the endpoint through the shared transport, then +reads the generation back from Langfuse and asserts its output reflects what +the caller received: the completion text, the transcript, the moderation +verdict, and for images and speech a bounded summary that never carries the +raw base64 or audio bytes. +""" + +from __future__ import annotations + +import base64 +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Final + +import pytest +from e2e_config import unique_marker +from e2e_http import unwrap +from lifecycle import ResourceManager +from logging_client import LangfuseCreds, LangfuseObservation, LoggingClient, load_langfuse_creds +from models import ( + CompletionBody, + CompletionResponse, + ImageGenerationBody, + ImageGenerationResponse, + LiteLLMParamsBody, + ModerationBody, + ModerationResponse, + SpeechBody, + TranscriptionForm, + TranscriptionResponse, +) +from pydantic import BaseModel, TypeAdapter, ValidationError + +pytestmark = [pytest.mark.e2e, pytest.mark.otel_v2] + +WEATHER_WAV: Final = ( + Path(__file__).resolve().parent.parent / "llm_translation" / "realtime" / "fixtures" / "weather_question_24k.wav" +) +BOUNDED_OUTPUT_CHARS: Final = 1024 + + +class _OutputMessage(BaseModel): + """One assistant message of the Langfuse generation output; only the text is read.""" + + content: str = "" + + +_OUTPUT_MESSAGES: Final = TypeAdapter(list[_OutputMessage]) + + +@pytest.fixture(scope="session") +def langfuse_creds() -> LangfuseCreds: + return load_langfuse_creds() + + +def _langfuse_key( + client: LoggingClient, creds: LangfuseCreds, resources: ResourceManager, params: LiteLLMParamsBody +) -> tuple[str, str, str]: + """A model registered for this run plus a key on a team whose Langfuse callback is `creds`.""" + model: Final = f"e2e-otel-out-{unique_marker()}" + model_id: Final = client.proxy.create_model(model, params) + resources.defer(lambda: client.proxy.delete_model(model_id)) + team_id: Final = client.create_team(f"otel-out-team-{unique_marker()}", models=[model]) + resources.defer(lambda: client.delete_team(team_id)) + client.add_team_langfuse_callback(team_id, creds) + alias: Final = f"otel-out-key-{unique_marker()}" + key: Final = client.key_with_alias(alias, models=[model], team_id=team_id) + resources.defer(lambda: client.delete_key(key)) + return model, key, alias + + +def _generation(client: LoggingClient, creds: LangfuseCreds, *, alias: str, started: datetime) -> LangfuseObservation: + since: Final = (started - timedelta(seconds=5)).isoformat() + observation: Final = client.poll_langfuse_generation(creds, key_alias=alias, from_start_time=since) + assert observation is not None, f"no OTel v2 generation reached Langfuse for key alias {alias!r}" + return observation + + +def _output_text(observation: LangfuseObservation) -> str: + assert observation.output not in (None, "", [], {}), f"generation output is empty: {observation!r}" + try: + messages: Final = _OUTPUT_MESSAGES.validate_python(observation.output) + except ValidationError: + pytest.fail(f"generation output is not a list of assistant messages: {observation!r}") + assert messages, f"generation output is empty: {observation!r}" + return "\n".join(message.content for message in messages) + + +def _openai(model: str) -> LiteLLMParamsBody: + return LiteLLMParamsBody(model=model, api_key="os.environ/OPENAI_API_KEY") + + +class TestOtelV2LangfuseGenerationOutput: + @pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["completions"]) + def test_completions_output_is_the_completion_text( + self, client: LoggingClient, langfuse_creds: LangfuseCreds, resources: ResourceManager + ) -> None: + model, key, alias = _langfuse_key(client, langfuse_creds, resources, _openai("openai/gpt-3.5-turbo-instruct")) + started: Final = datetime.now(timezone.utc) + response: Final = unwrap( + client.proxy.transport.post( + "/v1/completions", + headers=client.proxy.transport.bearer(key), + json=CompletionBody(model=model, prompt=f"Repeat exactly: {unique_marker()}", n=2), + response_type=CompletionResponse, + ) + ) + texts: Final = tuple(choice.text.strip() for choice in response.choices) + assert len(texts) == 2 and all(texts), f"/v1/completions returned no text: {response!r}" + + output: Final = _output_text(_generation(client, langfuse_creds, alias=alias, started=started)) + assert all(text in output for text in texts), ( + f"generation output lacks the completion texts {texts!r}: {output!r}" + ) + + @pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["images_generations"]) + def test_images_output_is_a_bounded_summary_without_base64( + self, client: LoggingClient, langfuse_creds: LangfuseCreds, resources: ResourceManager + ) -> None: + model, key, alias = _langfuse_key(client, langfuse_creds, resources, _openai("openai/gpt-image-1-mini")) + started: Final = datetime.now(timezone.utc) + response: Final = unwrap( + client.proxy.transport.post( + "/v1/images/generations", + headers=client.proxy.transport.bearer(key), + json=ImageGenerationBody(model=model, prompt=f"a plain red square {unique_marker()}"), + response_type=ImageGenerationResponse, + timeout=180.0, + ) + ) + assert response.data, f"/v1/images/generations returned no data: {response!r}" + encoded: Final = response.data[0].b64_json or "" + assert encoded, f"expected a b64_json image from gpt-image-1-mini: {response.data[0].url!r}" + image_bytes: Final = len(base64.b64decode(encoded)) + + output: Final = _output_text(_generation(client, langfuse_creds, alias=alias, started=started)) + assert len(output) <= BOUNDED_OUTPUT_CHARS, f"image generation output is not bounded ({len(output)} chars)" + assert encoded[:64] not in output, "image generation output leaks the raw base64 payload" + assert output == f"b64_json image ({image_bytes} bytes)", ( + f"image generation output does not report the {image_bytes} decoded bytes: {output!r}" + ) + + @pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["audio_speech"]) + def test_speech_output_is_a_bounded_summary_without_audio_bytes( + self, client: LoggingClient, langfuse_creds: LangfuseCreds, resources: ResourceManager + ) -> None: + model, key, alias = _langfuse_key(client, langfuse_creds, resources, _openai("openai/gpt-4o-mini-tts")) + started: Final = datetime.now(timezone.utc) + audio: Final = client.proxy.transport.stream_binary( + "/v1/audio/speech", + headers=client.proxy.transport.bearer(key), + json=SpeechBody(model=model, input=f"hello {unique_marker()}"), + ) + assert audio.ok and audio.total_bytes > 0, f"/v1/audio/speech returned no audio: {audio!r}" + + output: Final = _output_text(_generation(client, langfuse_creds, alias=alias, started=started)) + assert len(output) <= BOUNDED_OUTPUT_CHARS, f"speech output is not bounded ({len(output)} chars)" + assert output.endswith(f" ({audio.total_bytes} bytes)"), ( + f"speech output does not report the {audio.total_bytes} audio bytes the caller received: {output!r}" + ) + + @pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["audio_transcriptions"]) + def test_transcription_output_is_the_transcript( + self, client: LoggingClient, langfuse_creds: LangfuseCreds, resources: ResourceManager + ) -> None: + model, key, alias = _langfuse_key(client, langfuse_creds, resources, _openai("openai/gpt-4o-mini-transcribe")) + started: Final = datetime.now(timezone.utc) + response: Final = unwrap( + client.proxy.transport.upload( + "/v1/audio/transcriptions", + headers=client.proxy.transport.bearer(key), + form=TranscriptionForm(model=model), + filename=WEATHER_WAV.name, + content=WEATHER_WAV.read_bytes(), + file_content_type="audio/wav", + response_type=TranscriptionResponse, + ) + ) + transcript: Final = response.text.strip() + assert transcript, f"/v1/audio/transcriptions returned no text: {response!r}" + + output: Final = _output_text(_generation(client, langfuse_creds, alias=alias, started=started)) + assert transcript in output, f"generation output lacks the transcript {transcript!r}: {output!r}" + + @pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["moderations"]) + def test_moderations_output_is_the_verdict( + self, client: LoggingClient, langfuse_creds: LangfuseCreds, resources: ResourceManager + ) -> None: + model, key, alias = _langfuse_key(client, langfuse_creds, resources, _openai("openai/omni-moderation-latest")) + started: Final = datetime.now(timezone.utc) + response: Final = unwrap( + client.proxy.transport.post( + "/v1/moderations", + headers=client.proxy.transport.bearer(key), + json=ModerationBody(model=model, input=f"I will find you and hurt you badly {unique_marker()}"), + response_type=ModerationResponse, + ) + ) + assert response.results, f"/v1/moderations returned no results: {response!r}" + verdict: Final = "flagged: " if response.results[0].flagged else "not flagged" + + output: Final = _output_text(_generation(client, langfuse_creds, alias=alias, started=started)) + assert output.startswith(verdict), ( + f"generation output does not carry the moderation verdict {verdict!r}: {output!r}" + ) diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 2608da2e4fc..d0b1e8824da 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -760,6 +760,77 @@ class OcrResponse(BaseModel): pages: list[OcrPage] = [] +# ---------- completions ---------- + + +class CompletionBody(BaseModel): + model: str + prompt: str + max_tokens: int = 8 + n: int = 1 + + +class CompletionChoice(BaseModel): + text: str = "" + + +class CompletionResponse(BaseModel): + choices: list[CompletionChoice] = [] + + +# ---------- images ---------- + + +class ImageGenerationBody(BaseModel): + model: str + prompt: str + n: int = 1 + size: str = "1024x1024" + quality: str = "low" + + +class ImageDatum(BaseModel): + url: str | None = None + b64_json: str | None = None + + +class ImageGenerationResponse(BaseModel): + data: list[ImageDatum] = [] + + +# ---------- audio ---------- + + +class SpeechBody(BaseModel): + model: str + input: str + voice: str = "alloy" + + +class TranscriptionForm(BaseModel): + model: str + + +class TranscriptionResponse(BaseModel): + text: str = "" + + +# ---------- moderations ---------- + + +class ModerationBody(BaseModel): + model: str + input: str + + +class ModerationResult(BaseModel): + flagged: bool + + +class ModerationResponse(BaseModel): + results: list[ModerationResult] = [] + + # ---------- spend logs ---------- diff --git a/tests/e2e/pytest.ini b/tests/e2e/pytest.ini index c6acd449884..6f9f57d333e 100644 --- a/tests/e2e/pytest.ini +++ b/tests/e2e/pytest.ini @@ -14,3 +14,4 @@ markers = redis_chaos: load test that pauses the proxy's Redis outright mid-run; needs a proxy booted from gateway/redis_chaos_ci_config.yml on the same host, and is deselected unless E2E_REDIS_CHAOS is set mcp_oauth_live: real Linear OAuth consent via a captured browser session; deselected unless E2E_MCP_OAUTH_LIVE is set provider_edge_host: routes provider traffic through the pytest host's edge in every fixture mode, so the gateway must reach the pytest host; deselected unless E2E_PROVIDER_EDGE_HOST_REACHABLE is set + otel_v2: needs a proxy running with LITELLM_OTEL_V2=true; deselected unless E2E_OTEL_V2 is set diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py b/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py index 01f2a13d252..70ab4fe07de 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py @@ -4,6 +4,7 @@ and the typed StandardLoggingPayload adapter. These need no OTel SDK.""" import json import logging import re +from collections.abc import Mapping from pathlib import Path from typing import Final @@ -151,9 +152,7 @@ def test_llm_call_span_name(): def _all_constants(cls): return { - getattr(cls, name) - for name in vars(cls) - if not name.startswith("__") and isinstance(getattr(cls, name), str) + getattr(cls, name) for name in vars(cls) if not name.startswith("__") and isinstance(getattr(cls, name), str) } @@ -465,9 +464,7 @@ def test_mcp_tool_call_content_gated_off_by_default(): off = MCPToolCallSpanData.from_standard_logging_payload(_mcp_payload()) assert off.arguments_json is None and off.result_json is None - on = MCPToolCallSpanData.from_standard_logging_payload( - _mcp_payload(), capture_content=True - ) + on = MCPToolCallSpanData.from_standard_logging_payload(_mcp_payload(), capture_content=True) assert on.arguments_json is not None and '"Paris"' in on.arguments_json assert on.result_json is not None and "21" in on.result_json @@ -689,9 +686,7 @@ def test_content_capture_gated_off_by_default(): payload = _sample_payload( messages=[{"role": "user", "content": "secret prompt"}], ) - payload["response"]["choices"] = [ - {"finish_reason": "stop", "message": {"role": "assistant", "content": "secret"}} - ] + payload["response"]["choices"] = [{"finish_reason": "stop", "message": {"role": "assistant", "content": "secret"}}] data = LLMCallSpanData.from_standard_logging_payload(payload) assert data.messages_in == () assert data.choices_out == () @@ -911,6 +906,221 @@ def test_ocr_pages_without_markdown_stay_empty(): assert data.choices_out == () +def _assistant_choice(content: str, finish_reason: str | None = None) -> dict[str, object]: + return { + "message": {"role": "assistant", "content": content, "refusal": None, "tool_calls": None}, + "finish_reason": finish_reason, + } + + +def _route_payload(call_type: str, model: str, response: Mapping[str, object]) -> dict[str, object]: + return _sample_payload(call_type=call_type, model=model, messages=None, response=response) + + +def test_text_completion_choices_become_assistant_messages_in_choice_order() -> None: + data: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload( + "atext_completion", + "gpt-3.5-turbo-instruct", + { + "id": "cmpl-1", + "object": "text_completion", + "choices": [ + {"index": 0, "text": " first", "finish_reason": "length", "logprobs": None}, + {"index": 1, "text": " second", "finish_reason": "stop", "logprobs": None}, + ], + }, + ), + capture_content=True, + ) + + assert data.choices_out == (_assistant_choice(" first", "length"), _assistant_choice(" second", "stop")) + assert data.finish_reasons == ("length", "stop") + assert data.response_id == "cmpl-1" + + +def test_text_completion_choices_follow_the_content_capture_gate_but_finish_reasons_do_not() -> None: + data: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload( + "atext_completion", "gpt-3.5-turbo-instruct", {"choices": [{"text": "x", "finish_reason": "stop"}]} + ) + ) + + assert data.choices_out == () + assert data.finish_reasons == ("stop",) + + +def test_chat_choices_with_a_message_are_passed_through_untouched_even_beside_a_stray_text_key() -> None: + choice: Final = { + "index": 0, + "finish_reason": "stop", + "text": "no", + "message": {"role": "assistant", "content": "chat"}, + } + data: Final = LLMCallSpanData.from_standard_logging_payload( + _sample_payload(response={"choices": [choice]}), capture_content=True + ) + + assert data.choices_out == (choice,) + + +def test_transcription_text_becomes_one_assistant_choice() -> None: + data: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload("atranscription", "gpt-4o-mini-transcribe", {"text": "What is the weather like?", "task": "x"}), + capture_content=True, + ) + + assert data.choices_out == (_assistant_choice("What is the weather like?"),) + assert data.finish_reasons == () + + +def test_empty_transcription_text_stays_empty() -> None: + data: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload("atranscription", "gpt-4o-mini-transcribe", {"text": ""}), capture_content=True + ) + + assert data.choices_out == () + + +def test_moderation_results_become_one_verdict_per_input_naming_the_hit_categories() -> None: + data: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload( + "amoderation", + "omni-moderation-latest", + { + "id": "modr-1", + "results": [ + { + "flagged": True, + "categories": {"harassment": False, "violence": True, "self-harm": True}, + "category_scores": {"harassment": 0.01, "violence": 0.98, "self-harm": 0.7}, + }, + {"flagged": False, "categories": {"violence": False}}, + {"flagged": True}, + ], + }, + ), + capture_content=True, + ) + + assert data.choices_out == (_assistant_choice("flagged: violence, self-harm\n\nnot flagged\n\nflagged"),) + assert data.response_id == "modr-1" + + +def test_moderation_output_follows_the_content_capture_gate() -> None: + data: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload("amoderation", "omni-moderation-latest", {"results": [{"flagged": True}]}) + ) + + assert data.choices_out == () + + +def test_moderation_results_without_a_verdict_produce_no_output() -> None: + data: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload("amoderation", "omni-moderation-latest", {"results": [{"categories": {"violence": True}}]}), + capture_content=True, + ) + + assert data.choices_out == () + + +def test_image_data_becomes_a_size_summary_and_never_carries_the_base64_payload() -> None: + encoded: Final = "QUJDRA==" + data: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload( + "aimage_generation", + "gpt-image-1-mini", + { + "created": 1, + "data": [ + {"b64_json": encoded, "revised_prompt": "a red bicycle"}, + {"url": "https://images.example/cat.png"}, + {"b64_json": "QUJDREVGR0g="}, + ], + }, + ), + capture_content=True, + ) + + assert data.choices_out == ( + _assistant_choice( + "a red bicycle\nb64_json image (4 bytes)\n\nhttps://images.example/cat.png\n\nb64_json image (8 bytes)" + ), + ) + assert encoded not in json.dumps(data.choices_out) + + +def test_image_data_without_a_url_or_payload_stays_empty_and_embeddings_are_not_images() -> None: + images: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload("aimage_generation", "gpt-image-1-mini", {"data": [{"revised_prompt": "x"}]}), + capture_content=True, + ) + embeddings: Final = LLMCallSpanData.from_standard_logging_payload( + _embedding_payload([[0.1, 0.2]]), capture_content=True + ) + + assert images.choices_out == () + assert embeddings.choices_out == () + + +def test_speech_summary_becomes_a_media_type_and_byte_count_choice() -> None: + data: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload( + "aspeech", "gpt-4o-mini-tts", {"object": "binary", "content_type": "audio/mpeg", "num_bytes": 48210} + ), + capture_content=True, + ) + + assert data.choices_out == (_assistant_choice("audio/mpeg (48210 bytes)"),) + + +def test_speech_summary_without_a_media_type_is_the_byte_count_and_follows_the_capture_gate() -> None: + response: Final = {"object": "binary", "content_type": None, "num_bytes": 7} + shown: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload("aspeech", "gpt-4o-mini-tts", response), capture_content=True + ) + gated: Final = LLMCallSpanData.from_standard_logging_payload(_route_payload("aspeech", "gpt-4o-mini-tts", response)) + + assert shown.choices_out == (_assistant_choice("7 bytes"),) + assert gated.choices_out == () + + +def test_speech_response_without_a_byte_count_produces_no_output() -> None: + data: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload("aspeech", "gpt-4o-mini-tts", {"object": "binary", "content_type": "audio/mpeg"}), + capture_content=True, + ) + + assert data.choices_out == () + + +def test_speech_binary_response_is_logged_as_its_summary_not_dropped() -> None: + import httpx + + from litellm.litellm_core_utils.litellm_logging import _extract_response_obj_and_hidden_params + from litellm.types.llms.openai import HttpxBinaryResponseContent + + raw: Final = httpx.Response(200, headers={"content-type": "audio/mpeg"}, content=b"\x00" * 1234) + response_obj, hidden_params = _extract_response_obj_and_hidden_params(HttpxBinaryResponseContent(raw), None) + + assert response_obj == {"object": "binary", "content_type": "audio/mpeg", "num_bytes": 1234} + assert hidden_params is None + + +def test_speech_binary_response_still_streaming_reports_the_bytes_downloaded_so_far() -> None: + import httpx + + from litellm.types.llms.openai import HttpxBinaryResponseContent + + unread: Final = httpx.Response(200, stream=httpx.ByteStream(b"\x00" * 10)) + + assert HttpxBinaryResponseContent(unread).logging_summary() == { + "object": "binary", + "content_type": None, + "num_bytes": 0, + } + + def test_request_identity_prefers_canonical_team_keys(): from litellm.integrations.otel.model.payloads import RequestIdentity @@ -931,9 +1141,7 @@ def test_request_identity_prefers_canonical_team_keys(): def test_request_identity_falls_back_to_legacy_team_keys(): from litellm.integrations.otel.model.payloads import RequestIdentity - payload = _sample_payload( - metadata={"team_id": "legacy-team", "team_alias": "legacy"} - ) + payload = _sample_payload(metadata={"team_id": "legacy-team", "team_alias": "legacy"}) ident = RequestIdentity.from_payload(payload) assert ident.team_id == "legacy-team" assert ident.team_alias == "legacy" @@ -952,7 +1160,10 @@ def test_request_identity_falls_back_to_legacy_team_keys(): }, "from-header", ), - ({"proxy_server_request": {"headers": {"langfuse_trace_name": ""}}, "metadata": {"trace_name": "body"}}, "body"), + ( + {"proxy_server_request": {"headers": {"langfuse_trace_name": ""}}, "metadata": {"trace_name": "body"}}, + "body", + ), ({"proxy_server_request": {"headers": {}}, "metadata": {"user_api_key_team_id": "t1"}}, None), ({}, None), ], @@ -1002,7 +1213,15 @@ def test_caller_trace_name_prefers_the_langfuse_header_over_body_metadata(reques ), ({}, TraceControls()), ], - ids=["body", "headers-beat-body", "anthropic-body", "non-string-tags-dropped", "scalar-coercion", "mutation-controls-ignored", "empty"], + ids=[ + "body", + "headers-beat-body", + "anthropic-body", + "non-string-tags-dropped", + "scalar-coercion", + "mutation-controls-ignored", + "empty", + ], ) def test_caller_trace_controls_carry_user_session_and_tags(request_data, expected): assert caller_trace_controls({"litellm_params": request_data}) == expected @@ -1168,9 +1387,7 @@ def test_content_capture_opt_in_retains_bodies(): payload = _sample_payload( messages=[{"role": "user", "content": "secret prompt"}], ) - payload["response"]["choices"] = [ - {"finish_reason": "stop", "message": {"role": "assistant", "content": "hi"}} - ] + payload["response"]["choices"] = [{"finish_reason": "stop", "message": {"role": "assistant", "content": "hi"}}] data = LLMCallSpanData.from_standard_logging_payload(payload, capture_content=True) assert data.messages_in and data.messages_in[0]["content"] == "secret prompt" assert data.choices_out and data.choices_out[0]["message"]["content"] == "hi" @@ -1187,41 +1404,17 @@ def test_capture_span_content_resolves_modes(): # default (no_content) → off assert OpenTelemetryV2Config().capture_span_content is False + assert OpenTelemetryV2Config(capture_message_content=CaptureMessageContent.SPAN_ONLY).capture_span_content is True assert ( - OpenTelemetryV2Config( - capture_message_content=CaptureMessageContent.SPAN_ONLY - ).capture_span_content - is True - ) - assert ( - OpenTelemetryV2Config( - capture_message_content=CaptureMessageContent.SPAN_AND_EVENT - ).capture_span_content - is True + OpenTelemetryV2Config(capture_message_content=CaptureMessageContent.SPAN_AND_EVENT).capture_span_content is True ) # event-only does not authorize span-attribute content - assert ( - OpenTelemetryV2Config( - capture_message_content=CaptureMessageContent.EVENT_ONLY - ).capture_span_content - is False - ) + assert OpenTelemetryV2Config(capture_message_content=CaptureMessageContent.EVENT_ONLY).capture_span_content is False # V1 accepted UPPER_SNAKE_CASE; the env value is case-insensitive so an # operator carrying ``SPAN_AND_EVENT`` forward still enables capture. - assert ( - OpenTelemetryV2Config( - capture_message_content="SPAN_AND_EVENT" - ).capture_span_content - is True - ) - assert ( - OpenTelemetryV2Config(capture_message_content="SPAN_ONLY").capture_span_content - is True - ) - assert ( - OpenTelemetryV2Config(capture_message_content="NO_CONTENT").capture_span_content - is False - ) + assert OpenTelemetryV2Config(capture_message_content="SPAN_AND_EVENT").capture_span_content is True + assert OpenTelemetryV2Config(capture_message_content="SPAN_ONLY").capture_span_content is True + assert OpenTelemetryV2Config(capture_message_content="NO_CONTENT").capture_span_content is False def test_capture_message_content_normalizer_only_touches_strings(): diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_vendor_mappers.py b/tests/test_litellm/integrations/otel/test_otel_v2_vendor_mappers.py index 9fa198c4ec5..1e2ae24a329 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_vendor_mappers.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_vendor_mappers.py @@ -6,6 +6,8 @@ backends, so one trace lights up every configured destination. """ import json +from collections.abc import Mapping +from typing import Final import pytest @@ -249,6 +251,34 @@ def test_langfuse_mapper_renders_an_ocr_call_with_the_page_markdown_as_output(): assert attrs["langfuse.observation.type"] == "generation" +@pytest.mark.parametrize( + ("call_type", "response", "expected_content"), + [ + ("atext_completion", {"choices": [{"text": "Paris.", "finish_reason": "stop"}]}, "Paris."), + ("atranscription", {"text": "What is the weather like?"}, "What is the weather like?"), + ("amoderation", {"results": [{"flagged": True, "categories": {"violence": True}}]}, "flagged: violence"), + ("aimage_generation", {"data": [{"b64_json": "QUJDRA=="}]}, "b64_json image (4 bytes)"), + ("aspeech", {"object": "binary", "content_type": "audio/mpeg", "num_bytes": 9}, "audio/mpeg (9 bytes)"), + ], +) +def test_langfuse_mapper_renders_every_non_chat_route_output_as_an_assistant_message( + call_type: str, response: Mapping[str, object], expected_content: str +) -> None: + payload: Final[dict[str, object]] = { + "call_type": call_type, + "custom_llm_provider": "openai", + "model": "m", + "messages": None, + "response": response, + } + attrs: Final = LangfuseMapper().map(LLMCallSpanData.from_standard_logging_payload(payload, capture_content=True)) + + assert json.loads(attrs["langfuse.observation.output"]) == [ + {"role": "assistant", "content": expected_content, "refusal": None, "tool_calls": None} + ] + assert attrs["langfuse.observation.type"] == "generation" + + # --------------------------------------------------------------------------- # # Weave # --------------------------------------------------------------------------- # diff --git a/tests/test_litellm/litellm_core_utils/test_redact_messages.py b/tests/test_litellm/litellm_core_utils/test_redact_messages.py index 276a67e0bd4..22298c00219 100644 --- a/tests/test_litellm/litellm_core_utils/test_redact_messages.py +++ b/tests/test_litellm/litellm_core_utils/test_redact_messages.py @@ -304,6 +304,26 @@ class TestPerformRedaction: assert delta["thinking_blocks"] is None assert delta["audio"] is None + def test_redacts_text_completion_choices_in_standard_logging_object(self): + details = { + "standard_logging_object": { + "response": { + "object": "text_completion", + "choices": [ + {"text": " Paris.", "finish_reason": "stop", "index": 0}, + {"text": "\n\nBlue", "finish_reason": "length", "index": 1}, + ], + } + } + } + + perform_redaction(details, None) + + assert details["standard_logging_object"]["response"]["choices"] == [ + {"text": "redacted-by-litellm", "finish_reason": "stop", "index": 0}, + {"text": "redacted-by-litellm", "finish_reason": "length", "index": 1}, + ] + def test_redacts_object_choices_inside_model_response_dict(self): result = { "choices": [ From 8603d6e259fdcba31970b7b52e920282a57a194a Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 22:15:35 -0700 Subject: [PATCH 15/15] fix(cost): honor per-second custom pricing on chat completions for every provider (#42403) * fix(cost): honor per-second custom pricing on chat completions for every provider * test(cost): assert a per-second priced deployment bills instead of staying at $0 The zero-cost diagnostic tests from #42345 used a per-second-only entry as their misconfigured fixture, which this branch now bills. Switch that fixture to a per-query-only entry, which is still selected as the deployment's own pricing and still prices chat usage at $0, and add a per-second test asserting the call duration is billed with no diagnostic Also let a caller's explicit total_time outrank the logging window in completion_cost, so the SDK precedence stays stamped response, caller, logging * test(response_metadata): move the per-second pricing regression into the mapped tests/unit file * fix(cost_calculator): keep media-mode per-second rates off the wall-clock path A video, transcription, speech, or realtime entry's per-second rates price media seconds, which their dedicated cost paths bill from the media itself. The generic per-second branch now skips those modes, so a video status poll on a per-second video model bills nothing instead of the seconds the poll took to answer. --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/cost_calculator.py | 119 +++++++++----- litellm/litellm_core_utils/litellm_logging.py | 18 ++- .../llm_response_utils/response_metadata.py | 2 +- .../test_litellm_logging.py | 51 ++++-- tests/test_litellm/test_cost_calculator.py | 151 +++++++++++++++++- .../test_response_metadata.py | 58 ++++++- 6 files changed, 336 insertions(+), 63 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 37743a9ce33..7b02a5ede00 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -115,6 +115,7 @@ from litellm.types.utils import ( LlmProviders, LlmProvidersSet, ModelInfo, + ModelInfoBase, PromptTokensDetailsWrapper, ServiceTier, StandardBuiltInToolsParams, @@ -322,6 +323,48 @@ class OCRPricing(TypedDict, total=False): annotation_cost_per_page: ReadOnly[float | None] +_WALL_CLOCK_PRICED_MODES: Final = frozenset({"chat", "completion", "embedding", "responses"}) + + +def _has_token_or_tiered_pricing(model_info: ModelInfoBase) -> bool: + return ( + (model_info.get("input_cost_per_token") or 0.0) > 0 + or (model_info.get("output_cost_per_token") or 0.0) > 0 + or model_info.get("tiered_pricing") is not None + ) + + +def _bills_wall_clock_seconds(model_info: ModelInfoBase) -> bool: + mode: Final = model_info.get("mode") + return mode is None or mode in _WALL_CLOCK_PRICED_MODES + + +def _per_second_pricing_cost( + model: str, + custom_llm_provider: str | None, + response_time_ms: float | None, +) -> tuple[float, float] | None: + try: + model_info: Final = _cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider) + except Exception: # noqa: BLE001 # the lookup raises plain Exception for an unmapped model + return None + if _has_token_or_tiered_pricing(model_info) or not _bills_wall_clock_seconds(model_info): + return None + input_cost_per_second: Final = model_info.get("input_cost_per_second") + output_cost_per_second: Final = model_info.get("output_cost_per_second") + if input_cost_per_second is None and output_cost_per_second is None: + return None + seconds: Final = (response_time_ms or 0.0) / 1000 + verbose_logger.debug( + "For model=%s - input_cost_per_second: %s; output_cost_per_second: %s; response time: %s", + model, + input_cost_per_second, + output_cost_per_second, + response_time_ms, + ) + return (input_cost_per_second or 0.0) * seconds, (output_cost_per_second or 0.0) * seconds + + def cost_per_token( model: str = "", prompt_tokens: int = 0, @@ -448,9 +491,6 @@ def cost_per_token( if response_cost is not None: return response_cost[0], response_cost[1] - # given - prompt_tokens_cost_usd_dollar: float = 0 - completion_tokens_cost_usd_dollar: float = 0 model_cost_ref: Final = litellm.model_cost # Only callers that explicitly pass `custom_llm_provider` get the # dedup/prefix-join treatment. When provider is omitted, preserve legacy @@ -611,6 +651,14 @@ def cost_per_token( number_of_queries=number_of_queries or 1, optional_params=(getattr(response, "_hidden_params", None) if response else None), ) + elif ( + per_second_cost := _per_second_pricing_cost( + model=model, + custom_llm_provider=custom_llm_provider, + response_time_ms=response_time_ms, + ) + ) is not None: + return per_second_cost elif custom_llm_provider == "vertex_ai": cost_router: Final = google_cost_router( model=model_without_prefix, @@ -685,12 +733,7 @@ def cost_per_token( ) else: model_info: Final = _cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider) - - if ( - (model_info.get("input_cost_per_token") or 0.0) > 0 - or (model_info.get("output_cost_per_token") or 0.0) > 0 - or model_info.get("tiered_pricing") is not None - ): + if _has_token_or_tiered_pricing(model_info): return generic_cost_per_token( model=model, usage=usage_block, @@ -698,36 +741,8 @@ def cost_per_token( service_tier=service_tier, data_residency=data_residency, ) - - input_cost_per_second: Final = model_info.get("input_cost_per_second") - if input_cost_per_second is not None and response_time_ms is not None: - verbose_logger.debug( - "For model=%s - input_cost_per_second: %s; response time: %s", - model, - input_cost_per_second, - response_time_ms, - ) - ## COST PER SECOND ## - prompt_tokens_cost_usd_dollar = input_cost_per_second * response_time_ms / 1000 - - output_cost_per_second: Final = model_info.get("output_cost_per_second") - if output_cost_per_second is not None and response_time_ms is not None: - verbose_logger.debug( - "For model=%s - output_cost_per_second: %s; response time: %s", - model, - output_cost_per_second, - response_time_ms, - ) - ## COST PER SECOND ## - completion_tokens_cost_usd_dollar = output_cost_per_second * response_time_ms / 1000 - - verbose_logger.debug( - "Returned custom cost for model=%s - prompt_tokens_cost_usd_dollar: %s, completion_tokens_cost_usd_dollar: %s", - model, - prompt_tokens_cost_usd_dollar, - completion_tokens_cost_usd_dollar, - ) - return prompt_tokens_cost_usd_dollar, completion_tokens_cost_usd_dollar + verbose_logger.debug("No per-token, tiered, or per-second pricing for model=%s; cost is 0", model) + return 0.0, 0.0 def get_replicate_completion_pricing(completion_response: dict, total_time=0.0): @@ -1222,6 +1237,21 @@ def _split_responses_ws_logging_object_by_service_tier( ) +def _response_time_ms_for_cost( + completion_response: object, + litellm_logging_obj: LitellmLoggingObject | None, + total_time: float | None, +) -> float: + stamped: Final = getattr(completion_response, "_response_ms", None) + if isinstance(stamped, (int, float)): + return float(stamped) + if total_time: + return total_time + if litellm_logging_obj is not None: + return litellm_logging_obj.get_response_ms() + return 0.0 + + def completion_cost( completion_response: object | None = None, model: str | None = None, @@ -1443,8 +1473,6 @@ def completion_cost( prompt_tokens_details = _usage.get("prompt_tokens_details") or {} cache_read_input_tokens = prompt_tokens_details.get("cached_tokens", 0) - total_time = getattr(completion_response, "_response_ms", 0) - hidden_params = getattr(completion_response, "_hidden_params", None) if hidden_params is not None: custom_llm_provider = hidden_params.get("custom_llm_provider", custom_llm_provider or None) @@ -1676,6 +1704,11 @@ def completion_cost( ) return MCPCostCalculator.calculate_mcp_tool_call_cost(litellm_logging_obj=litellm_logging_obj) + response_time_ms = _response_time_ms_for_cost( + completion_response=completion_response, + litellm_logging_obj=litellm_logging_obj, + total_time=total_time, + ) # Calculate cost based on prompt_tokens, completion_tokens if ( "togethercomputer" in model or "together_ai" in model or custom_llm_provider == "together_ai" @@ -1686,7 +1719,7 @@ def completion_cost( # see https://replicate.com/pricing elif (model in litellm.replicate_models or "replicate" in model) and model not in litellm.model_cost: # for unmapped replicate model, default to replicate's time tracking logic - return get_replicate_completion_pricing(completion_response, total_time) + return get_replicate_completion_pricing(completion_response, response_time_ms) if model is None: raise ValueError( @@ -1718,7 +1751,7 @@ def completion_cost( prompt_tokens=prompt_tokens or 0, completion_tokens=completion_tokens or 0, custom_llm_provider=custom_llm_provider, - response_time_ms=total_time, + response_time_ms=response_time_ms, region_name=None if explicit_pricing else region_name, custom_cost_per_second=custom_cost_per_second, custom_cost_per_token=custom_cost_per_token, diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 0f14b46c8ba..d06780dda53 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -496,6 +496,14 @@ def mask_api_base_credentials(api_base: str) -> str: return api_base[:key_end] + "*" * 5 + api_base[-4:] +def _timestamp_seconds(moment: object) -> float | None: + if isinstance(moment, datetime.datetime): + return moment.timestamp() + if isinstance(moment, (int, float)): + return float(moment) + return None + + class Logging(LiteLLMLoggingBaseClass): global \ supabaseClient, \ @@ -1634,10 +1642,12 @@ class Logging(LiteLLMLoggingBaseClass): return response.mcp_tool_call_response def get_response_ms(self) -> float: - return ( - self.model_call_details.get("end_time", datetime.datetime.now()) - - self.model_call_details.get("start_time", datetime.datetime.now()) - ).total_seconds() * 1000 + now: Final = datetime.datetime.now() + start_seconds: Final = _timestamp_seconds(self.model_call_details.get("start_time", now)) + end_seconds: Final = _timestamp_seconds(self.model_call_details.get("end_time", now)) + if start_seconds is None or end_seconds is None: + return 0.0 + return (end_seconds - start_seconds) * 1000 def set_cost_breakdown( self, diff --git a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py index 4a3c8de78c5..503814cc143 100644 --- a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py +++ b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py @@ -251,6 +251,6 @@ def update_response_metadata( return metadata: Final = ResponseMetadata(result) - metadata.set_hidden_params(logging_obj, model, kwargs) metadata.set_timing_metrics(start_time, end_time, logging_obj, include_overhead) + metadata.set_hidden_params(logging_obj, model, kwargs) metadata.apply() diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 1bfc55d1486..021e012f29d 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -305,14 +305,15 @@ def test_response_cost_calculator_uses_router_model_id_from_litellm_metadata(): class TestZeroCostDiagnostic: - DEPLOYMENT_ID: Final = "lit7898-per-second-priced-deployment" - MODEL_GROUP: Final = "per-second-priced-chat" + DEPLOYMENT_ID: Final = "lit7898-query-only-priced-deployment" + MODEL_GROUP: Final = "query-only-priced-chat" + QUERY_ONLY_PRICING: Final = {"input_cost_per_query": 0.00042} PER_SECOND_PRICING: Final = {"input_cost_per_second": 0.00042, "output_cost_per_second": 0.00042} FREE_PRICING: Final = {"input_cost_per_token": 0, "output_cost_per_token": 0} - @pytest.fixture(params=["per_second", "free"]) + @pytest.fixture(params=["query_only", "free"]) def deployment_pricing(self, request: pytest.FixtureRequest) -> Iterator[Mapping[str, float]]: - pricing: Final = self.PER_SECOND_PRICING if request.param == "per_second" else self.FREE_PRICING + pricing: Final = self.QUERY_ONLY_PRICING if request.param == "query_only" else self.FREE_PRICING litellm.register_model(model_cost={self.DEPLOYMENT_ID: pricing}, persist_across_reloads=False) try: yield pricing @@ -557,11 +558,11 @@ class TestZeroCostDiagnostic: priced_pricing: Final = {"input_cost_per_token": 1e-06, "output_cost_per_token": 2e-06} usage: Final = litellm.Usage(prompt_tokens=10, completion_tokens=20, total_tokens=30) litellm.register_model( - model_cost={self.DEPLOYMENT_ID: self.PER_SECOND_PRICING, priced_id: priced_pricing}, + model_cost={self.DEPLOYMENT_ID: self.QUERY_ONLY_PRICING, priced_id: priced_pricing}, persist_across_reloads=False, ) try: - logging_obj: Final = self._logging_obj(self.PER_SECOND_PRICING) + logging_obj: Final = self._logging_obj(self.QUERY_ONLY_PRICING) with caplog.at_level(logging.WARNING, logger="LiteLLM"): assert logging_obj._response_cost_calculator(result=self._response(usage)) == 0.0 self._assert_flagged(logging_obj, caplog) @@ -570,7 +571,7 @@ class TestZeroCostDiagnostic: assert logging_obj._response_cost_calculator(result=self._response(usage)) == pytest.approx(5e-05) assert logging_obj.model_call_details["zero_cost_diagnostic"] is None - self._route_to_deployment(logging_obj, self.PER_SECOND_PRICING) + self._route_to_deployment(logging_obj, self.QUERY_ONLY_PRICING) assert logging_obj._response_cost_calculator(result=self._response(usage)) == 0.0 assert logging_obj.model_call_details["zero_cost_diagnostic"]["reason"] == "missing_pricing_key" @@ -585,7 +586,7 @@ class TestZeroCostDiagnostic: dated_model: Final = "lit7898-nano-2026-03-17" requested_model: Final = "lit7898-nano" usage: Final = litellm.Usage(prompt_tokens=10, completion_tokens=20, total_tokens=30) - cost_map_entry: Final = {"litellm_provider": "openai", "mode": "chat", **self.PER_SECOND_PRICING} + cost_map_entry: Final = {"litellm_provider": "openai", "mode": "chat", **self.QUERY_ONLY_PRICING} litellm.register_model( model_cost={dated_model: cost_map_entry, requested_model: cost_map_entry}, persist_across_reloads=False ) @@ -646,6 +647,24 @@ class TestZeroCostDiagnostic: assert logging_obj.model_call_details["zero_cost_diagnostic"] is None assert self._zero_cost_warnings(caplog) == [] + def test_per_second_priced_deployment_bills_the_call_duration_and_stays_silent( + self, caplog: pytest.LogCaptureFixture + ) -> None: + per_second_id: Final = "lit8315-per-second-priced-deployment" + usage: Final = litellm.Usage(prompt_tokens=10, completion_tokens=20, total_tokens=30) + litellm.register_model(model_cost={per_second_id: self.PER_SECOND_PRICING}, persist_across_reloads=False) + try: + logging_obj: Final = self._logging_obj(self.PER_SECOND_PRICING, deployment_id=per_second_id) + response: Final = self._response(usage) + response._response_ms = 1000.0 + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + assert logging_obj._response_cost_calculator(result=response) == pytest.approx(0.00084) + + assert logging_obj.model_call_details["zero_cost_diagnostic"] is None + assert self._zero_cost_warnings(caplog) == [] + finally: + litellm.model_cost.pop(per_second_id, None) + @pytest.mark.parametrize("spilled_over", [True, False]) def test_ptu_deployment_is_judged_by_the_entry_the_calculator_priced_with( self, spilled_over: bool, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture @@ -663,7 +682,7 @@ class TestZeroCostDiagnostic: litellm.register_model( model_cost={ router_model_id: {**self.FREE_PRICING, "litellm_provider": "azure", "mode": "chat"}, - served_model: {**self.PER_SECOND_PRICING, "litellm_provider": "azure", "mode": "chat"}, + served_model: {**self.QUERY_ONLY_PRICING, "litellm_provider": "azure", "mode": "chat"}, }, persist_across_reloads=False, ) @@ -3504,6 +3523,20 @@ def _make_logging_obj(stream: bool) -> LitellmLogging: ) +def test_get_response_ms_measures_a_float_start_time_against_a_datetime_end_time(): + """The files paths construct the logging object with ``time.time()`` while the success + handler stamps a datetime end, and the per-second cost path reads this window.""" + logging_obj = _make_logging_obj(stream=False) + logging_obj.update_environment_variables( + model="openai/codex-mini-latest", user="", optional_params={}, litellm_params={} + ) + start_seconds = logging_obj.model_call_details["start_time"] + assert isinstance(start_seconds, float) + logging_obj.model_call_details["end_time"] = datetime.datetime.fromtimestamp(start_seconds + 1.5) + + assert logging_obj.get_response_ms() == pytest.approx(1500) + + def test_get_assembled_streaming_response_returns_none_for_non_streaming(): """Non-streaming requests should return None so the streaming block is skipped.""" import datetime diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index 84c60e029cb..bdeedb5f38c 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -1,3 +1,4 @@ +import datetime import time from typing import Final @@ -29,6 +30,7 @@ from litellm.types.utils import ( PromptTokensDetailsWrapper, Usage, ) +from litellm.types.videos.main import VideoObject @pytest.fixture @@ -3038,21 +3040,23 @@ def test_completion_cost_logs_cache_and_reasoning_breakdown_for_custom_pricing() assert total == pytest.approx(100 * 1e-6 + 800 * 1e-7 + 100 * 1.25e-6 + 500 * 2e-6) -def test_cost_per_token_per_second_pricing(monkeypatch): +@pytest.mark.parametrize("custom_llm_provider", ["together_ai", "openai", "anthropic", "bedrock", "azure"]) +def test_cost_per_token_per_second_pricing(monkeypatch, custom_llm_provider: str): """ Models priced by duration (input/output_cost_per_second) with no per-token rates - must be billed as cost_per_second * response_time_ms / 1000 in cost_per_token. + must be billed as cost_per_second * response_time_ms / 1000 in cost_per_token, + whether or not the provider has its own cost calculator. """ monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) - model = "test-per-second-pricing-model" + model = f"test-per-second-pricing-{custom_llm_provider}" litellm.register_model( model_cost={ model: { "input_cost_per_second": 0.02, "output_cost_per_second": 0.04, - "litellm_provider": "together_ai", + "litellm_provider": custom_llm_provider, "mode": "chat", } } @@ -3060,7 +3064,7 @@ def test_cost_per_token_per_second_pricing(monkeypatch): prompt_cost, completion_cost_value = cost_per_token( model=model, - custom_llm_provider="together_ai", + custom_llm_provider=custom_llm_provider, prompt_tokens=10, completion_tokens=20, response_time_ms=1500.0, @@ -3070,6 +3074,143 @@ def test_cost_per_token_per_second_pricing(monkeypatch): assert completion_cost_value == pytest.approx(0.04 * 1.5) +def test_cost_per_token_keeps_token_pricing_when_per_second_rates_are_also_set(monkeypatch): + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + + model = "test-token-and-per-second-pricing-model" + litellm.register_model( + model_cost={ + model: { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "input_cost_per_second": 0.02, + "output_cost_per_second": 0.04, + "litellm_provider": "openai", + "mode": "chat", + } + } + ) + + prompt_cost, completion_cost_value = cost_per_token( + model=model, + custom_llm_provider="openai", + prompt_tokens=10, + completion_tokens=20, + response_time_ms=1500.0, + ) + + assert prompt_cost == pytest.approx(10 * 1e-6) + assert completion_cost_value == pytest.approx(20 * 2e-6) + + +def _logging_obj_with_call_window(duration_ms: float) -> Logging: + start_time: Final = datetime.datetime(2026, 9, 21, 12, 0, 0) + logging_obj: Final = Logging( + model="gpt-5.4-nano", + messages=[{"role": "user", "content": "Hello"}], + stream=False, + call_type="completion", + start_time=start_time, + litellm_call_id="per-second-call-window", + function_id="f", + ) + logging_obj.model_call_details["start_time"] = start_time + logging_obj.model_call_details["end_time"] = start_time + datetime.timedelta(milliseconds=duration_ms) + return logging_obj + + +@pytest.mark.parametrize( + ("stamped_response_ms", "total_time", "logged_duration_ms", "expected_seconds"), + [(None, 0.0, 1500.0, 1.5), (3000.0, 0.0, 1500.0, 3.0), (None, 2500.0, 1500.0, 2.5), (3000.0, 2500.0, 1500.0, 3.0)], +) +def test_completion_cost_per_second_deployment_bills_the_call_duration( + monkeypatch, + stamped_response_ms: float | None, + total_time: float, + logged_duration_ms: float, + expected_seconds: float, +): + """ + A deployment priced only per second bills the stamped ``_response_ms`` when there is one, + then the caller's explicit ``total_time``, and the logging object's start/end window otherwise + (a streamed response is never stamped). + """ + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + + deployment_id = "per-second-openai-deployment" + litellm.register_model( + model_cost={ + deployment_id: { + "input_cost_per_second": 0.02, + "output_cost_per_second": 0.04, + "litellm_provider": "openai", + "mode": "chat", + } + } + ) + response = ModelResponse( + model="gpt-5.4-nano", + usage=Usage(prompt_tokens=11, completion_tokens=7, total_tokens=18), + ) + response._response_ms = stamped_response_ms + + cost = completion_cost( + completion_response=response, + model="openai/gpt-5.4-nano", + custom_llm_provider="openai", + custom_pricing=True, + router_model_id=deployment_id, + total_time=total_time, + litellm_logging_obj=_logging_obj_with_call_window(logged_duration_ms), + ) + + assert cost == pytest.approx((0.02 + 0.04) * expected_seconds) + + +@pytest.mark.parametrize("mode", ["audio_transcription", "audio_speech", "video_generation", "realtime"]) +def test_cost_per_token_leaves_media_second_rates_to_their_dedicated_paths(monkeypatch, mode: str): + """ + A media-mode entry's per-second rates price audio or video seconds, which the dedicated + transcription, speech, video, and realtime paths bill from the media itself, so a call that + reaches the generic path with only a wall-clock duration must not bill them. + """ + model = f"test-media-per-second-{mode}" + monkeypatch.setitem( + litellm.model_cost, + model, + {"input_cost_per_second": 0.02, "output_cost_per_second": 0.4, "litellm_provider": "openai", "mode": mode}, + ) + + assert cost_per_token(model=model, custom_llm_provider="openai", response_time_ms=2000.0) == (0.0, 0.0) + + +def test_completion_cost_video_status_poll_bills_nothing_on_a_per_second_video_model(monkeypatch): + """ + Polling a video job returns a ``VideoObject`` with no stamped duration, so the cost path falls + back to the logging object's call window; on a video model priced per output second that + window must not be billed, or every status poll would charge for the seconds it took to answer. + """ + model = "test-veo-per-second-poll" + monkeypatch.setitem( + litellm.model_cost, + model, + {"output_cost_per_second": 0.4, "litellm_provider": "vertex_ai", "mode": "video_generation"}, + ) + video = VideoObject(id="video_1", object="video", status="completed", model=model, progress=100) + + cost = completion_cost( + completion_response=video, + model=model, + custom_llm_provider="vertex_ai", + call_type=CallTypes.video_retrieve.value, + litellm_logging_obj=_logging_obj_with_call_window(2000.0), + ) + + assert cost == 0.0 + + def _batch_cache_usage() -> Usage: return Usage( prompt_tokens=11000, diff --git a/tests/unit/litellm_core_utils/llm_response_utils/test_response_metadata.py b/tests/unit/litellm_core_utils/llm_response_utils/test_response_metadata.py index 50409b2ea2c..37201e8155b 100644 --- a/tests/unit/litellm_core_utils/llm_response_utils/test_response_metadata.py +++ b/tests/unit/litellm_core_utils/llm_response_utils/test_response_metadata.py @@ -7,10 +7,12 @@ through _hidden_params to the x-litellm-callback-duration-ms response header. import asyncio import datetime +from typing import Final from unittest.mock import MagicMock import pytest +import litellm import litellm.litellm_core_utils.llm_response_utils.response_metadata as response_metadata_mod import litellm.proxy.common_request_processing as common_request_processing_mod from litellm.litellm_core_utils.litellm_logging import Logging @@ -22,7 +24,7 @@ from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( ) from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing -from litellm.types.utils import ModelResponse +from litellm.types.utils import ModelResponse, Usage class TestCallbackDurationMs: @@ -583,3 +585,57 @@ class TestLoggingInitCallbackDuration: # Should still be set (deep copy of None is essentially a no-op) assert hasattr(obj, "callback_duration_ms") assert obj.callback_duration_ms >= 0 + + +def test_update_response_metadata_prices_per_second_deployment_from_its_stamped_duration(monkeypatch): + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + + deployment_id: Final = "per-second-deployment-response-metadata" + litellm.register_model( + model_cost={ + deployment_id: { + "input_cost_per_second": 0.02, + "output_cost_per_second": 0.04, + "litellm_provider": "openai", + "mode": "chat", + } + } + ) + start_time: Final = datetime.datetime(2026, 9, 21, 12, 0, 0) + logging_obj: Final = Logging( + model="gpt-5.4-nano", + messages=[{"role": "user", "content": "Hello"}], + stream=False, + call_type="completion", + start_time=start_time, + litellm_call_id="per-second-response-metadata", + function_id="f", + ) + logging_obj.update_environment_variables( + model="gpt-5.4-nano", + litellm_params={ + "input_cost_per_second": 0.02, + "output_cost_per_second": 0.04, + "metadata": {"model_info": {"id": deployment_id}}, + }, + optional_params={}, + custom_llm_provider="openai", + ) + logging_obj.model_call_details["end_time"] = start_time + datetime.timedelta(seconds=10) + result: Final = ModelResponse( + model="gpt-5.4-nano", + usage=Usage(prompt_tokens=11, completion_tokens=7, total_tokens=18), + ) + + update_response_metadata( + result=result, + logging_obj=logging_obj, + model="gpt-5.4-nano", + kwargs={"model_info": {"id": deployment_id}}, + start_time=start_time, + end_time=start_time + datetime.timedelta(seconds=2), + ) + + assert result._response_ms == pytest.approx(2000) + assert result._hidden_params["response_cost"] == pytest.approx((0.02 + 0.04) * 2)