diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index f9ce9e5dcb8..99f79c0b272 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -10,9 +10,9 @@ **Please complete all items before asking a LiteLLM maintainer to review your PR** -- [ ] I have Added testing in the [`tests/test_litellm/`](https://github.com/BerriAI/litellm/tree/main/tests/test_litellm) directory, **Adding at least 1 test is a hard requirement** - [see details](https://docs.litellm.ai/docs/extras/contributing_code) +- [ ] I have added meaningful tests - [ ] My PR passes all unit tests on [`make test-unit`](https://docs.litellm.ai/docs/extras/contributing_code) -- [ ] My PR's scope is as isolated as possible, it only solves 1 specific problem +- [ ] My PR's scope is as isolated as possible; it only solves 1 specific problem - [ ] I have requested a Greptile review by commenting `@greptileai` and received a **Confidence Score of at least 4/5** before requesting a maintainer review ## Delays in PR merge? diff --git a/.github/workflows/test_server_root_path.yml b/.github/workflows/test_server_root_path.yml index 155445acdf6..57ff746c9c8 100644 --- a/.github/workflows/test_server_root_path.yml +++ b/.github/workflows/test_server_root_path.yml @@ -101,6 +101,31 @@ jobs: docker logs litellm-test exit 1 + - name: Setup Node for Playwright + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: "20" + + - name: Install UI deps and Chromium + working-directory: ui/litellm-dashboard + run: | + npm ci + npx playwright install --with-deps chromium + + - name: Run SERVER_ROOT_PATH redirect e2e + working-directory: ui/litellm-dashboard + env: + SERVER_ROOT_PATH: ${{ matrix.root_path }} + run: npx playwright test --config=e2e_tests/serverRootPath.config.ts + + - name: Upload Playwright artifacts on failure + if: failure() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: playwright-trace-${{ strategy.job-index }} + path: ui/litellm-dashboard/test-results/ + retention-days: 7 + - name: Cleanup if: always() run: | diff --git a/AGENTS.md b/AGENTS.md index e99bf79d783..41921fdff4d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,293 +1 @@ -# INSTRUCTIONS FOR LITELLM - -This document provides comprehensive instructions for AI agents working in the LiteLLM repository. - -## OVERVIEW - -LiteLLM is a unified interface for 100+ LLMs that: -- Translates inputs to provider-specific completion, embedding, and image generation endpoints -- Provides consistent OpenAI-format output across all providers -- Includes retry/fallback logic across multiple deployments (Router) -- Offers a proxy server (LLM Gateway) with budgets, rate limits, and authentication -- Supports advanced features like function calling, streaming, caching, and observability - -## REPOSITORY STRUCTURE - -### Core Components -- `litellm/` - Main library code - - `llms/` - Provider-specific implementations (OpenAI, Anthropic, Azure, etc.) - - `proxy/` - Proxy server implementation (LLM Gateway) - - `router_utils/` - Load balancing and fallback logic - - `types/` - Type definitions and schemas - - `integrations/` - Third-party integrations (observability, caching, etc.) - -### Key Directories -- `tests/` - Comprehensive test suites -- `ui/litellm-dashboard/` - Admin dashboard UI -- `enterprise/` - Enterprise-specific features - -Documentation lives in the separate [BerriAI/litellm-docs](https://github.com/BerriAI/litellm-docs) repository and is served at [docs.litellm.ai](https://docs.litellm.ai). - -## DEVELOPMENT GUIDELINES - -### MAKING CODE CHANGES - -1. **Provider Implementations**: When adding/modifying LLM providers: - - Follow existing patterns in `litellm/llms/{provider}/` - - Implement proper transformation classes that inherit from `BaseConfig` - - Support both sync and async operations - - Handle streaming responses appropriately - - Include proper error handling with provider-specific exceptions - -2. **Type Safety**: - - Use proper type hints throughout - - Update type definitions in `litellm/types/` - - Ensure compatibility with both Pydantic v1 and v2 - -3. **Testing**: - - Add tests in appropriate `tests/` subdirectories - - Include both unit tests and integration tests - - Test provider-specific functionality thoroughly - - Consider adding load tests for performance-critical changes - -### MAKING CODE CHANGES FOR THE UI (IGNORE FOR BACKEND) - -1. **Always use `antd` for new UI components — Tremor is DEPRECATED** - - We are migrating off of `@tremor/react`. Do not introduce new `Badge`, `Text`, `Card`, `Grid`, `Title`, or other imports from `@tremor/react` in any new or modified file. - - Use `antd` equivalents: `Tag` for labels, plain ``/`
` with Tailwind classes (or `Typography.Text`) for text, `Card` from `antd`, etc. Note that `antd` has no `"yellow"` Tag color — use `"gold"` for amber/yellow. - - The only exception is the Tremor Table component and its required Tremor Table sub components. - -2. **Use Common Components as much as possible**: - - These are usually defined in the `common_components` directory - - Use these components as much as possible and avoid building new components unless needed - -3. **Testing**: - - The codebase uses **Vitest** and **React Testing Library** - - **Query Priority Order**: Use query methods in this order: `getByRole`, `getByLabelText`, `getByPlaceholderText`, `getByText`, `getByTestId` - - **Always use `screen`** instead of destructuring from `render()` (e.g., use `screen.getByText()` not `getByText`) - - **Wrap user interactions in `act()`**: Always wrap `fireEvent` calls with `act()` to ensure React state updates are properly handled - - **Use `query` methods for absence checks**: Use `queryBy*` methods (not `getBy*`) when expecting an element to NOT be present - - **Test names must start with "should"**: All test names should follow the pattern `it("should ...")` - - **Mock external dependencies**: Check `setupTests.ts` for global mocks and mock child components/networking calls as needed - - **Structure tests properly**: - - First test should verify the component renders successfully - - Subsequent tests should focus on functionality and user interactions - - Use `waitFor` for async operations that aren't already awaited - - **Avoid using `querySelector`**: Prefer React Testing Library queries over direct DOM manipulation - -### IMPORTANT PATTERNS - -1. **Function/Tool Calling**: - - LiteLLM standardizes tool calling across providers - - OpenAI format is the standard, with transformations for other providers - - See `litellm/llms/anthropic/chat/transformation.py` for complex tool handling - -2. **Streaming**: - - All providers should support streaming where possible - - Use consistent chunk formatting across providers - - Handle both sync and async streaming - -3. **Error Handling**: - - Use provider-specific exception classes - - Maintain consistent error formats across providers - - Include proper retry logic and fallback mechanisms - -4. **Configuration**: - - Support both environment variables and programmatic configuration - - Use `BaseConfig` classes for provider configurations - - Allow dynamic parameter passing - -## PROXY SERVER (LLM GATEWAY) - -The proxy server is a critical component that provides: -- Authentication and authorization -- Rate limiting and budget management -- Load balancing across multiple models/deployments -- Observability and logging -- Admin dashboard UI -- Enterprise features - -Key files: -- `litellm/proxy/proxy_server.py` - Main server implementation -- `litellm/proxy/auth/` - Authentication logic -- `litellm/proxy/management_endpoints/` - Admin API endpoints - -**Database (proxy)**: Use Prisma model methods (`prisma_client.db..upsert`, `.find_many`, `.find_unique`, etc.), not raw SQL (`execute_raw`/`query_raw`). See COMMON PITFALLS for details. - -## MCP (MODEL CONTEXT PROTOCOL) SUPPORT - -LiteLLM supports MCP for agent workflows: -- MCP server integration for tool calling -- Transformation between OpenAI and MCP tool formats -- Support for external MCP servers (Zapier, Jira, Linear, etc.) -- See `litellm/experimental_mcp_client/` and `litellm/proxy/_experimental/mcp_server/` - -## RUNNING SCRIPTS - -Use `uv run python script.py` to run Python scripts in the project environment (for non-test files). - -## GITHUB TEMPLATES - -When opening issues or pull requests, follow these templates: - -### Bug Reports (`.github/ISSUE_TEMPLATE/bug_report.yml`) -- Describe what happened vs. expected behavior -- Include relevant log output -- Specify LiteLLM version -- Indicate if you're part of an ML Ops team (helps with prioritization) - -### Feature Requests (`.github/ISSUE_TEMPLATE/feature_request.yml`) -- Clearly describe the feature -- Explain motivation and use case with concrete examples - -### Pull Requests (`.github/pull_request_template.md`) -- Add at least 1 test in `tests/litellm/` -- Ensure `make test-unit` passes - - -## TESTING CONSIDERATIONS - -1. **Provider Tests**: Test against real provider APIs when possible -2. **Proxy Tests**: Include authentication, rate limiting, and routing tests -3. **Performance Tests**: Load testing for high-throughput scenarios -4. **Integration Tests**: End-to-end workflows including tool calling - -## DOCUMENTATION - -- Keep documentation in sync with code changes -- Update provider documentation when adding new providers -- Include code examples for new features -- Update changelog and release notes - -## SECURITY CONSIDERATIONS - -- Handle API keys securely -- Validate all inputs, especially for proxy endpoints -- Consider rate limiting and abuse prevention -- Follow security best practices for authentication - -## ENTERPRISE FEATURES - -- Some features are enterprise-only -- Check `enterprise/` directory for enterprise-specific code -- Maintain compatibility between open-source and enterprise versions - -## COMMON PITFALLS TO AVOID - -1. **Breaking Changes**: LiteLLM has many users - avoid breaking existing APIs -2. **Provider Specifics**: Each provider has unique quirks - handle them properly -3. **Rate Limits**: Respect provider rate limits in tests -4. **Memory Usage**: Be mindful of memory usage in streaming scenarios -5. **Dependencies**: Keep dependencies minimal and well-justified -6. **UI/Backend Contract Mismatch**: When adding a new entity type to the UI, always check whether the backend endpoint accepts a single value or an array. Match the UI control accordingly (single-select vs. multi-select) to avoid silently dropping user selections -7. **Missing Tests for New Entity Types**: When adding a new entity type (e.g., in `EntityUsage`, `UsageViewSelect`), always add corresponding tests in the existing test files and update any icon/component mocks -8. **Raw SQL in proxy DB code**: Do not use `execute_raw` or `query_raw` for proxy database access. Use Prisma model methods (e.g. `prisma_client.db.litellm_tooltable.upsert()`, `.find_many()`, `.find_unique()`) so behavior stays consistent with the schema, the client stays mockable in tests, and you avoid the pitfalls of hand-written SQL (parameter ordering, type casting, schema drift) - -8. **Do not hardcode model-specific flags**: Put model-specific capability flags in `model_prices_and_context_window.json` and read them via `get_model_info` (or existing helpers like `supports_reasoning`). This prevents users from needing to upgrade LiteLLM each time a new model supports a feature. - - **Example of BAD** (hardcoded model checks): - - ```python - @staticmethod - def _is_effort_supported_model(model: str) -> bool: - """Check if the model supports the output_config.effort parameter...""" - model_lower = model.lower() - if AnthropicConfig._is_claude_4_6_model(model): - return True - return any( - v in model_lower for v in ("opus-4-5", "opus_4_5", "opus-4.5", "opus_4.5") - ) - ``` - - **Example of GOOD** (config-driven or helper that reads from config): - - ```python - if ( - "claude-3-7-sonnet" in model - or AnthropicConfig._is_claude_4_6_model(model) - or supports_reasoning( - model=model, - custom_llm_provider=self.custom_llm_provider, - ) - ): - ... - ``` - - Using helpers like `supports_reasoning` (which read from `model_prices_and_context_window.json` / `get_model_info`) allows future model updates to "just work" without code changes. - -9. **Never close HTTP/SDK clients on cache eviction**: Do not add `close()`, `aclose()`, or `create_task(close_fn())` inside `LLMClientCache._remove_key()` or any cache eviction path. Evicted clients may still be held by in-flight requests; closing them causes `RuntimeError: Cannot send a request, as the client has been closed.` in production after the cache TTL (1 hour) expires. Connection cleanup is handled at shutdown by `close_litellm_async_clients()`. See PR #22247 for the full incident history. - -## HELPFUL RESOURCES - -- Main documentation: https://docs.litellm.ai/ (source: [BerriAI/litellm-docs](https://github.com/BerriAI/litellm-docs)) -- Provider-specific docs: https://docs.litellm.ai/docs/providers/ -- Admin UI for testing proxy features - -## WHEN IN DOUBT - -- Follow existing patterns in the codebase -- Check similar provider implementations -- Ensure comprehensive test coverage -- Update documentation appropriately -- Consider backward compatibility impact - -## Cursor Cloud specific instructions - -### Environment - -- uv is installed in `~/.local/bin`; the update script ensures it is on `PATH`. -- Python 3.12, Node 22 are pre-installed. -- The project virtual environment lives under `.venv/`. - -### Running the proxy server - -Create a minimal config file and start the proxy: - -```yaml -# config.yaml -model_list: - - model_name: fake-openai-endpoint - litellm_params: - model: openai/fake-model - api_key: fake-key - api_base: https://fake-api.example.com - -general_settings: - master_key: sk-1234 - -litellm_settings: - drop_params: True - telemetry: False -``` - -```bash -uv run litellm --config config.yaml --port 4000 -``` - -The proxy takes ~15-20 seconds to fully start (it runs Prisma migrations on boot). Wait for `/health` to return before sending requests. Without a PostgreSQL `DATABASE_URL`, the proxy connects to a default Neon dev database embedded in the `litellm-proxy-extras` package. - -### Running tests - -See `CLAUDE.md` and the `Makefile` for standard commands. Key notes: - -- `uv sync --group proxy-dev --extra proxy` installs the Prisma and proxy-side test dependencies used by the standard local workflow. -- The `--timeout` pytest flag is NOT available; don't pass it. -- Unit tests: `uv run pytest tests/test_litellm/ -x -vv -n 4` -- **Before committing, always run `uv run black .` to format your code.** Black formatting is enforced in CI. -- If `uv sync` fails because the lockfile is outdated, run `uv lock` and retry. - -### Lint - -```bash -cd litellm && uv run ruff check . -``` - -Ruff is the primary fast linter. For the full lint suite (including mypy, black, circular imports), run `make lint` per `CLAUDE.md`. - -### UI Dashboard development - -- The UI is at `ui/litellm-dashboard/`. Run `npm run dev` from that directory for the Next.js dev server on port 3000. -- The proxy at port 4000 serves a **pre-built** static UI from `litellm/proxy/_experimental/out/`. After making UI code changes, you must run `npm run build` in the dashboard directory and copy the output: `cp -r ui/litellm-dashboard/out/* litellm/proxy/_experimental/out/` for the proxy to serve the updated UI. -- SVGs used as provider logos (loaded via `` tags) must NOT use `fill="currentColor"` — replace with an explicit color like `#000000` or use the `-color` variant from lobehub icons, since CSS color inheritance does not work inside `` elements. -- Provider logos live in `ui/litellm-dashboard/public/assets/logos/` (source) and `litellm/proxy/_experimental/out/assets/logos/` (pre-built). Both locations must have the file for it to work in dev and proxy-served modes. -- UI Vitest tests: `cd ui/litellm-dashboard && npx vitest run` +Read @CLAUDE.md for coding guidelines diff --git a/CLAUDE.md b/CLAUDE.md index baf23c90148..3477b71a621 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,181 +1,70 @@ -# CLAUDE.md +Do not write comments unless they are absolutely necessary to explain some very complex business logic. Please clean up if there are comments that are not absolutely necessary. Do not remove comments that are unrelated to the addition of the code of this PR -This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. +Explanation: code comments are, in a way, a violation of DRY code. You must update logic in two locations to change the code and "hard to change" is literally the definition of tech debt. We should instead aim to write code that is intuitive to the reader, while being both easy to maintain and high performance -## Documentation +Don't assume that the existing code is correct or the right way of doing things / good coding patterns. In fact, there are a lot of bad coding practices, overly complex code, code smells, etc. If something doesn't look right, speak up. Feel free to break existing patterns or question weird existing code to make new code high quality, as in: +- correct +- secure +- performant +- readable +- easy to maintain/change +- modern +In that order of importance -Documentation lives in a separate repository: [BerriAI/litellm-docs](https://github.com/BerriAI/litellm-docs). It is served at [docs.litellm.ai](https://docs.litellm.ai). Do not create or edit documentation files in this repository — open doc PRs against `BerriAI/litellm-docs` instead. +When adding new features, add meaningful tests. Don't add tests that don't check anything substantial and is there just to make the code coverage pass. Yes, code coverage is important, but I'd rather have no signal whether the code is working than tests that don't fail when code is broken. The goal is to have tests that would fail before the feature was added/if the code was mutated in a way that breaks the feature and succeed only when the feature is fully working. I should run mutation testing and see > 90% kill rate -## Development Commands +Same thing for bug fixes. The tests should make it so that this specific bug can never happen again without failing tests (i.e., regression) -### Installation -- `make install-dev` - Install core development dependencies -- `make install-proxy-dev` - Install proxy development dependencies with full feature set -- `make install-test-deps` - Install the full local test environment and generate the Prisma client +When creating PRs, don't set base to `main`. `litellm_internal_staging` serves that purpose -### Testing -- `make test` - Run all tests -- `make test-unit` - Run unit tests (tests/test_litellm) with 4 parallel workers -- `make test-integration` - Run integration tests (excludes unit tests) -- `pytest tests/` - Direct pytest execution +Always use @.github/pull_request_template.md as a guide for your PR body -### Code Quality -- `make lint` - Run all linting (Ruff, MyPy, Black, circular imports, import safety) -- `make format` - Apply Black code formatting -- `make lint-ruff` - Run Ruff linting only -- `make lint-mypy` - Run MyPy type checking only -- **Before committing, always run `uv run black .` to format your code.** Black formatting is enforced in CI. +Never use `pytest` commands or the like as "Screenshots / Proof of Fix". We prefer curl'ing a live proxy instance running on localhost:4000 (I like to run it with `python litellm/proxy/proxy_cli.py --config litellm/proxy/dev_config.yaml --detailed_debug --reload --use_v2_migration_resolver 2>&1 | tee litellm.log`) and showing both the command run and the output. Also, it should hit real LLM provider APIs, not mocks, and cost real $$$ because that is the most realistic test. The proof of fix should be exactly what the end user / customer would see / do. The run logs in PR #27703 is a prime example of how to do it (not a huge fan of using a python test script that future me and the team will have no visibility into; I prefer just curl commands or a short list of bash commands (e.g., using `for`)). If it's a UI thing, just tell me which URLs to go to (e.g., http://localhost:4000/ui/?page=logs), where to click, what fields to fill out, etc. along with the other commands to run in an ordered list, and I'll do it myself and post the screenshots after you make the PR -### Single Test Files -- `uv run pytest tests/path/to/test_file.py -v` - Run specific test file -- `uv run pytest tests/path/to/test_file.py::test_function -v` - Run specific test +If you ever make public-facing PR descriptions, comments, issues, commit messages, etc., always follow these guidelines to sound less AI-y: +- don't use emojis +- don't use "—". Instead, reach for ";", ".", etc. +- don't use the pattern "It's not X, it's Y", "You're not X, you're Y", etc. +- don't use bulleted or numbered lists unless it would be nonsensical not to. Instead, prefer prose -### Running Scripts -- `uv run python script.py` - Run Python scripts (use for non-test files) +Don't hesitate to use values in .env to get needed API keys and other secrets, as long as you never add them to conversation history, commit them, or include them in GitHub issues / PRs -### GitHub Issue & PR Templates -When contributing to the project, use the appropriate templates: +Run tests, format your code, and lint your code before each commit -**Bug Reports** (`.github/ISSUE_TEMPLATE/bug_report.yml`): -- Describe what happened vs. what you expected -- Include relevant log output -- Specify your LiteLLM version +Ask to commit and push your work when you're done (or if you're confident that your code is good and works, just do it) -**Feature Requests** (`.github/ISSUE_TEMPLATE/feature_request.yml`): -- Describe the feature clearly -- Explain the motivation and use case +When you must use real LLM models to, for example, write e2e tests, write a QA runbook, etc., make sure to use the latest models (doesn't have to be smartest, can also be a modern small, fast one. No strong preference for smart vs fast here, just use something modern) as of the year and month of the current date. Do a web search as necessary to figure that out -**Pull Requests** (`.github/pull_request_template.md`): -- Add at least 1 test in `tests/litellm/` -- Ensure `make test-unit` passes +If you're an internal contributor, when creating a new PR, the typical flow is to branch off litellm_internal_staging and create a branch prefixed with litellm_. Do not create a branch prefixed with claude/ and generally do not have / in your branch names -## Architecture Overview +Do not add `Co-Authored-By: Claude` or any Claude attribution to commit messages. Never use a `claude/` prefix or put a `/` in a branch name. Do not add "Generated with Claude Code" (or any similar attribution) to PR descriptions. Do not create a new PR/branch off the existing PR to fix/add something that is related and could've just been committed directly to the existing PR's branch -LiteLLM is a unified interface for 100+ LLM providers with two main components: +When working on a PR, keep the PR description in sync with new commits being made -### Core Library (`litellm/`) -- **Main entry point**: `litellm/main.py` - Contains core completion() function -- **Provider implementations**: `litellm/llms/` - Each provider has its own subdirectory -- **Router system**: `litellm/router.py` + `litellm/router_utils/` - Load balancing and fallback logic -- **Type definitions**: `litellm/types/` - Pydantic models and type hints -- **Integrations**: `litellm/integrations/` - Third-party observability, caching, logging -- **Caching**: `litellm/caching/` - Multiple cache backends (Redis, in-memory, S3, etc.) +Monkeypatching attributes of a class to do testing is an anti-pattern. Prefer dependency-injecting things into classes. That way, at unit test time, you can pass a mocked dependency in -### Proxy Server (`litellm/proxy/`) -- **Main server**: `proxy_server.py` - FastAPI application -- **Authentication**: `auth/` - API key management, JWT, OAuth2 -- **Database**: `db/` - Prisma ORM with PostgreSQL/SQLite support -- **Management endpoints**: `management_endpoints/` - Admin APIs for keys, teams, models -- **Pass-through endpoints**: `pass_through_endpoints/` - Provider-specific API forwarding -- **Guardrails**: `guardrails/` - Safety and content filtering hooks -- **UI Dashboard**: Served from `_experimental/out/` (Next.js build) +Do not put names of customers or customer company names in code, PRs, and issues. The codebase is public -## Key Patterns +CI supply-chain safety: Never pipe a remote script into a shell (`curl ... | bash`, `wget ... | sh`); download the artifact to a file, verify its SHA-256 checksum, then install. Pin every external tool to a specific version with a full URL (not `latest` or `stable`). Verify checksums for all downloaded binaries, using the provider's official `.sha256` / `.sha256sum` sidecar when available. These rules apply to every download in CI -### Provider Implementation -- Providers inherit from base classes in `litellm/llms/base.py` -- Each provider has transformation functions for input/output formatting -- Support both sync and async operations -- Handle streaming responses and function calling +## Think Before Coding -### Error Handling -- Provider-specific exceptions mapped to OpenAI-compatible errors -- Fallback logic handled by Router system -- Comprehensive logging through `litellm/_logging.py` +**Don't assume. Don't hide confusion. Surface tradeoffs.** -### Configuration -- YAML config files for proxy server (see `proxy/example_config_yaml/`) -- Environment variables for API keys and settings -- Database schema managed via Prisma (`proxy/schema.prisma`) +Before implementing: +- State your assumptions explicitly. If uncertain, ask. +- If multiple interpretations exist, present them. Don't pick silently. +- If a simpler approach exists, say so. Push back when warranted. +- If something is unclear, stop. Name what's confusing. Ask. -## Development Notes +## Simplicity First -### Code Style -- Uses Black formatter, Ruff linter, MyPy type checker -- Pydantic v2 for data validation -- Async/await patterns throughout -- Type hints required for all public APIs -- **Avoid imports within methods** — place all imports at the top of the file (module-level). Inline imports inside functions/methods make dependencies harder to trace and hurt readability. The only exception is avoiding circular imports where absolutely necessary. -- **Use dict spread for immutable copies** — prefer `{**original, "key": new_value}` over `dict(obj)` + mutation. The spread produces the final dict in one step and makes intent clear. -- **Guard at resolution time** — when resolving an optional value through a fallback chain (`a or b or ""`), raise immediately if the resolved result being empty is an error. Don't pass empty strings or sentinel values downstream for the callee to deal with. -- **Extract complex comprehensions to named helpers** — a set/dict comprehension that calls into the DB or manager (e.g. "which of these server IDs are OAuth2?") belongs in a named helper function, not inline in the caller. -- **FastAPI parameter declarations** — mark required query/form params with `= Query(...)` / `= Form(...)` explicitly when other params in the same handler are optional. Mixing `str` (required) with `Optional[str] = None` in the same signature causes silent 422s when the required param is missing. +**Minimum code that solves the problem. Nothing speculative.** -### Testing Strategy -- Unit tests in `tests/test_litellm/` -- Integration tests for each provider in `tests/llm_translation/` -- Proxy tests in `tests/proxy_unit_tests/` -- Load tests in `tests/load_tests/` -- **Always add tests when adding new entity types or features** — if the existing test file covers other entity types, add corresponding tests for the new one -- **Keep monkeypatch stubs in sync with real signatures** — when a function gains a new optional parameter, update every `fake_*` / `stub_*` in tests that patch it to also accept that kwarg (even as `**kwargs`). Stale stubs fail with `unexpected keyword argument` and mask real bugs. -- **Test all branches of name→ID resolution** — when adding server/resource lookup that resolves names to UUIDs, test: (1) name resolves and UUID is allowed, (2) name resolves but UUID is not allowed, (3) name does not resolve at all. The silent-fallback path is where access-control bugs hide. +- No features beyond what was asked. +- No abstractions for single-use code. +- No "flexibility" or "configurability" that wasn't requested. +- No error handling for impossible scenarios. +- If you write 200 lines and it could be 50, rewrite it. -### UI / Backend Consistency -- When wiring a new UI entity type to an existing backend endpoint, verify the backend API contract (single value vs. array, required vs. optional params) and ensure the UI controls match — e.g., use a single-select dropdown when the backend accepts a single value, not a multi-select - -### UI Component Library -- **Always use `antd` for new UI components** — we are migrating off of `@tremor/react`. Do not introduce new `Badge`, `Text`, `Card`, `Grid`, `Title`, or other imports from `@tremor/react` in any new or modified file. Use `antd` equivalents: `Tag` for labels, `Typography.Text` / `Typography.Title` / `Typography.Paragraph` for textual content (avoid plain text-only ``, `

`, `` when Typography fits), and `Card` from `antd`. Note that `antd` has no `"yellow"` Tag color — use `"gold"` for amber/yellow. - -### MCP OAuth / OpenAPI Transport Mapping -- **`available_on_public_internet: false` with `delegate_auth_to_upstream: true` (oauth2, interactive — not `client_credentials`)** — LiteLLM still allows the anonymous upstream PKCE path (no proxy API key for `/authorize` and matching MCP routes). The internal-only flag mainly affects other surfaces (e.g. IP-based discovery). Rely on the upstream IdP and network policy; the dashboard shows a warning when both are set, and the proxy logs a warning when the server is loaded from config or the database. -- `TRANSPORT.OPENAPI` is a UI-only concept. The backend only accepts `"http"`, `"sse"`, or `"stdio"`. Always map it to `"http"` before any API call (including pre-OAuth temp-session calls). -- FastAPI validation errors return `detail` as an array of `{loc, msg, type}` objects. Error extractors must handle: array (map `.msg`), string, nested `{error: string}`, and fallback. -- When an MCP server already has `authorization_url` stored, skip OAuth discovery (`_discovery_metadata`) — the server URL for OpenAPI MCPs is the spec file, not the API base, and fetching it causes timeouts. -- `client_id` should be optional in the `/authorize` endpoint — if the server has a stored `client_id` in credentials, use that. Never require callers to re-supply it. - -### MCP Credential Storage -- OAuth credentials and BYOK credentials share the `litellm_mcpusercredentials` table, distinguished by a `"type"` field in the JSON payload (`"oauth2"` vs plain string). -- When deleting OAuth credentials, check type before deleting to avoid accidentally deleting a BYOK credential for the same `(user_id, server_id)` pair. -- Always pass the raw `expires_at` timestamp to the client — never set it to `None` for expired credentials. Let the frontend compute the "Expired" display state from the timestamp. -- Use `RecordNotFoundError` (not bare `except Exception`) when catching "already deleted" in credential delete endpoints. - -### Browser Storage Safety (UI) -- Never write LiteLLM access tokens or API keys to `localStorage` — use `sessionStorage` only. `localStorage` survives browser close and is readable by any injected script (XSS). -- Shared utility functions (e.g. `extractErrorMessage`) belong in `src/utils/` — never define them inline in hooks or duplicate them across files. - -### Database Migrations -- Prisma handles schema migrations -- Migration files auto-generated with `prisma migrate dev` -- Always test migrations against both PostgreSQL and SQLite - -### Proxy database access -- **Do not write raw SQL** for proxy DB operations. Use Prisma model methods instead of `execute_raw` / `query_raw`. -- Use the generated client: `prisma_client.db.` (e.g. `litellm_tooltable`, `litellm_usertable`) with `.upsert()`, `.find_many()`, `.find_unique()`, `.update()`, `.update_many()` as appropriate. This avoids schema/client drift, keeps code testable with simple mocks, and matches patterns used in spend logs and other proxy code. -- **No N+1 queries.** Never query the DB inside a loop. Batch-fetch with `{"in": ids}` and distribute in-memory. -- **Batch writes.** Use `create_many`/`update_many`/`delete_many` instead of individual calls (these return counts only; `update_many`/`delete_many` no-op silently on missing rows). When multiple separate writes target the same table (e.g. in `batch_()`), order by primary key to avoid deadlocks. -- **Push work to the DB.** Filter, sort, group, and aggregate in SQL, not Python. Verify Prisma generates the expected SQL — e.g. prefer `group_by` over `find_many(distinct=...)` which does client-side processing. -- **Bound large result sets.** Prisma materializes full results in memory. For results over ~10 MB, paginate with `take`/`skip` or `cursor`/`take`, always with an explicit `order`. Prefer cursor-based pagination (`skip` is O(n)). Don't paginate naturally small result sets. -- **Limit fetched columns on wide tables.** Use `select` to fetch only needed fields — returns a partial object, so downstream code must not access unselected fields. -- **Check index coverage.** For new or modified queries, check `schema.prisma` for a supporting index. Prefer extending an existing index (e.g. `@@index([a])` → `@@index([a, b])`) over adding a new one, unless it's a `@@unique`. Only add indexes for large/frequent queries. -- **Keep schema files in sync.** Apply schema changes to all `schema.prisma` copies (`schema.prisma`, `litellm/proxy/`, `litellm-proxy-extras/`) with a migration under `litellm-proxy-extras/litellm_proxy_extras/migrations/`. - -### Setup Wizard (`litellm/setup_wizard.py`) -- The wizard is implemented as a single `SetupWizard` class with `@staticmethod` methods — keep it that way. No module-level functions except `run_setup_wizard()` (the public entrypoint) and pure helpers (color, ANSI). -- Use `litellm.utils.check_valid_key(model, api_key)` for credential validation — never roll a custom completion call. -- Do not hardcode provider env-key names or model lists that already exist in the codebase. Add a `test_model` field to each provider entry to drive `check_valid_key`; set it to `None` for providers that can't be validated with a single API key (Azure, Bedrock, Ollama). - -### Enterprise Features -- Enterprise-specific code in `enterprise/` directory -- Optional features enabled via environment variables -- Separate licensing and authentication for enterprise features - -### CI Supply-Chain Safety -- **Never pipe a remote script into a shell** (`curl ... | bash`, `wget ... | sh`). Download the artifact to a file, verify its SHA-256 checksum, then install. -- **Pin every external tool to a specific version** with a full URL (not `latest` or `stable`). Unversioned downloads silently change under you. -- **Verify checksums for all downloaded binaries.** Use the provider's official `.sha256` / `.sha256sum` sidecar file when available; otherwise compute and hardcode the digest. -- **Prefer reusable CircleCI commands** (`commands:` section) so a tool is installed and verified in exactly one place, then referenced everywhere with `- install_` or `- wait_for_service`. -- **Don't add tools just because they were there before.** Audit whether an external dependency is still needed. If it can be replaced with a shell one-liner or a tool already in the image, remove it. -- These rules apply to every download in CI: binaries, install scripts, language version managers, package repos. No exceptions. - -### HTTP Client Cache Safety -- **Never close HTTP/SDK clients on cache eviction.** `LLMClientCache._remove_key()` must not call `close()`/`aclose()` on evicted clients — they may still be used by in-flight requests. Doing so causes `RuntimeError: Cannot send a request, as the client has been closed.` after the 1-hour TTL expires. Cleanup happens at shutdown via `close_litellm_async_clients()`. - -### Troubleshooting: DB schema out of sync after proxy restart -`litellm-proxy-extras` runs `prisma migrate deploy` on startup using **its own** bundled migration files, which may lag behind schema changes in the current worktree. Symptoms: `Unknown column`, `Invalid prisma invocation`, or missing data on new fields. - -**Diagnose:** Run `\d "TableName"` in psql and compare against `schema.prisma` — missing columns confirm the issue. - -**Fix options:** -1. **Create a Prisma migration** (permanent) — run `prisma migrate dev --name ` in the worktree. The generated file will be picked up by `prisma migrate deploy` on next startup. -2. **Apply manually for local dev** — `psql -d litellm -c "ALTER TABLE ... ADD COLUMN IF NOT EXISTS ..."` after each proxy start. Fine for dev, not for production. -3. **Update litellm-proxy-extras** — if the package is installed from PyPI, its migration directory must include the new file. Either update the package or run the migration manually until the next release ships it. +Ask yourself: "Would a senior engineer say this is overcomplicated?" If yes, simplify. diff --git a/GEMINI.md b/GEMINI.md index 9e950d89b33..41921fdff4d 100644 --- a/GEMINI.md +++ b/GEMINI.md @@ -1,108 +1 @@ -# GEMINI.md - -This file provides guidance to Gemini when working with code in this repository. - -## Development Commands - -### Installation -- `make install-dev` - Install core development dependencies -- `make install-proxy-dev` - Install proxy development dependencies with full feature set -- `make install-test-deps` - Install all test dependencies - -### Testing -- `make test` - Run all tests -- `make test-unit` - Run unit tests (tests/test_litellm) with 4 parallel workers -- `make test-integration` - Run integration tests (excludes unit tests) -- `pytest tests/` - Direct pytest execution - -### Code Quality -- `make lint` - Run all linting (Ruff, MyPy, Black, circular imports, import safety) -- `make format` - Apply Black code formatting -- `make lint-ruff` - Run Ruff linting only -- `make lint-mypy` - Run MyPy type checking only - -### Single Test Files -- `uv run pytest tests/path/to/test_file.py -v` - Run specific test file -- `uv run pytest tests/path/to/test_file.py::test_function -v` - Run specific test - -### Running Scripts -- `uv run python script.py` - Run Python scripts (use for non-test files) - -### GitHub Issue & PR Templates -When contributing to the project, use the appropriate templates: - -**Bug Reports** (`.github/ISSUE_TEMPLATE/bug_report.yml`): -- Describe what happened vs. what you expected -- Include relevant log output -- Specify your LiteLLM version - -**Feature Requests** (`.github/ISSUE_TEMPLATE/feature_request.yml`): -- Describe the feature clearly -- Explain the motivation and use case - -**Pull Requests** (`.github/pull_request_template.md`): -- Add at least 1 test in `tests/litellm/` -- Ensure `make test-unit` passes - -## Architecture Overview - -LiteLLM is a unified interface for 100+ LLM providers with two main components: - -### Core Library (`litellm/`) -- **Main entry point**: `litellm/main.py` - Contains core completion() function -- **Provider implementations**: `litellm/llms/` - Each provider has its own subdirectory -- **Router system**: `litellm/router.py` + `litellm/router_utils/` - Load balancing and fallback logic -- **Type definitions**: `litellm/types/` - Pydantic models and type hints -- **Integrations**: `litellm/integrations/` - Third-party observability, caching, logging -- **Caching**: `litellm/caching/` - Multiple cache backends (Redis, in-memory, S3, etc.) - -### Proxy Server (`litellm/proxy/`) -- **Main server**: `proxy_server.py` - FastAPI application -- **Authentication**: `auth/` - API key management, JWT, OAuth2 -- **Database**: `db/` - Prisma ORM with PostgreSQL/SQLite support -- **Management endpoints**: `management_endpoints/` - Admin APIs for keys, teams, models -- **Pass-through endpoints**: `pass_through_endpoints/` - Provider-specific API forwarding -- **Guardrails**: `guardrails/` - Safety and content filtering hooks -- **UI Dashboard**: Served from `_experimental/out/` (Next.js build) - -## Key Patterns - -### Provider Implementation -- Providers inherit from base classes in `litellm/llms/base.py` -- Each provider has transformation functions for input/output formatting -- Support both sync and async operations -- Handle streaming responses and function calling - -### Error Handling -- Provider-specific exceptions mapped to OpenAI-compatible errors -- Fallback logic handled by Router system -- Comprehensive logging through `litellm/_logging.py` - -### Configuration -- YAML config files for proxy server (see `proxy/example_config_yaml/`) -- Environment variables for API keys and settings -- Database schema managed via Prisma (`proxy/schema.prisma`) - -## Development Notes - -### Code Style -- Uses Black formatter, Ruff linter, MyPy type checker -- Pydantic v2 for data validation -- Async/await patterns throughout -- Type hints required for all public APIs - -### Testing Strategy -- Unit tests in `tests/test_litellm/` -- Integration tests for each provider in `tests/llm_translation/` -- Proxy tests in `tests/proxy_unit_tests/` -- Load tests in `tests/load_tests/` - -### Database Migrations -- Prisma handles schema migrations -- Migration files auto-generated with `prisma migrate dev` -- Always test migrations against both PostgreSQL and SQLite - -### Enterprise Features -- Enterprise-specific code in `enterprise/` directory -- Optional features enabled via environment variables -- Separate licensing and authentication for enterprise features +Read @CLAUDE.md for coding guidelines diff --git a/backend/Dockerfile b/backend/Dockerfile index c08014fc0ef..2cfdde8a517 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -12,17 +12,27 @@ USER root COPY --from=uvbin /uv /uvx /usr/local/bin/ -RUN apk add --no-cache bash gcc python3 python3-dev openssl openssl-dev libsndfile +# nodejs/npm so `prisma generate` uses Wolfi's Node via PRISMA_USE_GLOBAL_NODE +# instead of nodeenv downloading one whose dynamic deps may not be in Wolfi +# (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes. +RUN for i in 1 2 3; do \ + apk add --no-cache bash gcc python3 python3-dev openssl openssl-dev libsndfile nodejs npm && break; \ + [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \ + sleep 5; \ + done # UV_COMPILE_BYTECODE=1 precompiles .pyc at install time → faster cold start. # UV_LINK_MODE=copy avoids hardlink warnings when uv installs from a # BuildKit cache mount (different filesystem). # UV_PYTHON_DOWNLOADS=0 force uv to use the apk-installed CPython instead of # silently pulling a managed interpreter. +# PRISMA_USE_GLOBAL_NODE explicit (matches default) so an env override can't +# silently re-enable nodeenv's Node download. ENV UV_PROJECT_ENVIRONMENT=/app/.venv \ UV_LINK_MODE=copy \ UV_COMPILE_BYTECODE=1 \ UV_PYTHON_DOWNLOADS=0 \ + PRISMA_USE_GLOBAL_NODE=true \ PATH="/app/.venv/bin:${PATH}" # Stage 1 — install dependencies only. @@ -58,7 +68,11 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime USER root -RUN apk add --no-cache bash openssl tzdata python3 libsndfile libatomic +RUN for i in 1 2 3; do \ + apk add --no-cache bash openssl tzdata python3 libsndfile libatomic && break; \ + [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \ + sleep 5; \ + done # wolfi-base ships an unprivileged `nonroot` account (UID/GID 65532) with # /home/nonroot. We run the backend as that user diff --git a/cookbook/gollem_go_agent_framework/go.mod b/cookbook/gollem_go_agent_framework/go.mod index 89d9033aa22..a8dc9365d7f 100644 --- a/cookbook/gollem_go_agent_framework/go.mod +++ b/cookbook/gollem_go_agent_framework/go.mod @@ -1,5 +1,5 @@ module github.com/BerriAI/litellm/cookbook/gollem_go_agent_framework -go 1.25.1 +go 1.26.3 require github.com/fugue-labs/gollem v0.1.0 diff --git a/gateway/Dockerfile b/gateway/Dockerfile index a2ca3d3f83f..19c8a10fdfe 100644 --- a/gateway/Dockerfile +++ b/gateway/Dockerfile @@ -12,17 +12,27 @@ USER root COPY --from=uvbin /uv /uvx /usr/local/bin/ -RUN apk add --no-cache bash gcc python3 python3-dev openssl openssl-dev libsndfile +# nodejs/npm so `prisma generate` uses Wolfi's Node via PRISMA_USE_GLOBAL_NODE +# instead of nodeenv downloading one whose dynamic deps may not be in Wolfi +# (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes. +RUN for i in 1 2 3; do \ + apk add --no-cache bash gcc python3 python3-dev openssl openssl-dev libsndfile nodejs npm && break; \ + [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \ + sleep 5; \ + done # UV_COMPILE_BYTECODE=1 precompiles .pyc at install time → faster cold start. # UV_LINK_MODE=copy avoids hardlink warnings when uv installs from a # BuildKit cache mount (different filesystem). # UV_PYTHON_DOWNLOADS=0 force uv to use the apk-installed CPython instead of # silently pulling a managed interpreter. +# PRISMA_USE_GLOBAL_NODE explicit (matches default) so an env override can't +# silently re-enable nodeenv's Node download. ENV UV_PROJECT_ENVIRONMENT=/app/.venv \ UV_LINK_MODE=copy \ UV_COMPILE_BYTECODE=1 \ UV_PYTHON_DOWNLOADS=0 \ + PRISMA_USE_GLOBAL_NODE=true \ PATH="/app/.venv/bin:${PATH}" # Stage 1 — install dependencies only. @@ -58,7 +68,11 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime USER root -RUN apk add --no-cache bash openssl tzdata python3 libsndfile libatomic +RUN for i in 1 2 3; do \ + apk add --no-cache bash openssl tzdata python3 libsndfile libatomic && break; \ + [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \ + sleep 5; \ + done # wolfi-base ships an unprivileged `nonroot` account (UID/GID 65532) with # /home/nonroot. We run the proxy as that user. diff --git a/helm/litellm/templates/_helpers.tpl b/helm/litellm/templates/_helpers.tpl index e2faf42b766..4319907883e 100644 --- a/helm/litellm/templates/_helpers.tpl +++ b/helm/litellm/templates/_helpers.tpl @@ -56,16 +56,34 @@ app.kubernetes.io/component: ui {{- end -}} {{/* -Shared ServiceAccount name used by all three component Deployments. When -`serviceAccount.create` is true and `serviceAccount.name` is empty, default -to the chart fullname. When `create` is false, fall back to the provided -name or the namespace's `default` SA. +Per-component ServiceAccount name helpers. + +Each component (gateway, backend, ui) has its own SA config under +.Values.serviceAccounts.. When `create` is true and `name` is +empty the chart defaults to "-litellm-". When `create` +is false the chart uses the provided name, or the namespace `default` SA. */}} -{{- define "litellm.serviceAccountName" -}} -{{- if .Values.serviceAccount.create -}} -{{ default (include "litellm.fullname" .) .Values.serviceAccount.name }} +{{- define "litellm.gateway.serviceAccountName" -}} +{{- if .Values.serviceAccounts.gateway.create -}} +{{ default (include "litellm.gateway.fullname" .) .Values.serviceAccounts.gateway.name }} {{- else -}} -{{ default "default" .Values.serviceAccount.name }} +{{ default "default" .Values.serviceAccounts.gateway.name }} +{{- end -}} +{{- end -}} + +{{- define "litellm.backend.serviceAccountName" -}} +{{- if .Values.serviceAccounts.backend.create -}} +{{ default (include "litellm.backend.fullname" .) .Values.serviceAccounts.backend.name }} +{{- else -}} +{{ default "default" .Values.serviceAccounts.backend.name }} +{{- end -}} +{{- end -}} + +{{- define "litellm.ui.serviceAccountName" -}} +{{- if .Values.serviceAccounts.ui.create -}} +{{ default (include "litellm.ui.fullname" .) .Values.serviceAccounts.ui.name }} +{{- else -}} +{{ default "default" .Values.serviceAccounts.ui.name }} {{- end -}} {{- end -}} diff --git a/helm/litellm/templates/backend/deployment.yaml b/helm/litellm/templates/backend/deployment.yaml index e761409f8c4..3b59c58c8bf 100644 --- a/helm/litellm/templates/backend/deployment.yaml +++ b/helm/litellm/templates/backend/deployment.yaml @@ -19,7 +19,8 @@ spec: labels: {{- include "litellm.backend.selectorLabels" . | nindent 8 }} spec: - serviceAccountName: {{ include "litellm.serviceAccountName" . }} + serviceAccountName: {{ include "litellm.backend.serviceAccountName" . }} + automountServiceAccountToken: {{ .Values.serviceAccounts.backend.automount }} {{- with .Values.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} diff --git a/helm/litellm/templates/gateway/deployment.yaml b/helm/litellm/templates/gateway/deployment.yaml index 935d432342e..05ea4052159 100644 --- a/helm/litellm/templates/gateway/deployment.yaml +++ b/helm/litellm/templates/gateway/deployment.yaml @@ -22,7 +22,8 @@ spec: labels: {{- include "litellm.gateway.selectorLabels" . | nindent 8 }} spec: - serviceAccountName: {{ include "litellm.serviceAccountName" . }} + serviceAccountName: {{ include "litellm.gateway.serviceAccountName" . }} + automountServiceAccountToken: {{ .Values.serviceAccounts.gateway.automount }} {{- with .Values.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} diff --git a/helm/litellm/templates/migrations-job.yaml b/helm/litellm/templates/migrations-job.yaml index f3dc2ae0236..92671388546 100644 --- a/helm/litellm/templates/migrations-job.yaml +++ b/helm/litellm/templates/migrations-job.yaml @@ -28,7 +28,7 @@ spec: app.kubernetes.io/component: migrations spec: restartPolicy: Never - serviceAccountName: {{ include "litellm.serviceAccountName" . }} + serviceAccountName: {{ include "litellm.backend.serviceAccountName" . }} {{- with .Values.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} diff --git a/helm/litellm/templates/serviceaccount.yaml b/helm/litellm/templates/serviceaccount.yaml index 3c998448ae5..a2fc52f47c0 100644 --- a/helm/litellm/templates/serviceaccount.yaml +++ b/helm/litellm/templates/serviceaccount.yaml @@ -1,13 +1,51 @@ -{{- if .Values.serviceAccount.create -}} +{{- $prev := false -}} +{{- if .Values.serviceAccounts.gateway.create -}} +{{- $prev = true }} apiVersion: v1 kind: ServiceAccount metadata: - name: {{ include "litellm.serviceAccountName" . }} + name: {{ include "litellm.gateway.serviceAccountName" . }} labels: {{- include "litellm.commonLabels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} + app.kubernetes.io/component: gateway + {{- with .Values.serviceAccounts.gateway.annotations }} annotations: {{- toYaml . | nindent 4 }} {{- end }} -automountServiceAccountToken: {{ .Values.serviceAccount.automount }} +automountServiceAccountToken: {{ .Values.serviceAccounts.gateway.automount }} +{{- end }} +{{- if .Values.serviceAccounts.backend.create }} +{{- if $prev }} +--- +{{- end }} +{{- $prev = true }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "litellm.backend.serviceAccountName" . }} + labels: + {{- include "litellm.commonLabels" . | nindent 4 }} + app.kubernetes.io/component: backend + {{- with .Values.serviceAccounts.backend.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: {{ .Values.serviceAccounts.backend.automount }} +{{- end }} +{{- if .Values.serviceAccounts.ui.create }} +{{- if $prev }} +--- +{{- end }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "litellm.ui.serviceAccountName" . }} + labels: + {{- include "litellm.commonLabels" . | nindent 4 }} + app.kubernetes.io/component: ui + {{- with .Values.serviceAccounts.ui.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: {{ .Values.serviceAccounts.ui.automount }} {{- end }} diff --git a/helm/litellm/templates/ui/deployment.yaml b/helm/litellm/templates/ui/deployment.yaml index 549bf61a0dd..b40b44cca53 100644 --- a/helm/litellm/templates/ui/deployment.yaml +++ b/helm/litellm/templates/ui/deployment.yaml @@ -19,7 +19,8 @@ spec: labels: {{- include "litellm.ui.selectorLabels" . | nindent 8 }} spec: - serviceAccountName: {{ include "litellm.serviceAccountName" . }} + serviceAccountName: {{ include "litellm.ui.serviceAccountName" . }} + automountServiceAccountToken: {{ .Values.serviceAccounts.ui.automount }} {{- with .Values.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} diff --git a/helm/litellm/values.yaml b/helm/litellm/values.yaml index 92477616a9a..934661643bd 100644 --- a/helm/litellm/values.yaml +++ b/helm/litellm/values.yaml @@ -14,16 +14,33 @@ ingress: host: "" # optional; if set, becomes the rule's host tls: [] -# Shared ServiceAccount used by all three component Deployments. Set -# `create: true` to have the chart provision it (e.g. when wiring an EKS -# Pod Identity association by SA name). Set `name` to use an existing SA -# (chart-created or out-of-band). When both are empty / false, pods run -# with the namespace's `default` SA. -serviceAccount: - create: false - automount: true - annotations: {} - name: "" +# Per-component ServiceAccounts for gateway, backend, and ui. +# +# Each section mirrors the old shared serviceAccount shape. Set `create: +# true` to have the chart provision the SA (useful for EKS Pod Identity / +# GKE Workload Identity annotations). Set `name` to bind an existing SA. +# When both are unset the component pod runs with the namespace `default` SA. +# +# The UI SA deliberately defaults to `automount: false` — the static nginx +# container does not need the K8s API and should not carry a projected +# ServiceAccount token that a compromised container could use to call the +# cloud-provider metadata service or the K8s API. +serviceAccounts: + gateway: + create: false + automount: true + annotations: {} + name: "" + backend: + create: false + automount: true + annotations: {} + name: "" + ui: + create: false + automount: false + annotations: {} + name: "" # Pre-install / pre-upgrade Helm hook that runs `prisma migrate deploy` # against the writer database, creating the LiteLLM schema (tables that diff --git a/litellm/__init__.py b/litellm/__init__.py index 7c92623358d..56d516536e8 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -225,6 +225,11 @@ use_chat_completions_url_for_anthropic_messages: bool = bool( route_all_chat_openai_to_responses: bool = ( os.getenv("LITELLM_ROUTE_ALL_CHAT_OPENAI_TO_RESPONSES", "false").lower() == "true" ) # When True, routes all OpenAI /chat/completions requests through the Responses API bridge +# When True, Gemini/Vertex Live setup is deferred until client `session.update`. +# Default False preserves historical behavior (auto-send setup on connect). +gemini_live_defer_setup: bool = ( + os.getenv("LITELLM_GEMINI_LIVE_DEFER_SETUP", "false").lower() == "true" +) use_legacy_interactions_schema: bool = ( os.getenv("LITELLM_USE_LEGACY_INTERACTIONS_SCHEMA", "false").lower() == "true" ) # When True, sends Api-Revision: 2026-05-07 to Google so responses use the legacy `outputs` diff --git a/litellm/constants.py b/litellm/constants.py index fb765c0226c..f72528eb170 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1147,6 +1147,7 @@ BEDROCK_CONVERSE_MODELS = [ "openai.gpt-oss-120b-1:0", "anthropic.claude-haiku-4-5-20251001-v1:0", "anthropic.claude-sonnet-4-5-20250929-v1:0", + "anthropic.claude-opus-4-8", "anthropic.claude-opus-4-7", "anthropic.claude-opus-4-6-v1:0", "anthropic.claude-opus-4-6-v1", diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 1d4c57df42d..9a4b158b622 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -133,6 +133,8 @@ _VIDEO_CALL_TYPES = frozenset( { CallTypes.create_video.value, CallTypes.acreate_video.value, + CallTypes.video_edit.value, + CallTypes.avideo_edit.value, CallTypes.video_remix.value, CallTypes.avideo_remix.value, } diff --git a/litellm/integrations/datadog/datadog_cost_management.py b/litellm/integrations/datadog/datadog_cost_management.py index a961d4f9244..0f954eb1ce0 100644 --- a/litellm/integrations/datadog/datadog_cost_management.py +++ b/litellm/integrations/datadog/datadog_cost_management.py @@ -2,10 +2,17 @@ import asyncio import os import time from datetime import datetime -from typing import Dict, List, Optional, Tuple +from typing import Any, Dict, List, Optional, Tuple, cast from litellm._logging import verbose_logger from litellm.integrations.custom_batch_logger import CustomBatchLogger +from litellm.integrations.datadog.datadog_handler import ( + get_datadog_env, + get_datadog_hostname, + get_datadog_pod_name, + get_datadog_service, +) +from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, @@ -15,9 +22,30 @@ from litellm.types.integrations.datadog_cost_management import ( ) from litellm.types.utils import StandardLoggingPayload +# Reserved tag keys whose values come from trusted sources (infra env, LiteLLM +# core payload fields, or proxy-controlled auth metadata). User-supplied +# request_tags / metadata cannot overwrite these, even when the key is +# allowlisted via cost_tag_keys, because that would let an authenticated caller +# spoof cost attribution (e.g. request_tags=["team:victim-team"]). +_RESERVED_TAG_KEYS: frozenset = frozenset( + { + "env", + "service", + "host", + "pod_name", + "provider", + "model", + "model_id", + "team", + "user", + "model_group", + } +) + class DatadogCostManagementLogger(CustomBatchLogger): - def __init__(self, **kwargs): + def __init__(self, cost_tag_keys: Optional[List[str]] = None, **kwargs): + self.cost_tag_keys: List[str] = list(cost_tag_keys) if cost_tag_keys else [] self.dd_api_key = os.getenv("DD_API_KEY") self.dd_app_key = os.getenv("DD_APP_KEY") self.dd_site = os.getenv("DD_SITE", "datadoghq.com") @@ -68,20 +96,21 @@ class DatadogCostManagementLogger(CustomBatchLogger): if not self.log_queue: return + batch_to_send = self.log_queue[:] + self.log_queue = [] + try: - # Aggregate costs from the batch - aggregated_entries = self._aggregate_costs(self.log_queue) - + aggregated_entries = self._aggregate_costs(batch_to_send) if not aggregated_entries: + verbose_logger.debug( + "Datadog Cost Management: batch produced no aggregable entries; " + "dropping %d log(s) from queue.", + len(batch_to_send), + ) return - - # Send to Datadog await self._upload_to_datadog(aggregated_entries) - - # Clear queue only on success (or if we decide to drop on failure) - # CustomBatchLogger clears queue in flush_queue, so we just process here - except Exception as e: + self.log_queue = batch_to_send + self.log_queue verbose_logger.exception( f"Datadog Cost Management: Error in async_send_batch: {str(e)}" ) @@ -151,45 +180,81 @@ class DatadogCostManagementLogger(CustomBatchLogger): return list(aggregator.values()) def _extract_tags(self, log: StandardLoggingPayload) -> Dict[str, str]: - from litellm.integrations.datadog.datadog_handler import ( - get_datadog_env, - get_datadog_hostname, - get_datadog_pod_name, - get_datadog_service, - ) - - tags = { + tags: Dict[str, str] = { "env": get_datadog_env(), "service": get_datadog_service(), "host": get_datadog_hostname(), "pod_name": get_datadog_pod_name(), } - # Add metadata as tags - metadata = log.get("metadata", {}) - if metadata: - # Add user info - # Add user info - if metadata.get("user_api_key_alias"): - tags["user"] = str(metadata["user_api_key_alias"]) + # Always-on canonical FOCUS dimensions from top-level payload fields. + # Non-sensitive and required for Datadog Custom Costs per-model attribution. + self._add_tag(tags, "provider", log.get("custom_llm_provider")) + self._add_tag(tags, "model", log.get("model")) + self._add_tag(tags, "model_id", log.get("model_id")) - # Add Team Tag - team_tag = ( - metadata.get("user_api_key_team_alias") - or metadata.get("team_alias") # type: ignore - or metadata.get("user_api_key_team_id") - or metadata.get("team_id") # type: ignore - ) + # cast because StandardLoggingMetadata is a TypedDict; we iterate it + # as a generic mapping below. + metadata: Dict[str, Any] = cast(Dict[str, Any], log.get("metadata") or {}) - if team_tag: - tags["team"] = str(team_tag) - # model_group is not in StandardLoggingMetadata TypedDict, so we need to access it via dict.get() - model_group = metadata.get("model_group") # type: ignore[misc] - if model_group: - tags["model_group"] = str(model_group) + # Backwards-compat: team/user/model_group preserved regardless of allowlist. + if metadata.get("user_api_key_alias"): + tags["user"] = str(metadata["user_api_key_alias"]) + team_tag = ( + metadata.get("user_api_key_team_alias") + or metadata.get("team_alias") + or metadata.get("user_api_key_team_id") + or metadata.get("team_id") + ) + if team_tag: + tags["team"] = str(team_tag) + if metadata.get("model_group"): + tags["model_group"] = str(metadata["model_group"]) + + # Allowlist-gated: request_tags (split on `:`) and arbitrary metadata.*. + # Reserved keys are hard-blocked here regardless of allowlist membership — + # see _RESERVED_TAG_KEYS for the rationale. + if self.cost_tag_keys: + allow = set(self.cost_tag_keys) + for rt in log.get("request_tags") or []: + if not isinstance(rt, str) or ":" not in rt: + continue + k, _, v = rt.partition(":") + if k in allow and v: + self._set_custom_tag(tags, k, v) + for k, v in metadata.items(): + if k in allow and v is not None and not isinstance(v, (dict, list)): + self._set_custom_tag(tags, k, str(v)) + for nested_key in ("spend_logs_metadata", "requester_metadata"): + nested = metadata.get(nested_key) + if isinstance(nested, dict): + for k, v in nested.items(): + if ( + k in allow + and v is not None + and not isinstance(v, (dict, list)) + ): + self._set_custom_tag(tags, k, str(v)) return tags + @staticmethod + def _set_custom_tag(tags: Dict[str, str], key: str, value: str) -> None: + if key in _RESERVED_TAG_KEYS: + verbose_logger.debug( + "Datadog Cost Management: dropping user-supplied tag %r=%r — " + "key is reserved for trusted cost attribution.", + key, + value, + ) + return + tags[key] = value + + @staticmethod + def _add_tag(tags: Dict[str, str], key: str, value: Any) -> None: + if value: + tags[key] = str(value) + async def _upload_to_datadog(self, payload: List[Dict]): if not self.dd_api_key or not self.dd_app_key: return @@ -201,8 +266,6 @@ class DatadogCostManagementLogger(CustomBatchLogger): } # The API endpoint expects a list of objects directly in the body (file content behavior) - from litellm.litellm_core_utils.safe_json_dumps import safe_dumps - data_json = safe_dumps(payload) response = await self.async_client.put( diff --git a/litellm/integrations/galileo.py b/litellm/integrations/galileo.py index e99d5f23a4c..a598124f612 100644 --- a/litellm/integrations/galileo.py +++ b/litellm/integrations/galileo.py @@ -1,18 +1,29 @@ +import json import os -from typing import Any, Dict, List, Optional +import re +from typing import Any, Dict, List, Optional, Tuple, cast from pydantic import BaseModel, Field import litellm from litellm._logging import verbose_logger from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + convert_content_list_to_str, + get_content_from_model_response, +) from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, ) +from litellm.types.llms.openai import AllMessageValues + +GALILEO_CLOUD_API_BASE_URL = "https://api.galileo.ai" +# Cap the in-memory buffer so persistent flush failures (e.g. Galileo +# unavailable, invalid credentials) cannot leak memory unboundedly. +GALILEO_MAX_IN_MEMORY_RECORDS = 1000 -# from here: https://docs.rungalileo.io/galileo/gen-ai-studio-products/galileo-observe/how-to/logging-data-via-restful-apis#structuring-your-records class LLMResponse(BaseModel): latency_ms: int status_code: int @@ -37,65 +48,190 @@ class GalileoObserve(CustomLogger): def __init__(self) -> None: self.in_memory_records: List[dict] = [] self.batch_size = 1 - self.base_url = os.getenv("GALILEO_BASE_URL", None) - self.project_id = os.getenv("GALILEO_PROJECT_ID", None) + self.api_key = os.getenv("GALILEO_API_KEY") + self.project_id = os.getenv("GALILEO_PROJECT_ID") + self.log_stream_id = os.getenv("GALILEO_LOG_STREAM_ID") + self.username = os.getenv("GALILEO_USERNAME") + self.password = os.getenv("GALILEO_PASSWORD") + self.base_url = self._normalize_base_url(os.getenv("GALILEO_BASE_URL")) + if self.api_key and not self.base_url: + self.base_url = GALILEO_CLOUD_API_BASE_URL + self.use_v2_api = bool(self.api_key) self.headers: Optional[Dict[str, str]] = None self.async_httpx_handler = get_async_httpx_client( llm_provider=httpxSpecialProvider.LoggingCallback ) - pass - def set_galileo_headers(self): - # following https://docs.rungalileo.io/galileo/gen-ai-studio-products/galileo-observe/how-to/logging-data-via-restful-apis#logging-your-records + @staticmethod + def _normalize_base_url(base_url: Optional[str]) -> Optional[str]: + if base_url: + return base_url.rstrip("/") + return None - headers = { - "accept": "application/json", - "Content-Type": "application/x-www-form-urlencoded", - } - galileo_login_response = litellm.module_level_client.post( + def _is_configured(self) -> bool: + if not self.project_id or not self.base_url: + return False + if self.use_v2_api: + return bool(self.api_key) + return bool(self.username and self.password) + + async def async_set_galileo_headers(self) -> None: + galileo_login_response = await self.async_httpx_handler.post( url=f"{self.base_url}/login", - headers=headers, + headers={ + "accept": "application/json", + "Content-Type": "application/x-www-form-urlencoded", + }, data={ - "username": os.getenv("GALILEO_USERNAME"), - "password": os.getenv("GALILEO_PASSWORD"), + "username": self.username, + "password": self.password, }, ) - + galileo_login_response.raise_for_status() access_token = galileo_login_response.json()["access_token"] - self.headers = { "accept": "application/json", "Content-Type": "application/json", "Authorization": f"Bearer {access_token}", } - def get_output_str_from_response(self, response_obj, kwargs): - output = None - if response_obj is not None and ( - kwargs.get("call_type", None) == "embedding" - or isinstance(response_obj, litellm.EmbeddingResponse) - ): - output = None - elif response_obj is not None and isinstance( - response_obj, litellm.ModelResponse - ): - output = response_obj["choices"][0]["message"].json() - elif response_obj is not None and isinstance( - response_obj, litellm.TextCompletionResponse - ): - output = response_obj.choices[0].text - elif response_obj is not None and isinstance( - response_obj, litellm.ImageResponse - ): - output = response_obj["data"] + async def _ensure_headers(self) -> bool: + if self.headers is not None: + return True - return output + if self.use_v2_api: + if not self.api_key: + return False + self.headers = { + "accept": "application/json", + "Content-Type": "application/json", + "Galileo-API-Key": self.api_key, + } + return True + + if not (self.username and self.password and self.base_url): + return False + + try: + await self.async_set_galileo_headers() + return True + except Exception as e: + verbose_logger.debug("Galileo Logger: failed to authenticate: %s", e) + return False + + @staticmethod + def _galileo_input_messages( + messages: Optional[List[Any]], input_text: str + ) -> List[Dict[str, str]]: + if not messages: + return [{"role": "user", "content": input_text}] + + galileo_messages: List[Dict[str, str]] = [] + for message in messages: + if not isinstance(message, dict): + continue + role = message.get("role") + if not role: + continue + galileo_messages.append( + { + "role": str(role), + "content": convert_content_list_to_str( + message=cast(AllMessageValues, message) + ), + } + ) + + if galileo_messages: + return galileo_messages + return [{"role": "user", "content": input_text}] + + @staticmethod + def _record_to_v2_span(record: Dict[str, Any]) -> Dict[str, Any]: + created_at = record.get("created_at", "") + if created_at and not re.search(r"(Z|[+-]\d{2}:?\d{2})$", created_at): + created_at = f"{created_at}Z" + + span: Dict[str, Any] = { + "type": "llm", + "name": record.get("node_type", "litellm"), + "created_at": created_at, + "input": GalileoObserve._galileo_input_messages( + record.get("messages"), record.get("input_text", "") + ), + "output": { + "role": "assistant", + "content": record.get("output_text", ""), + }, + "status_code": record.get("status_code", 200), + "model": record.get("model"), + "metrics": { + "duration_ns": int(record.get("latency_ms", 0)) * 1_000_000, + "num_input_tokens": record.get("num_input_tokens"), + "num_output_tokens": record.get("num_output_tokens"), + }, + } + if record.get("tags"): + span["tags"] = record["tags"] + return span + + def _get_ingest_request(self) -> Optional[Tuple[str, Dict[str, Any]]]: + if not self.base_url or not self.project_id: + return None + + # Snapshot the records to be sent into a new list so concurrent appends + # during the network round-trip (across the await points in + # flush_in_memory_records) aren't silently dropped when we later clear + # the in-memory buffer. + records = list(self.in_memory_records) + + if self.use_v2_api: + payload: Dict[str, Any] = { + "spans": [self._record_to_v2_span(record) for record in records], + "reliable": False, + } + if self.log_stream_id: + payload["log_stream_id"] = self.log_stream_id + return ( + f"{self.base_url}/v2/projects/{self.project_id}/spans", + payload, + ) + + return ( + f"{self.base_url}/projects/{self.project_id}/observe/ingest", + {"records": records}, + ) + + def get_output_str_from_response( + self, response_obj: Any, kwargs: Dict[str, Any] + ) -> Optional[str]: + if response_obj is None: + return None + if kwargs.get("call_type", None) == "embedding" or isinstance( + response_obj, litellm.EmbeddingResponse + ): + return None + if isinstance(response_obj, litellm.TextCompletionResponse): + return response_obj.choices[0].text + if isinstance(response_obj, litellm.ImageResponse): + return json.dumps(response_obj["data"], default=str) + if isinstance(response_obj, (litellm.ModelResponse, dict)): + return get_content_from_model_response(response_obj) + return None async def async_log_success_event( self, kwargs: Any, response_obj: Any, start_time: Any, end_time: Any ): verbose_logger.debug("On Async Success") + if not self._is_configured(): + verbose_logger.debug( + "Galileo Logger: skipping flush — set GALILEO_PROJECT_ID and " + "either GALILEO_API_KEY (hosted) or GALILEO_USERNAME/GALILEO_PASSWORD " + "(enterprise Observe)." + ) + return + _latency_ms = int((end_time - start_time).total_seconds() * 1000) _call_type = kwargs.get("call_type", "litellm") input_text = litellm.utils.get_formatted_prompt( @@ -125,26 +261,69 @@ class GalileoObserve(CustomLogger): ), # timestamp str constructed in "%Y-%m-%dT%H:%M:%S" format ) - # dump to dict request_dict = request_record.model_dump() + messages = kwargs.get("messages") + if messages: + request_dict["messages"] = messages self.in_memory_records.append(request_dict) + # Bound the buffer so persistent flush failures cannot grow it + # without limit. Drop the oldest records once we exceed the cap. + if len(self.in_memory_records) > GALILEO_MAX_IN_MEMORY_RECORDS: + dropped = len(self.in_memory_records) - GALILEO_MAX_IN_MEMORY_RECORDS + self.in_memory_records = self.in_memory_records[ + -GALILEO_MAX_IN_MEMORY_RECORDS: + ] + verbose_logger.warning( + "Galileo Logger: in-memory buffer exceeded %s records; " + "dropped %s oldest record(s). Check Galileo connectivity/credentials.", + GALILEO_MAX_IN_MEMORY_RECORDS, + dropped, + ) + if len(self.in_memory_records) >= self.batch_size: await self.flush_in_memory_records() async def flush_in_memory_records(self): - verbose_logger.debug("flushing in memory records") - response = await self.async_httpx_handler.post( - url=f"{self.base_url}/projects/{self.project_id}/observe/ingest", - headers=self.headers, - json={"records": self.in_memory_records}, - ) + if not self.in_memory_records: + return - if response.status_code == 200: + # Capture the number of records that will be sent BEFORE any await so + # that concurrent appends made by other asyncio tasks during the + # network round-trip aren't silently dropped on the success-clear. + records_in_payload = len(self.in_memory_records) + + ingest_request = self._get_ingest_request() + if ingest_request is None: verbose_logger.debug( - "Galileo Logger:successfully flushed in memory records" + "Galileo Logger: missing GALILEO_BASE_URL or GALILEO_PROJECT_ID" ) - self.in_memory_records = [] + return + + if not await self._ensure_headers(): + verbose_logger.debug("Galileo Logger: could not set request headers") + return + + url, payload = ingest_request + verbose_logger.debug("flushing in memory records to %s", url) + + try: + response = await self.async_httpx_handler.post( + url=url, + headers=self.headers, + json=payload, + ) + except Exception as e: + verbose_logger.debug( + "Galileo Logger: failed to flush in memory records: %s", e + ) + return + + if response.is_success: + verbose_logger.debug( + "Galileo Logger: successfully flushed in memory records" + ) + del self.in_memory_records[:records_in_payload] else: verbose_logger.debug("Galileo Logger: failed to flush in memory records") verbose_logger.debug( @@ -152,6 +331,13 @@ class GalileoObserve(CustomLogger): response.text, response.status_code, ) + # Legacy enterprise auth caches a bearer token obtained from + # /login. If the request was rejected for auth reasons, drop the + # cached headers so the next flush re-authenticates instead of + # silently failing forever on a stale token. The v2 API key path + # uses a long-lived static key, so leave its headers in place. + if not self.use_v2_api and response.status_code in (401, 403): + self.headers = None async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): verbose_logger.debug("On Async Failure") diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index f9f1bf5c56b..33bb6d7d2ea 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -86,6 +86,12 @@ class RealTimeStreaming: # When a text message is blocked, hold the guardrail reason so the next # response.create can be rewritten to include the failure context. self._pending_guardrail_message: Optional[str] = None + # Track whether session.created has already been sent to the client + # (e.g. synthetic event in deferred setup mode). + self._session_created_sent_to_client: bool = False + # Track whether we have already sent the guardrail turn-detection update + # that disables provider auto-response for transcription guardrails. + self._guardrail_turn_detection_update_sent: bool = False _SESSION_EVENT_TYPES = frozenset(["session.created", "session.updated"]) _AUDIO_FORMAT_MAP: Dict[str, Dict[str, Any]] = { @@ -248,22 +254,52 @@ class RealTimeStreaming: ## SYNC LOGGING executor.submit(self.logging_obj.success_handler(self.messages)) - async def _send_to_backend(self, message: str) -> None: + async def _send_to_backend(self, message: str) -> bool: """Send a message to the backend WebSocket. If a provider_config is set the message is first passed through transform_realtime_request so that provider-specific translation (e.g. dropping session.update for Vertex AI) is applied even for guardrail-injected messages. + + Returns True if at least one message was actually delivered to the + backend, False if the provider transformation produced no output and + the message was effectively dropped. """ if self.provider_config: transformed = self.provider_config.transform_realtime_request( message, self.model, self.session_configuration_request ) + sent = False for msg in transformed: + # Send first; only cache the setup payload once the backend + # has actually accepted it. Caching before send would leave + # ``session_configuration_request`` populated after a failed + # send, causing subsequent client session.update messages to + # be treated as "subsequent" and dropped even though the + # backend never received the original setup. await self.backend_ws.send(msg) # type: ignore[union-attr, attr-defined] - else: - await self.backend_ws.send(message) # type: ignore[union-attr, attr-defined] + self._cache_session_configuration_request(msg) + sent = True + return sent + await self.backend_ws.send(message) # type: ignore[union-attr, attr-defined] + return True + + def _cache_session_configuration_request(self, transformed_message: str) -> None: + """Store setup payload once sent to backend. + + Updates the cached setup on every successful setup send so follow-up + ``session.update`` messages (which produce a merged setup with new + ``generationConfig`` / ``systemInstruction`` / etc.) are reflected in + the cache used by downstream readers (``transform_session_created_event``, + ``return_new_content_delta_events`` modality lookup, ...). + """ + try: + message_obj = json.loads(transformed_message) + if "setup" in message_obj: + self.session_configuration_request = transformed_message + except (json.JSONDecodeError, TypeError): + return def _make_disable_auto_response_message(self) -> str: """Return a session.update that disables VAD auto-response.""" @@ -280,6 +316,20 @@ class RealTimeStreaming: } return json.dumps({"type": "session.update", "session": session}) + async def _maybe_send_guardrail_turn_detection_update(self) -> None: + """Disable provider auto-response once when transcription guardrails are enabled.""" + if self._guardrail_turn_detection_update_sent: + return + if not self._has_audio_transcription_guardrails(): + return + sent = await self._send_to_backend(self._make_disable_auto_response_message()) + # Only mark as sent when the provider transformation actually delivered + # the update to the backend. Otherwise (e.g. Gemini drops session.update + # after the initial setup), leave the flag unset so future opportunities + # — such as a duplicate session.created — can retry. + if sent: + self._guardrail_turn_detection_update_sent = True + def _has_realtime_guardrails(self) -> bool: """Return True if any callback is registered for realtime guardrail event types.""" from litellm.integrations.custom_guardrail import CustomGuardrail @@ -318,12 +368,20 @@ class RealTimeStreaming: self, transcript: str, item_id: Optional[str] = None, + pre_block_backend_message: Optional[str] = None, ) -> bool: """ Run registered guardrails on a completed speech transcription. Returns True if blocked (synthetic warning already sent to client). Returns False if clean (caller should send response.create to the backend). + + ``pre_block_backend_message`` (if provided) is sent to the backend + BEFORE any of the guardrail's own backend messages when a block is + triggered. This is needed for protocol contracts that require a + specific message to be sent first — e.g. Gemini Live requires a + matching ``toolResponse`` immediately after a ``toolCall`` before any + other client messages can be accepted. """ from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.types.guardrails import GuardrailEventHooks @@ -383,6 +441,13 @@ class RealTimeStreaming: getattr(callback, "realtime_violation_message", None) or safe_msg ) + # Deliver any caller-supplied backend message FIRST so that + # protocol contracts requiring a specific ordering (e.g. + # Gemini Live's mandatory ``toolResponse`` after a + # ``toolCall``) are honored before the guardrail's own + # clientContent / cancel messages are sent. + if pre_block_backend_message is not None: + await self._send_to_backend(pre_block_backend_message) # Cancel any in-progress LLM response (e.g. VAD auto-response). await self._send_to_backend(json.dumps({"type": "response.cancel"})) # Send the policy violation hint (shows as small gray status text in UI). @@ -478,16 +543,34 @@ class RealTimeStreaming: else [transformed_response] ) for event in events: + is_session_created_event = ( + isinstance(event, dict) and event.get("type") == "session.created" + ) + if is_session_created_event: + if self._session_created_sent_to_client: + # A synthetic session.created (with placeholder defaults) was + # already forwarded to the client when we connected. The + # provider's real session.created (e.g. emitted from Gemini + # `setupComplete`) carries the authoritative modalities/model + # from the client's session.update. Re-emit it as + # `session.updated` so the client learns the corrected + # configuration without seeing two `session.created` events. + event = {**event, "type": "session.updated"} + else: + self._session_created_sent_to_client = True event_str = json.dumps(event) - ## For audio/VAD guardrail path: forward session.created first, then inject. - if ( - isinstance(event, dict) - and event.get("type") == "session.created" - and self._has_audio_transcription_guardrails() - ): + ## For audio/VAD guardrail path: forward the (possibly retyped) + ## session.created first, then invoke the one-time guardrail + ## turn-detection update. ``_maybe_send_guardrail_turn_detection_update`` + ## is idempotent (gated by ``_guardrail_turn_detection_update_sent``), + ## so duplicate session.created events — including those emitted + ## after a synthetic session.created from ``llm_http_handler`` in + ## deferred-setup mode — still get a single chance to inject the + ## update if a prior attempt was dropped by the provider transform. + if is_session_created_event and self._has_audio_transcription_guardrails(): self.store_message(event_str) await self.websocket.send_text(event_str) - await self._send_to_backend(self._make_disable_auto_response_message()) + await self._maybe_send_guardrail_turn_detection_update() continue ## GUARDRAIL: run on transcription events in provider_config path too if ( @@ -790,12 +873,13 @@ class RealTimeStreaming: item["content"] = new_content return item - async def client_ack_messages(self): + async def client_ack_messages(self): # noqa: PLR0915 try: while True: message = await self.websocket.receive_text() ## GUARDRAIL: intercept conversation.item.create for text-based injection. + guardrail_turn_detection_injected = False try: msg_obj = json.loads(message) msg_type = msg_obj.get("type") @@ -803,7 +887,68 @@ class RealTimeStreaming: if msg_type == "conversation.item.create": # Check user text messages for prompt injection item = msg_obj.get("item", {}) - if item.get("role") == "user": + # Check function_call_output first so a client cannot + # bypass the tool-result guardrail by also setting + # role="user" on a function_call_output item. + if item.get("type") == "function_call_output": + # Tool results are client-controlled and fed to the + # model; check them with the same guardrail used for + # user text so an attacker cannot smuggle blocked + # content into a function_call_output. + output = item.get("output", "") + output_text = ( + output + if isinstance(output, str) + else json.dumps(output) + ) + if output_text: + # Build the sanitized function_call_output up + # front so we can hand it to the guardrail + # runner as the pre-block message. Providers + # that pair every toolCall with a toolResponse + # (e.g. Gemini/Vertex Live) require the + # toolResponse to arrive BEFORE any other + # client message — otherwise the guardrail's + # own clientContent would violate the + # pending-tool-call protocol contract and the + # backend could close the connection before + # the sanitized response ever lands. Dropping + # the blocked item outright would similarly + # leave such providers waiting indefinitely. + # The sanitized payload carries no blocked + # content — only a generic policy marker. + sanitized_msg = json.dumps( + { + **msg_obj, + "item": { + **item, + "output": json.dumps( + { + "error": "Tool output blocked by content policy", + } + ), + }, + } + ) + blocked = await self.run_realtime_guardrails( + output_text, + pre_block_backend_message=sanitized_msg, + ) + if blocked: + # ``_pending_guardrail_message`` is + # intentionally NOT set here. That flag + # exists to swallow the reflexive + # ``response.create`` an OpenAI client + # sends immediately after a user text + # message. In a tool-calling flow the + # client may not send a ``response.create`` + # at all (e.g. Gemini SDKs auto-respond), + # so leaving the flag set would + # incorrectly drop an unrelated + # ``response.create`` from a later + # interaction turn. + continue + elif item.get("role") == "user": content_list = item.get("content", []) texts = [ c.get("text", "") @@ -831,6 +976,89 @@ class RealTimeStreaming: self._pending_guardrail_message = None continue + ## GUARDRAIL: Inject turn_detection into first session.update + # if needed. Done BEFORE the GA remap so the injected + # ``create_response`` rides along with any client-provided + # turn_detection fields (e.g. silence_duration_ms) into the + # nested ``audio.input.turn_detection`` path produced by the + # remap. Doing this after the remap would create a separate + # minimal root-level ``turn_detection`` and silently drop + # the client's nested settings. + if ( + msg_type == "session.update" + and self.session_configuration_request is None + and not self._guardrail_turn_detection_update_sent + and self._has_audio_transcription_guardrails() + ): + session = msg_obj.setdefault("session", {}) + if isinstance(session, dict): + existing_td = session.get("turn_detection") + if not isinstance(existing_td, dict): + existing_td = {} + existing_td["create_response"] = False + session["turn_detection"] = existing_td + message = json.dumps(msg_obj) + guardrail_turn_detection_injected = True + verbose_logger.debug( + "Injected turn_detection into first session.update for audio transcription guardrails" + ) + + ## GUARDRAIL: Force ``create_response`` to False in any + # client-provided ``turn_detection`` so a later + # ``session.update`` cannot re-enable VAD auto-response + # and bypass the transcription guardrail after the + # initial disable. Covers both the flat beta key and the + # nested GA ``audio.input.turn_detection`` shape, since + # the GA remap below also accepts either form. Skipped + # when the injection block above already ran for this + # message, to avoid redundant double-serialization. + if ( + msg_type == "session.update" + and not guardrail_turn_detection_injected + and self._has_audio_transcription_guardrails() + ): + session = msg_obj.get("session") + if isinstance(session, dict): + td_overridden = False + flat_td = session.get("turn_detection") + flat_td_present = flat_td is not None + if flat_td_present: + if not isinstance(flat_td, dict): + flat_td = {} + if flat_td.get("create_response") is not False: + flat_td["create_response"] = False + session["turn_detection"] = flat_td + td_overridden = True + nested_td_present = False + audio = session.get("audio") + if isinstance(audio, dict): + audio_input = audio.get("input") + if isinstance(audio_input, dict): + nested_td = audio_input.get("turn_detection") + if nested_td is not None: + nested_td_present = True + if not isinstance(nested_td, dict): + nested_td = {} + if ( + nested_td.get("create_response") + is not False + ): + nested_td["create_response"] = False + audio_input["turn_detection"] = nested_td + td_overridden = True + # Symmetric with the first-update injection block: + # if the client omitted turn_detection entirely on + # a subsequent session.update, still inject the + # ``create_response: False`` override so the + # transcription guardrail cannot be re-enabled by + # any downstream merge that drops the original + # disable. + if not flat_td_present and not nested_td_present: + session["turn_detection"] = {"create_response": False} + td_overridden = True + if td_overridden: + message = json.dumps(msg_obj) + # GA compatibility: remap beta-style session fields only when # the upstream is in GA mode. Beta upstreams expect the flat # session shape unchanged. @@ -848,17 +1076,20 @@ class RealTimeStreaming: pass ## LOGGING + # Log after any in-place modifications (GA remap, guardrail + # turn_detection injection) so audit logs reflect what we + # actually forward to the backend. self.store_input(message=message) - ## FORWARD TO BACKEND - if self.provider_config: - message = self.provider_config.transform_realtime_request( - message, self.model - ) - for msg in message: - await self.backend_ws.send(msg) # type: ignore[union-attr] - else: - await self.backend_ws.send(message) # type: ignore[union-attr] + ## FORWARD TO BACKEND + # Only mark the guardrail turn_detection update as sent after the + # backend actually accepted the message. Setting the flag earlier + # would permanently disable the injection if ``_send_to_backend`` + # raised — neither this loop nor + # ``_maybe_send_guardrail_turn_detection_update`` would retry. + sent = await self._send_to_backend(message) + if guardrail_turn_detection_injected and sent: + self._guardrail_turn_detection_update_sent = True except Exception as e: verbose_logger.debug(f"Error in client ack messages: {e}") diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index fa7faf3035d..4642201ca67 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -59,6 +59,8 @@ FUNCTION_CALL_ATTRIBUTE = "function_call" _SYNC_ITER_EXHAUSTED = object() +_GCHUNK_FIELDS: frozenset = frozenset(GChunk.__annotations__) + def _next_sync_or_exhausted(it: Any) -> Any: """ @@ -181,6 +183,30 @@ class CustomStreamWrapper: self.created: Optional[int] = None self._last_returned_hidden_params: Optional[dict] = None + _cached_logging_provider = self.logging_obj.model_call_details.get( + "custom_llm_provider", None + ) + self._cached_logging_llm_provider: Optional[str] = _cached_logging_provider + _effective_model = model or "" + if ( + custom_llm_provider == "openai" + and custom_llm_provider != _cached_logging_provider + ): + _effective_model = "{}/{}".format( + _cached_logging_provider, _effective_model + ) + self._cached_model_name: str = _effective_model + + # Snapshot assumes self._hidden_params is populated from litellm_params + # at init and never mutated during the stream. If that ever changes, + # this cache must be removed. + self._base_hidden_params: Dict[str, Any] = { + **self._hidden_params, + "response_cost": None, + } + + self._post_streaming_hooks: Optional[List] = None + def _check_max_streaming_duration(self) -> None: """Raise litellm.Timeout if the stream has exceeded LITELLM_MAX_STREAMING_DURATION_SECONDS.""" from litellm.constants import LITELLM_MAX_STREAMING_DURATION_SECONDS @@ -681,29 +707,16 @@ class CustomStreamWrapper: def model_response_creator( self, chunk: Optional[dict] = None, hidden_params: Optional[dict] = None ): - _model = self.model - _received_llm_provider = self.custom_llm_provider - _logging_obj_llm_provider = self.logging_obj.model_call_details.get("custom_llm_provider", None) # type: ignore - if ( - _received_llm_provider == "openai" - and _received_llm_provider != _logging_obj_llm_provider - ): - _model = "{}/{}".format(_logging_obj_llm_provider, _model) + _model = self._cached_model_name + _logging_obj_llm_provider = self._cached_logging_llm_provider + if chunk is None: - chunk = {} + args: Dict[str, Any] = {"model": _model} else: - # pop model keyword chunk.pop("model", None) - - chunk_dict = {} - for key, value in chunk.items(): - if key != "stream": - chunk_dict[key] = value - - args = { - "model": _model, - **chunk_dict, - } + args = {"model": _model} + if chunk: + args.update({k: v for k, v in chunk.items() if k != "stream"}) model_response = ModelResponseStream(**args) if self.response_id is not None: @@ -717,15 +730,23 @@ class CustomStreamWrapper: model_response.created = self.created else: self.created = model_response.created + + # Spread order is load-bearing: _base_hidden_params (model_id, api_base, ...) + # must win over both caller-supplied hidden_params and the computed + # custom_llm_provider/created_at values, so it comes last. if hidden_params is not None: - model_response._hidden_params = hidden_params - model_response._hidden_params["custom_llm_provider"] = _logging_obj_llm_provider - model_response._hidden_params["created_at"] = time.time() - model_response._hidden_params = { - **model_response._hidden_params, - **self._hidden_params, - "response_cost": None, - } + model_response._hidden_params = { + **hidden_params, + "custom_llm_provider": _logging_obj_llm_provider, + "created_at": time.time(), + **self._base_hidden_params, + } + else: + model_response._hidden_params = { + "custom_llm_provider": _logging_obj_llm_provider, + "created_at": time.time(), + **self._base_hidden_params, + } if ( len(model_response.choices) > 0 @@ -1627,7 +1648,17 @@ class CustomStreamWrapper: from litellm.integrations.custom_logger import CustomLogger from litellm.types.utils import CallTypes - # Get request kwargs from logging object + if self._post_streaming_hooks is None: + self._post_streaming_hooks = [ + cb + for cb in litellm.callbacks + if isinstance(cb, CustomLogger) + and hasattr(cb, "async_post_call_streaming_deployment_hook") + ] + + if not self._post_streaming_hooks: + return chunk + request_data = self.logging_obj.model_call_details call_type_str = self.logging_obj.call_type @@ -1636,18 +1667,14 @@ class CustomStreamWrapper: except ValueError: typed_call_type = None - # Call hooks for all callbacks - for callback in litellm.callbacks: - if isinstance(callback, CustomLogger) and hasattr( - callback, "async_post_call_streaming_deployment_hook" - ): - result = await callback.async_post_call_streaming_deployment_hook( - request_data=request_data, - response_chunk=chunk, - call_type=typed_call_type, - ) - if result is not None: - chunk = result + for callback in self._post_streaming_hooks: + result = await callback.async_post_call_streaming_deployment_hook( + request_data=request_data, + response_chunk=chunk, + call_type=typed_call_type, + ) + if result is not None: + chunk = result return chunk except Exception as e: @@ -1888,17 +1915,15 @@ class CustomStreamWrapper: response = self._add_mcp_list_tools_to_first_chunk(response) self.sent_first_chunk = True - if hasattr( - response, "usage" - ): # remove usage from chunk, only send on final chunk - # Convert the object to a dictionary + # ModelResponseStream declares `usage` as a field, so + # hasattr(response, "usage") is always True — must check + # `is not None` to avoid running this path on every chunk. + if getattr(response, "usage", None) is not None: obj_dict = response.model_dump() - # Remove an attribute (e.g., 'attr2') if "usage" in obj_dict: del obj_dict["usage"] - # Create a new object without the removed attribute response = self.model_response_creator( chunk=obj_dict, hidden_params=response._hidden_params ) @@ -2398,10 +2423,7 @@ def generic_chunk_has_all_required_fields(chunk: dict) -> bool: :param chunk: The dictionary to check. :return: True if all required fields are present, False otherwise. """ - _all_fields = GChunk.__annotations__ - - decision = all(key in _all_fields for key in chunk) - return decision + return all(key in _GCHUNK_FIELDS for key in chunk) def convert_generic_chunk_to_model_response_stream( diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 0b56eb86d9c..57609cfcd26 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -337,13 +337,12 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ) @staticmethod - def _supports_effort_level(model: str, level: str) -> bool: - """Check ``supports_{level}_reasoning_effort`` in the model map. + def _supports_model_capability(model: str, key: str) -> bool: + """Check a boolean capability ``key`` in the model map. Strips bedrock/vertex prefixes so a provider-routed Claude still resolves to the Anthropic model-map entry. """ - key = f"supports_{level}_reasoning_effort" try: if _supports_factory( model=model, @@ -372,8 +371,6 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): except Exception: pass try: - import litellm - for cand in candidates: if cand in litellm.model_cost and ( litellm.model_cost[cand].get(key) is True @@ -383,6 +380,13 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): pass return False + @staticmethod + def _supports_effort_level(model: str, level: str) -> bool: + """Check ``supports_{level}_reasoning_effort`` in the model map.""" + return AnthropicConfig._supports_model_capability( + model, f"supports_{level}_reasoning_effort" + ) + @staticmethod def _validate_effort_for_model(model: str, effort: Optional[str]) -> Optional[str]: """Return ``None`` if ``effort`` is allowed on ``model``, else an error message.""" @@ -400,7 +404,15 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): @staticmethod def _model_supports_effort_param(model: str) -> bool: - """Whether the model accepts ``output_config.effort`` at all.""" + """Whether the model accepts ``output_config.effort`` at all. + + A model qualifies if its map entry advertises ``supports_output_config`` + or any ``supports_*_reasoning_effort`` flag. The two are independent + signals: e.g. Claude Opus 4.5 supports ``output_config`` without + advertising a non-default (max/xhigh) effort level. + """ + if AnthropicConfig._supports_model_capability(model, "supports_output_config"): + return True return any( AnthropicConfig._supports_effort_level(model, level) for level in ("low", "minimal", "medium", "high", "xhigh", "max") @@ -1793,7 +1805,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): self._ensure_context_management_beta_header( headers, optional_params["context_management"] ) - if optional_params.get("output_format") is not None: + output_config = optional_params.get("output_config") + if optional_params.get("output_format") is not None or ( + isinstance(output_config, dict) and output_config.get("format") is not None + ): self._ensure_beta_header( headers, ANTHROPIC_BETA_HEADER_VALUES.STRUCTURED_OUTPUT_2025_09_25.value ) diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py index 15f404d3f53..f94232fa451 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py @@ -427,8 +427,13 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value ) - # Check for structured outputs - if optional_params.get("output_format") is not None: + # Check for structured outputs. Anthropic's newer request shape nests + # the schema under output_config.format; the older top-level + # output_format remains supported for backwards compatibility. + output_config = optional_params.get("output_config") + if optional_params.get("output_format") is not None or ( + isinstance(output_config, dict) and output_config.get("format") is not None + ): beta_values.add( ANTHROPIC_BETA_HEADER_VALUES.STRUCTURED_OUTPUT_2025_09_25.value ) diff --git a/litellm/llms/base_llm/realtime/transformation.py b/litellm/llms/base_llm/realtime/transformation.py index d5531a532b9..0f239b4ad45 100644 --- a/litellm/llms/base_llm/realtime/transformation.py +++ b/litellm/llms/base_llm/realtime/transformation.py @@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Any, List, Optional, Union import httpx +from litellm.types.llms.openai import OpenAIRealtimeStreamSessionEvents from litellm.types.realtime import ( RealtimeResponseTransformInput, RealtimeResponseTypedDict, @@ -69,6 +70,20 @@ class BaseRealtimeConfig(ABC): ) -> Optional[str]: # message sent to setup the realtime session return None + def transform_session_created_event( + self, + model: str, + logging_session_id: str, + session_configuration_request: Optional[str] = None, + ) -> Optional[Union[dict, OpenAIRealtimeStreamSessionEvents]]: + """ + Optional hook for providers that defer session setup until client `session.update`. + + Return an OpenAI-compatible `session.created` payload when the proxy should + emit a synthetic event immediately after backend websocket connection. + """ + return None + @abstractmethod def transform_realtime_response( self, diff --git a/litellm/llms/base_llm/videos/transformation.py b/litellm/llms/base_llm/videos/transformation.py index 87289ad6a0c..9b4cf777280 100644 --- a/litellm/llms/base_llm/videos/transformation.py +++ b/litellm/llms/base_llm/videos/transformation.py @@ -321,6 +321,23 @@ class BaseVideoConfig(ABC): "video get character is not supported for this provider" ) + def get_video_edit_prefetch_params( + self, + video_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Optional[Tuple[str, Dict]]: + """ + Return (url, body) for a pre-fetch HTTP call that must be made before + transform_video_edit_request, or None if no pre-fetch is required. + + Providers that need to retrieve the source video before constructing the + edit request (e.g. Vertex AI) should override this method. The handler + uses the existing shared httpx client so the call is properly async. + """ + return None + def transform_video_edit_request( self, prompt: str, @@ -329,6 +346,7 @@ class BaseVideoConfig(ABC): litellm_params: GenericLiteLLMParams, headers: dict, extra_body: Optional[Dict[str, Any]] = None, + prefetched_source_data: Optional[Dict[str, Any]] = None, ) -> Tuple[str, Dict]: """ Transform the video edit request into a URL and JSON data. @@ -343,6 +361,7 @@ class BaseVideoConfig(ABC): raw_response: httpx.Response, logging_obj: LiteLLMLoggingObj, custom_llm_provider: Optional[str] = None, + request_data: Optional[Dict] = None, ) -> VideoObject: raise NotImplementedError("video edit is not supported for this provider") diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index ae100eda8d4..d58d2e27595 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -78,6 +78,7 @@ from ..common_utils import ( get_anthropic_beta_from_headers, get_bedrock_tool_name, is_claude_4_5_on_bedrock, + normalize_bedrock_opus_output_config_effort, ) # Computer use tool prefixes supported by Bedrock @@ -448,10 +449,20 @@ class AmazonConverseConfig(BaseConfig): value=reasoning_effort, llm_provider="bedrock_converse", ) + existing_output_config = optional_params.get("output_config") + if not isinstance(existing_output_config, dict): + existing_output_config = {} + existing_output_config.setdefault("effort", mapped_effort) + normalize_bedrock_opus_output_config_effort( + model=model, + output_config=existing_output_config, + ) + mapped_effort = existing_output_config["effort"] self._validate_anthropic_adaptive_effort( model=model, effort=mapped_effort ) - optional_params["output_config"] = {"effort": mapped_effort} + optional_params["output_config"] = existing_output_config + optional_params["_output_config_normalized"] = True @staticmethod def _validate_anthropic_adaptive_effort(model: str, effort: str) -> None: @@ -1201,6 +1212,12 @@ class AmazonConverseConfig(BaseConfig): self, optional_params: dict, model: str ) -> Tuple[dict, dict, dict, Optional[OutputConfigBlock]]: """Prepare and separate request parameters.""" + # Consume the internal ``_output_config_normalized`` marker set by + # ``_handle_reasoning_effort_parameter`` so it does not linger on the + # caller's ``optional_params`` after the transformation returns. + anthropic_output_config_already_normalized = bool( + optional_params.pop("_output_config_normalized", False) + ) # Filter out exception objects before deepcopy to prevent deepcopy failures # Exceptions should not be stored in optional_params (this is a defensive fix) cleaned_params = filter_exceptions_from_params(optional_params) @@ -1219,8 +1236,17 @@ class AmazonConverseConfig(BaseConfig): # Anthropic-only ``output_config`` (snake_case) — re-attached to # ``additionalModelRequestFields`` for Anthropic models below. The - # Bedrock-native ``outputConfig`` (camelCase) is handled separately. + # structured-output ``format`` subfield is consumed into Bedrock's + # native ``outputConfig`` (camelCase), which is handled separately. anthropic_output_config = inference_params.pop("output_config", None) + output_config_format = None + if isinstance(anthropic_output_config, dict): + anthropic_output_config = dict(anthropic_output_config) + candidate_output_config_format = anthropic_output_config.pop("format", None) + if isinstance(candidate_output_config_format, dict): + output_config_format = candidate_output_config_format + if not anthropic_output_config: + anthropic_output_config = None # Extract requestMetadata before processing other parameters request_metadata = inference_params.pop("requestMetadata", None) @@ -1230,6 +1256,30 @@ class AmazonConverseConfig(BaseConfig): output_config: Optional[OutputConfigBlock] = inference_params.pop( "outputConfig", None ) + base_model = BedrockModelInfo.get_base_model(model) + if ( + output_config is None + and output_config_format is not None + and output_config_format.get("type") == "json_schema" + and base_model.startswith("anthropic") + and self._supports_native_structured_outputs( + model, self.custom_llm_provider + ) + ): + output_config = self._create_output_config_for_response_format( + json_schema=output_config_format.get("schema"), + name=output_config_format.get("name"), + description=output_config_format.get("description"), + ) + elif output_config is None and output_config_format is not None: + litellm.verbose_logger.warning( + "Bedrock Converse: dropping `output_config.format` for model=%s — " + "model does not advertise `supports_native_structured_output` in " + "model_prices_and_context_window.json. The schema will not be " + "enforced; pass `response_format` to use the synthetic tool-call " + "fallback.", + model, + ) # keep supported params in 'inference_params', and set all model-specific params in 'additional_request_params' additional_request_params = { @@ -1275,7 +1325,6 @@ class AmazonConverseConfig(BaseConfig): if anthropic_output_config is not None and isinstance( anthropic_output_config, dict ): - base_model = BedrockModelInfo.get_base_model(model) if base_model.startswith("anthropic"): if ( litellm.drop_params is True @@ -1286,6 +1335,11 @@ class AmazonConverseConfig(BaseConfig): model, ) else: + if not anthropic_output_config_already_normalized: + normalize_bedrock_opus_output_config_effort( + model=model, + output_config=anthropic_output_config, + ) effort = anthropic_output_config.get("effort") if effort is not None: self._validate_anthropic_adaptive_effort( diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py index d9599b8b9c4..a13336b6c88 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py @@ -16,8 +16,11 @@ from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation AmazonInvokeConfig, ) from litellm.llms.bedrock.common_utils import ( + convert_bedrock_invoke_output_format_to_inline_schema, get_anthropic_beta_from_headers, + normalize_bedrock_opus_output_config_effort, normalize_tool_input_schema_types_for_bedrock_invoke, + pop_bedrock_invoke_output_config_format, remove_custom_field_from_tools, ) from litellm.types.llms.anthropic import ANTHROPIC_TOOL_SEARCH_BETA_HEADER @@ -75,6 +78,17 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): # Use a model name that forces tool-based approach model = "claude-3-sonnet-20240229" + # Clamp ``reasoning_effort`` to the Bedrock effort ceiling before the + # parent mapping converts it to ``output_config.effort`` and the + # downstream effort gate runs. Mirrors the converse path's + # ``_handle_reasoning_effort_parameter`` and the messages path's + # ``_clamp_adaptive_reasoning_effort_for_bedrock`` so adaptive Claude + # requests degrade ``xhigh`` -> ``max`` rather than 400-ing on + # models like Opus 4.6 that don't natively advertise xhigh. + self._clamp_adaptive_reasoning_effort_for_bedrock( + model=original_model, params=non_default_params + ) + optional_params = AnthropicConfig.map_openai_params( self, non_default_params, @@ -88,6 +102,27 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): return optional_params + @staticmethod + def _clamp_adaptive_reasoning_effort_for_bedrock(model: str, params: dict) -> None: + """Lower ``reasoning_effort`` to the Bedrock effort ceiling before mapping. + + Bedrock's adaptive Claude models accept the OpenAI-style + ``reasoning_effort`` tier, but the request validator can reject tiers + the model does not natively advertise (e.g. ``xhigh`` on Opus 4.6). + Clamp the raw tier to the model's + ``bedrock_output_config_effort_ceiling`` so Claude Code "goal mode" + keeps working. Non-adaptive models and models without a ceiling are + left untouched. + """ + if not AnthropicConfig._is_adaptive_thinking_model(model): + return + effort = params.get("reasoning_effort") + if not isinstance(effort, str): + return + clamped = {"effort": effort} + normalize_bedrock_opus_output_config_effort(model=model, output_config=clamped) + params["reasoning_effort"] = clamped["effort"] + def transform_request( self, model: str, @@ -157,6 +192,13 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): for k, v in optional_params.items() if k not in self.aws_authentication_params } + output_config = filtered_params.get("output_config") + if isinstance(output_config, dict): + filtered_params["output_config"] = dict(output_config) + normalize_bedrock_opus_output_config_effort( + model=model, + output_config=filtered_params["output_config"], + ) filtered_params = self._normalize_bedrock_tool_search_tools(filtered_params) anthropic_request = AnthropicConfig.transform_request( @@ -170,7 +212,20 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): anthropic_request.pop("model", None) anthropic_request.pop("stream", None) - anthropic_request.pop("output_format", None) + output_format = anthropic_request.pop("output_format", None) + output_config_format = pop_bedrock_invoke_output_config_format( + anthropic_request + ) + if output_format: + convert_bedrock_invoke_output_format_to_inline_schema( + output_format=output_format, + request_body=anthropic_request, + ) + elif output_config_format: + convert_bedrock_invoke_output_format_to_inline_schema( + output_format=output_config_format, + request_body=anthropic_request, + ) if not ( _supports_factory( model=model, diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 4f4729e4019..bdc5da321c6 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -34,6 +34,15 @@ class BedrockError(BaseLLMException): # Lazy import cache to avoid circular imports and performance impact _get_model_info = None +BedrockOutputConfigEffort = Literal["low", "medium", "high", "max", "xhigh"] +_BEDROCK_OUTPUT_CONFIG_EFFORT_ORDER: Dict[BedrockOutputConfigEffort, int] = { + "low": 0, + "medium": 1, + "high": 2, + "max": 3, + "xhigh": 4, +} + def get_cached_model_info(): """ @@ -51,6 +60,79 @@ def get_cached_model_info(): return _get_model_info +@functools.lru_cache(maxsize=1) +def _get_local_model_cost_map() -> Dict: + from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap + + return GetModelCostMap.load_local_model_cost_map() + + +def pop_bedrock_invoke_output_config_format(request_body: Dict) -> Optional[Dict]: + """ + Remove and return Anthropic's nested ``output_config.format`` field. + + Bedrock Invoke paths convert the schema to inline message text. Any remaining + ``output_config`` keys, such as ``effort``, are left in place. + """ + output_config = request_body.get("output_config") + if not isinstance(output_config, dict): + return None + + output_format = output_config.pop("format", None) + if not output_config: + request_body.pop("output_config", None) + + if isinstance(output_format, dict): + return output_format + return None + + +def convert_bedrock_invoke_output_format_to_inline_schema( + output_format: Dict, + request_body: Dict, +) -> None: + """ + Embed an Anthropic structured-output schema into the last user message. + + Bedrock Invoke does not support ``output_format`` directly, so the schema is + appended to the final user message for prompt-engineered structured output. + The caller's ``messages`` list, message dict, and content list are not + mutated; a fresh ``messages`` list with a copied final user message is + written back to ``request_body``. + """ + schema = output_format.get("schema") + if not schema: + return + + messages = request_body.get("messages") + if not isinstance(messages, list) or not messages: + return + + last_user_idx = None + for i in range(len(messages) - 1, -1, -1): + message = messages[i] + if isinstance(message, dict) and message.get("role") == "user": + last_user_idx = i + break + + if last_user_idx is None: + return + + original = messages[last_user_idx] + content = original.get("content", []) + schema_block = {"type": "text", "text": json.dumps(schema)} + if isinstance(content, str): + new_content = [{"type": "text", "text": content}, schema_block] + elif isinstance(content, list): + new_content = [*content, schema_block] + else: + return + + new_messages = list(messages) + new_messages[last_user_idx] = {**original, "content": new_content} + request_body["messages"] = new_messages + + def remove_custom_field_from_tools(request_body: dict) -> None: """ Remove ``custom`` field from each tool in the request body. @@ -603,6 +685,62 @@ def is_claude_4_5_on_bedrock(model: str) -> bool: return any(pattern in model_lower for pattern in claude_4_5_patterns) +def normalize_bedrock_opus_output_config_effort(model: str, output_config: Any) -> None: + """ + Normalize Anthropic ``output_config.effort`` values for Bedrock Opus ids. + + Bedrock's Claude Opus request validator can accept a narrower effort + vocabulary than Anthropic's compatibility surface. The Bedrock ceiling is + read from ``model_prices_and_context_window.json`` via + ``bedrock_output_config_effort_ceiling``. + + Mutates ``output_config`` in place so callers can accept Claude Code's + ``xhigh`` input without forwarding a provider-invalid value. + """ + if not isinstance(output_config, dict): + return + + effort = output_config.get("effort") + if effort not in _BEDROCK_OUTPUT_CONFIG_EFFORT_ORDER: + return + + ceiling = _get_bedrock_output_config_effort_ceiling(model) + if ceiling is None: + return + + if ( + _BEDROCK_OUTPUT_CONFIG_EFFORT_ORDER[effort] + > _BEDROCK_OUTPUT_CONFIG_EFFORT_ORDER[ceiling] + ): + output_config["effort"] = ceiling + + +def _get_bedrock_output_config_effort_ceiling( + model: str, +) -> Optional[BedrockOutputConfigEffort]: + try: + model_info = get_cached_model_info()( + model=model, + custom_llm_provider="bedrock", + ) + except Exception: + return None + + ceiling = model_info.get("bedrock_output_config_effort_ceiling") + if isinstance(ceiling, str) and ceiling in _BEDROCK_OUTPUT_CONFIG_EFFORT_ORDER: + return ceiling # type: ignore[return-value] + + model_cost_key = model_info.get("key") + if not isinstance(model_cost_key, str): + return None + + local_model_info = _get_local_model_cost_map().get(model_cost_key, {}) + ceiling = local_model_info.get("bedrock_output_config_effort_ceiling") + if isinstance(ceiling, str) and ceiling in _BEDROCK_OUTPUT_CONFIG_EFFORT_ORDER: + return ceiling # type: ignore[return-value] + return None + + # Import after standalone functions to avoid circular imports from litellm.llms.bedrock.count_tokens.bedrock_token_counter import BedrockTokenCounter diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py index 69b61298d33..b223f4534fa 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py @@ -32,10 +32,13 @@ from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation AmazonInvokeConfig, ) from litellm.llms.bedrock.common_utils import ( + convert_bedrock_invoke_output_format_to_inline_schema, ensure_bedrock_anthropic_messages_tool_names, get_anthropic_beta_from_headers, is_claude_4_5_on_bedrock, + normalize_bedrock_opus_output_config_effort, normalize_tool_input_schema_types_for_bedrock_invoke, + pop_bedrock_invoke_output_config_format, remove_custom_field_from_tools, ) from litellm.types.llms.anthropic import ANTHROPIC_TOOL_SEARCH_BETA_HEADER @@ -450,145 +453,15 @@ class AmazonAnthropicClaudeMessagesConfig( else: anthropic_messages_request.pop("context_management", None) - def _convert_output_format_to_inline_schema( - self, - output_format: Dict, - anthropic_messages_request: Dict, - ) -> None: - """ - Convert Anthropic output_format to inline schema in message content. - - Bedrock Invoke doesn't support the output_format parameter, so we embed - the schema directly into the user message content as text instructions. - - This approach adds the schema to the last user message, instructing the model - to respond in the specified JSON format. - - Args: - output_format: The output_format dict with 'type' and 'schema' - anthropic_messages_request: The request dict to modify in-place - - Ref: https://aws.amazon.com/blogs/machine-learning/structured-data-response-with-amazon-bedrock-prompt-engineering-and-tool-use/ - """ - import json - - # Extract schema from output_format - schema = output_format.get("schema") - if not schema: - return - - # Get messages from the request - messages = anthropic_messages_request.get("messages", []) - if not messages: - return - - # Find the last user message - last_user_message_idx = None - for idx in range(len(messages) - 1, -1, -1): - if messages[idx].get("role") == "user": - last_user_message_idx = idx - break - - if last_user_message_idx is None: - return - - last_user_message = messages[last_user_message_idx] - content = last_user_message.get("content", []) - - # Ensure content is a list - if isinstance(content, str): - content = [{"type": "text", "text": content}] - last_user_message["content"] = content - - # Add schema as text content to the message - schema_text = {"type": "text", "text": json.dumps(schema)} - content.append(schema_text) - - def transform_anthropic_messages_request( + def _get_bedrock_invoke_anthropic_beta_headers( self, model: str, messages: List[Dict], anthropic_messages_optional_request_params: Dict, - litellm_params: GenericLiteLLMParams, headers: dict, - ) -> Dict: - anthropic_messages_request = AnthropicMessagesConfig.transform_anthropic_messages_request( - self=self, - model=model, - messages=messages, - anthropic_messages_optional_request_params=anthropic_messages_optional_request_params, - litellm_params=litellm_params, - headers=headers, - ) - ######################################################### - ############## BEDROCK Invoke SPECIFIC TRANSFORMATION ### - ######################################################### - - # 1. anthropic_version is required for all claude models - if "anthropic_version" not in anthropic_messages_request: - anthropic_messages_request["anthropic_version"] = ( - self.DEFAULT_BEDROCK_ANTHROPIC_API_VERSION - ) - - # 2. `stream` is not allowed in request body for bedrock invoke - if "stream" in anthropic_messages_request: - anthropic_messages_request.pop("stream", None) - - # 3. `model` is not allowed in request body for bedrock invoke - if "model" in anthropic_messages_request: - anthropic_messages_request.pop("model", None) - - injected_thinking_for_clear_thinking = ( - self._ensure_thinking_for_clear_thinking_context_management( - anthropic_messages_request=anthropic_messages_request, - model=model, - ) - ) - - # 4. Remove `ttl` field from cache_control in messages (Bedrock doesn't support it for older models) - self._remove_ttl_from_cache_control( - anthropic_messages_request=anthropic_messages_request, model=model - ) - - # 5. Convert `output_format` to inline schema (Bedrock invoke doesn't support output_format) - output_format = anthropic_messages_request.pop("output_format", None) - if output_format: - self._convert_output_format_to_inline_schema( - output_format=output_format, - anthropic_messages_request=anthropic_messages_request, - ) - - # 5a. Bedrock Invoke supports output_config (effort) for Claude 4.6+ models, - # but older models do not — strip it to avoid request rejection. - # Ref: https://github.com/BerriAI/litellm/issues/22797 - if not ( - _supports_factory( - model=model, - custom_llm_provider="bedrock", - key="supports_output_config", - ) - or AnthropicConfig._model_supports_effort_param(model) - ): - if anthropic_messages_request.pop("output_config", None) is not None: - verbose_logger.warning( - "Bedrock Invoke: stripping unsupported `output_config` for " - "model=%s — neither `supports_output_config` nor any " - "`supports_*_reasoning_effort` flag is set in " - "model_prices_and_context_window.json. Add the capability " - "flag to the model JSON entry if this model accepts " - "`output_config`.", - model, - ) - - # 5b. Remove `custom` field from tools (Bedrock doesn't support it) - # Claude Code sends `custom: {defer_loading: true}` on tool definitions, - # which causes Bedrock to reject the request with "Extra inputs are not permitted" - # Ref: https://github.com/BerriAI/litellm/issues/22847 - remove_custom_field_from_tools(anthropic_messages_request) - normalize_tool_input_schema_types_for_bedrock_invoke(anthropic_messages_request) - ensure_bedrock_anthropic_messages_tool_names(anthropic_messages_request) - - # 6. AUTO-INJECT beta headers based on features used + anthropic_messages_request: Dict, + injected_thinking_for_clear_thinking: bool, + ) -> List[str]: anthropic_model_info = AnthropicModelInfo() tools = anthropic_messages_optional_request_params.get("tools") messages_typed = cast(List[AllMessageValues], messages) @@ -651,6 +524,160 @@ class AmazonAnthropicClaudeMessagesConfig( dropped_user_betas, ) + return filtered_betas + + def _strip_unsupported_bedrock_invoke_fields( + self, + anthropic_messages_request: Dict, + ) -> Dict: + allowed = self.BEDROCK_INVOKE_ALLOWED_TOP_LEVEL_FIELDS + stripped = sorted(k for k in anthropic_messages_request if k not in allowed) + if stripped: + verbose_logger.debug( + "Bedrock Invoke: stripping unsupported top-level request fields: %s", + stripped, + ) + return {k: v for k, v in anthropic_messages_request.items() if k in allowed} + + @staticmethod + def _clamp_adaptive_reasoning_effort_for_bedrock( + model: str, optional_params: Dict + ) -> None: + """Lower ``reasoning_effort`` to the Bedrock effort ceiling before validation. + + The shared ``/v1/messages`` effort gate rejects tiers a model does not + natively support (e.g. ``xhigh`` on Opus 4.6). Bedrock's chat paths instead + clamp the tier to the model's ``bedrock_output_config_effort_ceiling`` so + Claude Code "goal mode" keeps working; mirror that here so the messages + path degrades ``xhigh`` -> ``max`` rather than 400-ing. Non-adaptive models + and models without a ceiling are left untouched. + """ + if not AnthropicModelInfo._is_adaptive_thinking_model(model): + return + effort = optional_params.get("reasoning_effort") + if not isinstance(effort, str): + return + clamped = {"effort": effort} + normalize_bedrock_opus_output_config_effort(model=model, output_config=clamped) + optional_params["reasoning_effort"] = clamped["effort"] + + def transform_anthropic_messages_request( + self, + model: str, + messages: List[Dict], + anthropic_messages_optional_request_params: Dict, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Dict: + self._clamp_adaptive_reasoning_effort_for_bedrock( + model=model, + optional_params=anthropic_messages_optional_request_params, + ) + anthropic_messages_request = AnthropicMessagesConfig.transform_anthropic_messages_request( + self=self, + model=model, + messages=messages, + anthropic_messages_optional_request_params=anthropic_messages_optional_request_params, + litellm_params=litellm_params, + headers=headers, + ) + ######################################################### + ############## BEDROCK Invoke SPECIFIC TRANSFORMATION ### + ######################################################### + + # 1. anthropic_version is required for all claude models + if "anthropic_version" not in anthropic_messages_request: + anthropic_messages_request["anthropic_version"] = ( + self.DEFAULT_BEDROCK_ANTHROPIC_API_VERSION + ) + + # 2. `stream` is not allowed in request body for bedrock invoke + if "stream" in anthropic_messages_request: + anthropic_messages_request.pop("stream", None) + + # 3. `model` is not allowed in request body for bedrock invoke + if "model" in anthropic_messages_request: + anthropic_messages_request.pop("model", None) + + injected_thinking_for_clear_thinking = ( + self._ensure_thinking_for_clear_thinking_context_management( + anthropic_messages_request=anthropic_messages_request, + model=model, + ) + ) + + # 4. Remove `ttl` field from cache_control in messages (Bedrock doesn't support it for older models) + self._remove_ttl_from_cache_control( + anthropic_messages_request=anthropic_messages_request, model=model + ) + + # 5. Convert structured-output params to inline schema. + # Bedrock Invoke doesn't support top-level `output_format`; its + # accepted `output_config` subset is also narrower than Anthropic's, so + # consume the newer `output_config.format` shape here instead of + # forwarding it as an unknown nested key. + existing_output_config = anthropic_messages_request.get("output_config") + if isinstance(existing_output_config, dict): + anthropic_messages_request["output_config"] = dict(existing_output_config) + output_format = anthropic_messages_request.pop("output_format", None) + output_config_format = pop_bedrock_invoke_output_config_format( + anthropic_messages_request + ) + if output_format: + convert_bedrock_invoke_output_format_to_inline_schema( + output_format=output_format, + request_body=anthropic_messages_request, + ) + elif output_config_format: + convert_bedrock_invoke_output_format_to_inline_schema( + output_format=output_config_format, + request_body=anthropic_messages_request, + ) + normalize_bedrock_opus_output_config_effort( + model=model, + output_config=anthropic_messages_request.get("output_config"), + ) + + # 5a. Bedrock Invoke supports output_config (effort) for Claude 4.6+ models, + # but older models do not — strip it to avoid request rejection. + # Ref: https://github.com/BerriAI/litellm/issues/22797 + if not ( + _supports_factory( + model=model, + custom_llm_provider="bedrock", + key="supports_output_config", + ) + or AnthropicConfig._model_supports_effort_param(model) + ): + if anthropic_messages_request.pop("output_config", None) is not None: + verbose_logger.warning( + "Bedrock Invoke: stripping unsupported `output_config` for " + "model=%s — neither `supports_output_config` nor any " + "`supports_*_reasoning_effort` flag is set in " + "model_prices_and_context_window.json. Add the capability " + "flag to the model JSON entry if this model accepts " + "`output_config`.", + model, + ) + + # 5b. Remove `custom` field from tools (Bedrock doesn't support it) + # Claude Code sends `custom: {defer_loading: true}` on tool definitions, + # which causes Bedrock to reject the request with "Extra inputs are not permitted" + # Ref: https://github.com/BerriAI/litellm/issues/22847 + remove_custom_field_from_tools(anthropic_messages_request) + normalize_tool_input_schema_types_for_bedrock_invoke(anthropic_messages_request) + ensure_bedrock_anthropic_messages_tool_names(anthropic_messages_request) + + # 6. AUTO-INJECT beta headers based on features used + filtered_betas = self._get_bedrock_invoke_anthropic_beta_headers( + model=model, + messages=messages, + anthropic_messages_optional_request_params=anthropic_messages_optional_request_params, + headers=headers, + anthropic_messages_request=anthropic_messages_request, + injected_thinking_for_clear_thinking=injected_thinking_for_clear_thinking, + ) + if filtered_betas: anthropic_messages_request["anthropic_beta"] = filtered_betas @@ -669,16 +696,9 @@ class AmazonAnthropicClaudeMessagesConfig( # Catches Anthropic-only extensions (output_config, speed, mcp_servers, ...) # and any future additions Claude Code may start sending. ``context_management`` # has already been pre-filtered to its Bedrock-supported subset above. - allowed = self.BEDROCK_INVOKE_ALLOWED_TOP_LEVEL_FIELDS - stripped = sorted(k for k in anthropic_messages_request if k not in allowed) - if stripped: - verbose_logger.debug( - "Bedrock Invoke: stripping unsupported top-level request fields: %s", - stripped, - ) - anthropic_messages_request = { - k: v for k, v in anthropic_messages_request.items() if k in allowed - } + anthropic_messages_request = self._strip_unsupported_bedrock_invoke_fields( + anthropic_messages_request + ) return anthropic_messages_request diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index c9ab3c648ac..941fe59e825 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -5316,6 +5316,28 @@ class BaseLLMHTTPHandler: ) if _session_config: realtime_streaming.session_configuration_request = _session_config + + # For providers that defer setup until client session.update, optionally + # send synthetic session.created to unblock clients waiting on connect. + if not provider_config.requires_session_configuration(): + synthetic_session = provider_config.transform_session_created_event( + model=model, + logging_session_id=logging_obj.litellm_trace_id, + session_configuration_request=None, + ) + if synthetic_session is not None: + synthetic_session_str = json.dumps(synthetic_session) + # Record before sending so the synthetic session.created is + # captured in the session log alongside provider-driven + # events; without this it would be silently absent from + # success_handler / async_success_handler payloads. + realtime_streaming.store_message(synthetic_session_str) + await websocket.send_text(synthetic_session_str) + realtime_streaming._session_created_sent_to_client = True + verbose_logger.debug( + "Sent synthetic session.created to client to unblock connection" + ) + await realtime_streaming.bidirectional_forward() except websockets.exceptions.InvalidStatusCode as e: # type: ignore @@ -6538,6 +6560,7 @@ class BaseLLMHTTPHandler: api_key=api_key or litellm_params.get("api_key", None), headers=extra_headers or {}, model="", + litellm_params=litellm_params, ) if extra_headers: @@ -6620,6 +6643,7 @@ class BaseLLMHTTPHandler: api_key=api_key or litellm_params.get("api_key", None), headers=extra_headers or {}, model="", + litellm_params=litellm_params, ) if extra_headers: @@ -6712,6 +6736,7 @@ class BaseLLMHTTPHandler: api_key=api_key or litellm_params.get("api_key", None), headers=extra_headers or {}, model="", + litellm_params=litellm_params, ) if extra_headers: headers.update(extra_headers) @@ -6783,6 +6808,7 @@ class BaseLLMHTTPHandler: api_key=api_key or litellm_params.get("api_key", None), headers=extra_headers or {}, model="", + litellm_params=litellm_params, ) if extra_headers: headers.update(extra_headers) @@ -6866,6 +6892,7 @@ class BaseLLMHTTPHandler: api_key=api_key or litellm_params.get("api_key", None), headers=extra_headers or {}, model="", + litellm_params=litellm_params, ) if extra_headers: headers.update(extra_headers) @@ -6923,6 +6950,7 @@ class BaseLLMHTTPHandler: api_key=api_key or litellm_params.get("api_key", None), headers=extra_headers or {}, model="", + litellm_params=litellm_params, ) if extra_headers: headers.update(extra_headers) @@ -6999,6 +7027,7 @@ class BaseLLMHTTPHandler: api_key=api_key or litellm_params.get("api_key", None), headers=extra_headers or {}, model="", + litellm_params=litellm_params, ) if extra_headers: headers.update(extra_headers) @@ -7009,27 +7038,49 @@ class BaseLLMHTTPHandler: litellm_params=dict(litellm_params), ) - url, data = video_provider_config.transform_video_edit_request( - prompt=prompt, + prefetched_source_data = None + prefetch_params = video_provider_config.get_video_edit_prefetch_params( video_id=video_id, api_base=api_base, litellm_params=litellm_params, headers=headers, - extra_body=extra_body, - ) - - logging_obj.pre_call( - input=prompt, - api_key="", - additional_args={ - "complete_input_dict": data, - "api_base": url, - "headers": headers, - "video_id": video_id, - }, ) + if prefetch_params is not None: + prefetch_url, prefetch_body = prefetch_params + try: + prefetch_resp = sync_httpx_client.post( + url=prefetch_url, + headers=headers, + json=prefetch_body, + timeout=timeout, + ) + prefetch_resp.raise_for_status() + except Exception as e: + raise self._handle_error(e=e, provider_config=video_provider_config) + prefetched_source_data = prefetch_resp.json() try: + url, data = video_provider_config.transform_video_edit_request( + prompt=prompt, + video_id=video_id, + api_base=api_base, + litellm_params=litellm_params, + headers=headers, + extra_body=extra_body, + prefetched_source_data=prefetched_source_data, + ) + + logging_obj.pre_call( + input=prompt, + api_key="", + additional_args={ + "complete_input_dict": data, + "api_base": url, + "headers": headers, + "video_id": video_id, + }, + ) + response = sync_httpx_client.post( url=url, headers=headers, @@ -7041,6 +7092,7 @@ class BaseLLMHTTPHandler: raw_response=response, logging_obj=logging_obj, custom_llm_provider=custom_llm_provider, + request_data=data, ) except Exception as e: raise self._handle_error(e=e, provider_config=video_provider_config) @@ -7071,6 +7123,7 @@ class BaseLLMHTTPHandler: api_key=api_key or litellm_params.get("api_key", None), headers=extra_headers or {}, model="", + litellm_params=litellm_params, ) if extra_headers: headers.update(extra_headers) @@ -7081,27 +7134,49 @@ class BaseLLMHTTPHandler: litellm_params=dict(litellm_params), ) - url, data = video_provider_config.transform_video_edit_request( - prompt=prompt, + prefetched_source_data = None + prefetch_params = video_provider_config.get_video_edit_prefetch_params( video_id=video_id, api_base=api_base, litellm_params=litellm_params, headers=headers, - extra_body=extra_body, - ) - - logging_obj.pre_call( - input=prompt, - api_key="", - additional_args={ - "complete_input_dict": data, - "api_base": url, - "headers": headers, - "video_id": video_id, - }, ) + if prefetch_params is not None: + prefetch_url, prefetch_body = prefetch_params + try: + prefetch_resp = await async_httpx_client.post( + url=prefetch_url, + headers=headers, + json=prefetch_body, + timeout=timeout, + ) + prefetch_resp.raise_for_status() + except Exception as e: + raise self._handle_error(e=e, provider_config=video_provider_config) + prefetched_source_data = prefetch_resp.json() try: + url, data = video_provider_config.transform_video_edit_request( + prompt=prompt, + video_id=video_id, + api_base=api_base, + litellm_params=litellm_params, + headers=headers, + extra_body=extra_body, + prefetched_source_data=prefetched_source_data, + ) + + logging_obj.pre_call( + input=prompt, + api_key="", + additional_args={ + "complete_input_dict": data, + "api_base": url, + "headers": headers, + "video_id": video_id, + }, + ) + response = await async_httpx_client.post( url=url, headers=headers, @@ -7113,6 +7188,7 @@ class BaseLLMHTTPHandler: raw_response=response, logging_obj=logging_obj, custom_llm_provider=custom_llm_provider, + request_data=data, ) except Exception as e: raise self._handle_error(e=e, provider_config=video_provider_config) @@ -7160,6 +7236,7 @@ class BaseLLMHTTPHandler: api_key=api_key or litellm_params.get("api_key", None), headers=extra_headers or {}, model="", + litellm_params=litellm_params, ) if extra_headers: headers.update(extra_headers) @@ -7234,6 +7311,7 @@ class BaseLLMHTTPHandler: api_key=api_key or litellm_params.get("api_key", None), headers=extra_headers or {}, model="", + litellm_params=litellm_params, ) if extra_headers: headers.update(extra_headers) @@ -7445,6 +7523,7 @@ class BaseLLMHTTPHandler: api_key=api_key, headers=extra_headers or {}, model="", + litellm_params=litellm_params, ) if extra_headers: diff --git a/litellm/llms/gemini/realtime/transformation.py b/litellm/llms/gemini/realtime/transformation.py index 4378db06358..cf1fc75ef10 100644 --- a/litellm/llms/gemini/realtime/transformation.py +++ b/litellm/llms/gemini/realtime/transformation.py @@ -3,8 +3,10 @@ This file contains the transformation logic for the Gemini realtime API. """ import json +from collections import OrderedDict from typing import Any, Dict, List, Optional, Union, cast +import litellm from litellm import verbose_logger from litellm._uuid import uuid from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -29,6 +31,7 @@ from litellm.types.llms.openai import ( OpenAIRealtimeDoneEvent, OpenAIRealtimeEvents, OpenAIRealtimeEventTypes, + OpenAIRealtimeFunctionCallArgumentsDone, OpenAIRealtimeOutputItemDone, OpenAIRealtimeResponseAudioDone, OpenAIRealtimeResponseContentPartAdded, @@ -36,10 +39,12 @@ from litellm.types.llms.openai import ( OpenAIRealtimeResponseDoneObject, OpenAIRealtimeResponseTextDone, OpenAIRealtimeStreamResponseBaseObject, + OpenAIRealtimeStreamResponseOutputItem, OpenAIRealtimeStreamResponseOutputItemAdded, OpenAIRealtimeStreamSession, OpenAIRealtimeStreamSessionEvents, OpenAIRealtimeTurnDetection, + ResponsesAPIStreamEvents, ) from litellm.types.llms.vertex_ai import ( GeminiResponseModalities, @@ -56,15 +61,43 @@ from litellm.utils import get_empty_usage from ..common_utils import encode_unserializable_types, get_api_key_from_env -MAP_GEMINI_FIELD_TO_OPENAI_EVENT: Dict[str, OpenAIRealtimeEventTypes] = { +MAP_GEMINI_FIELD_TO_OPENAI_EVENT: Dict[ + str, Union[OpenAIRealtimeEventTypes, ResponsesAPIStreamEvents] +] = { "setupComplete": OpenAIRealtimeEventTypes.SESSION_CREATED, "serverContent.generationComplete": OpenAIRealtimeEventTypes.RESPONSE_TEXT_DONE, "serverContent.turnComplete": OpenAIRealtimeEventTypes.RESPONSE_DONE, "serverContent.interrupted": OpenAIRealtimeEventTypes.RESPONSE_DONE, + "toolCall": ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DONE, +} + +# Top-level keys in a Gemini realtime message that map_openai_event knows how +# to handle. Other keys (e.g. ``usageMetadata``) can appear alongside these as +# siblings and must be skipped by the main transform loop — otherwise +# map_openai_event raises ``ValueError`` and the WebSocket session terminates. +_KNOWN_GEMINI_TOP_LEVEL_KEYS: set = { + map_key.split(".", 1)[0] for map_key in MAP_GEMINI_FIELD_TO_OPENAI_EVENT } class GeminiRealtimeConfig(BaseRealtimeConfig): + # Cap the LRU of in-flight tool calls so long sessions with many tool + # calls don't grow the dict without bound. Sized large enough to cover + # bursts of pending tool responses; the oldest entry is evicted when a + # new call beyond the cap arrives. + _TOOL_CALL_ID_TO_NAME_MAX = 256 + + def __init__(self): + super().__init__() + # Store call_id → function_name mapping for tool call round-trip + self._tool_call_id_to_name: "OrderedDict[str, str]" = OrderedDict() + # Buffer ``usageMetadata`` that Gemini Live emits as a standalone + # frame (between turns) so the next ``response.done`` attributes the + # tokens consumed. Without this an authenticated client can drive + # tool-call or normal turns whose token usage is recorded as zero, + # bypassing spend and budget accounting. + self._pending_usage_metadata: Optional[dict] = None + def validate_environment( self, headers: dict, model: str, api_key: Optional[str] = None ) -> dict: @@ -190,10 +223,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): ) vertex_gemini_config = VertexGeminiConfig() - optional_params["generationConfig"]["tools"] = ( - vertex_gemini_config._map_function( - value=value, optional_params=optional_params - ) + # Tools should be at the top level of setup, not inside generationConfig + optional_params["tools"] = vertex_gemini_config._map_function( + value=value, optional_params=optional_params ) elif key == "input_audio_transcription" and value is not None: optional_params["inputAudioTranscription"] = {} @@ -214,6 +246,272 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): optional_params.pop("generationConfig") return optional_params + @staticmethod + def _extract_turn_detection(session: dict) -> Optional[dict]: + """Extract turn_detection from a session.update payload. + + Handles both the flat beta shape (``session.turn_detection``) and the + GA shape (``session.audio.input.turn_detection``). + """ + if not isinstance(session, dict): + return None + td = session.get("turn_detection") + if isinstance(td, dict): + return td + audio = session.get("audio") + if isinstance(audio, dict): + input_cfg = audio.get("input") + if isinstance(input_cfg, dict): + td = input_cfg.get("turn_detection") + if isinstance(td, dict): + return td + return None + + @staticmethod + def _normalize_session_payload_for_mapping(session: dict) -> dict: + """Normalize GA-remapped session fields back to their beta keys. + + ``map_openai_params`` only recognises the flat OpenAI-beta key names + (``modalities``, ``input_audio_transcription``, ``turn_detection``). + For GA clients the upstream shim renames these into the nested GA + schema (``output_modalities``, ``audio.input.transcription``, + ``audio.input.turn_detection``), which would otherwise be silently + dropped here. Surface them back at the top level so the existing + mapping logic picks them up without duplicating provider-specific + knowledge of the GA schema in ``map_openai_params``. + """ + if not isinstance(session, dict): + return session + + normalized = dict(session) + + if "modalities" not in normalized and "output_modalities" in normalized: + normalized["modalities"] = normalized["output_modalities"] + + audio = normalized.get("audio") + if isinstance(audio, dict): + input_cfg = audio.get("input") + if isinstance(input_cfg, dict): + if ( + "input_audio_transcription" not in normalized + and "transcription" in input_cfg + ): + normalized["input_audio_transcription"] = input_cfg["transcription"] + + extracted_turn_detection = GeminiRealtimeConfig._extract_turn_detection( + normalized + ) + if extracted_turn_detection is not None and not isinstance( + normalized.get("turn_detection"), dict + ): + normalized["turn_detection"] = extracted_turn_detection + + return normalized + + def _handle_session_update( + self, + json_message: dict, + model: str, + session_configuration_request: Optional[str], + ) -> List[str]: + """ + Handle session.update by sending setup to Gemini. + + On the FIRST session.update (when session_configuration_request is None), + the full setup with all configuration is sent. + + Subsequent session.update messages are forwarded as a follow-up setup + with the new fields merged into the original setup. Gemini Live treats + a follow-up BidiGenerateContentSetup as a full session replacement + rather than a partial merge, so we carry forward the previous setup + (tools, generationConfig, inputAudioTranscription, systemInstruction, + ...) and overlay the new fields on top. This preserves the old + behavior where clients could refine the session via session.update + (e.g. add tools after the auto-setup on connect), and also keeps the + guardrail-driven turn_detection update working. + """ + session_payload = json_message.get("session") or {} + # Normalize GA-remapped fields (``output_modalities``, + # nested ``audio.input.transcription``, + # ``audio.input.turn_detection``) back to their flat beta keys so + # ``map_openai_params`` picks them up. Without this, GA clients' + # explicit modality / transcription / turn-detection settings + # would be silently dropped because ``map_openai_params`` only + # recognises the flat OpenAI-beta key names. + session_payload = self._normalize_session_payload_for_mapping(session_payload) + new_overrides = self.map_openai_params( + optional_params={}, non_default_params=session_payload + ) + + if session_configuration_request is None: + generation_config = new_overrides.setdefault("generationConfig", {}) + generation_config.setdefault("responseModalities", ["AUDIO"]) + new_overrides.setdefault("inputAudioTranscription", {}) + new_overrides["model"] = f"models/{model}" + verbose_logger.debug( + "Gemini Realtime: Sending initial setup with tools to backend" + ) + return [json.dumps({"setup": new_overrides})] + + if not new_overrides: + verbose_logger.debug( + "Gemini Realtime: Ignoring session.update (no mappable fields)" + ) + return [] + + try: + original_setup = cast( + BidiGenerateContentSetup, + json.loads(session_configuration_request).get("setup", {}), + ) + except (json.JSONDecodeError, AttributeError): + original_setup = {} + + # Deep-merge ``generationConfig`` and ``realtimeInputConfig`` so a + # partial session.update (e.g. only ``temperature`` or only + # ``modalities``) does not silently drop unrelated sub-keys + # (``responseModalities``, ``maxOutputTokens``, ...) from the original + # setup. + follow_up_setup: BidiGenerateContentSetup = { + **original_setup, + **new_overrides, + "model": f"models/{model}", + } + original_generation_config = original_setup.get("generationConfig") + new_generation_config = new_overrides.get("generationConfig") + if isinstance(original_generation_config, dict) and isinstance( + new_generation_config, dict + ): + follow_up_setup["generationConfig"] = { + **original_generation_config, + **new_generation_config, + } + original_realtime_input_config = original_setup.get("realtimeInputConfig") + new_realtime_input_config = new_overrides.get("realtimeInputConfig") + if isinstance(original_realtime_input_config, dict) and isinstance( + new_realtime_input_config, dict + ): + merged_realtime_input_config = { + **original_realtime_input_config, + **new_realtime_input_config, + } + # Deep-merge ``automaticActivityDetection`` so a partial VAD + # update (e.g. the guardrail-injected ``disabled: True`` from + # ``create_response: False``) does not silently drop unrelated + # knobs like ``silenceDurationMs`` / ``prefixPaddingMs`` from + # the original setup. + original_automatic_activity_detection = original_realtime_input_config.get( + "automaticActivityDetection" + ) + new_automatic_activity_detection = new_realtime_input_config.get( + "automaticActivityDetection" + ) + if isinstance(original_automatic_activity_detection, dict) and isinstance( + new_automatic_activity_detection, dict + ): + merged_realtime_input_config["automaticActivityDetection"] = { + **original_automatic_activity_detection, + **new_automatic_activity_detection, + } + follow_up_setup["realtimeInputConfig"] = cast( + BidiGenerateContentRealtimeInputConfig, + merged_realtime_input_config, + ) + verbose_logger.debug( + "Gemini Realtime: Forwarding session.update as follow-up setup" + ) + return [json.dumps({"setup": follow_up_setup})] + + def _handle_conversation_item(self, json_message: dict) -> List[str]: + """ + Handle conversation.item.create for user text or function call output. + + Converts OpenAI format to Gemini's clientContent (for user text) or + toolResponse (for function outputs). + """ + item = json_message.get("item", {}) + item_type = item.get("type") + + # Handle function call output (tool response) + if item_type == "function_call_output": + return self._handle_function_call_output(item) + + # Handle regular text content + return self._handle_user_text_content(item) + + def _handle_function_call_output(self, item: dict) -> List[str]: + """Transform function_call_output to Gemini toolResponse format.""" + call_id = item.get("call_id", "") + output = item.get("output", "{}") + + verbose_logger.debug( + f"Gemini Realtime: Transforming function_call_output for call_id={call_id}" + ) + + # Parse the output to get the result. Gemini's + # functionResponses[].response field is a Struct, so it must be a + # dict; wrap any non-dict (primitives, lists, invalid JSON) under a + # `result` key. + try: + parsed_output = json.loads(output) if isinstance(output, str) else output + except json.JSONDecodeError: + parsed_output = output + output_dict = ( + parsed_output + if isinstance(parsed_output, dict) + else {"result": parsed_output} + ) + + # Look up the function name from stored mapping. Keep the entry so a + # client SDK that retries function_call_output (or sends it twice for + # the same tool call) still produces a Gemini toolResponse with the + # required ``name`` field; refresh the LRU position so an active + # call_id stays warm across long sessions. + function_name = self._tool_call_id_to_name.get(call_id) + if function_name: + self._tool_call_id_to_name.move_to_end(call_id) + else: + verbose_logger.warning( + f"Gemini Realtime: Function name not found for call_id={call_id}. " + "This may cause Gemini to reject the response." + ) + + # Build Gemini toolResponse format + function_response = { + "id": call_id, + "response": output_dict, + } + if function_name: + function_response["name"] = function_name + + tool_response_message = { + "toolResponse": {"functionResponses": [function_response]} + } + + return [json.dumps(tool_response_message)] + + def _handle_user_text_content(self, item: dict) -> List[str]: + """Transform user text content to Gemini clientContent format.""" + content_list = item.get("content", []) + text_parts = [ + c.get("text", "") + for c in content_list + if isinstance(c, dict) and c.get("type") == "input_text" + ] + text = " ".join(filter(None, text_parts)) + if not text: + return [] + + # Build clientContent message with turns (proper Gemini Live API format) + client_content_message = { + "clientContent": { + "turns": [{"role": "user", "parts": [{"text": text}]}], + "turnComplete": True, + } + } + + return [json.dumps(client_content_message)] + def transform_realtime_request( self, message: str, @@ -233,55 +531,42 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): messages: List[str] = [] msg_type = json_message.get("type") - ## HANDLE SESSION UPDATE — translate to Gemini setup; no realtime_input needed ## + ## HANDLE SESSION UPDATE — translate to Gemini setup ## if msg_type == "session.update": - client_session_configuration_request = self.map_openai_params( - optional_params={}, non_default_params=json_message["session"] + return self._handle_session_update( + json_message, model, session_configuration_request ) - client_session_configuration_request["model"] = f"models/{model}" - messages.append(json.dumps({"setup": client_session_configuration_request})) - return messages ## HANDLE response.create — Gemini responds automatically; nothing to forward ## if msg_type == "response.create": return [] - ## HANDLE INPUT AUDIO BUFFER ## + ## HANDLE conversation.item.create — extract user text or function call output ## + if msg_type == "conversation.item.create": + return self._handle_conversation_item(json_message) + + ## HANDLE INPUT AUDIO BUFFER - use realtimeInput for audio streaming ## if msg_type == "input_audio_buffer.append": realtime_input_dict["audio"] = HttpxBlobType( mimeType=self.get_audio_mime_type(), data=json_message["audio"] ) - ## HANDLE conversation.item.create — extract actual user text ## - elif msg_type == "conversation.item.create": - item = json_message.get("item", {}) - content_list = item.get("content", []) - text_parts = [ - c.get("text", "") - for c in content_list - if isinstance(c, dict) and c.get("type") == "input_text" - ] - text = " ".join(filter(None, text_parts)) - if not text: - return [] - realtime_input_dict["text"] = text - else: - # Unknown/unsupported OpenAI event type — drop silently rather than - # forwarding raw JSON as text input to the model. - return [] - if len(realtime_input_dict) != 1: - raise ValueError( - f"Only one argument can be set, got {len(realtime_input_dict)}:" - f" {list(realtime_input_dict.keys())}" + realtime_input_dict = cast( + BidiGenerateContentRealtimeInput, + encode_unserializable_types( + cast(Dict[str, object], realtime_input_dict) + ), ) - realtime_input_dict = cast( - BidiGenerateContentRealtimeInput, - encode_unserializable_types(cast(Dict[str, object], realtime_input_dict)), - ) - - messages.append(json.dumps({"realtime_input": realtime_input_dict})) - return messages + gemini_msg = json.dumps({"realtimeInput": realtime_input_dict}) + verbose_logger.debug( + "Gemini Realtime: Sending audio realtimeInput to backend" + ) + messages.append(gemini_msg) + return messages + # Unknown/unsupported OpenAI event type — drop silently rather than + # forwarding raw JSON as text input to the model. + return [] def transform_session_created_event( self, @@ -300,7 +585,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): generation_config = ( session_configuration_request_dict.get("generationConfig", {}) or {} ) - gemini_modalities = generation_config.get("responseModalities", ["TEXT"]) + gemini_modalities = generation_config.get("responseModalities", ["AUDIO"]) _modalities = [ modality.lower() for modality in cast(List[str], gemini_modalities) ] @@ -352,18 +637,18 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): delta_type: ALL_DELTA_TYPES, session_configuration_request: Optional[str] = None, ) -> List[OpenAIRealtimeEvents]: - if session_configuration_request is None: - raise ValueError( - "session_configuration_request is required for Gemini API calls" - ) - - session_configuration_request_dict: BidiGenerateContentSetup = json.loads( - session_configuration_request - ).get("setup", {}) + session_configuration_request_dict: BidiGenerateContentSetup = {} + if session_configuration_request is not None: + try: + session_configuration_request_dict = json.loads( + session_configuration_request + ).get("setup", {}) + except json.JSONDecodeError: + session_configuration_request_dict = {} generation_config = session_configuration_request_dict.get( "generationConfig", {} ) - gemini_modalities = generation_config.get("responseModalities", ["TEXT"]) + gemini_modalities = generation_config.get("responseModalities", ["AUDIO"]) _modalities = [ modality.lower() for modality in cast(List[str], gemini_modalities) ] @@ -576,6 +861,86 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): returned_items.append(response_output_item_done) return returned_items + def _consume_usage_metadata_for_response_done(self, frame: dict) -> Optional[dict]: + """Return the ``usageMetadata`` to attribute to a ``response.done``. + + Gemini Live emits ``usageMetadata`` either alongside the closing + frame (``serverContent.turnComplete`` / ``toolCall``) or as a + standalone frame between turns. The standalone form would otherwise + be discarded by the no-op branch in ``transform_realtime_response`` + and the consumed tokens silently dropped from spend/budget + accounting. ``_pending_usage_metadata`` buffers any such standalone + frames so the next emitted ``response.done`` carries the deferred + token counts. + + Returns the in-frame ``usageMetadata`` if present (and clears the + buffer since the in-frame counts are the authoritative attribution + for this turn), otherwise returns the buffered counts. ``None`` is + returned when neither is available so the caller can fall back to + ``get_empty_usage()``. + """ + # ``pop`` (rather than ``get``) so a single Gemini frame containing + # multiple closing keys (e.g. both ``toolCall`` and + # ``serverContent.turnComplete``) cannot attribute the same + # ``usageMetadata`` to two ``response.done`` events and double-count + # tokens in spend/budget accounting. + in_frame = frame.pop("usageMetadata", None) if isinstance(frame, dict) else None + if isinstance(in_frame, dict): + self._pending_usage_metadata = None + return in_frame + buffered = self._pending_usage_metadata + self._pending_usage_metadata = None + return buffered + + def transform_tool_call_events( + self, + tool_call_message: dict, + response_id: Optional[str] = None, + output_item_id: Optional[str] = None, + ) -> List[OpenAIRealtimeFunctionCallArgumentsDone]: + """ + Transform Gemini toolCall message to OpenAI function call events. + + Converts Gemini's functionCalls format to OpenAI's response.function_call_arguments.done events. + Also stores call_id → name mapping for later use in function_call_output responses. + """ + function_calls = tool_call_message.get("functionCalls", []) + resolved_response_id = response_id or f"resp_{uuid.uuid4()}" + resolved_output_item_id = output_item_id or f"item_{uuid.uuid4()}" + + verbose_logger.debug( + f"Gemini Realtime: Transforming {len(function_calls)} tool call(s) to OpenAI format" + ) + + events: List[OpenAIRealtimeFunctionCallArgumentsDone] = [] + for idx, fc in enumerate(function_calls): + call_id = fc.get("id", "") + name = fc.get("name", "") + + # Store call_id → name mapping for round-trip. Use an LRU so + # repeated function_call_output lookups (retries) still hit, while + # sessions with many tool calls don't grow the dict unboundedly. + if call_id and name: + self._tool_call_id_to_name[call_id] = name + self._tool_call_id_to_name.move_to_end(call_id) + while len(self._tool_call_id_to_name) > self._TOOL_CALL_ID_TO_NAME_MAX: + self._tool_call_id_to_name.popitem(last=False) + + events.append( + OpenAIRealtimeFunctionCallArgumentsDone( + type="response.function_call_arguments.done", + event_id=f"event_{uuid.uuid4()}", + response_id=resolved_response_id, + item_id=f"{resolved_output_item_id}_tool_{idx}", + output_index=idx, + call_id=call_id, + name=name, + arguments=json.dumps(fc.get("args", {})), + ) + ) + + return events + @staticmethod def get_nested_value(obj: dict, path: str) -> Any: keys = path.split(".") @@ -681,14 +1046,20 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): "generationConfig", {} ) temperature = generation_config.get("temperature") - max_output_tokens = generation_config.get("max_output_tokens") - gemini_modalities = generation_config.get("responseModalities", ["TEXT"]) + max_output_tokens = generation_config.get("maxOutputTokens") + gemini_modalities = generation_config.get("responseModalities", ["AUDIO"]) _modalities = [ modality.lower() for modality in cast(List[str], gemini_modalities) ] - if "usageMetadata" in message: + resolved_usage_metadata = self._consume_usage_metadata_for_response_done( + cast(dict, message) + ) + if resolved_usage_metadata is not None: _chat_completion_usage = VertexGeminiConfig._calculate_usage( - completion_response=message, + completion_response=cast( + BidiGenerateContentServerMessage, + {**cast(dict, message), "usageMetadata": resolved_usage_metadata}, + ), ) else: _chat_completion_usage = get_empty_usage() @@ -716,7 +1087,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): if temperature is not None: response_done_event["response"]["temperature"] = temperature if max_output_tokens is not None: - response_done_event["response"]["max_output_tokens"] = max_output_tokens + response_done_event["response"]["max_output_tokens"] = cast( + int, max_output_tokens + ) return response_done_event @@ -808,13 +1181,18 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): def map_openai_event( self, key: str, - value: dict, + value: Any, current_delta_type: Optional[ALL_DELTA_TYPES], - json_message: dict, - ) -> OpenAIRealtimeEventTypes: - model_turn_event = value.get("modelTurn") - generation_complete_event = value.get("generationComplete") - openai_event: Optional[OpenAIRealtimeEventTypes] = None + ) -> Union[OpenAIRealtimeEventTypes, ResponsesAPIStreamEvents]: + if isinstance(value, dict): + model_turn_event = value.get("modelTurn") + generation_complete_event = value.get("generationComplete") + else: + model_turn_event = None + generation_complete_event = None + openai_event: Optional[ + Union[OpenAIRealtimeEventTypes, ResponsesAPIStreamEvents] + ] = None if model_turn_event: # check if model turn event openai_event = self.map_model_turn_event(model_turn_event) elif generation_complete_event: @@ -822,15 +1200,27 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): delta_type=current_delta_type ) else: - # Check if this key or any nested key matches our mapping - for map_key, openai_event in MAP_GEMINI_FIELD_TO_OPENAI_EVENT.items(): - if map_key == key or ( - "." in map_key - and GeminiRealtimeConfig.get_nested_value(json_message, map_key) - is not None - ): - openai_event = openai_event + # Check if this key or any nested key matches our mapping. Use a + # distinct loop variable so we don't shadow ``openai_event`` and + # leak the last dict value when no entry matches. Scope dotted-key + # lookups to the current ``key``/``value`` pair — checking the + # whole ``json_message`` would let a sibling key (e.g. + # ``serverContent.turnComplete``) misclassify the event currently + # being processed (e.g. ``toolCall``). + for map_key, candidate_event in MAP_GEMINI_FIELD_TO_OPENAI_EVENT.items(): + if map_key == key: + openai_event = candidate_event break + if "." in map_key: + prefix, _, nested_path = map_key.partition(".") + if ( + prefix == key + and isinstance(value, dict) + and GeminiRealtimeConfig.get_nested_value(value, nested_path) + is not None + ): + openai_event = candidate_event + break if openai_event is None: raise ValueError(f"Unknown openai event: {key}, value: {value}") return openai_event @@ -854,6 +1244,15 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): message_str = str(message) raise ValueError(f"Invalid JSON message: {message_str}") + verbose_logger.debug( + "Realtime Response Transform: Gemini frame keys=%s", + ( + sorted(json_message.keys()) + if isinstance(json_message, dict) + else type(json_message).__name__ + ), + ) + logging_session_id = logging_obj.litellm_trace_id current_output_item_id = realtime_response_transform_input[ @@ -913,32 +1312,44 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): ) # If serverContent only contained transcription(s) and no model - # content, return early — the main loop would fail on unknown keys. + # content, mark it as already handled so the main loop skips it + # (map_openai_event would raise on an unknown serverContent + # subkey). Fall through so sibling top-level keys such as + # ``toolCall`` are still processed in the main loop. _model_content_keys = { "modelTurn", "turnComplete", "interrupted", "generationComplete", } - if not any(k in server_content for k in _model_content_keys): - return { - "response": returned_message, - "current_output_item_id": current_output_item_id, - "current_response_id": current_response_id, - "current_delta_chunks": current_delta_chunks, - "current_conversation_id": current_conversation_id, - "current_item_chunks": current_item_chunks, - "current_delta_type": current_delta_type, - "session_configuration_request": session_configuration_request, - } + server_content_handled = not any( + k in server_content for k in _model_content_keys + ) + else: + server_content_handled = False - for key, value in json_message.items(): + tool_call_handled = False + # Snapshot the items so handlers below can safely mutate + # ``json_message`` (e.g. ``_consume_usage_metadata_for_response_done`` + # pops ``usageMetadata`` to prevent a single frame from attributing + # the same token counts to two ``response.done`` events). + for key, value in list(json_message.items()): + # Skip sibling metadata keys (e.g. ``usageMetadata``) that can + # accompany a primary payload like ``toolCall`` or ``serverContent``. + # ``map_openai_event`` raises ValueError on unknown keys, which + # would otherwise terminate the WebSocket session. + if key not in _KNOWN_GEMINI_TOP_LEVEL_KEYS: + continue + # serverContent was a transcription-only payload already emitted + # above; skip it here so map_openai_event doesn't raise on the + # missing model-content subkeys. + if key == "serverContent" and server_content_handled: + continue # Check if this key or any nested key matches our mapping openai_event = self.map_openai_event( key=key, value=value, current_delta_type=current_delta_type, - json_message=json_message, ) if openai_event == OpenAIRealtimeEventTypes.SESSION_CREATED: @@ -947,8 +1358,226 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): logging_session_id, realtime_response_transform_input["session_configuration_request"], ) - session_configuration_request = json.dumps(transformed_message) returned_message.append(transformed_message) + elif openai_event == ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DONE: + # Handle toolCall from Gemini. If the payload has no function + # calls, emit nothing — an orphaned response.created/done pair + # with no output items would confuse OpenAI-compatible clients. + # Mark the key as intentionally consumed (mirroring + # ``server_content_handled``) so any sibling keys in the same + # frame are still processed by the rest of the loop and the + # post-loop guard doesn't treat the no-op as fatal. + if not value.get("functionCalls"): + tool_call_handled = True + continue + + if current_conversation_id is None: + current_conversation_id = f"conv_{uuid.uuid4()}" + + # Extract session-level response metadata once so both + # response.created and response.done can include matching + # modalities/temperature/max_output_tokens fields. + session_setup: BidiGenerateContentSetup = {} + if session_configuration_request is not None: + try: + session_setup = json.loads(session_configuration_request).get( + "setup", {} + ) + except (json.JSONDecodeError, TypeError): + session_setup = {} + tool_call_generation_config = ( + session_setup.get("generationConfig", {}) or {} + ) + tool_call_modalities = [ + modality.lower() + for modality in cast( + List[str], + tool_call_generation_config.get( + "responseModalities", ["AUDIO"] + ), + ) + ] + + # Emit response.created preamble if this is the first event in the response + if current_response_id is None: + current_response_id = f"resp_{uuid.uuid4()}" + current_output_item_id = f"item_{uuid.uuid4()}" + + # Mirror the audio/text path: include modalities, + # temperature, and max_output_tokens on response.created so + # spec-compliant clients see consistent response metadata + # regardless of whether the response starts with content or + # a tool call. + returned_message.append( + { + "type": "response.created", + "event_id": f"event_{uuid.uuid4()}", + "response": { + "object": "realtime.response", + "id": current_response_id, + "status": "in_progress", + "output": [], + "conversation_id": current_conversation_id, + "modalities": tool_call_modalities, + "temperature": tool_call_generation_config.get( + "temperature" + ), + "max_output_tokens": tool_call_generation_config.get( + "maxOutputTokens" + ), + }, + } + ) + + tool_call_events = self.transform_tool_call_events( + value, + response_id=current_response_id, + output_item_id=current_output_item_id, + ) + # Emit output_item.added and conversation.item.created for each function call + for idx, tool_call in enumerate(tool_call_events): + item_id = tool_call["item_id"] + function_call_item: OpenAIRealtimeStreamResponseOutputItem = { + "id": item_id, + "object": "realtime.item", + "type": "function_call", + "status": "completed", + "call_id": tool_call["call_id"], + "name": tool_call["name"], + "arguments": tool_call["arguments"], + } + # response.output_item.added + returned_message.append( + OpenAIRealtimeStreamResponseOutputItemAdded( + type="response.output_item.added", + event_id=f"event_{uuid.uuid4()}", + response_id=current_response_id, + output_index=idx, + item={ + **function_call_item, + "status": "in_progress", + "arguments": "", + }, + ) + ) + # response.function_call_arguments.delta — Gemini delivers + # the full arguments string in a single toolCall frame + # rather than streaming partial chunks, so emit one delta + # carrying the complete payload before the matching + # ``.done`` event. Spec-compliant OpenAI Realtime SDK + # clients accumulate ``delta.delta`` and rely on at least + # one delta before ``.done``. + returned_message.append( + cast( + OpenAIRealtimeEvents, + { + "type": "response.function_call_arguments.delta", + "event_id": f"event_{uuid.uuid4()}", + "response_id": current_response_id, + "item_id": item_id, + "output_index": idx, + "call_id": tool_call["call_id"], + "delta": tool_call["arguments"], + }, + ) + ) + # response.function_call_arguments.done + returned_message.append(tool_call) + # response.output_item.done — pass a fresh copy so + # downstream handlers that mutate the item dict (e.g. the + # beta-protocol translator) don't corrupt the references + # used by sibling events sharing the same function_call_item. + returned_message.append( + OpenAIRealtimeOutputItemDone( + type="response.output_item.done", + event_id=f"event_{uuid.uuid4()}", + response_id=current_response_id, + output_index=idx, + item={**function_call_item}, + ) + ) + # conversation.item.created + returned_message.append( + OpenAIRealtimeConversationItemCreated( + type="conversation.item.created", + event_id=f"event_{uuid.uuid4()}", + item={**function_call_item}, + ) + ) + + # response.done - close the response so clients can submit tool + # results. Mirror the non-tool-call RESPONSE_DONE path: if Gemini + # delivered ``usageMetadata`` alongside this ``toolCall`` frame, + # propagate the real token counts so spend/budget accounting + # records the tokens consumed by the tool-call turn. Standalone + # ``usageMetadata`` frames emitted in a separate WebSocket frame + # are buffered on the instance so the next ``response.done`` + # picks them up (otherwise an authenticated client could drive + # tool-call turns whose token usage is recorded as zero, + # bypassing budgets). Falls back to an empty usage block when + # neither is available (OpenAI-compatible clients expect + # ``usage`` to always be present on response.done). + resolved_tool_call_usage_metadata = ( + self._consume_usage_metadata_for_response_done(json_message) + ) + if resolved_tool_call_usage_metadata is not None: + _tool_call_chat_completion_usage = ( + VertexGeminiConfig._calculate_usage( + completion_response=cast( + BidiGenerateContentServerMessage, + { + **json_message, + "usageMetadata": resolved_tool_call_usage_metadata, + }, + ), + ) + ) + else: + _tool_call_chat_completion_usage = get_empty_usage() + tool_call_responses_api_usage = LiteLLMCompletionResponsesConfig._transform_chat_completion_usage_to_responses_usage( + _tool_call_chat_completion_usage, + ) + tool_call_done_event = OpenAIRealtimeDoneEvent( + type="response.done", + event_id=f"event_{uuid.uuid4()}", + response=OpenAIRealtimeResponseDoneObject( + id=current_response_id, + object="realtime.response", + status="completed", + output=[ + { + "id": te["item_id"], + "object": "realtime.item", + "type": "function_call", + "status": "completed", + "call_id": te["call_id"], + "name": te["name"], + "arguments": te["arguments"], + } + for te in tool_call_events + ], + conversation_id=current_conversation_id, + modalities=tool_call_modalities, + usage=tool_call_responses_api_usage.model_dump(), + ), + ) + tool_call_temperature = tool_call_generation_config.get("temperature") + if tool_call_temperature is not None: + tool_call_done_event["response"][ + "temperature" + ] = tool_call_temperature + tool_call_max_output_tokens = tool_call_generation_config.get( + "maxOutputTokens" + ) + if tool_call_max_output_tokens is not None: + tool_call_done_event["response"]["max_output_tokens"] = cast( + int, tool_call_max_output_tokens + ) + returned_message.append(tool_call_done_event) + # Reset IDs so the next model turn (after tool results) starts a + # fresh response with its own response.created preamble. + current_output_item_id = None + current_response_id = None elif openai_event == OpenAIRealtimeEventTypes.RESPONSE_DONE: transformed_response_done_event = self.transform_response_done_event( message=BidiGenerateContentServerMessage(**json_message), # type: ignore @@ -958,16 +1587,37 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): output_items=None, ) returned_message.append(transformed_response_done_event) + # Reset IDs so a subsequent turn (e.g. a `toolCall` arriving in + # a later WebSocket frame after `turnComplete`) starts a fresh + # response with its own `response.created` preamble instead of + # reusing the just-completed response ID. + current_output_item_id = None + current_response_id = None elif ( openai_event == OpenAIRealtimeEventTypes.RESPONSE_TEXT_DELTA or openai_event == OpenAIRealtimeEventTypes.RESPONSE_TEXT_DONE or openai_event == OpenAIRealtimeEventTypes.RESPONSE_AUDIO_DELTA or openai_event == OpenAIRealtimeEventTypes.RESPONSE_AUDIO_DONE ): + # Pass the locally-updated state (rather than the original + # input snapshot) so that prior iterations of this loop — + # e.g. a tool-call or response.done that just reset + # current_response_id/current_output_item_id to None — are + # honoured by the modality handler. + _modality_input: RealtimeResponseTransformInput = { + **realtime_response_transform_input, + "current_output_item_id": current_output_item_id, + "current_response_id": current_response_id, + "current_conversation_id": current_conversation_id, + "current_delta_chunks": current_delta_chunks, + "current_item_chunks": current_item_chunks, + "current_delta_type": current_delta_type, + "session_configuration_request": session_configuration_request, + } _returned_message = self.handle_openai_modality_event( openai_event, json_message, - realtime_response_transform_input, + _modality_input, delta_type="text" if "text" in openai_event.value else "audio", ) returned_message.extend(_returned_message["returned_message"]) @@ -979,6 +1629,41 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): else: raise ValueError(f"Unknown openai event: {openai_event}") if len(returned_message) == 0: + # A frame whose only top-level keys are sibling metadata (e.g. + # a standalone ``{"usageMetadata": {...}}`` emitted by Gemini + # Live between turns) is not an error — there is just nothing + # to forward to the OpenAI-shaped client. Returning the + # unchanged state keeps the WebSocket alive; raising would + # terminate the session for a benign no-op frame. + # serverContent already consumed by the transcription handler is + # a benign no-op for downstream — treat it like a metadata-only + # key when deciding whether to raise. + unhandled_known_keys = [ + key + for key in json_message + if key in _KNOWN_GEMINI_TOP_LEVEL_KEYS + and not (key == "serverContent" and server_content_handled) + and not (key == "toolCall" and tool_call_handled) + ] + # Buffer standalone usage metadata so the next response.done can + # attribute the token counts. Without this, an authenticated + # client driving turns whose usageMetadata is emitted in a + # separate frame would have those tokens recorded as zero spend, + # bypassing budget enforcement. + standalone_usage_metadata = json_message.get("usageMetadata") + if isinstance(standalone_usage_metadata, dict): + self._pending_usage_metadata = standalone_usage_metadata + if not unhandled_known_keys: + return { + "response": returned_message, + "current_output_item_id": current_output_item_id, + "current_response_id": current_response_id, + "current_delta_chunks": current_delta_chunks, + "current_conversation_id": current_conversation_id, + "current_item_chunks": current_item_chunks, + "current_delta_type": current_delta_type, + "session_configuration_request": session_configuration_request, + } if isinstance(message, bytes): message_str = message.decode("utf-8", errors="replace") else: @@ -993,6 +1678,13 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): transformed_message=returned_message, current_item_chunks=current_item_chunks, ) + + for msg in returned_message: + event_type = msg.get("type") if isinstance(msg, dict) else "unknown" + verbose_logger.debug( + "Realtime Response Transform: OpenAI event=%s", event_type + ) + return { "response": returned_message, "current_output_item_id": current_output_item_id, @@ -1005,7 +1697,10 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): } def requires_session_configuration(self) -> bool: - return True + # Default behavior is backwards-compatible: send setup on connect. + # Opt-in to deferred setup for tool-injection flow via: + # litellm.gemini_live_defer_setup = True + return not litellm.gemini_live_defer_setup def session_configuration_request(self, model: str) -> str: """ diff --git a/litellm/llms/gemini/videos/transformation.py b/litellm/llms/gemini/videos/transformation.py index 9714c8a3923..77a95bfa5ab 100644 --- a/litellm/llms/gemini/videos/transformation.py +++ b/litellm/llms/gemini/videos/transformation.py @@ -581,12 +581,23 @@ class GeminiVideoConfig(BaseVideoConfig): raise NotImplementedError("video get character is not supported for Gemini") def transform_video_edit_request( - self, prompt, video_id, api_base, litellm_params, headers, extra_body=None + self, + prompt, + video_id, + api_base, + litellm_params, + headers, + extra_body=None, + prefetched_source_data=None, ): raise NotImplementedError("video edit is not supported for Gemini") def transform_video_edit_response( - self, raw_response, logging_obj, custom_llm_provider=None + self, + raw_response, + logging_obj, + custom_llm_provider=None, + request_data=None, ): raise NotImplementedError("video edit is not supported for Gemini") diff --git a/litellm/llms/openai/videos/transformation.py b/litellm/llms/openai/videos/transformation.py index 2d165a7d7df..520a42e9dd1 100644 --- a/litellm/llms/openai/videos/transformation.py +++ b/litellm/llms/openai/videos/transformation.py @@ -534,6 +534,7 @@ class OpenAIVideoConfig(BaseVideoConfig): litellm_params: GenericLiteLLMParams, headers: dict, extra_body: Optional[Dict[str, Any]] = None, + prefetched_source_data: Optional[Dict[str, Any]] = None, ) -> Tuple[str, Dict]: original_video_id = extract_original_video_id(video_id) url = f"{api_base.rstrip('/')}/edits" @@ -547,6 +548,7 @@ class OpenAIVideoConfig(BaseVideoConfig): raw_response: httpx.Response, logging_obj: Any, custom_llm_provider: Optional[str] = None, + request_data: Optional[Dict] = None, ) -> VideoObject: video_obj = VideoObject(**raw_response.json()) if custom_llm_provider and video_obj.id: diff --git a/litellm/llms/runwayml/videos/transformation.py b/litellm/llms/runwayml/videos/transformation.py index 4f84816a2bc..b1723f494ec 100644 --- a/litellm/llms/runwayml/videos/transformation.py +++ b/litellm/llms/runwayml/videos/transformation.py @@ -623,12 +623,23 @@ class RunwayMLVideoConfig(BaseVideoConfig): raise NotImplementedError("video get character is not supported for RunwayML") def transform_video_edit_request( - self, prompt, video_id, api_base, litellm_params, headers, extra_body=None + self, + prompt, + video_id, + api_base, + litellm_params, + headers, + extra_body=None, + prefetched_source_data=None, ): raise NotImplementedError("video edit is not supported for RunwayML") def transform_video_edit_response( - self, raw_response, logging_obj, custom_llm_provider=None + self, + raw_response, + logging_obj, + custom_llm_provider=None, + request_data=None, ): raise NotImplementedError("video edit is not supported for RunwayML") diff --git a/litellm/llms/vertex_ai/realtime/transformation.py b/litellm/llms/vertex_ai/realtime/transformation.py index 2b4746b174e..ea4dbccc8c8 100644 --- a/litellm/llms/vertex_ai/realtime/transformation.py +++ b/litellm/llms/vertex_ai/realtime/transformation.py @@ -14,6 +14,7 @@ Auth: OAuth2 Bearer token (not an API key). import json from typing import List, Optional +from litellm import verbose_logger from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig @@ -26,6 +27,7 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig): """ def __init__(self, access_token: str, project: str, location: str) -> None: + super().__init__() self._access_token = access_token self._project = project self._location = location @@ -138,6 +140,62 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig): # Request translation # ------------------------------------------------------------------ + def _vertex_model_path(self, model: str) -> str: + """Return the fully-qualified Vertex AI model resource path.""" + return ( + f"projects/{self._project}" + f"/locations/{self._location}" + f"/publishers/google/models/{model}" + ) + + def _build_vertex_ai_setup_config(self, model: str, session_params: dict) -> dict: + """Build Vertex AI setup configuration with proper model path and defaults.""" + # Normalize GA-remapped fields (``output_modalities``, nested + # ``audio.input.transcription``, ``audio.input.turn_detection``) back to + # their flat beta keys so ``map_openai_params`` picks them up. Without + # this, GA clients' explicit modality / transcription / turn-detection + # settings would be silently dropped because ``map_openai_params`` only + # recognises the flat OpenAI-beta key names. + session_params = self._normalize_session_payload_for_mapping(session_params) + setup_config = self.map_openai_params( + optional_params={}, non_default_params=session_params + ) + + # Use full Vertex AI model path + setup_config["model"] = self._vertex_model_path(model) + + # Add Vertex AI specific defaults if not provided + generation_config = setup_config.setdefault("generationConfig", {}) + generation_config.setdefault("responseModalities", ["AUDIO"]) + + # Ensure Vertex defaults for realtimeInputConfig apply even when + # the client provided a partial ``turn_detection`` (e.g. only + # ``silence_duration_ms``). ``map_automatic_turn_detection`` sets + # ``disabled=True`` whenever ``create_response`` is absent or + # ``False``. Force ``disabled=False`` only when the client did + # not explicitly request ``create_response: False`` — that path + # is how transcription guardrails suppress automatic responses, + # and overriding it here would silently bypass the guardrail. + # Vertex Live has no "VAD on, no auto-response" mode, so callers + # that need that behaviour must accept that VAD is off. + client_turn_detection = session_params.get("turn_detection") + client_disabled_auto_response = ( + isinstance(client_turn_detection, dict) + and client_turn_detection.get("create_response") is False + ) + realtime_input_config = setup_config.setdefault("realtimeInputConfig", {}) + automatic_detection = realtime_input_config.setdefault( + "automaticActivityDetection", {} + ) + if not client_disabled_auto_response: + automatic_detection["disabled"] = False + automatic_detection.setdefault("silenceDurationMs", 800) + + setup_config.setdefault("inputAudioTranscription", {}) + setup_config.setdefault("outputAudioTranscription", {}) + + return setup_config + def transform_realtime_request( self, message: str, @@ -147,16 +205,50 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig): """ Translate OpenAI realtime client messages to Vertex AI format. - ``session.update`` is intentionally ignored (returns []) because - Vertex AI only accepts a single ``setup`` message at the start of - the connection — sending a second one causes a 1007 close error. - The initial setup (sent automatically before bidirectional_forward) - already includes AUDIO modality and server VAD, so there is nothing - more to configure. + On the first ``session.update`` (when no setup has been sent yet) the + full ``BidiGenerateContentSetup`` is built with Vertex AI's model path + and forwarded. Any later ``session.update`` is dropped: Vertex AI + documents ``setup`` as the first-and-only client message, and a second + ``setup`` closes the connection with a 1007 policy error. """ json_message = json.loads(message) - if json_message.get("type") == "session.update": - # Do not forward as a second setup — Vertex AI rejects it. + msg_type = json_message.get("type") + + if msg_type == "session.update": + if session_configuration_request is None: + setup_config = self._build_vertex_ai_setup_config( + model, json_message.get("session") or {} + ) + gemini_setup_msg = json.dumps({"setup": setup_config}) + + verbose_logger.debug( + "Vertex AI Realtime: Sending initial setup with tools to backend" + ) + return [gemini_setup_msg] + + # A follow-up session.update can't be forwarded as a second setup + # (Vertex Live closes the WebSocket with 1007). If this drop is + # silencing the audio-transcription guardrail's create_response + # disable, surface a warning so operators know the model will + # auto-respond before the guardrail can gate it on Vertex AI. + client_turn_detection = GeminiRealtimeConfig._extract_turn_detection( + json_message.get("session") or {} + ) + if ( + isinstance(client_turn_detection, dict) + and client_turn_detection.get("create_response") is False + ): + verbose_logger.warning( + "Vertex AI Realtime: Dropping subsequent session.update " + "(turn_detection.create_response=False) — Vertex Live " + "rejects a second setup message. Audio-transcription " + "guardrails cannot suppress the model's auto-response on " + "Vertex AI in non-deferred mode." + ) + else: + verbose_logger.debug( + "Vertex AI Realtime: Ignoring session.update (setup already sent)" + ) return [] return super().transform_realtime_request( diff --git a/litellm/llms/vertex_ai/videos/transformation.py b/litellm/llms/vertex_ai/videos/transformation.py index ed6176cef05..b84966354b8 100644 --- a/litellm/llms/vertex_ai/videos/transformation.py +++ b/litellm/llms/vertex_ai/videos/transformation.py @@ -40,6 +40,29 @@ else: BaseLLMException = Any +def _build_vertex_video_usage_from_request_data( + request_data: Optional[Dict[str, Any]], +) -> Dict[str, Any]: + """Build usage metadata (duration, resolution) for video cost calculation.""" + usage_data: Dict[str, Any] = {} + if not request_data: + return usage_data + + parameters = request_data.get("parameters", {}) + duration = ( + parameters.get("durationSeconds") or DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS + ) + if duration is not None: + try: + usage_data["duration_seconds"] = float(duration) + except (ValueError, TypeError): + pass + res = parameters.get("resolution") + if res is not None and str(res).strip() != "": + usage_data["video_resolution"] = str(res).strip().lower() + return usage_data + + def _convert_image_to_vertex_format(image_file) -> Dict[str, str]: """ Convert image file to Vertex AI format with base64 encoding and MIME type. @@ -363,23 +386,7 @@ class VertexAIVideoConfig(BaseVideoConfig, VertexBase): id=video_id, object="video", status="processing", model=model ) - usage_data: Dict[str, Any] = {} - if request_data: - parameters = request_data.get("parameters", {}) - duration = ( - parameters.get("durationSeconds") - or DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS - ) - if duration is not None: - try: - usage_data["duration_seconds"] = float(duration) - except (ValueError, TypeError): - pass - res = parameters.get("resolution") - if res is not None and str(res).strip() != "": - usage_data["video_resolution"] = str(res).strip().lower() - - video_obj.usage = usage_data + video_obj.usage = _build_vertex_video_usage_from_request_data(request_data) return video_obj def transform_video_status_retrieve_request( @@ -647,15 +654,123 @@ class VertexAIVideoConfig(BaseVideoConfig, VertexBase): def transform_video_get_character_response(self, raw_response, logging_obj): raise NotImplementedError("video get character is not supported for Vertex AI") + def get_video_edit_prefetch_params( + self, + video_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[str, Dict]: + """Return the fetchPredictOperation URL and body needed to retrieve the source video.""" + return self.transform_video_status_retrieve_request( + video_id=video_id, + api_base=api_base, + litellm_params=litellm_params, + headers=headers, + ) + def transform_video_edit_request( - self, prompt, video_id, api_base, litellm_params, headers, extra_body=None - ): - raise NotImplementedError("video edit is not supported for Vertex AI") + self, + prompt: str, + video_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + extra_body: Optional[Dict[str, Any]] = None, + prefetched_source_data: Optional[Dict[str, Any]] = None, + ) -> Tuple[str, Dict]: + """ + Build a predictLongRunning edit request from the pre-fetched source video. + + The actual fetchPredictOperation HTTP call is hoisted into the handler so + it can use the shared async/sync httpx client instead of blocking the loop. + """ + if prefetched_source_data is None: + raise ValueError( + "prefetched_source_data is required for Vertex AI video edit. " + "Ensure get_video_edit_prefetch_params is called by the handler." + ) + + if not prefetched_source_data.get("done", False): + raise ValueError( + "Source video generation is not complete yet. " + "Check the video status before editing." + ) + + videos = prefetched_source_data.get("response", {}).get("videos", []) + if not videos: + raise ValueError("No videos found in the completed operation. Cannot edit.") + + source_video = videos[0] + video_input: Dict[str, Any] = {} + if "gcsUri" in source_video: + video_input["gcsUri"] = source_video["gcsUri"] + elif "bytesBase64Encoded" in source_video: + video_input["bytesBase64Encoded"] = source_video["bytesBase64Encoded"] + video_input["mimeType"] = source_video.get("mimeType", "video/mp4") + else: + raise ValueError( + "Source video has neither gcsUri nor bytesBase64Encoded. Cannot edit." + ) + + operation_name = extract_original_video_id(video_id) + model = self.extract_model_from_operation_name(operation_name) or "" + + instance_dict: Dict[str, Any] = {"prompt": prompt, "video": video_input} + request_data: Dict[str, Any] = {"instances": [instance_dict]} + + if extra_body: + extra_body_copy = dict(extra_body) + nested_params = extra_body_copy.pop("parameters", None) + vertex_params: Dict[str, Any] = {} + if isinstance(nested_params, dict): + vertex_params.update(nested_params) + vertex_params.update(extra_body_copy) + if vertex_params: + request_data["parameters"] = vertex_params + + edit_url = f"{api_base.rstrip('/')}/{model}:predictLongRunning" + return edit_url, request_data def transform_video_edit_response( - self, raw_response, logging_obj, custom_llm_provider=None - ): - raise NotImplementedError("video edit is not supported for Vertex AI") + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + custom_llm_provider: Optional[str] = None, + request_data: Optional[Dict] = None, + ) -> VideoObject: + """ + Transform the Veo video edit response. + + Veo returns the same operation response as video generation: + {"name": "projects/.../operations/OPERATION_ID"} + + usage includes duration_seconds and optional video_resolution from the + edit request parameters for cost calculation. + """ + response_data = raw_response.json() + + operation_name = response_data.get("name") + if not operation_name: + raise ValueError(f"No operation name in Veo edit response: {response_data}") + + model = self.extract_model_from_operation_name(operation_name) or "" + + if custom_llm_provider: + video_id = encode_video_id_with_provider( + operation_name, custom_llm_provider, model + ) + else: + video_id = operation_name + + video_obj = VideoObject( + id=video_id, + object="video", + status="processing", + model=model, + ) + video_obj.usage = _build_vertex_video_usage_from_request_data(request_data) + return video_obj def transform_video_extension_request( self, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index ec16f19799b..ce6d4ac824c 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -731,7 +731,6 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true }, "anthropic.claude-haiku-4-5@20251001": { @@ -755,7 +754,6 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_streaming": true, "supports_native_structured_output": true }, @@ -926,8 +924,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "anthropic.claude-opus-4-20250514-v1:0": { "cache_creation_input_token_cost": 1.875e-05, @@ -952,8 +949,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "anthropic.claude-opus-4-5-20251101-v1:0": { "cache_creation_input_token_cost": 6.25e-06, @@ -977,12 +973,12 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_reasoning": true, - "supports_minimal_reasoning_effort": true, "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 159, - "supports_native_structured_output": true + "supports_native_structured_output": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "high" }, "anthropic.claude-opus-4-6-v1": { "cache_creation_input_token_cost": 6.25e-06, @@ -1009,11 +1005,10 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true, "supports_output_config": true, "supports_max_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "bedrock_output_config_effort_ceiling": "max" }, "global.anthropic.claude-opus-4-6-v1": { "cache_creation_input_token_cost": 6.25e-06, @@ -1040,11 +1035,10 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true, "supports_output_config": true, "supports_max_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "bedrock_output_config_effort_ceiling": "max" }, "us.anthropic.claude-opus-4-6-v1": { "cache_creation_input_token_cost": 6.875e-06, @@ -1071,11 +1065,10 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true, "supports_output_config": true, "supports_max_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "bedrock_output_config_effort_ceiling": "max" }, "eu.anthropic.claude-opus-4-6-v1": { "cache_creation_input_token_cost": 6.875e-06, @@ -1101,11 +1094,10 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true, "supports_output_config": true, "supports_max_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "bedrock_output_config_effort_ceiling": "max" }, "au.anthropic.claude-opus-4-6-v1": { "cache_creation_input_token_cost": 6.875e-06, @@ -1131,11 +1123,10 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true, "supports_output_config": true, "supports_max_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "bedrock_output_config_effort_ceiling": "max" }, "anthropic.claude-opus-4-7": { "cache_creation_input_token_cost": 6.25e-06, @@ -1163,10 +1154,10 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true, "supports_max_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh" }, "anthropic.claude-mythos-preview": { "input_cost_per_token": 0, @@ -1180,8 +1171,8 @@ "supports_vision": true, "supports_prompt_caching": false, "supports_reasoning": true, - "supports_minimal_reasoning_effort": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_output_config": true }, "global.anthropic.claude-opus-4-7": { "cache_creation_input_token_cost": 6.25e-06, @@ -1209,10 +1200,10 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true, "supports_max_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh" }, "us.anthropic.claude-opus-4-7": { "cache_creation_input_token_cost": 6.875e-06, @@ -1240,10 +1231,10 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true, "supports_max_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh" }, "eu.anthropic.claude-opus-4-7": { "cache_creation_input_token_cost": 6.875e-06, @@ -1270,10 +1261,10 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true, "supports_max_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh" }, "au.anthropic.claude-opus-4-7": { "cache_creation_input_token_cost": 6.875e-06, @@ -1300,10 +1291,165 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true, "supports_max_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh" + }, + "anthropic.claude-opus-4-8": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": true, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh" + }, + "global.anthropic.claude-opus-4-8": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": true, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh" + }, + "us.anthropic.claude-opus-4-8": { + "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_1hr": 1.1e-05, + "cache_read_input_token_cost": 5.5e-07, + "input_cost_per_token": 5.5e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.75e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": true, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh" + }, + "eu.anthropic.claude-opus-4-8": { + "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_1hr": 1.1e-05, + "cache_read_input_token_cost": 5.5e-07, + "input_cost_per_token": 5.5e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.75e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": true, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh" + }, + "au.anthropic.claude-opus-4-8": { + "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_1hr": 1.1e-05, + "cache_read_input_token_cost": 5.5e-07, + "input_cost_per_token": 5.5e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.75e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": true, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh" }, "anthropic.claude-sonnet-4-6": { "cache_creation_input_token_cost": 3.75e-06, @@ -1331,10 +1477,8 @@ "supports_max_reasoning_effort": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true, - "supports_output_config": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true }, "global.anthropic.claude-sonnet-4-6": { "cache_creation_input_token_cost": 3.75e-06, @@ -1362,10 +1506,8 @@ "supports_max_reasoning_effort": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true, - "supports_output_config": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true }, "us.anthropic.claude-sonnet-4-6": { "cache_creation_input_token_cost": 4.125e-06, @@ -1393,10 +1535,8 @@ "supports_max_reasoning_effort": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true, - "supports_output_config": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true }, "eu.anthropic.claude-sonnet-4-6": { "cache_creation_input_token_cost": 4.125e-06, @@ -1423,10 +1563,8 @@ "supports_max_reasoning_effort": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true, - "supports_output_config": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true }, "au.anthropic.claude-sonnet-4-6": { "cache_creation_input_token_cost": 4.125e-06, @@ -1453,10 +1591,8 @@ "supports_max_reasoning_effort": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true, - "supports_output_config": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true }, "jp.anthropic.claude-sonnet-4-6": { "cache_creation_input_token_cost": 4.125e-06, @@ -1483,10 +1619,8 @@ "supports_max_reasoning_effort": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true, - "supports_output_config": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true }, "anthropic.claude-sonnet-4-20250514-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -1515,8 +1649,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -1548,7 +1681,6 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 159, "supports_native_structured_output": true }, "anthropic.claude-v1": { @@ -1799,7 +1931,6 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true }, "apac.anthropic.claude-3-sonnet-20240229-v1:0": { @@ -1845,8 +1976,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "assemblyai/best": { "input_cost_per_second": 3.333e-05, @@ -1888,7 +2018,6 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true }, "azure/ada": { @@ -1976,10 +2105,10 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_reasoning": true, - "supports_minimal_reasoning_effort": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "supports_output_config": true }, "azure_ai/claude-opus-4-6": { "input_cost_per_token": 5e-06, @@ -2006,10 +2135,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 159, "supports_output_config": true, - "supports_max_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_max_reasoning_effort": true }, "azure_ai/claude-opus-4-7": { "input_cost_per_token": 5e-06, @@ -2037,9 +2164,35 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "tool_use_system_prompt_tokens": 159, - "supports_max_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_max_reasoning_effort": true + }, + "azure_ai/claude-opus-4-8": { + "input_cost_per_token": 5e-06, + "output_cost_per_token": 2.5e-05, + "litellm_provider": "azure_ai", + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_read_input_token_cost": 5e-07, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true }, "azure_ai/claude-opus-4-1": { "cache_creation_input_token_cost": 1.875e-05, @@ -2104,9 +2257,7 @@ "supports_max_reasoning_effort": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, - "supports_output_config": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true }, "azure/computer-use-preview": { "input_cost_per_token": 3e-06, @@ -9480,8 +9631,7 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_web_search": true, - "tool_use_system_prompt_tokens": 159 + "supports_web_search": true }, "claude-3-haiku-20240307": { "cache_creation_input_token_cost": 3e-07, @@ -9499,8 +9649,7 @@ "supports_prompt_caching": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 264 + "supports_vision": true }, "claude-3-opus-20240229": { "cache_creation_input_token_cost": 1.875e-05, @@ -9519,8 +9668,7 @@ "supports_prompt_caching": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 395 + "supports_vision": true }, "claude-4-opus-20250514": { "cache_creation_input_token_cost": 1.875e-05, @@ -9545,8 +9693,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "claude-4-sonnet-20250514": { "cache_creation_input_token_cost": 3.75e-06, @@ -9576,8 +9723,7 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_web_search": true, - "tool_use_system_prompt_tokens": 159 + "supports_web_search": true }, "claude-sonnet-4-5": { "cache_creation_input_token_cost": 3.75e-06, @@ -9606,8 +9752,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 346 + "supports_vision": true }, "claude-sonnet-4-5-20250929": { "cache_creation_input_token_cost": 3.75e-06, @@ -9637,8 +9782,7 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_web_search": true, - "tool_use_system_prompt_tokens": 346 + "supports_web_search": true }, "claude-sonnet-4-6": { "cache_creation_input_token_cost": 3.75e-06, @@ -9666,9 +9810,7 @@ "supports_max_reasoning_effort": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, - "supports_output_config": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true }, "claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -9692,8 +9834,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "claude-opus-4-1": { "cache_creation_input_token_cost": 1.875e-05, @@ -9719,8 +9860,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "claude-opus-4-1-20250805": { "cache_creation_input_token_cost": 1.875e-05, @@ -9747,8 +9887,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "claude-opus-4-20250514": { "cache_creation_input_token_cost": 1.875e-05, @@ -9775,8 +9914,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "claude-opus-4-5-20251101": { "cache_creation_input_token_cost": 6.25e-06, @@ -9800,11 +9938,10 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_reasoning": true, - "supports_minimal_reasoning_effort": true, "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_output_config": true }, "claude-opus-4-5": { "cache_creation_input_token_cost": 6.25e-06, @@ -9828,11 +9965,10 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_reasoning": true, - "supports_minimal_reasoning_effort": true, "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_output_config": true }, "claude-opus-4-6": { "cache_creation_input_token_cost": 6.25e-06, @@ -9860,14 +9996,12 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "provider_specific_entry": { "us": 1.1, "fast": 6.0 }, "supports_output_config": true, - "supports_max_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_max_reasoning_effort": true }, "claude-opus-4-6-20260205": { "cache_creation_input_token_cost": 6.25e-06, @@ -9895,13 +10029,11 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "provider_specific_entry": { "us": 1.1, "fast": 6.0 }, "supports_max_reasoning_effort": true, - "supports_minimal_reasoning_effort": true, "supports_output_config": true }, "claude-opus-4-7": { @@ -9932,12 +10064,10 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, - "tool_use_system_prompt_tokens": 346, "provider_specific_entry": { "us": 1.1, "fast": 6.0 }, - "supports_minimal_reasoning_effort": true, "supports_output_config": true }, "claude-opus-4-7-20260416": { @@ -9968,12 +10098,44 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, - "tool_use_system_prompt_tokens": 346, "provider_specific_entry": { "us": 1.1, "fast": 6.0 }, - "supports_minimal_reasoning_effort": true, + "supports_output_config": true + }, + "claude-opus-4-8": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "anthropic", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "provider_specific_entry": { + "us": 1.1, + "fast": 2.0 + }, "supports_output_config": true }, "claude-sonnet-4-20250514": { @@ -10005,8 +10167,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "cloudflare/@cf/meta/llama-2-7b-chat-fp16": { "input_cost_per_token": 1.923e-06, @@ -11251,8 +11412,8 @@ "supports_assistant_prefill": true, "supports_function_calling": true, "supports_reasoning": true, - "supports_minimal_reasoning_effort": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_output_config": true }, "databricks/databricks-claude-sonnet-4": { "input_cost_per_token": 2.9999900000000002e-06, @@ -13418,7 +13579,6 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true }, "eu.anthropic.claude-3-5-sonnet-20240620-v1:0": { @@ -13546,8 +13706,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "eu.anthropic.claude-opus-4-20250514-v1:0": { "cache_creation_input_token_cost": 1.875e-05, @@ -13572,8 +13731,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "eu.anthropic.claude-sonnet-4-20250514-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -13602,8 +13760,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "eu.anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.125e-06, @@ -13633,7 +13790,6 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true }, "eu.meta.llama3-2-1b-instruct-v1:0": { @@ -17943,8 +18099,8 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_vision": true, - "supports_minimal_reasoning_effort": true, - "supports_reasoning": true + "supports_reasoning": true, + "supports_output_config": true }, "github_copilot/claude-opus-4.6-fast": { "litellm_provider": "github_copilot", @@ -18666,7 +18822,7 @@ "output_cost_per_token": 2.5e-05, "supports_function_calling": true, "supports_vision": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true }, "gmi/anthropic/claude-sonnet-4.5": { "input_cost_per_token": 3e-06, @@ -18966,7 +19122,6 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true }, "global.anthropic.claude-sonnet-4-20250514-v1:0": { @@ -18996,8 +19151,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "global.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.25e-06, @@ -19020,7 +19174,6 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true }, "global.amazon.nova-2-lite-v1:0": { @@ -23128,7 +23281,6 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true }, "jp.anthropic.claude-haiku-4-5-20251001-v1:0": { @@ -23151,7 +23303,6 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true }, "crusoe/deepseek-ai/DeepSeek-R1-0528": { @@ -27093,8 +27244,7 @@ "supports_computer_use": true, "supports_function_calling": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "openrouter/anthropic/claude-3.7-sonnet": { "input_cost_per_image": 0.0048, @@ -27110,8 +27260,7 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "openrouter/anthropic/claude-opus-4": { "input_cost_per_image": 0.0048, @@ -27130,8 +27279,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "openrouter/anthropic/claude-opus-4.1": { "input_cost_per_image": 0.0048, @@ -27151,8 +27299,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "openrouter/anthropic/claude-sonnet-4": { "input_cost_per_image": 0.0048, @@ -27175,8 +27322,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "openrouter/anthropic/claude-sonnet-4.6": { "cache_creation_input_token_cost": 3.75e-06, @@ -27200,9 +27346,7 @@ "supports_reasoning": true, "supports_max_reasoning_effort": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159, - "supports_minimal_reasoning_effort": true + "supports_vision": true }, "openrouter/anthropic/claude-opus-4.5": { "cache_creation_input_token_cost": 6.25e-06, @@ -27217,12 +27361,11 @@ "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, - "supports_minimal_reasoning_effort": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_output_config": true }, "openrouter/anthropic/claude-opus-4.6": { "cache_creation_input_token_cost": 6.25e-06, @@ -27241,9 +27384,7 @@ "supports_reasoning": true, "supports_max_reasoning_effort": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 346, - "supports_minimal_reasoning_effort": true + "supports_vision": true }, "openrouter/anthropic/claude-sonnet-4.5": { "input_cost_per_image": 0.0048, @@ -27266,8 +27407,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "openrouter/anthropic/claude-haiku-4.5": { "cache_creation_input_token_cost": 1.25e-06, @@ -27285,8 +27425,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 346 + "supports_vision": true }, "openrouter/anthropic/claude-opus-4.7": { "cache_creation_input_token_cost": 6.25e-06, @@ -27308,8 +27447,7 @@ "supports_max_reasoning_effort": true, "supports_tool_choice": true, "supports_vision": true, - "supports_xhigh_reasoning_effort": true, - "tool_use_system_prompt_tokens": 346 + "supports_xhigh_reasoning_effort": true }, "openrouter/bytedance/ui-tars-1.5-7b": { "input_cost_per_token": 1e-07, @@ -29290,7 +29428,7 @@ "supports_web_search": true, "supports_reasoning": false, "supports_function_calling": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true }, "perplexity/anthropic/claude-sonnet-4-5": { "litellm_provider": "perplexity", @@ -31543,7 +31681,6 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true }, "us.anthropic.claude-3-5-sonnet-20240620-v1:0": { @@ -31671,8 +31808,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "us.anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.125e-06, @@ -31704,7 +31840,6 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true }, "us-gov.anthropic.claude-sonnet-4-5-20250929-v1:0": { @@ -31730,7 +31865,6 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true }, "au.anthropic.claude-haiku-4-5-20251001-v1:0": { @@ -31752,7 +31886,6 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true }, "us.anthropic.claude-opus-4-20250514-v1:0": { @@ -31778,8 +31911,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "us.anthropic.claude-opus-4-5-20251101-v1:0": { "cache_creation_input_token_cost": 6.875e-06, @@ -31800,15 +31932,15 @@ "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, - "supports_minimal_reasoning_effort": true, "supports_pdf_input": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 159, - "supports_native_structured_output": true + "supports_native_structured_output": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "high" }, "global.anthropic.claude-opus-4-5-20251101-v1:0": { "cache_creation_input_token_cost": 6.25e-06, @@ -31829,15 +31961,15 @@ "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, - "supports_minimal_reasoning_effort": true, "supports_pdf_input": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 159, - "supports_native_structured_output": true + "supports_native_structured_output": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "high" }, "eu.anthropic.claude-opus-4-5-20251101-v1:0": { "cache_creation_input_token_cost": 6.25e-06, @@ -31857,15 +31989,15 @@ "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, - "supports_minimal_reasoning_effort": true, "supports_pdf_input": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 159, - "supports_native_structured_output": true + "supports_native_structured_output": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "high" }, "us.anthropic.claude-sonnet-4-20250514-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -31894,8 +32026,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "us.deepseek.r1-v1:0": { "input_cost_per_token": 1.35e-06, @@ -32438,13 +32569,13 @@ "output_cost_per_token": 2.5e-05, "supports_assistant_prefill": true, "supports_computer_use": true, - "supports_minimal_reasoning_effort": true, "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "supports_output_config": true }, "vercel_ai_gateway/anthropic/claude-opus-4.6": { "cache_creation_input_token_cost": 6.25e-06, @@ -32464,7 +32595,7 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true }, "vercel_ai_gateway/anthropic/claude-sonnet-4": { "cache_creation_input_token_cost": 3.75e-06, @@ -33472,8 +33603,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "vertex_ai/claude-3-haiku": { "input_cost_per_token": 2.5e-07, @@ -33576,8 +33706,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "vertex_ai/claude-opus-4-1": { "cache_creation_input_token_cost": 1.875e-05, @@ -33631,14 +33760,13 @@ "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, - "supports_minimal_reasoning_effort": true, "supports_pdf_input": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_output_config": true }, "vertex_ai/claude-opus-4-5@20251101": { "cache_creation_input_token_cost": 6.25e-06, @@ -33658,15 +33786,14 @@ "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, - "supports_minimal_reasoning_effort": true, "supports_pdf_input": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 159, - "supports_native_streaming": true + "supports_native_streaming": true, + "supports_output_config": true }, "vertex_ai/claude-opus-4-6": { "cache_creation_input_token_cost": 6.25e-06, @@ -33692,10 +33819,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_output_config": true, - "supports_max_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_max_reasoning_effort": true }, "vertex_ai/claude-opus-4-6@default": { "cache_creation_input_token_cost": 6.25e-06, @@ -33721,10 +33846,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_output_config": true, - "supports_max_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_max_reasoning_effort": true }, "vertex_ai/claude-opus-4-7": { "cache_creation_input_token_cost": 6.25e-06, @@ -33751,9 +33874,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "tool_use_system_prompt_tokens": 346, - "supports_max_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_max_reasoning_effort": true }, "vertex_ai/claude-opus-4-7@default": { "cache_creation_input_token_cost": 6.25e-06, @@ -33780,9 +33901,63 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "tool_use_system_prompt_tokens": 346, - "supports_max_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_max_reasoning_effort": true + }, + "vertex_ai/claude-opus-4-8": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true + }, + "vertex_ai/claude-opus-4-8@default": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true }, "vertex_ai/claude-sonnet-4-5": { "cache_creation_input_token_cost": 3.75e-06, @@ -33830,14 +34005,12 @@ "supports_max_reasoning_effort": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_output_config": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true }, "vertex_ai/claude-sonnet-4-5@20250929": { "cache_creation_input_token_cost": 3.75e-06, @@ -33889,8 +34062,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "vertex_ai/claude-sonnet-4": { "cache_creation_input_token_cost": 3.75e-06, @@ -33919,8 +34091,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "vertex_ai/claude-sonnet-4@20250514": { "cache_creation_input_token_cost": 3.75e-06, @@ -33949,8 +34120,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "vertex_ai/mistralai/codestral-2@001": { "input_cost_per_token": 3e-07, @@ -40943,14 +41113,12 @@ "supports_max_reasoning_effort": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_output_config": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true }, "duckduckgo/search": { "litellm_provider": "duckduckgo", @@ -41266,7 +41434,6 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true, "supports_pdf_input": true }, @@ -41289,7 +41456,6 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true, "supports_pdf_input": true } diff --git a/litellm/proxy/_experimental/mcp_server/CLAUDE.md b/litellm/proxy/_experimental/mcp_server/CLAUDE.md new file mode 100644 index 00000000000..0ba8f73315f --- /dev/null +++ b/litellm/proxy/_experimental/mcp_server/CLAUDE.md @@ -0,0 +1 @@ +MCP note: **`available_on_public_internet: false` with `delegate_auth_to_upstream: true` (oauth2, interactive - not `client_credentials`)** - LiteLLM still allows the anonymous upstream PKCE path (no proxy API key for `/authorize` and matching MCP routes). The internal-only flag mainly affects other surfaces (e.g. IP-based discovery). Rely on the upstream IdP and network policy; the dashboard shows a warning when both are set, and the proxy logs a warning when the server is loaded from config or the database diff --git a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py index 39fda7074cc..97d3a8cf5cc 100644 --- a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py +++ b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py @@ -992,42 +992,78 @@ class MCPRequestHandler: """ Get allowed MCP servers for a team. - Note: object_permission is automatically loaded by get_team_object() in main auth flow. + Unions two sources: + - Legacy team.object_permission (mcp_servers, mcp_access_groups, + mcp_tool_permissions). + - Unified team.access_group_ids → access_group.access_mcp_server_ids. + Mirrors the model-side pattern in can_team_access_model — the group + is already attached to the team, so the team relationship is itself + the gate (no assigned_team_ids check needed here). """ try: - # Get team object permission (already loaded in main auth flow) - object_permissions = await MCPRequestHandler._get_team_object_permission( - user_api_key_auth - ) - - if object_permissions is None: - return [] - - # Permission entries may be server_ids OR names/aliases — expand to ids. from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) + from litellm.proxy.auth.auth_checks import ( + _get_mcp_server_ids_from_access_groups, + get_team_object, + ) + from litellm.proxy.proxy_server import ( + prisma_client, + proxy_logging_obj, + user_api_key_cache, + ) + + if ( + user_api_key_auth is None + or not user_api_key_auth.team_id + or prisma_client is None + ): + return [] + + team_obj: Optional[LiteLLM_TeamTable] = await get_team_object( + team_id=user_api_key_auth.team_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=user_api_key_auth.parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + ) + if team_obj is None: + return [] + + team_access_group_servers = await _get_mcp_server_ids_from_access_groups( + access_group_ids=team_obj.access_group_ids or [], + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + + object_permissions = team_obj.object_permission + if object_permissions is None: + return list(set(team_access_group_servers)) direct_mcp_servers = global_mcp_server_manager.expand_permission_list( object_permissions.mcp_servers or [] ) - # Get MCP servers from access groups - access_group_servers = ( + legacy_access_group_servers = ( await MCPRequestHandler._get_mcp_servers_from_access_groups( object_permissions.mcp_access_groups or [] ) ) - # servers referenced in tool permissions should also be accessible tool_perm_servers = list( global_mcp_server_manager.expand_tool_permissions( object_permissions.mcp_tool_permissions ).keys() ) - # Combine all lists - all_servers = direct_mcp_servers + access_group_servers + tool_perm_servers + all_servers = ( + direct_mcp_servers + + legacy_access_group_servers + + tool_perm_servers + + team_access_group_servers + ) return list(set(all_servers)) except Exception as e: verbose_logger.warning( diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 9046d522280..522e85632dc 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1067,6 +1067,7 @@ class KeyRequestBase(GenerateRequestBase): key: Optional[str] = None budget_id: Optional[str] = None tags: Optional[List[str]] = None + disable_global_guardrails: Optional[bool] = None enforced_params: Optional[List[str]] = None allowed_routes: Optional[list] = [] allowed_passthrough_routes: Optional[list] = None @@ -1832,6 +1833,7 @@ class NewTeamRequest(TeamBase): prompts: Optional[List[str]] = None object_permission: Optional[LiteLLM_ObjectPermissionBase] = None allowed_passthrough_routes: Optional[list] = None + disable_global_guardrails: Optional[bool] = None secret_manager_settings: Optional[dict] = None model_rpm_limit: Optional[Dict[str, int]] = None rpm_limit_type: Optional[ @@ -1900,6 +1902,7 @@ class UpdateTeamRequest(LiteLLMPydanticObjectBase): guardrails: Optional[List[str]] = None policies: Optional[List[str]] = None object_permission: Optional[LiteLLM_ObjectPermissionBase] = None + disable_global_guardrails: Optional[bool] = None team_member_budget: Optional[float] = None team_member_budget_duration: Optional[str] = None team_member_rpm_limit: Optional[int] = None @@ -4281,6 +4284,7 @@ LiteLLM_ManagementEndpoint_MetadataFields = [ ] LiteLLM_ManagementEndpoint_MetadataFields_Premium = [ + "disable_global_guardrails", "guardrails", "policies", "tags", diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index e291cdbbfb3..38976f79aa3 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -619,6 +619,9 @@ async def common_checks( # noqa: PLR0915 proxy_logging_obj=proxy_logging_obj, ) + # Run before apply_key_tags_pre_auth injects key metadata.tags into request_body. + _reject_clientside_metadata_tags_check(general_settings, request_body, route) + # If this is a free model, skip all budget checks if not skip_budget_checks: # 3. If team is in budget @@ -660,6 +663,14 @@ async def common_checks( # noqa: PLR0915 proxy_logging_obj=proxy_logging_obj, ) + if valid_token is not None: + from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup + + LiteLLMProxyRequestSetup.apply_key_tags_pre_auth( + request_data=request_body, + user_api_key_dict=valid_token, + ) + with tracer.trace("litellm.proxy.auth.common_checks.tag_max_budget_check"): await _tag_max_budget_check( request_body=request_body, @@ -709,7 +720,6 @@ async def common_checks( # noqa: PLR0915 await _check_end_user_budget(end_user_obj=end_user_object, route=route) _enforce_user_param_check(general_settings, request, request_body, route) - _reject_clientside_metadata_tags_check(general_settings, request_body, route) _global_proxy_budget_check(global_proxy_spend, skip_budget_checks, route) _guardrail_modification_check(request_body, team_object) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index ef1d64335b4..6782208458e 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -839,6 +839,7 @@ class ProxyBaseLLMRequestProcessing: "aget_run", "acancel_run", "adelete_run", + "apply_guardrail", ], version: Optional[str] = None, user_model: Optional[str] = None, @@ -1368,6 +1369,21 @@ class ProxyBaseLLMRequestProcessing: user_api_key_dict=user_api_key_dict, request_data=self.data, ) + if route_type == "aresponses": + # Streaming /v1/responses returns here without + # reaching the non-streaming ownership tail below. + # Wrap the SSE generator so container ownership is + # written once the upstream iterator finishes + # assembling ``completed_response`` — otherwise + # code-interpreter containers created during the + # stream stay unregistered and follow-up file API + # calls 403. Covers the background-polling path + # too, which loops ``body_iterator`` end-to-end. + selected_data_generator = ProxyBaseLLMRequestProcessing._wrap_responses_stream_for_container_ownership( + original_stream_response=response, + wrapped_generator=selected_data_generator, + user_api_key_dict=user_api_key_dict, + ) return await create_response( generator=selected_data_generator, media_type="text/event-stream", @@ -1483,8 +1499,93 @@ class ProxyBaseLLMRequestProcessing: await check_response_size_is_safe(response=response) + if route_type in {"aresponses", "aget_responses"}: + await ProxyBaseLLMRequestProcessing._record_container_owners_from_responses_if_needed( + response=response, + user_api_key_dict=user_api_key_dict, + ) + return response + @staticmethod + async def _record_container_owners_from_responses_if_needed( + response: Any, + user_api_key_dict: UserAPIKeyAuth, + ) -> None: + """Register code-interpreter containers so follow-up file APIs pass ownership checks.""" + from litellm.proxy.container_endpoints.ownership import ( + record_container_owners_from_responses_response, + ) + + if response is None: + return + + try: + await record_container_owners_from_responses_response( + response=response, + user_api_key_dict=user_api_key_dict, + ) + except Exception as e: + verbose_proxy_logger.exception( + "Container ownership recording failed after responses call: %s", + e, + ) + + @staticmethod + def _extract_completed_responses_response(stream_response: Any) -> Any: + """Pull the assembled ``ResponsesAPIResponse`` off a streaming iterator. + + ``ResponsesAPIStreamingIterator`` stores the terminal stream event + (``response.completed`` / ``response.incomplete`` / ``response.failed``) + in ``completed_response``; the actual response body hangs off + that event's ``.response`` attribute. Some iterators store the + ``ResponsesAPIResponse`` directly. Handle both shapes so the + container-ownership recording path can walk ``.output`` either way. + """ + completed = getattr(stream_response, "completed_response", None) + if completed is None: + return None + response_obj = getattr(completed, "response", None) + if response_obj is not None: + return response_obj + return completed + + @staticmethod + async def _wrap_responses_stream_for_container_ownership( + original_stream_response: Any, + wrapped_generator: Any, + user_api_key_dict: UserAPIKeyAuth, + ): + """Forward SSE chunks, then record container ownership at stream end. + + Streaming ``/v1/responses`` short-circuits out of + ``base_process_llm_request`` before the non-streaming ownership + tail runs, so without this wrap the + ``LiteLLM_ManagedObjectTable`` row for any container created + during the stream is never written and follow-up file API calls + return 403. + """ + try: + async for chunk in wrapped_generator: + yield chunk + finally: + try: + completed_obj = ( + ProxyBaseLLMRequestProcessing._extract_completed_responses_response( + original_stream_response + ) + ) + if completed_obj is not None: + await ProxyBaseLLMRequestProcessing._record_container_owners_from_responses_if_needed( + response=completed_obj, + user_api_key_dict=user_api_key_dict, + ) + except Exception as e: + verbose_proxy_logger.exception( + "Container ownership recording failed after streaming responses call: %s", + e, + ) + async def base_passthrough_process_llm_request( self, request: Request, diff --git a/litellm/proxy/common_utils/callback_utils.py b/litellm/proxy/common_utils/callback_utils.py index c5b97db07a0..a65e737f248 100644 --- a/litellm/proxy/common_utils/callback_utils.py +++ b/litellm/proxy/common_utils/callback_utils.py @@ -317,7 +317,15 @@ def initialize_callbacks_on_proxy( # noqa: PLR0915 DatadogCostManagementLogger, ) - datadog_cost_management_obj = DatadogCostManagementLogger() + init_params = {} + if ( + "datadog_cost_management" in callback_specific_params + and isinstance( + callback_specific_params["datadog_cost_management"], dict + ) + ): + init_params = callback_specific_params["datadog_cost_management"] + datadog_cost_management_obj = DatadogCostManagementLogger(**init_params) imported_list.append(datadog_cost_management_obj) elif isinstance(callback, CustomLogger): imported_list.append(callback) diff --git a/litellm/proxy/container_endpoints/ownership.py b/litellm/proxy/container_endpoints/ownership.py index 57de6c4a63d..e0015e112e1 100644 --- a/litellm/proxy/container_endpoints/ownership.py +++ b/litellm/proxy/container_endpoints/ownership.py @@ -117,6 +117,58 @@ async def _get_prisma_client(): return prisma_client +def _custom_llm_provider_from_responses_response( + response: Any, + default: str = "openai", +) -> str: + hidden_params: Dict[str, Any] = {} + if isinstance(response, dict): + hidden_params = response.get("_hidden_params") or {} + else: + hidden_params = getattr(response, "_hidden_params", None) or {} + + provider = hidden_params.get("custom_llm_provider") + if isinstance(provider, str) and provider: + return provider + return default + + +async def record_container_owners_from_responses_response( + response: Any, + user_api_key_dict: UserAPIKeyAuth, + custom_llm_provider: Optional[str] = None, +) -> None: + """Track containers created implicitly by code interpreter in /v1/responses.""" + container_ids = ( + ResponsesAPIRequestUtils.collect_container_ids_from_responses_response(response) + ) + if not container_ids: + return + + resolved_provider = ( + custom_llm_provider or _custom_llm_provider_from_responses_response(response) + ) + + for container_id in container_ids: + try: + await record_container_owner( + response={"id": container_id, "object": "container"}, + user_api_key_dict=user_api_key_dict, + custom_llm_provider=resolved_provider, + ) + except Exception as e: + # Per-container errors (including ``HTTPException`` from + # conflicting/forbidden ownership rows) must not abort the + # batch — other containers in the same response should still + # get recorded so their follow-up file API calls don't 403. + verbose_proxy_logger.exception( + "Failed to record container ownership from responses output " + "for container_id=%s: %s", + container_id, + e, + ) + + async def record_container_owner( response: Any, user_api_key_dict: UserAPIKeyAuth, @@ -151,6 +203,8 @@ async def record_container_owner( file_object = _dump_response(response) file_object["custom_llm_provider"] = resolved_provider file_object["provider_container_id"] = original_container_id + # Prisma Python requires Json fields to be serialized as a JSON string. + file_object_json: str = json.dumps(file_object) prisma_client = await _get_prisma_client() if prisma_client is None: @@ -172,7 +226,7 @@ async def record_container_owner( where={"model_object_id": model_object_id}, data={ "unified_object_id": container_id, - "file_object": file_object, + "file_object": file_object_json, "updated_by": owner, }, ) @@ -181,7 +235,7 @@ async def record_container_owner( data={ "unified_object_id": container_id, "model_object_id": model_object_id, - "file_object": file_object, + "file_object": file_object_json, "file_purpose": CONTAINER_OBJECT_PURPOSE, "created_by": owner, "updated_by": owner, diff --git a/litellm/proxy/example_config_yaml/oai_misc_config.yaml b/litellm/proxy/example_config_yaml/oai_misc_config.yaml index 551043ec76b..0b647de8a08 100644 --- a/litellm/proxy/example_config_yaml/oai_misc_config.yaml +++ b/litellm/proxy/example_config_yaml/oai_misc_config.yaml @@ -23,11 +23,11 @@ model_list: model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0 ######################################################### ########## batch specific params ######################## - s3_bucket_name: litellm-proxy-123456789012 + s3_bucket_name: litellm-proxy-941277531214 s3_region_name: us-west-2 s3_access_key_id: os.environ/AWS_ACCESS_KEY_ID s3_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY - aws_batch_role_arn: arn:aws:iam::123456789012:role/service-role/AmazonBedrockExecutionRoleForAgents_EXAMPLE + aws_batch_role_arn: arn:aws:iam::941277531214:role/service-role/AmazonBedrockExecutionRoleForAgents_BB9HNW6V4CV model_info: mode: batch diff --git a/litellm/proxy/guardrails/guardrail_endpoints.py b/litellm/proxy/guardrails/guardrail_endpoints.py index e55f3b6e16b..e0e4bdcf4a4 100644 --- a/litellm/proxy/guardrails/guardrail_endpoints.py +++ b/litellm/proxy/guardrails/guardrail_endpoints.py @@ -10,7 +10,7 @@ from datetime import datetime, timezone from typing import Any, Dict, List, Literal, Optional, Type, TypeVar, Union, cast from urllib.parse import urlparse -from fastapi import APIRouter, Depends, HTTPException +from fastapi import APIRouter, Depends, HTTPException, Request from pydantic import BaseModel from litellm.proxy.common_utils.path_utils import safe_join @@ -2187,9 +2187,97 @@ async def test_custom_code_guardrail( ) +def _resolve_guardrail_input_type( + active_guardrail: CustomGuardrail, input_type: str +) -> Literal["request", "response"]: + """Return the effective input_type, auto-upgrading to 'response' for post_call guardrails.""" + if input_type == "request": + hook = getattr(active_guardrail, "event_hook", None) + if hook == GuardrailEventHooks.post_call or hook == "post_call": + return "response" + return "response" if input_type == "response" else "request" + + +def _patch_logging_obj_for_guardrail( + litellm_logging_obj: Any, request: ApplyGuardrailRequest +) -> None: + """Configure the logging object so Langfuse/OTEL extract input and output correctly.""" + litellm_logging_obj.call_type = "pass_through_endpoint" + litellm_logging_obj.model_call_details["call_type"] = "pass_through_endpoint" + litellm_logging_obj.update_messages( + request.messages + if request.messages + else [{"role": "user", "content": request.text}] + ) + + +async def _emit_guardrail_success_logs( + proxy_logging_obj: Any, + litellm_logging_obj: Any, + data: dict, + user_api_key_dict: UserAPIKeyAuth, + response: ApplyGuardrailResponse, + start_time: datetime, +) -> ApplyGuardrailResponse: + """Fire proxy and LiteLLM success hooks after a successful guardrail run. + + Each hook is wrapped defensively so a callback failure never prevents the + caller from receiving the guardrail response. Returns the (possibly + hook-modified) response. + """ + from litellm.litellm_core_utils.thread_pool_executor import ( + executor as thread_pool_executor, + ) + + try: + modified = await proxy_logging_obj.post_call_success_hook( + data=data, + user_api_key_dict=user_api_key_dict, + response=response, + ) + if isinstance(modified, ApplyGuardrailResponse): + response = modified + except Exception: + verbose_proxy_logger.exception("apply_guardrail: post_call_success_hook failed") + + # Build the logging payload after post_call_success_hook so that logged + # data matches what the caller actually receives if the hook modified + # the response. + response_for_logging = {"response": response.model_dump(exclude_none=True)} + + if litellm_logging_obj is not None: + end_time = datetime.now(timezone.utc) + try: + await litellm_logging_obj.async_success_handler( + result=response_for_logging, + start_time=start_time, + end_time=end_time, + cache_hit=False, + ) + except Exception: + verbose_proxy_logger.exception( + "apply_guardrail: async_success_handler failed" + ) + try: + thread_pool_executor.submit( + litellm_logging_obj.success_handler, + response_for_logging, + start_time, + end_time, + False, + ) + except Exception: + verbose_proxy_logger.exception( + "apply_guardrail: success_handler submit failed" + ) + + return response + + @router.post("/guardrails/apply_guardrail", response_model=ApplyGuardrailResponse) @router.post("/apply_guardrail", response_model=ApplyGuardrailResponse) async def apply_guardrail( + fastapi_request: Request, request: ApplyGuardrailRequest, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): @@ -2198,8 +2286,29 @@ async def apply_guardrail( This endpoint allows testing guardrails by applying them to custom text inputs. """ + import traceback + + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + from litellm.litellm_core_utils.thread_pool_executor import ( + executor as thread_pool_executor, + ) + from litellm.proxy.proxy_server import ( + general_settings, + proxy_config, + proxy_logging_obj, + version, + ) from litellm.proxy.utils import handle_exception_on_proxy + data: dict = { + "guardrail_name": request.guardrail_name, + "input": [request.text], + "messages": request.messages or [], + "metadata": {"route": "/apply_guardrail"}, + } + litellm_logging_obj = None + start_time = datetime.now(timezone.utc) + try: active_guardrail: Optional[CustomGuardrail] = ( GUARDRAIL_REGISTRY.get_initialized_guardrail_callback( @@ -2212,23 +2321,25 @@ async def apply_guardrail( detail=f"Guardrail '{request.guardrail_name}' not found. Please ensure the guardrail is configured in your LiteLLM proxy.", ) - request_data: dict = {} - if request.messages: - request_data["messages"] = request.messages + request_processor = ProxyBaseLLMRequestProcessing(data=data) + data, litellm_logging_obj = ( + await request_processor.common_processing_pre_call_logic( + request=fastapi_request, + general_settings=general_settings, + user_api_key_dict=user_api_key_dict, + version=version, + proxy_logging_obj=proxy_logging_obj, + proxy_config=proxy_config, + route_type="apply_guardrail", + ) + ) - # Auto-detect input_type: if the caller didn't specify "response" but the - # guardrail only runs post_call (e.g. LLM-as-a-judge), use "response" so - # the test actually exercises the guardrail logic. - from litellm.types.guardrails import GuardrailEventHooks + if litellm_logging_obj is not None: + _patch_logging_obj_for_guardrail(litellm_logging_obj, request) - resolved_input_type = request.input_type - if resolved_input_type == "request": - hook = getattr(active_guardrail, "event_hook", None) - if hook == GuardrailEventHooks.post_call or hook == "post_call": - resolved_input_type = "response" - - _input_type: Literal["request", "response"] = ( - "response" if resolved_input_type == "response" else "request" + request_data: dict = {"messages": request.messages} if request.messages else {} + _input_type = _resolve_guardrail_input_type( + active_guardrail, request.input_type ) guardrailed_inputs = await active_guardrail.apply_guardrail( inputs={"texts": [request.text]}, @@ -2236,13 +2347,55 @@ async def apply_guardrail( input_type=_input_type, ) response_text = guardrailed_inputs.get("texts", []) - - return ApplyGuardrailResponse( + response = ApplyGuardrailResponse( response_text=response_text[0] if response_text else request.text ) except Exception as e: + if litellm_logging_obj is not None and not isinstance(e, HTTPException): + try: + await litellm_logging_obj.async_failure_handler( + exception=e, + traceback_exception=traceback.format_exc(), + ) + except Exception: + verbose_proxy_logger.exception( + "apply_guardrail: async_failure_handler failed" + ) + try: + thread_pool_executor.submit( + litellm_logging_obj.failure_handler, + e, + traceback.format_exc(), + ) + except Exception: + verbose_proxy_logger.exception( + "apply_guardrail: failure_handler submit failed" + ) + try: + transformed_exception = await proxy_logging_obj.post_call_failure_hook( + user_api_key_dict=user_api_key_dict, + original_exception=e, + request_data=data, + ) + if isinstance(transformed_exception, Exception): + e = transformed_exception + except Exception: + verbose_proxy_logger.exception( + "apply_guardrail: post_call_failure_hook failed" + ) raise handle_exception_on_proxy(e) + # Success logging outside except so a hook error never triggers failure handlers. + response = await _emit_guardrail_success_logs( + proxy_logging_obj=proxy_logging_obj, + litellm_logging_obj=litellm_logging_obj, + data=data, + user_api_key_dict=user_api_key_dict, + response=response, + start_time=start_time, + ) + return response + # Usage (dashboard) endpoints: overview, detail, logs router.include_router(guardrails_usage_router) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 0d27b283c47..7666b23f2af 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -1193,6 +1193,36 @@ class LiteLLMProxyRequestSetup: return tags + @staticmethod + def apply_key_tags_pre_auth( + request_data: dict, + user_api_key_dict: UserAPIKeyAuth, + ) -> None: + """Merge key metadata tags into request_data before _tag_max_budget_check.""" + key_metadata = user_api_key_dict.metadata + if not key_metadata: + return + + key_tags = key_metadata.get("tags") + if not key_tags or not isinstance(key_tags, list): + return + + _metadata_variable_name = get_metadata_variable_name_from_kwargs(request_data) + metadata = request_data.get(_metadata_variable_name) + if isinstance(metadata, str): + parsed = safe_json_loads(metadata) + metadata = parsed if isinstance(parsed, dict) else {} + request_data[_metadata_variable_name] = metadata + elif not isinstance(metadata, dict): + metadata = {} + request_data[_metadata_variable_name] = metadata + + existing_tags = metadata.get("tags") + metadata["tags"] = LiteLLMProxyRequestSetup._merge_tags( + request_tags=existing_tags if isinstance(existing_tags, list) else None, + tags_to_add=key_tags, + ) + @staticmethod def apply_client_tag_policy_pre_auth( request: Request, @@ -1513,10 +1543,16 @@ async def add_litellm_data_to_request( # noqa: PLR0915 # spend_tracking_utils, streaming_iterator) read `body` to audit the # request; taking the snapshot here ensures they see cleaned metadata. # - # Exclude secret_fields (which contains raw_headers with Authorization - # tokens) from the snapshot — they must never be persisted in spend logs - # or any other audit trail. - _body_snapshot = {k: v for k, v in data.items() if k != "secret_fields"} + # Exclude: + # - secret_fields: contains raw_headers with Authorization tokens; must + # never be persisted in spend logs or any other audit trail. + # - proxy_server_request: already a key on `data` at this point (set + # earlier in this function); including it would make the snapshot + # self-reference — body.proxy_server_request.body would be the same + # dict as body, producing an infinite traversal loop for any consumer + # that walks the structure. + _body_snapshot_exclude = {"secret_fields", "proxy_server_request"} + _body_snapshot = {k: v for k, v in data.items() if k not in _body_snapshot_exclude} data["proxy_server_request"]["body"] = _body_snapshot # Snapshot the requester-supplied metadata for downstream consumers. diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 0d34974fbef..8a8e703831b 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -3978,11 +3978,13 @@ async def _batch_resolve_access_group_resources( def _convert_teams_to_response_models( teams: list, use_deleted_table: bool, + keys_count_by_team: Optional[Dict[str, int]] = None, ) -> List[Union[TeamListItem, LiteLLM_TeamTable, LiteLLM_DeletedTeamTable]]: """Convert raw Prisma team rows to response models.""" team_list: List[ Union[TeamListItem, LiteLLM_TeamTable, LiteLLM_DeletedTeamTable] ] = [] + counts = keys_count_by_team or {} for team in teams: try: team_dict = team.model_dump() @@ -3997,10 +3999,45 @@ def _convert_teams_to_response_models( members_with_roles = [] team_dict["members_with_roles"] = members_with_roles members_count = len(members_with_roles) - team_list.append(TeamListItem(**team_dict, members_count=members_count)) + keys_count = counts.get(team_dict.get("team_id") or "", 0) + team_list.append( + TeamListItem( + **team_dict, + members_count=members_count, + keys_count=keys_count, + ) + ) return team_list +async def _get_keys_count_by_team( + prisma_client: Any, + teams: list, +) -> Dict[str, int]: + """Aggregate virtual-key counts per team for the given page of teams. + + Runs a single GROUP BY against LiteLLM_VerificationToken. The IN clause is + bounded by page_size and uses the existing @@index([team_id]), so this is + one DB round-trip per page. Returns an empty map when the page has no teams. + """ + page_team_ids = [ + getattr(t, "team_id", None) for t in teams if getattr(t, "team_id", None) + ] + if not page_team_ids: + return {} + + grouped = await prisma_client.db.litellm_verificationtoken.group_by( + by=["team_id"], + where={"team_id": {"in": page_team_ids}}, + count={"team_id": True}, + ) + return { + row["team_id"]: row.get("_count", {}).get("team_id", 0) + for row in grouped + if row.get("team_id") + } + + async def _enforce_list_team_v2_access( user_api_key_dict: UserAPIKeyAuth, user_id: Optional[str], @@ -4228,8 +4265,16 @@ async def list_team_v2( # Calculate total pages total_pages = -(-total_count // page_size) # Ceiling division - # Convert Prisma models to response models with members_count - team_list = _convert_teams_to_response_models(teams, use_deleted_table) + # Aggregate virtual-key counts per team for the current page. The deleted + # table does not carry keys_count, so it is skipped. + keys_count_by_team: Dict[str, int] = {} + if not use_deleted_table: + keys_count_by_team = await _get_keys_count_by_team(prisma_client, teams) + + # Convert Prisma models to response models with members_count and keys_count + team_list = _convert_teams_to_response_models( + teams, use_deleted_table, keys_count_by_team=keys_count_by_team + ) # Resolve resources inherited from access groups (single batch query) if not use_deleted_table: diff --git a/litellm/proxy/management_helpers/utils.py b/litellm/proxy/management_helpers/utils.py index 549d9e469f9..888ccc78188 100644 --- a/litellm/proxy/management_helpers/utils.py +++ b/litellm/proxy/management_helpers/utils.py @@ -568,14 +568,22 @@ def management_endpoint_wrapper(func): ) parent_otel_span = getattr(user_api_key_dict, "parent_otel_span", None) if parent_otel_span is not None: - await _emit_management_endpoint_otel_span( - func=func, - kwargs=kwargs, - parent_otel_span=parent_otel_span, - start_time=start_time, - end_time=end_time, - exception=e, - ) + try: + await _emit_management_endpoint_otel_span( + func=func, + kwargs=kwargs, + parent_otel_span=parent_otel_span, + start_time=start_time, + end_time=end_time, + exception=e, + ) + except Exception as otel_exc: + # Non-Blocking Exception - never let OTEL failures swallow + # the original management-endpoint exception. + verbose_logger.debug( + "Error emitting OTEL span in management endpoint wrapper failure path: %s", + str(otel_exc), + ) raise e diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 814111762b6..8fbe6d97dbc 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1071,6 +1071,7 @@ app = FastAPI( root_path=server_root_path, lifespan=proxy_startup_event, # type: ignore[reportGeneralTypeIssues] generate_unique_id_function=_generate_stable_operation_id, + strict_content_type=False, ) vertex_live_passthrough_vertex_base = VertexBase() diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index 74e4d7a533a..46a2894bd10 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -738,6 +738,98 @@ class ResponsesAPIRequestUtils: model_id, ) + @staticmethod + def _collect_container_ids_from_annotations( + annotations: Any, + collected: set[str], + ) -> None: + if not annotations or not isinstance(annotations, list): + return + for ann in annotations: + ResponsesAPIRequestUtils._collect_container_ids_from_output_item( + ann, collected + ) + + @staticmethod + def _collect_container_ids_from_message_content( + content: Any, + collected: set[str], + ) -> None: + if not content: + return + if isinstance(content, list): + for part in content: + if isinstance(part, dict): + ResponsesAPIRequestUtils._collect_container_ids_from_annotations( + part.get("annotations"), + collected, + ) + else: + ResponsesAPIRequestUtils._collect_container_ids_from_annotations( + getattr(part, "annotations", None), + collected, + ) + + @staticmethod + def _collect_container_ids_from_output_item( + item: Any, + collected: set[str], + ) -> None: + """Collect managed or raw ``container_id`` values from one output item.""" + if item is None: + return + + if isinstance(item, dict): + cid = item.get("container_id") + if isinstance(cid, str) and cid: + collected.add(cid) + nested = item.get("code_interpreter_call") + if isinstance(nested, dict): + nc = nested.get("container_id") + if isinstance(nc, str) and nc: + collected.add(nc) + if item.get("type") == "message": + ResponsesAPIRequestUtils._collect_container_ids_from_message_content( + item.get("content"), + collected, + ) + return + + cid_attr = getattr(item, "container_id", None) + if isinstance(cid_attr, str) and cid_attr: + collected.add(cid_attr) + + nested_obj = getattr(item, "code_interpreter_call", None) + if nested_obj is not None: + ResponsesAPIRequestUtils._collect_container_ids_from_output_item( + nested_obj, collected + ) + + if getattr(item, "type", None) == "message": + ResponsesAPIRequestUtils._collect_container_ids_from_message_content( + getattr(item, "content", None), + collected, + ) + + @staticmethod + def collect_container_ids_from_responses_response(response: Any) -> list[str]: + """Return unique container IDs referenced in a Responses API payload.""" + if response is None: + return [] + + if isinstance(response, dict): + output = response.get("output", []) + else: + output = getattr(response, "output", []) or [] + + collected: set[str] = set() + if output: + for item in output: + ResponsesAPIRequestUtils._collect_container_ids_from_output_item( + item, collected + ) + return list(collected) + @staticmethod def _update_container_ids_in_response( responses_api_response: Union[ResponsesAPIResponse, Dict[str, Any]], diff --git a/litellm/setup_wizard.py b/litellm/setup_wizard.py index f70cfad7fb5..862ca13e7ba 100644 --- a/litellm/setup_wizard.py +++ b/litellm/setup_wizard.py @@ -52,11 +52,12 @@ PROVIDERS: List[Dict] = [ { "id": "anthropic", "name": "Anthropic", - "description": "Claude Opus 4.7, Opus 4.6, Sonnet 4.6, Haiku 4.5", + "description": "Claude Opus 4.8, Opus 4.7, Opus 4.6, Sonnet 4.6, Haiku 4.5", "env_key": "ANTHROPIC_API_KEY", "key_hint": "sk-ant-...", "test_model": "claude-haiku-4-5-20251001", "models": [ + "claude-opus-4-8", "claude-opus-4-7", "claude-opus-4-6", "claude-sonnet-4-6", diff --git a/litellm/types/integrations/datadog_cost_management.py b/litellm/types/integrations/datadog_cost_management.py index fe04f43ea03..08744d2f52e 100644 --- a/litellm/types/integrations/datadog_cost_management.py +++ b/litellm/types/integrations/datadog_cost_management.py @@ -1,4 +1,4 @@ -from typing import Dict, Optional, TypedDict +from typing import Dict, List, Optional, TypedDict from litellm.types.integrations.custom_logger import StandardCustomLoggerInitParams @@ -9,7 +9,7 @@ class DatadogCostManagementInitParams(StandardCustomLoggerInitParams): Init params for Datadog Cost Management """ - datadog_cost_management_params: Optional[Dict] = None + cost_tag_keys: Optional[List[str]] = None class DatadogFOCUSCostEntry(TypedDict): diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index 1c4d31d21ad..bbb892a0276 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -39,7 +39,8 @@ class AnthropicOutputSchema(TypedDict, total=False): class AnthropicOutputConfig(TypedDict, total=False): """Configuration for controlling Claude's output behavior.""" - effort: Literal["high", "medium", "low"] + effort: Literal["high", "medium", "low", "xhigh", "max"] + format: AnthropicOutputSchema class AnthropicMessagesTool(TypedDict, total=False): diff --git a/litellm/types/llms/gemini.py b/litellm/types/llms/gemini.py index 9e3fea1bbbb..8763544facc 100644 --- a/litellm/types/llms/gemini.py +++ b/litellm/types/llms/gemini.py @@ -133,7 +133,7 @@ class BidiGenerateContentSetup(TypedDict, total=False): tools: List[Tools] """The tools to be used for the realtime session.""" - realtimeInputConfig: dict + realtimeInputConfig: BidiGenerateContentRealtimeInputConfig """The realtime config to be used for the realtime session.""" sessionResumption: dict diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index abe58199dfd..14114b22f39 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -79,7 +79,14 @@ from pydantic import ( field_serializer, field_validator, ) -from typing_extensions import Annotated, Dict, Required, TypedDict, override +from typing_extensions import ( + Annotated, + Dict, + NotRequired, + Required, + TypedDict, + override, +) from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject from litellm.types.responses.main import ( @@ -1935,6 +1942,7 @@ class OpenAIRealtimeStreamResponseOutputItemAdded(TypedDict): response_id: str output_index: int item: OpenAIRealtimeStreamResponseOutputItem + event_id: NotRequired[str] class OpenAIRealtimeStreamResponseBaseObject(TypedDict): @@ -2061,6 +2069,17 @@ class OpenAIRealtimeContentPartDone(TypedDict): type: Literal["response.content_part.done"] +class OpenAIRealtimeFunctionCallArgumentsDone(TypedDict): + type: Literal["response.function_call_arguments.done"] + event_id: str + response_id: str + item_id: str + output_index: int + call_id: str + name: str + arguments: str + + class OpenAIRealtimeOutputItemDone(TypedDict): event_id: str item: OpenAIRealtimeStreamResponseOutputItem @@ -2126,6 +2145,7 @@ OpenAIRealtimeEvents = Union[ OpenAIRealtimeResponseAudioDone, OpenAIRealtimeContentPartDone, OpenAIRealtimeOutputItemDone, + OpenAIRealtimeFunctionCallArgumentsDone, OpenAIRealtimeDoneEvent, ] diff --git a/litellm/types/proxy/management_endpoints/team_endpoints.py b/litellm/types/proxy/management_endpoints/team_endpoints.py index cb27fd52300..0e555535874 100644 --- a/litellm/types/proxy/management_endpoints/team_endpoints.py +++ b/litellm/types/proxy/management_endpoints/team_endpoints.py @@ -69,6 +69,7 @@ class TeamListItem(LiteLLM_TeamTable): """A team item in the paginated list response, enriched with computed fields.""" members_count: int = 0 + keys_count: int = 0 # Resources inherited from access groups (separate from direct assignments) access_group_models: Optional[List[str]] = None access_group_mcp_server_ids: Optional[List[str]] = None diff --git a/litellm/types/utils.py b/litellm/types/utils.py index e7bce27170b..8f471b62b5e 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -148,6 +148,9 @@ class ProviderSpecificModelInfo(TypedDict, total=False): supports_xhigh_reasoning_effort: Optional[bool] supports_max_reasoning_effort: Optional[bool] supports_output_config: Optional[bool] + bedrock_output_config_effort_ceiling: Optional[ + Literal["low", "medium", "high", "max", "xhigh"] + ] class SearchContextCostPerQuery(TypedDict, total=False): diff --git a/litellm/utils.py b/litellm/utils.py index 760a615664e..5a9dccc089e 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6036,6 +6036,9 @@ def _get_model_info_helper( # noqa: PLR0915 supports_max_reasoning_effort=_model_info.get( "supports_max_reasoning_effort", None ), + bedrock_output_config_effort_ceiling=_model_info.get( + "bedrock_output_config_effort_ceiling", None + ), supports_computer_use=_model_info.get("supports_computer_use", None), search_context_cost_per_query=_model_info.get( "search_context_cost_per_query", None diff --git a/migrations/Dockerfile b/migrations/Dockerfile index 2160514251a..a78a4e2225a 100644 --- a/migrations/Dockerfile +++ b/migrations/Dockerfile @@ -31,12 +31,20 @@ USER root COPY --from=uvbin /uv /uvx /usr/local/bin/ -RUN apk add --no-cache bash gcc python3 python3-dev openssl openssl-dev libsndfile +# nodejs/npm so `prisma generate` uses Wolfi's Node via PRISMA_USE_GLOBAL_NODE +# instead of nodeenv downloading one whose dynamic deps may not be in Wolfi +# (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes. +RUN for i in 1 2 3; do \ + apk add --no-cache bash gcc python3 python3-dev openssl openssl-dev libsndfile nodejs npm && break; \ + [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \ + sleep 5; \ + done ENV UV_PROJECT_ENVIRONMENT=/app/.venv \ UV_LINK_MODE=copy \ UV_COMPILE_BYTECODE=1 \ UV_PYTHON_DOWNLOADS=0 \ + PRISMA_USE_GLOBAL_NODE=true \ PATH="/app/.venv/bin:${PATH}" # Stage 1 — install third-party deps only (cached by pyproject.toml/uv.lock). @@ -78,7 +86,11 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime USER root -RUN apk add --no-cache bash openssl tzdata python3 libsndfile libatomic +RUN for i in 1 2 3; do \ + apk add --no-cache bash openssl tzdata python3 libsndfile libatomic && break; \ + [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \ + sleep 5; \ + done # wolfi-base ships an unprivileged `nonroot` account (UID/GID 65532). The # Prisma engine binaries are dynamically linked against libssl/libcrypto, so diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 6e1c79c4e39..80c2f32dc70 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -731,7 +731,6 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true }, "anthropic.claude-haiku-4-5@20251001": { @@ -755,7 +754,6 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_streaming": true, "supports_native_structured_output": true }, @@ -926,8 +924,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "anthropic.claude-opus-4-20250514-v1:0": { "cache_creation_input_token_cost": 1.875e-05, @@ -952,8 +949,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "anthropic.claude-opus-4-5-20251101-v1:0": { "cache_creation_input_token_cost": 6.25e-06, @@ -977,12 +973,12 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_reasoning": true, - "supports_minimal_reasoning_effort": true, "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 159, - "supports_native_structured_output": true + "supports_native_structured_output": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "high" }, "anthropic.claude-opus-4-6-v1": { "cache_creation_input_token_cost": 6.25e-06, @@ -1009,11 +1005,10 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true, "supports_output_config": true, "supports_max_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "bedrock_output_config_effort_ceiling": "max" }, "global.anthropic.claude-opus-4-6-v1": { "cache_creation_input_token_cost": 6.25e-06, @@ -1040,11 +1035,10 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true, "supports_output_config": true, "supports_max_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "bedrock_output_config_effort_ceiling": "max" }, "us.anthropic.claude-opus-4-6-v1": { "cache_creation_input_token_cost": 6.875e-06, @@ -1071,11 +1065,10 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true, "supports_output_config": true, "supports_max_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "bedrock_output_config_effort_ceiling": "max" }, "eu.anthropic.claude-opus-4-6-v1": { "cache_creation_input_token_cost": 6.875e-06, @@ -1101,11 +1094,10 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true, "supports_output_config": true, "supports_max_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "bedrock_output_config_effort_ceiling": "max" }, "au.anthropic.claude-opus-4-6-v1": { "cache_creation_input_token_cost": 6.875e-06, @@ -1131,11 +1123,10 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true, "supports_output_config": true, "supports_max_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "bedrock_output_config_effort_ceiling": "max" }, "anthropic.claude-opus-4-7": { "cache_creation_input_token_cost": 6.25e-06, @@ -1163,10 +1154,10 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true, "supports_max_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh" }, "anthropic.claude-mythos-preview": { "input_cost_per_token": 0, @@ -1180,8 +1171,8 @@ "supports_vision": true, "supports_prompt_caching": false, "supports_reasoning": true, - "supports_minimal_reasoning_effort": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_output_config": true }, "global.anthropic.claude-opus-4-7": { "cache_creation_input_token_cost": 6.25e-06, @@ -1209,10 +1200,10 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true, "supports_max_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh" }, "us.anthropic.claude-opus-4-7": { "cache_creation_input_token_cost": 6.875e-06, @@ -1240,10 +1231,10 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true, "supports_max_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh" }, "eu.anthropic.claude-opus-4-7": { "cache_creation_input_token_cost": 6.875e-06, @@ -1270,10 +1261,10 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true, "supports_max_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh" }, "au.anthropic.claude-opus-4-7": { "cache_creation_input_token_cost": 6.875e-06, @@ -1300,10 +1291,165 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true, "supports_max_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh" + }, + "anthropic.claude-opus-4-8": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": true, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh" + }, + "global.anthropic.claude-opus-4-8": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": true, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh" + }, + "us.anthropic.claude-opus-4-8": { + "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_1hr": 1.1e-05, + "cache_read_input_token_cost": 5.5e-07, + "input_cost_per_token": 5.5e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.75e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": true, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh" + }, + "eu.anthropic.claude-opus-4-8": { + "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_1hr": 1.1e-05, + "cache_read_input_token_cost": 5.5e-07, + "input_cost_per_token": 5.5e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.75e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": true, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh" + }, + "au.anthropic.claude-opus-4-8": { + "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_1hr": 1.1e-05, + "cache_read_input_token_cost": 5.5e-07, + "input_cost_per_token": 5.5e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.75e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": true, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh" }, "anthropic.claude-sonnet-4-6": { "cache_creation_input_token_cost": 3.75e-06, @@ -1331,10 +1477,8 @@ "supports_max_reasoning_effort": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true, - "supports_output_config": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true }, "global.anthropic.claude-sonnet-4-6": { "cache_creation_input_token_cost": 3.75e-06, @@ -1362,10 +1506,8 @@ "supports_max_reasoning_effort": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true, - "supports_output_config": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true }, "us.anthropic.claude-sonnet-4-6": { "cache_creation_input_token_cost": 4.125e-06, @@ -1393,10 +1535,8 @@ "supports_max_reasoning_effort": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true, - "supports_output_config": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true }, "eu.anthropic.claude-sonnet-4-6": { "cache_creation_input_token_cost": 4.125e-06, @@ -1423,10 +1563,8 @@ "supports_max_reasoning_effort": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true, - "supports_output_config": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true }, "au.anthropic.claude-sonnet-4-6": { "cache_creation_input_token_cost": 4.125e-06, @@ -1453,10 +1591,8 @@ "supports_max_reasoning_effort": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true, - "supports_output_config": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true }, "jp.anthropic.claude-sonnet-4-6": { "cache_creation_input_token_cost": 4.125e-06, @@ -1483,10 +1619,8 @@ "supports_max_reasoning_effort": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true, - "supports_output_config": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true }, "anthropic.claude-sonnet-4-20250514-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -1515,8 +1649,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -1548,7 +1681,6 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 159, "supports_native_structured_output": true }, "anthropic.claude-v1": { @@ -1799,7 +1931,6 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true }, "apac.anthropic.claude-3-sonnet-20240229-v1:0": { @@ -1845,8 +1976,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "assemblyai/best": { "input_cost_per_second": 3.333e-05, @@ -1888,7 +2018,6 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true }, "azure/ada": { @@ -1976,10 +2105,10 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_reasoning": true, - "supports_minimal_reasoning_effort": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "supports_output_config": true }, "azure_ai/claude-opus-4-6": { "input_cost_per_token": 5e-06, @@ -2006,10 +2135,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 159, "supports_output_config": true, - "supports_max_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_max_reasoning_effort": true }, "azure_ai/claude-opus-4-7": { "input_cost_per_token": 5e-06, @@ -2037,9 +2164,35 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "tool_use_system_prompt_tokens": 159, - "supports_max_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_max_reasoning_effort": true + }, + "azure_ai/claude-opus-4-8": { + "input_cost_per_token": 5e-06, + "output_cost_per_token": 2.5e-05, + "litellm_provider": "azure_ai", + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_read_input_token_cost": 5e-07, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true }, "azure_ai/claude-opus-4-1": { "cache_creation_input_token_cost": 1.875e-05, @@ -2104,9 +2257,7 @@ "supports_max_reasoning_effort": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, - "supports_output_config": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true }, "azure/computer-use-preview": { "input_cost_per_token": 3e-06, @@ -9480,8 +9631,7 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_web_search": true, - "tool_use_system_prompt_tokens": 159 + "supports_web_search": true }, "claude-3-haiku-20240307": { "cache_creation_input_token_cost": 3e-07, @@ -9499,8 +9649,7 @@ "supports_prompt_caching": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 264 + "supports_vision": true }, "claude-3-opus-20240229": { "cache_creation_input_token_cost": 1.875e-05, @@ -9519,8 +9668,7 @@ "supports_prompt_caching": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 395 + "supports_vision": true }, "claude-4-opus-20250514": { "cache_creation_input_token_cost": 1.875e-05, @@ -9545,8 +9693,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "claude-4-sonnet-20250514": { "cache_creation_input_token_cost": 3.75e-06, @@ -9576,8 +9723,7 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_web_search": true, - "tool_use_system_prompt_tokens": 159 + "supports_web_search": true }, "claude-sonnet-4-5": { "cache_creation_input_token_cost": 3.75e-06, @@ -9606,8 +9752,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 346 + "supports_vision": true }, "claude-sonnet-4-5-20250929": { "cache_creation_input_token_cost": 3.75e-06, @@ -9637,8 +9782,7 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_web_search": true, - "tool_use_system_prompt_tokens": 346 + "supports_web_search": true }, "claude-sonnet-4-6": { "cache_creation_input_token_cost": 3.75e-06, @@ -9666,9 +9810,7 @@ "supports_max_reasoning_effort": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, - "supports_output_config": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true }, "claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -9692,8 +9834,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "claude-opus-4-1": { "cache_creation_input_token_cost": 1.875e-05, @@ -9719,8 +9860,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "claude-opus-4-1-20250805": { "cache_creation_input_token_cost": 1.875e-05, @@ -9747,8 +9887,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "claude-opus-4-20250514": { "cache_creation_input_token_cost": 1.875e-05, @@ -9775,8 +9914,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "claude-opus-4-5-20251101": { "cache_creation_input_token_cost": 6.25e-06, @@ -9800,11 +9938,10 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_reasoning": true, - "supports_minimal_reasoning_effort": true, "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_output_config": true }, "claude-opus-4-5": { "cache_creation_input_token_cost": 6.25e-06, @@ -9828,11 +9965,10 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_reasoning": true, - "supports_minimal_reasoning_effort": true, "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_output_config": true }, "claude-opus-4-6": { "cache_creation_input_token_cost": 6.25e-06, @@ -9860,14 +9996,12 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "provider_specific_entry": { "us": 1.1, "fast": 6.0 }, "supports_output_config": true, - "supports_max_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_max_reasoning_effort": true }, "claude-opus-4-6-20260205": { "cache_creation_input_token_cost": 6.25e-06, @@ -9895,13 +10029,11 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "provider_specific_entry": { "us": 1.1, "fast": 6.0 }, "supports_max_reasoning_effort": true, - "supports_minimal_reasoning_effort": true, "supports_output_config": true }, "claude-opus-4-7": { @@ -9932,12 +10064,10 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, - "tool_use_system_prompt_tokens": 346, "provider_specific_entry": { "us": 1.1, "fast": 6.0 }, - "supports_minimal_reasoning_effort": true, "supports_output_config": true }, "claude-opus-4-7-20260416": { @@ -9968,12 +10098,44 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, - "tool_use_system_prompt_tokens": 346, "provider_specific_entry": { "us": 1.1, "fast": 6.0 }, - "supports_minimal_reasoning_effort": true, + "supports_output_config": true + }, + "claude-opus-4-8": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "anthropic", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "provider_specific_entry": { + "us": 1.1, + "fast": 2.0 + }, "supports_output_config": true }, "claude-sonnet-4-20250514": { @@ -10005,8 +10167,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "cloudflare/@cf/meta/llama-2-7b-chat-fp16": { "input_cost_per_token": 1.923e-06, @@ -11251,8 +11412,8 @@ "supports_assistant_prefill": true, "supports_function_calling": true, "supports_reasoning": true, - "supports_minimal_reasoning_effort": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_output_config": true }, "databricks/databricks-claude-sonnet-4": { "input_cost_per_token": 2.9999900000000002e-06, @@ -13418,7 +13579,6 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true }, "eu.anthropic.claude-3-5-sonnet-20240620-v1:0": { @@ -13546,8 +13706,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "eu.anthropic.claude-opus-4-20250514-v1:0": { "cache_creation_input_token_cost": 1.875e-05, @@ -13572,8 +13731,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "eu.anthropic.claude-sonnet-4-20250514-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -13602,8 +13760,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "eu.anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.125e-06, @@ -13633,7 +13790,6 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true }, "eu.meta.llama3-2-1b-instruct-v1:0": { @@ -17960,7 +18116,7 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_vision": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true }, "github_copilot/claude-opus-4.6-fast": { "litellm_provider": "github_copilot", @@ -18474,7 +18630,7 @@ "output_cost_per_token": 2.5e-05, "supports_function_calling": true, "supports_vision": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true }, "gmi/anthropic/claude-sonnet-4.5": { "input_cost_per_token": 3e-06, @@ -18774,7 +18930,6 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true }, "global.anthropic.claude-sonnet-4-20250514-v1:0": { @@ -18804,8 +18959,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "global.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.25e-06, @@ -18828,7 +18982,6 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true }, "global.amazon.nova-2-lite-v1:0": { @@ -22936,7 +23089,6 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true }, "jp.anthropic.claude-haiku-4-5-20251001-v1:0": { @@ -22959,7 +23111,6 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true }, "crusoe/deepseek-ai/DeepSeek-R1-0528": { @@ -26968,8 +27119,7 @@ "supports_computer_use": true, "supports_function_calling": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "openrouter/anthropic/claude-3.7-sonnet": { "input_cost_per_image": 0.0048, @@ -26985,8 +27135,7 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "openrouter/anthropic/claude-opus-4": { "input_cost_per_image": 0.0048, @@ -27005,8 +27154,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "openrouter/anthropic/claude-opus-4.1": { "input_cost_per_image": 0.0048, @@ -27026,8 +27174,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "openrouter/anthropic/claude-sonnet-4": { "input_cost_per_image": 0.0048, @@ -27050,8 +27197,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "openrouter/anthropic/claude-sonnet-4.6": { "cache_creation_input_token_cost": 3.75e-06, @@ -27075,9 +27221,7 @@ "supports_reasoning": true, "supports_max_reasoning_effort": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159, - "supports_minimal_reasoning_effort": true + "supports_vision": true }, "openrouter/anthropic/claude-opus-4.5": { "cache_creation_input_token_cost": 6.25e-06, @@ -27092,12 +27236,11 @@ "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, - "supports_minimal_reasoning_effort": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_output_config": true }, "openrouter/anthropic/claude-opus-4.6": { "cache_creation_input_token_cost": 6.25e-06, @@ -27116,9 +27259,7 @@ "supports_reasoning": true, "supports_max_reasoning_effort": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 346, - "supports_minimal_reasoning_effort": true + "supports_vision": true }, "openrouter/anthropic/claude-sonnet-4.5": { "input_cost_per_image": 0.0048, @@ -27141,8 +27282,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "openrouter/anthropic/claude-haiku-4.5": { "cache_creation_input_token_cost": 1.25e-06, @@ -27160,8 +27300,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 346 + "supports_vision": true }, "openrouter/anthropic/claude-opus-4.7": { "cache_creation_input_token_cost": 6.25e-06, @@ -27183,8 +27322,7 @@ "supports_max_reasoning_effort": true, "supports_tool_choice": true, "supports_vision": true, - "supports_xhigh_reasoning_effort": true, - "tool_use_system_prompt_tokens": 346 + "supports_xhigh_reasoning_effort": true }, "openrouter/bytedance/ui-tars-1.5-7b": { "input_cost_per_token": 1e-07, @@ -29165,7 +29303,7 @@ "supports_web_search": true, "supports_reasoning": false, "supports_function_calling": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true }, "perplexity/anthropic/claude-sonnet-4-5": { "litellm_provider": "perplexity", @@ -31418,7 +31556,6 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true }, "us.anthropic.claude-3-5-sonnet-20240620-v1:0": { @@ -31546,8 +31683,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "us.anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.125e-06, @@ -31579,7 +31715,6 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true }, "us-gov.anthropic.claude-sonnet-4-5-20250929-v1:0": { @@ -31605,7 +31740,6 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true }, "au.anthropic.claude-haiku-4-5-20251001-v1:0": { @@ -31627,7 +31761,6 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true }, "us.anthropic.claude-opus-4-20250514-v1:0": { @@ -31653,8 +31786,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "us.anthropic.claude-opus-4-5-20251101-v1:0": { "cache_creation_input_token_cost": 6.875e-06, @@ -31675,15 +31807,15 @@ "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, - "supports_minimal_reasoning_effort": true, "supports_pdf_input": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 159, - "supports_native_structured_output": true + "supports_native_structured_output": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "high" }, "global.anthropic.claude-opus-4-5-20251101-v1:0": { "cache_creation_input_token_cost": 6.25e-06, @@ -31704,15 +31836,15 @@ "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, - "supports_minimal_reasoning_effort": true, "supports_pdf_input": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 159, - "supports_native_structured_output": true + "supports_native_structured_output": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "high" }, "eu.anthropic.claude-opus-4-5-20251101-v1:0": { "cache_creation_input_token_cost": 6.25e-06, @@ -31732,15 +31864,15 @@ "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, - "supports_minimal_reasoning_effort": true, "supports_pdf_input": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 159, - "supports_native_structured_output": true + "supports_native_structured_output": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "high" }, "us.anthropic.claude-sonnet-4-20250514-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -31769,8 +31901,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "us.deepseek.r1-v1:0": { "input_cost_per_token": 1.35e-06, @@ -32313,13 +32444,13 @@ "output_cost_per_token": 2.5e-05, "supports_assistant_prefill": true, "supports_computer_use": true, - "supports_minimal_reasoning_effort": true, "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "supports_output_config": true }, "vercel_ai_gateway/anthropic/claude-opus-4.6": { "cache_creation_input_token_cost": 6.25e-06, @@ -32339,7 +32470,7 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true }, "vercel_ai_gateway/anthropic/claude-sonnet-4": { "cache_creation_input_token_cost": 3.75e-06, @@ -33347,8 +33478,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "vertex_ai/claude-3-haiku": { "input_cost_per_token": 2.5e-07, @@ -33451,8 +33581,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "vertex_ai/claude-opus-4-1": { "cache_creation_input_token_cost": 1.875e-05, @@ -33506,14 +33635,13 @@ "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, - "supports_minimal_reasoning_effort": true, "supports_pdf_input": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_output_config": true }, "vertex_ai/claude-opus-4-5@20251101": { "cache_creation_input_token_cost": 6.25e-06, @@ -33533,15 +33661,14 @@ "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, - "supports_minimal_reasoning_effort": true, "supports_pdf_input": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 159, - "supports_native_streaming": true + "supports_native_streaming": true, + "supports_output_config": true }, "vertex_ai/claude-opus-4-6": { "cache_creation_input_token_cost": 6.25e-06, @@ -33567,10 +33694,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_output_config": true, - "supports_max_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_max_reasoning_effort": true }, "vertex_ai/claude-opus-4-6@default": { "cache_creation_input_token_cost": 6.25e-06, @@ -33596,10 +33721,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_output_config": true, - "supports_max_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_max_reasoning_effort": true }, "vertex_ai/claude-opus-4-7": { "cache_creation_input_token_cost": 6.25e-06, @@ -33626,9 +33749,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "tool_use_system_prompt_tokens": 346, - "supports_max_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_max_reasoning_effort": true }, "vertex_ai/claude-opus-4-7@default": { "cache_creation_input_token_cost": 6.25e-06, @@ -33655,9 +33776,63 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "tool_use_system_prompt_tokens": 346, - "supports_max_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_max_reasoning_effort": true + }, + "vertex_ai/claude-opus-4-8": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true + }, + "vertex_ai/claude-opus-4-8@default": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true }, "vertex_ai/claude-sonnet-4-5": { "cache_creation_input_token_cost": 3.75e-06, @@ -33705,14 +33880,12 @@ "supports_max_reasoning_effort": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_output_config": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true }, "vertex_ai/claude-sonnet-4-5@20250929": { "cache_creation_input_token_cost": 3.75e-06, @@ -33764,8 +33937,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "vertex_ai/claude-sonnet-4": { "cache_creation_input_token_cost": 3.75e-06, @@ -33794,8 +33966,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "vertex_ai/claude-sonnet-4@20250514": { "cache_creation_input_token_cost": 3.75e-06, @@ -33824,8 +33995,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "vertex_ai/mistralai/codestral-2@001": { "input_cost_per_token": 3e-07, @@ -40827,14 +40997,12 @@ "supports_max_reasoning_effort": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_output_config": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true }, "duckduckgo/search": { "litellm_provider": "duckduckgo", @@ -41150,7 +41318,6 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true, "supports_pdf_input": true }, @@ -41173,7 +41340,6 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true, "supports_pdf_input": true } diff --git a/pyproject.toml b/pyproject.toml index 8dedca241ad..6e84afad17e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm" -version = "1.87.0" +version = "1.88.0" description = "Library to easily interface with LLM API providers" readme = "README.md" requires-python = ">=3.10, <3.14" @@ -33,62 +33,66 @@ Homepage = "https://litellm.ai" Repository = "https://github.com/BerriAI/litellm" Documentation = "https://docs.litellm.ai" -# Optional extras retain exact pins because they are consumed by Docker images -# where exact reproducibility matters. The core SDK uses ranges so downstream -# consumers can coexist with other packages without forced downgrades. +# Optional extras use compatible ranges (like the core SDK above) so downstream +# consumers can coexist with other packages and pick up security patches without +# forking. Reproducibility for our Docker/CI comes from `uv.lock` (images install +# via `uv sync --frozen`). A few deps stay exact-pinned: litellm's own +# sub-packages and the opentelemetry trio move in lockstep, and grpcio is +# supply-chain-pinned to a vetted, aged release. [project.optional-dependencies] proxy = [ - "gunicorn==23.0.0", - "uvicorn==0.33.0", - "granian==2.5.7", - "uvloop==0.21.0; sys_platform != 'win32'", - "fastapi==0.124.4", - "backoff==2.2.1", - "pyyaml==6.0.3", - "rq==2.7.0", - "orjson==3.11.6", - "apscheduler==3.11.2", - "fastapi-sso==0.19.0", - "PyJWT==2.12.0", - "python-multipart==0.0.27", - "cryptography==46.0.7", - "pynacl==1.6.2", - "websockets==15.0.1", - "boto3==1.43.1", - "azure-identity==1.25.2", - "azure-storage-blob==12.28.0", - "mcp==1.26.0", + "gunicorn>=23.0.0,<24.0", + "uvicorn>=0.33.0,<1.0", + "granian>=2.7.4,<3.0", + "uvloop>=0.21.0,<1.0; sys_platform != 'win32'", + "fastapi>=0.136.3,<1.0", + "starlette>=1.0.1,<2.0", + "backoff>=2.2.1,<3.0", + "pyyaml>=6.0.3,<7.0", + "rq>=2.7.0,<3.0", + "orjson>=3.11.6,<4.0", + "apscheduler>=3.11.2,<4.0", + "fastapi-sso>=0.19.0,<1.0", + "PyJWT>=2.12.0,<3.0", + "python-multipart>=0.0.27,<1.0", + "cryptography>=46.0.7,<47.0", + "pynacl>=1.6.2,<2.0", + "websockets>=15.0.1,<16.0", + "boto3>=1.43.1,<2.0", + "azure-identity>=1.25.2,<2.0", + "azure-storage-blob>=12.28.0,<13.0", + "mcp>=1.26.0,<2.0", "litellm-proxy-extras==0.4.73", "litellm-enterprise==0.1.41", - "RestrictedPython==8.1", - "rich==13.9.4", - "polars==1.38.1", - "soundfile==0.12.1", - "pyroscope-io==0.8.16; sys_platform != 'win32'", - "pydantic-settings>=2.14.1", + "RestrictedPython>=8.1,<9.0", + "rich>=13.9.4,<14.0", + "polars>=1.38.1,<2.0", + "soundfile>=0.12.1,<1.0", + "pyroscope-io>=0.8.16,<1.0; sys_platform != 'win32'", + "pydantic-settings>=2.14.1,<3.0", ] extra_proxy = [ - "prisma==0.11.0", - "azure-identity==1.25.2", - "azure-keyvault-secrets==4.10.0", + "prisma>=0.11.0,<1.0", + "azure-identity>=1.25.2,<2.0", + "azure-keyvault-secrets>=4.10.0,<5.0", # Not in PyPI proxy extra. - "google-cloud-kms==2.24.2", - "google-cloud-iam==2.19.1", + "google-cloud-kms>=2.24.2,<3.0", + "google-cloud-iam>=2.19.1,<3.0", # Not in PyPI proxy extra. - "resend==2.23.0", - "redisvl==0.4.1; python_version < '3.14'", - "a2a-sdk==0.3.24", + "resend>=2.23.0,<3.0", + "redisvl>=0.4.1,<1.0; python_version < '3.14'", + "a2a-sdk>=0.3.24,<1.0", ] utils = [ # Not in Docker or PyPI proxy extra. - "numpydoc==1.8.0", + "numpydoc>=1.8.0,<2.0", ] -caching = ["diskcache==5.6.3"] +caching = ["diskcache>=5.6.3,<6.0"] semantic-router = [ - "semantic-router==0.1.12; python_version < '3.14'", - "aurelio-sdk==0.0.19; python_version < '3.14'", + "semantic-router>=0.1.15,<1.0; python_version < '3.14'", + "aurelio-sdk>=0.0.19,<1.0; python_version < '3.14'", ] -mlflow = ["mlflow==3.11.1"] +mlflow = ["mlflow>=3.11.1,<4.0"] grpc = [ # Newest non-yanked release older than the 30-day cutoff. "grpcio==1.78.0", @@ -101,28 +105,28 @@ stt-nvidia-riva = [ "audioread>=3.0.1", "numpy>=1.26.0", ] -google = ["google-cloud-aiplatform==1.133.0"] +google = ["google-cloud-aiplatform>=1.133.0,<2.0"] proxy-runtime = [ # Historically bundled in the proxy Docker images via requirements.txt. # Keep these in a dedicated extra so uv-based images preserve the same # feature surface without forcing the base SDK install to grow. - "google-cloud-aiplatform==1.133.0", - "google-genai==1.37.0", - "anthropic[vertex]==0.84.0", + "google-cloud-aiplatform>=1.133.0,<2.0", + "google-genai>=1.37.0,<2.0", + "anthropic[vertex]>=0.84.0,<1.0", "grpcio==1.78.0", - "prometheus-client==0.20.0", - "langfuse==2.59.7", + "prometheus-client>=0.20.0,<1.0", + "langfuse>=2.59.7,<3.0", "opentelemetry-api==1.28.0", "opentelemetry-sdk==1.28.0", "opentelemetry-exporter-otlp==1.28.0", - "ddtrace==2.19.0", - "sentry-sdk==2.21.0", - "mangum==0.17.0", - "azure-ai-contentsafety==1.0.0", - "azure-storage-file-datalake==12.20.0", - "pypdf==6.10.2; python_version < '3.14'", - "llm-sandbox==0.3.39", - "detect-secrets==1.5.0", + "ddtrace>=2.19.0,<3.0", + "sentry-sdk>=2.21.0,<3.0", + "mangum>=0.17.0,<1.0", + "azure-ai-contentsafety>=1.0.0,<2.0", + "azure-storage-file-datalake>=12.20.0,<13.0", + "pypdf>=6.10.2,<7.0; python_version < '3.14'", + "llm-sandbox>=0.3.39,<1.0", + "detect-secrets>=1.5.0,<2.0", ] [project.scripts] @@ -188,7 +192,7 @@ ci = [ "psycopg2-binary==2.9.11", "pytest-codspeed==4.3.0", "pytest-retry==1.7.0", - "pyarrow==22.0.0", + "pyarrow==23.0.1", "langchain==1.2.10", "lunary==1.4.36; python_version == '3.10'", "lunary==1.4.37; python_version >= '3.11'", @@ -253,7 +257,7 @@ source-exclude = [ profile = "black" [tool.commitizen] -version = "1.87.0" +version = "1.88.0" version_files = [ "pyproject.toml:^version", ] diff --git a/scripts/benchmark_model_response_creator.py b/scripts/benchmark_model_response_creator.py new file mode 100644 index 00000000000..881870d3854 --- /dev/null +++ b/scripts/benchmark_model_response_creator.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python3 +"""Tight microbenchmark for CustomStreamWrapper.model_response_creator. + +Calls model_response_creator() in a tight loop on a pre-built wrapper to +isolate per-call cost. Driving the full wrapper adds threadpool logging, +gc, and other noise that swamps microsecond-scale changes here. + +Example: + uv run python scripts/benchmark_model_response_creator.py --label baseline + uv run python scripts/benchmark_model_response_creator.py --label optimized +""" + +from __future__ import annotations + +import argparse +import gc +import json +import logging +import os +import statistics +import time +from dataclasses import asdict, dataclass +from typing import List +from unittest.mock import MagicMock + +os.environ.setdefault("LITELLM_LOG", "ERROR") +logging.getLogger("LiteLLM").setLevel(logging.ERROR) + +import litellm # noqa: E402 + +litellm.suppress_debug_info = True + +from litellm.litellm_core_utils.streaming_handler import ( + CustomStreamWrapper, +) # noqa: E402 + + +def _make_logging_obj(provider: str) -> MagicMock: + logging_obj = MagicMock() + logging_obj.model_call_details = { + "custom_llm_provider": provider, + "litellm_params": {}, + } + logging_obj.call_type = "completion" + logging_obj.stream_options = None + logging_obj.messages = [{"role": "user", "content": "hi"}] + logging_obj.completion_start_time = None + logging_obj._llm_caching_handler = None + return logging_obj + + +def _make_wrapper(provider: str, model: str) -> CustomStreamWrapper: + return CustomStreamWrapper( + completion_stream=iter([]), + model=model, + logging_obj=_make_logging_obj(provider), + custom_llm_provider=provider, + ) + + +@dataclass +class Result: + label: str + scenario: str + iterations: int + elapsed_min_s: float + elapsed_median_s: float + per_call_us: float + calls_per_sec: float + + +SCENARIOS = { + "no_chunk": { + "description": "model_response_creator() — no chunk arg (most common path)", + "chunk_factory": lambda i: None, + }, + "text_chunk": { + "description": "model_response_creator(chunk={'text': '...'}) — text delta path", + "chunk_factory": lambda i: {"text": f"token{i}"}, + }, + "rich_chunk": { + "description": "model_response_creator(chunk={...}) — full chunk dict path", + "chunk_factory": lambda i: { + "id": f"id-{i}", + "object": "chat.completion.chunk", + "created": 1234567890, + }, + }, +} + + +def bench_no_chunk(wrapper: CustomStreamWrapper, iterations: int) -> float: + gc.collect() + gc.disable() + try: + start = time.perf_counter() + for _ in range(iterations): + wrapper.model_response_creator() + elapsed = time.perf_counter() - start + finally: + gc.enable() + return elapsed + + +def bench_with_chunk(wrapper: CustomStreamWrapper, factory, iterations: int) -> float: + # Pre-build chunks so we don't measure their construction cost. + chunks = [factory(i) for i in range(iterations)] + gc.collect() + gc.disable() + try: + start = time.perf_counter() + for chunk in chunks: + wrapper.model_response_creator(chunk=dict(chunk)) # copy because mutated + elapsed = time.perf_counter() - start + finally: + gc.enable() + return elapsed + + +def run_scenario( + label: str, + scenario_key: str, + iterations: int, + repeats: int, + warmup: int, +) -> Result: + spec = SCENARIOS[scenario_key] + wrapper = _make_wrapper(provider="anthropic", model="claude-3-5-sonnet") + + if scenario_key == "no_chunk": + runner = lambda: bench_no_chunk(wrapper, iterations) # noqa: E731 + else: + runner = lambda: bench_with_chunk( + wrapper, spec["chunk_factory"], iterations + ) # noqa: E731 + + for _ in range(warmup): + runner() + samples = [runner() for _ in range(repeats)] + + elapsed_min = min(samples) + elapsed_median = statistics.median(samples) + per_call_us = (elapsed_min * 1_000_000) / iterations + calls_per_sec = iterations / elapsed_min if elapsed_min > 0 else 0.0 + + return Result( + label=label, + scenario=scenario_key, + iterations=iterations, + elapsed_min_s=elapsed_min, + elapsed_median_s=elapsed_median, + per_call_us=per_call_us, + calls_per_sec=calls_per_sec, + ) + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--label", required=True) + ap.add_argument("--iterations", type=int, default=200_000) + ap.add_argument("--warmup", type=int, default=2) + ap.add_argument("--repeats", type=int, default=8) + ap.add_argument("--json", dest="json_out") + args = ap.parse_args() + + print( + f"\n=== label={args.label} iterations={args.iterations:,} " + f"warmup={args.warmup} repeats={args.repeats} (min reported) ===" + ) + results: List[Result] = [] + for scenario in SCENARIOS: + r = run_scenario( + args.label, scenario, args.iterations, args.repeats, args.warmup + ) + results.append(r) + print( + f" {r.scenario:12s}: " + f"min={r.elapsed_min_s*1000:8.2f} ms " + f"median={r.elapsed_median_s*1000:8.2f} ms " + f"per-call={r.per_call_us:7.3f} μs " + f"calls/s={r.calls_per_sec:>12,.0f}" + ) + + if args.json_out: + with open(args.json_out, "w", encoding="utf-8") as f: + json.dump([asdict(r) for r in results], f, indent=2) + print(f"\nWrote {len(results)} results to {args.json_out}") + + +if __name__ == "__main__": + main() diff --git a/scripts/benchmark_streaming_chunk_overhead.py b/scripts/benchmark_streaming_chunk_overhead.py new file mode 100644 index 00000000000..948be096bec --- /dev/null +++ b/scripts/benchmark_streaming_chunk_overhead.py @@ -0,0 +1,369 @@ +#!/usr/bin/env python3 +"""Benchmark CustomStreamWrapper per-chunk overhead. + +Drives CustomStreamWrapper directly with synthetic in-memory chunks for +Anthropic (GenericStreamingChunk), Bedrock Invoke (GenericStreamingChunk), +and Bedrock Converse (ModelResponseStream). A full proxy benchmark adds +FastAPI, HTTP, and TCP latency, which dilutes the per-chunk CPU signal. + +Example: + uv run python scripts/benchmark_streaming_chunk_overhead.py \\ + --streams 500 --chunks 200 --warmup 50 --repeats 5 +""" + +from __future__ import annotations + +import argparse +import asyncio +import gc +import json +import logging +import os +import statistics +import time +from dataclasses import asdict, dataclass +from typing import Callable, List, Optional +from unittest.mock import MagicMock + +# Silence litellm's "Provider List" warnings emitted by get_llm_provider +# when it sees synthetic model names — we're not exercising provider +# routing, only the per-chunk wrapper hot path. +os.environ.setdefault("LITELLM_LOG", "ERROR") +logging.getLogger("LiteLLM").setLevel(logging.ERROR) + +import litellm # noqa: E402 + +litellm.suppress_debug_info = True + +from litellm.litellm_core_utils.streaming_handler import ( + CustomStreamWrapper, +) # noqa: E402 +from litellm.types.utils import ( # noqa: E402 + Delta, + GenericStreamingChunk as GChunk, + ModelResponseStream, + StreamingChoices, + Usage, +) + +# --------------------------------------------------------------------------- +# Synthetic chunk fixtures +# --------------------------------------------------------------------------- + + +def _make_logging_obj(provider: str) -> MagicMock: + logging_obj = MagicMock() + logging_obj.model_call_details = { + "custom_llm_provider": provider, + "litellm_params": {}, + } + logging_obj.call_type = "completion" + logging_obj.stream_options = None + logging_obj.messages = [{"role": "user", "content": "hi"}] + logging_obj.completion_start_time = None + logging_obj._llm_caching_handler = None + return logging_obj + + +def _make_generic_chunk( + text: str, + is_finished: bool = False, + finish_reason: str = "", + usage: Optional[dict] = None, +) -> GChunk: + return GChunk( + text=text, + is_finished=is_finished, + finish_reason=finish_reason, + usage=usage, + index=0, + tool_use=None, + ) + + +def _make_converse_chunk( + text: str = "", + finish_reason: str = "", + usage: Optional[Usage] = None, +) -> ModelResponseStream: + return ModelResponseStream( + choices=[ + StreamingChoices( + finish_reason=finish_reason or None, + index=0, + delta=Delta(content=text, role="assistant"), + ) + ], + id="msg-bench", + model="anthropic.claude-3-5-sonnet", + usage=usage, + ) + + +# --------------------------------------------------------------------------- +# Provider stream factories +# --------------------------------------------------------------------------- + + +def anthropic_chunks(n: int) -> List[GChunk]: + out: List[GChunk] = [_make_generic_chunk(f"tok{i} ") for i in range(n)] + out.append( + _make_generic_chunk( + "", + is_finished=True, + finish_reason="stop", + usage={"prompt_tokens": 10, "completion_tokens": n, "total_tokens": 10 + n}, + ) + ) + return out + + +def bedrock_invoke_chunks(n: int) -> List[GChunk]: + # Bedrock Invoke surfaces GChunk-shaped dicts, same shape as Anthropic. + return anthropic_chunks(n) + + +def bedrock_converse_chunks(n: int) -> List[ModelResponseStream]: + out: List[ModelResponseStream] = [ + _make_converse_chunk(f"tok{i} ") for i in range(n) + ] + out.append( + _make_converse_chunk( + text="", + finish_reason="stop", + usage=Usage(prompt_tokens=10, completion_tokens=n, total_tokens=10 + n), + ) + ) + return out + + +PROVIDERS: dict[str, tuple[str, Callable[[int], list]]] = { + "anthropic": ("anthropic", anthropic_chunks), + "bedrock_invoke": ("bedrock", bedrock_invoke_chunks), + "bedrock_converse": ("bedrock", bedrock_converse_chunks), +} + + +# --------------------------------------------------------------------------- +# Drive a single stream end-to-end +# --------------------------------------------------------------------------- + + +def _make_wrapper( + chunks: list, provider: str, async_stream: bool +) -> CustomStreamWrapper: + logging_obj = _make_logging_obj(provider) + if async_stream: + + async def _agen(): + for c in chunks: + yield c + + stream = _agen() + else: + stream = iter(chunks) + return CustomStreamWrapper( + completion_stream=stream, + model="claude-3-5-sonnet", + logging_obj=logging_obj, + custom_llm_provider=provider, + ) + + +def drive_sync(provider_key: str, chunks_per_stream: int, n_streams: int) -> float: + provider, factory = PROVIDERS[provider_key] + # Pre-build the chunk lists; we only measure wrapper iteration cost. + chunk_lists = [factory(chunks_per_stream) for _ in range(n_streams)] + gc.collect() + gc.disable() + try: + start = time.perf_counter() + for chunks in chunk_lists: + wrapper = _make_wrapper(chunks, provider, async_stream=False) + for _ in wrapper: + pass + elapsed = time.perf_counter() - start + finally: + gc.enable() + return elapsed + + +async def drive_async( + provider_key: str, chunks_per_stream: int, n_streams: int +) -> float: + provider, factory = PROVIDERS[provider_key] + chunk_lists = [factory(chunks_per_stream) for _ in range(n_streams)] + gc.collect() + gc.disable() + try: + start = time.perf_counter() + for chunks in chunk_lists: + wrapper = _make_wrapper(chunks, provider, async_stream=True) + async for _ in wrapper: + pass + elapsed = time.perf_counter() - start + finally: + gc.enable() + return elapsed + + +# --------------------------------------------------------------------------- +# Repeat × take-min runner +# --------------------------------------------------------------------------- + + +@dataclass +class Result: + label: str + provider: str + mode: str + streams: int + chunks_per_stream: int + total_chunks: int + elapsed_min_s: float + elapsed_median_s: float + per_chunk_us: float + chunks_per_sec: float + streams_per_sec: float + + +def run_case( + label: str, + provider_key: str, + mode: str, + chunks_per_stream: int, + n_streams: int, + repeats: int, + warmup: int, +) -> Result: + if mode == "sync": + # Warmup runs amortize import-time and JIT-y caches. + for _ in range(warmup): + drive_sync(provider_key, chunks_per_stream, max(1, n_streams // 10)) + samples = [ + drive_sync(provider_key, chunks_per_stream, n_streams) + for _ in range(repeats) + ] + elif mode == "async": + + async def _warm(): + for _ in range(warmup): + await drive_async( + provider_key, chunks_per_stream, max(1, n_streams // 10) + ) + + asyncio.run(_warm()) + samples = [ + asyncio.run(drive_async(provider_key, chunks_per_stream, n_streams)) + for _ in range(repeats) + ] + else: + raise ValueError(f"unknown mode {mode!r}") + + elapsed_min = min(samples) + elapsed_median = statistics.median(samples) + # Each stream emits chunks_per_stream text chunks + 1 finish/usage chunk. + total_chunks = n_streams * (chunks_per_stream + 1) + per_chunk_us = (elapsed_min * 1_000_000) / total_chunks + chunks_per_sec = total_chunks / elapsed_min if elapsed_min > 0 else 0.0 + streams_per_sec = n_streams / elapsed_min if elapsed_min > 0 else 0.0 + + return Result( + label=label, + provider=provider_key, + mode=mode, + streams=n_streams, + chunks_per_stream=chunks_per_stream, + total_chunks=total_chunks, + elapsed_min_s=elapsed_min, + elapsed_median_s=elapsed_median, + per_chunk_us=per_chunk_us, + chunks_per_sec=chunks_per_sec, + streams_per_sec=streams_per_sec, + ) + + +def format_result(r: Result) -> str: + return ( + f" {r.provider:18s} {r.mode:5s}: " + f"min={r.elapsed_min_s*1000:8.2f} ms " + f"median={r.elapsed_median_s*1000:8.2f} ms " + f"per-chunk={r.per_chunk_us:7.2f} μs " + f"chunks/s={r.chunks_per_sec:>10,.0f} " + f"streams/s={r.streams_per_sec:>8,.1f}" + ) + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument( + "--label", required=True, help="Run label (e.g. baseline / optimized)" + ) + ap.add_argument("--streams", type=int, default=500, help="Streams per run") + ap.add_argument( + "--chunks", + type=int, + default=200, + help="Text chunks per stream (excl. finish chunk)", + ) + ap.add_argument("--warmup", type=int, default=2, help="Warmup runs") + ap.add_argument( + "--repeats", type=int, default=5, help="Measured runs (we report min)" + ) + ap.add_argument( + "--providers", + default="anthropic,bedrock_invoke,bedrock_converse", + help="Comma-separated provider list", + ) + ap.add_argument( + "--modes", + default="sync,async", + help="Comma-separated iteration modes (sync/async)", + ) + ap.add_argument( + "--json", dest="json_out", help="Write results as JSON to this path" + ) + args = ap.parse_args() + + providers = [p.strip() for p in args.providers.split(",") if p.strip()] + modes = [m.strip() for m in args.modes.split(",") if m.strip()] + + for p in providers: + if p not in PROVIDERS: + raise SystemExit(f"unknown provider {p!r}; choose from {list(PROVIDERS)}") + for m in modes: + if m not in {"sync", "async"}: + raise SystemExit(f"unknown mode {m!r}; choose from sync/async") + + print( + f"\n=== label={args.label} streams={args.streams} chunks/stream={args.chunks} " + f"warmup={args.warmup} repeats={args.repeats} (min reported) ===" + ) + results: List[Result] = [] + for provider_key in providers: + for mode in modes: + r = run_case( + label=args.label, + provider_key=provider_key, + mode=mode, + chunks_per_stream=args.chunks, + n_streams=args.streams, + repeats=args.repeats, + warmup=args.warmup, + ) + results.append(r) + print(format_result(r)) + + if args.json_out: + with open(args.json_out, "w", encoding="utf-8") as f: + json.dump([asdict(r) for r in results], f, indent=2) + print(f"\nWrote {len(results)} results to {args.json_out}") + + +if __name__ == "__main__": + main() diff --git a/tests/_vcr_conftest_common.py b/tests/_vcr_conftest_common.py index 6d3d34d4acf..d08b87bd580 100644 --- a/tests/_vcr_conftest_common.py +++ b/tests/_vcr_conftest_common.py @@ -644,6 +644,23 @@ def _should_drop_telemetry_record(request) -> bool: return not _current_test_records_telemetry() +def _should_passthrough_credential_exchange(request) -> bool: + """Force the Google OAuth2/STS token mint to run live, never from cassette. + + The mint returns a short-lived ``ya29.*`` access token. Recording it lets a + *stale* token replay on a later run; litellm caches it (the recorded + ``expires_in`` keeps ``credentials.expired`` False, so it is never + refreshed) and sends it to a live Vertex/Gemini endpoint, which rejects it + with ``ACCESS_TOKEN_EXPIRED``. The token body carries nothing a test asserts + on, so always mint it live: returning ``None`` from ``before_record_request`` + makes vcrpy neither store nor replay the call. Inert during + ``Cassette._load`` for the same reason as ``_should_drop_telemetry_record``. + """ + if _vcr_load_in_progress(): + return False + return _is_credential_exchange_request(request) + + # Google APIs (Vertex AI, Gemini, OAuth2/STS). Auth is a ``ya29.*`` OAuth2 # access token minted fresh on every run, so the per-request key fingerprint # rotates and never matches a recording. The logical credential — the GCP @@ -931,6 +948,8 @@ def _before_record_request(request): # store the interaction; the request passes through live (fire-and-forget). if _should_drop_telemetry_record(request): return None + if _should_passthrough_credential_exchange(request): + return None headers = getattr(request, "headers", None) if headers is None: return request diff --git a/tests/code_coverage_tests/check_licenses.py b/tests/code_coverage_tests/check_licenses.py index 5fb2b495c24..389e534b1ff 100644 --- a/tests/code_coverage_tests/check_licenses.py +++ b/tests/code_coverage_tests/check_licenses.py @@ -376,8 +376,23 @@ class LicenseChecker: all_compliant = True for req in requirements: + # Prefer a lower-bound/exact version (a real released version) for the + # PyPI license lookup. ``next(iter(req.specifier))`` returns an + # arbitrary clause; for a range like ``>=1.0,<2.0`` that can be the + # upper bound (``2.0``) — a version that may not exist on PyPI and + # would 404 to an "unknown" license. try: - version = next(iter(req.specifier)).version if req.specifier else None + floor_versions = [ + spec.version + for spec in req.specifier + if spec.operator in (">=", "==", "===", "~=", ">") + ] + if floor_versions: + version = floor_versions[0] + else: + version = ( + next(iter(req.specifier)).version if req.specifier else None + ) except StopIteration: version = None diff --git a/tests/enterprise/litellm_enterprise/proxy/guardrails/conftest.py b/tests/enterprise/litellm_enterprise/proxy/guardrails/conftest.py new file mode 100644 index 00000000000..4dd5c3d88ca --- /dev/null +++ b/tests/enterprise/litellm_enterprise/proxy/guardrails/conftest.py @@ -0,0 +1,42 @@ +"""Shared fixtures for guardrail apply_guardrail tests.""" + +from contextlib import contextmanager +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + + +@contextmanager +def _mock_proxy_logging(): + """Patch the proxy-server globals that apply_guardrail imports at call time.""" + mock_proxy_logging = MagicMock() + mock_proxy_logging.post_call_success_hook = AsyncMock(return_value=None) + mock_proxy_logging.post_call_failure_hook = AsyncMock(return_value=None) + mock_logging_obj = MagicMock() + mock_logging_obj.async_success_handler = AsyncMock(return_value=None) + mock_logging_obj.async_failure_handler = AsyncMock(return_value=None) + mock_logging_obj.success_handler = MagicMock(return_value=None) + mock_logging_obj.failure_handler = MagicMock(return_value=None) + mock_logging_obj.model_call_details = {} + + with ( + patch( + "litellm.proxy.common_request_processing.ProxyBaseLLMRequestProcessing" + ) as mock_proc_cls, + patch("litellm.proxy.proxy_server.proxy_logging_obj", mock_proxy_logging), + patch("litellm.proxy.proxy_server.general_settings", {}), + patch("litellm.proxy.proxy_server.proxy_config", MagicMock()), + patch("litellm.proxy.proxy_server.version", "0.0.0"), + ): + mock_proc = MagicMock() + mock_proc.common_processing_pre_call_logic = AsyncMock( + return_value=({}, mock_logging_obj) + ) + mock_proc_cls.return_value = mock_proc + yield mock_proxy_logging + + +@pytest.fixture +def mock_proxy_logging_ctx(): + """Return the proxy-logging context manager factory for use as `with ctx():`.""" + return _mock_proxy_logging diff --git a/tests/enterprise/litellm_enterprise/proxy/guardrails/test_apply_guardrail_endpoint.py b/tests/enterprise/litellm_enterprise/proxy/guardrails/test_apply_guardrail_endpoint.py index 0d27df50d15..e5074c44210 100644 --- a/tests/enterprise/litellm_enterprise/proxy/guardrails/test_apply_guardrail_endpoint.py +++ b/tests/enterprise/litellm_enterprise/proxy/guardrails/test_apply_guardrail_endpoint.py @@ -18,14 +18,19 @@ from litellm.types.guardrails import ApplyGuardrailRequest, ApplyGuardrailRespon @pytest.mark.asyncio -async def test_apply_guardrail_endpoint_returns_correct_response(): +async def test_apply_guardrail_endpoint_returns_correct_response( + mock_proxy_logging_ctx, +): """Test that apply_guardrail endpoint returns ApplyGuardrailResponse object""" from litellm.proxy.guardrails.guardrail_endpoints import apply_guardrail # Mock the guardrail registry - with patch( - "litellm.proxy.guardrails.guardrail_endpoints.GUARDRAIL_REGISTRY" - ) as mock_registry: + with ( + patch( + "litellm.proxy.guardrails.guardrail_endpoints.GUARDRAIL_REGISTRY" + ) as mock_registry, + mock_proxy_logging_ctx(), + ): # Create a mock guardrail mock_guardrail = Mock(spec=CustomGuardrail) # Apply guardrail returns GenericGuardrailAPIInputs (dict with texts key) @@ -49,7 +54,9 @@ async def test_apply_guardrail_endpoint_returns_correct_response(): # Call the endpoint response = await apply_guardrail( - request=request, user_api_key_dict=user_api_key_dict + fastapi_request=Mock(), + request=request, + user_api_key_dict=user_api_key_dict, ) # Verify the response is of the correct type @@ -65,15 +72,18 @@ async def test_apply_guardrail_endpoint_returns_correct_response(): @pytest.mark.asyncio -async def test_apply_guardrail_endpoint_guardrail_not_found(): +async def test_apply_guardrail_endpoint_guardrail_not_found(mock_proxy_logging_ctx): """Test that apply_guardrail endpoint raises exception when guardrail not found""" from litellm.proxy._types import ProxyException from litellm.proxy.guardrails.guardrail_endpoints import apply_guardrail # Mock the guardrail registry to return None - with patch( - "litellm.proxy.guardrails.guardrail_endpoints.GUARDRAIL_REGISTRY" - ) as mock_registry: + with ( + patch( + "litellm.proxy.guardrails.guardrail_endpoints.GUARDRAIL_REGISTRY" + ) as mock_registry, + mock_proxy_logging_ctx(), + ): mock_registry.get_initialized_guardrail_callback.return_value = None # Create the request @@ -86,26 +96,35 @@ async def test_apply_guardrail_endpoint_guardrail_not_found(): # Verify exception is raised with pytest.raises(ProxyException) as exc_info: - await apply_guardrail(request=request, user_api_key_dict=user_api_key_dict) + await apply_guardrail( + fastapi_request=Mock(), + request=request, + user_api_key_dict=user_api_key_dict, + ) assert "non-existent-guardrail" in exc_info.value.message assert "not found" in exc_info.value.message @pytest.mark.asyncio -async def test_apply_guardrail_endpoint_with_presidio_guardrail(): +async def test_apply_guardrail_endpoint_with_presidio_guardrail(mock_proxy_logging_ctx): """Test apply_guardrail endpoint with a Presidio-like guardrail""" from litellm.proxy.guardrails.guardrail_endpoints import apply_guardrail # Mock the guardrail registry - with patch( - "litellm.proxy.guardrails.guardrail_endpoints.GUARDRAIL_REGISTRY" - ) as mock_registry: + with ( + patch( + "litellm.proxy.guardrails.guardrail_endpoints.GUARDRAIL_REGISTRY" + ) as mock_registry, + mock_proxy_logging_ctx(), + ): # Create a mock guardrail that simulates Presidio behavior mock_guardrail = Mock(spec=CustomGuardrail) # Simulate masking PII entities - returns GenericGuardrailAPIInputs (dict with texts key) mock_guardrail.apply_guardrail = AsyncMock( - return_value={"texts": ["My name is [PERSON] and my email is [EMAIL_ADDRESS]"]} + return_value={ + "texts": ["My name is [PERSON] and my email is [EMAIL_ADDRESS]"] + } ) # Configure the registry to return our mock guardrail @@ -124,7 +143,9 @@ async def test_apply_guardrail_endpoint_with_presidio_guardrail(): # Call the endpoint response = await apply_guardrail( - request=request, user_api_key_dict=user_api_key_dict + fastapi_request=Mock(), + request=request, + user_api_key_dict=user_api_key_dict, ) # Verify the response is of the correct type @@ -138,14 +159,17 @@ async def test_apply_guardrail_endpoint_with_presidio_guardrail(): @pytest.mark.asyncio -async def test_apply_guardrail_endpoint_without_optional_params(): +async def test_apply_guardrail_endpoint_without_optional_params(mock_proxy_logging_ctx): """Test apply_guardrail endpoint without optional language and entities parameters""" from litellm.proxy.guardrails.guardrail_endpoints import apply_guardrail # Mock the guardrail registry - with patch( - "litellm.proxy.guardrails.guardrail_endpoints.GUARDRAIL_REGISTRY" - ) as mock_registry: + with ( + patch( + "litellm.proxy.guardrails.guardrail_endpoints.GUARDRAIL_REGISTRY" + ) as mock_registry, + mock_proxy_logging_ctx(), + ): # Create a mock guardrail mock_guardrail = Mock(spec=CustomGuardrail) # Returns GenericGuardrailAPIInputs (dict with texts key) @@ -166,7 +190,9 @@ async def test_apply_guardrail_endpoint_without_optional_params(): # Call the endpoint response = await apply_guardrail( - request=request, user_api_key_dict=user_api_key_dict + fastapi_request=Mock(), + request=request, + user_api_key_dict=user_api_key_dict, ) # Verify the response is of the correct type diff --git a/tests/enterprise/litellm_enterprise/proxy/guardrails/test_bedrock_apply_guardrail.py b/tests/enterprise/litellm_enterprise/proxy/guardrails/test_bedrock_apply_guardrail.py index dff444168c2..d1caf398540 100644 --- a/tests/enterprise/litellm_enterprise/proxy/guardrails/test_bedrock_apply_guardrail.py +++ b/tests/enterprise/litellm_enterprise/proxy/guardrails/test_bedrock_apply_guardrail.py @@ -4,7 +4,7 @@ Test the Bedrock guardrail apply_guardrail functionality import os import sys -from unittest.mock import AsyncMock, patch +from unittest.mock import AsyncMock, Mock, patch import pytest @@ -153,7 +153,7 @@ async def test_bedrock_apply_guardrail_api_failure(): @pytest.mark.asyncio -async def test_bedrock_apply_guardrail_endpoint_integration(): +async def test_bedrock_apply_guardrail_endpoint_integration(mock_proxy_logging_ctx): """Test the full endpoint integration with Bedrock guardrail""" from litellm.proxy.guardrails.guardrail_endpoints import apply_guardrail @@ -165,9 +165,12 @@ async def test_bedrock_apply_guardrail_endpoint_integration(): ) # Mock the guardrail registry - with patch( - "litellm.proxy.guardrails.guardrail_endpoints.GUARDRAIL_REGISTRY" - ) as mock_registry: + with ( + patch( + "litellm.proxy.guardrails.guardrail_endpoints.GUARDRAIL_REGISTRY" + ) as mock_registry, + mock_proxy_logging_ctx(), + ): # Mock the make_bedrock_api_request method with patch.object( guardrail, "make_bedrock_api_request", new_callable=AsyncMock @@ -194,7 +197,9 @@ async def test_bedrock_apply_guardrail_endpoint_integration(): # Call the endpoint response = await apply_guardrail( - request=request, user_api_key_dict=user_api_key_dict + fastapi_request=Mock(), + request=request, + user_api_key_dict=user_api_key_dict, ) # Verify the response diff --git a/tests/llm_translation/reasoning_effort_grid/grid_spec.py b/tests/llm_translation/reasoning_effort_grid/grid_spec.py index 993643e0fc1..2f9735274c6 100644 --- a/tests/llm_translation/reasoning_effort_grid/grid_spec.py +++ b/tests/llm_translation/reasoning_effort_grid/grid_spec.py @@ -22,6 +22,7 @@ class ModelEntry: required_env: FrozenSet[str] = field(default_factory=frozenset) caps: FrozenSet[str] = field(default_factory=frozenset) fail_reason: Optional[str] = None + bedrock_effort_ceiling: Optional[str] = None def params(self) -> Dict[str, str]: return dict(self.extra_params) @@ -59,9 +60,31 @@ _ADAPTIVE_EFFORT_LABEL: Dict[str, str] = { "max": "max", } +_EFFORT_RANK: Dict[str, int] = { + "low": 0, + "medium": 1, + "high": 2, + "max": 3, + "xhigh": 4, +} + _BAD_REQUEST_EFFORTS: FrozenSet[str] = frozenset({"disabled", "invalid", ""}) +def _bedrock_clamps_effort(model: "ModelEntry", effort: str) -> bool: + """Whether Bedrock will clamp ``effort`` down to ``bedrock_effort_ceiling``. + + Bedrock chat/messages paths clamp unsupported high tiers (e.g. ``xhigh`` + on Opus 4.6) to the model's ceiling rather than rejecting them, so the + missing native capability is OK — the wire effort just degrades. + """ + if model.bedrock_effort_ceiling is None: + return False + if effort not in _EFFORT_RANK or model.bedrock_effort_ceiling not in _EFFORT_RANK: + return False + return _EFFORT_RANK[effort] > _EFFORT_RANK[model.bedrock_effort_ceiling] + + def expected(model: ModelEntry, effort: str) -> CellExpectation: if effort in ("__omit__", "none"): if model.mode == "budget": @@ -73,14 +96,20 @@ def expected(model: ModelEntry, effort: str) -> CellExpectation: if effort in ("xhigh", "max"): cap = f"supports_{effort}_reasoning_effort" - if cap not in model.caps: + if cap not in model.caps and not _bedrock_clamps_effort(model, effort): return CellExpectation(status=400, thinking_type=OMIT) if model.mode == "adaptive": + wire_effort = _ADAPTIVE_EFFORT_LABEL[effort] + if model.bedrock_effort_ceiling is not None: + wire_rank = _EFFORT_RANK[wire_effort] + ceiling_rank = _EFFORT_RANK[model.bedrock_effort_ceiling] + if wire_rank > ceiling_rank: + wire_effort = model.bedrock_effort_ceiling return CellExpectation( status=200, thinking_type="adaptive", - output_config_effort=_ADAPTIVE_EFFORT_LABEL[effort], + output_config_effort=wire_effort, ) return CellExpectation( @@ -219,6 +248,7 @@ BEDROCK_CONVERSE_MODELS: Tuple[ModelEntry, ...] = ( extra_params=(("aws_region_name", "us-east-1"),), required_env=_BEDROCK_REQ, caps=_CAPS_4_6, + bedrock_effort_ceiling="max", ), ModelEntry( alias="bedrock-claude-sonnet-4-6", @@ -247,6 +277,7 @@ BEDROCK_INVOKE_CHAT_MODELS: Tuple[ModelEntry, ...] = ( extra_params=(("aws_region_name", "us-east-1"),), required_env=_BEDROCK_REQ, caps=_CAPS_4_6, + bedrock_effort_ceiling="max", ), ModelEntry( alias="bedrock-invoke-claude-sonnet-4-6", diff --git a/tests/llm_translation/test_vcr_filters.py b/tests/llm_translation/test_vcr_filters.py index 03891682781..2b5a6b32a72 100644 --- a/tests/llm_translation/test_vcr_filters.py +++ b/tests/llm_translation/test_vcr_filters.py @@ -21,11 +21,13 @@ sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", from tests._vcr_conftest_common import ( # noqa: E402 VCR_FIXED_MULTIPART_BOUNDARY, VCR_IMAGE_B64_PLACEHOLDER, + _before_record_request, _normalize_multipart_boundary, + _should_passthrough_credential_exchange, _strip_image_b64_payloads, + _vcr_load_guard, ) - # --------------------------------------------------------------------------- # Image b64 stripper # --------------------------------------------------------------------------- @@ -218,3 +220,55 @@ def test_normalize_multipart_handles_quoted_boundary(): _normalize_multipart_boundary(req) assert b"quoted-boundary" not in req.body assert VCR_FIXED_MULTIPART_BOUNDARY.encode("utf-8") in req.body + + +# --------------------------------------------------------------------------- +# Credential-exchange passthrough (Google OAuth2/STS token mint must run live) +# --------------------------------------------------------------------------- + + +def _oauth_token_request() -> Request: + return Request( + method="POST", + uri="https://oauth2.googleapis.com/token", + body=b"assertion=eyJhbGciOiJSUzI1NiJ9.signed-jwt&grant_type=urn", + headers={"content-type": "application/x-www-form-urlencoded"}, + ) + + +def test_before_record_request_drops_oauth_token_mint(): + # The token mint must never be stored or replayed, else a stale ya29.* token + # gets sent to a live Vertex/Gemini endpoint -> ACCESS_TOKEN_EXPIRED. + assert _before_record_request(_oauth_token_request()) is None + + +def test_before_record_request_keeps_normal_request(): + req = Request( + method="POST", + uri="https://api.openai.com/v1/chat/completions", + body=b'{"model":"gpt-4o"}', + headers={"content-type": "application/json"}, + ) + assert _before_record_request(req) is req + + +def test_credential_exchange_passthrough_inert_during_cassette_load(): + # During Cassette._load stored episodes are replayed through this hook; + # dropping there would mutate the cassette on read. The guard makes it inert. + _vcr_load_guard.active = True + try: + assert _should_passthrough_credential_exchange(_oauth_token_request()) is False + assert _before_record_request(_oauth_token_request()) is not None + finally: + _vcr_load_guard.active = False + + +def test_credential_exchange_passthrough_covers_sts_and_metadata_hosts(): + for host in ("sts.googleapis.com", "metadata.google.internal", "169.254.169.254"): + req = Request( + method="POST", + uri=f"https://{host}/token", + body=b"grant_type=urn", + headers={}, + ) + assert _should_passthrough_credential_exchange(req) is True diff --git a/tests/proxy_admin_ui_tests/test_key_management.py b/tests/proxy_admin_ui_tests/test_key_management.py index 933c75e4d38..4c5a045509a 100644 --- a/tests/proxy_admin_ui_tests/test_key_management.py +++ b/tests/proxy_admin_ui_tests/test_key_management.py @@ -853,6 +853,18 @@ def test_personal_key_generation_check(): {"tags": ["old_tag"]}, {"metadata": {"tags": ["old_tag"], "enforced_params": ["metadata.tags"]}}, ), + ( + {"disable_global_guardrails": True}, + {}, + {}, + {"metadata": {"disable_global_guardrails": True}}, + ), + ( + {"disable_global_guardrails": False}, + {}, + {"disable_global_guardrails": True}, + {"metadata": {"disable_global_guardrails": False}}, + ), ], ) def test_prepare_metadata_fields( diff --git a/tests/test_litellm/containers/test_container_proxy_ownership.py b/tests/test_litellm/containers/test_container_proxy_ownership.py index c295805bdb3..176405bb9ca 100644 --- a/tests/test_litellm/containers/test_container_proxy_ownership.py +++ b/tests/test_litellm/containers/test_container_proxy_ownership.py @@ -1,3 +1,4 @@ +import json import sys from types import SimpleNamespace from unittest.mock import AsyncMock @@ -91,8 +92,9 @@ async def test_should_not_mutate_dict_container_response_when_recording_owner( assert returned == {"id": "cntr_provider", "object": "container"} data = table.create.await_args.kwargs["data"] - assert data["file_object"]["custom_llm_provider"] == "openai" - assert data["file_object"]["provider_container_id"] == "cntr_provider" + file_obj = json.loads(data["file_object"]) + assert file_obj["custom_llm_provider"] == "openai" + assert file_obj["provider_container_id"] == "cntr_provider" @pytest.mark.asyncio @@ -913,3 +915,195 @@ async def test_admin_with_identity_records_container_ownership(monkeypatch): table.create.assert_awaited_once() created_data = table.create.await_args.kwargs["data"] assert created_data["created_by"] == "proxy-admin" + + +@pytest.mark.asyncio +async def test_should_record_containers_from_responses_output_for_service_account( + monkeypatch, +): + table = AsyncMock() + table.find_unique.return_value = None + prisma_client = SimpleNamespace( + db=SimpleNamespace(litellm_managedobjecttable=table) + ) + monkeypatch.setattr( + ownership, + "_get_prisma_client", + AsyncMock(return_value=prisma_client), + ) + auth = UserAPIKeyAuth(team_id="team-1") + encoded_container_id = ( + "cntr_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOmF6dXJlO21vZGVsX2lkOmR" + "lZi0xMjM7Y29udGFpbmVyX2lkOmNudHJfbmF0aXZl" + ) + responses_payload = { + "output": [ + { + "type": "message", + "content": [ + { + "type": "output_text", + "annotations": [ + { + "type": "container_file_citation", + "container_id": encoded_container_id, + "file_id": "cfile_abc", + } + ], + } + ], + } + ], + "_hidden_params": {"custom_llm_provider": "azure"}, + } + + await ownership.record_container_owners_from_responses_response( + response=responses_payload, + user_api_key_dict=auth, + ) + + table.create.assert_awaited_once() + created_data = table.create.await_args.kwargs["data"] + assert created_data["created_by"] == "team:team-1" + assert created_data["unified_object_id"] == encoded_container_id + + +@pytest.mark.asyncio +async def test_service_account_can_access_container_after_responses_tracking( + monkeypatch, +): + encoded_container_id = ( + "cntr_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOmF6dXJlO21vZGVsX2lkOmR" + "lZi0xMjM7Y29udGFpbmVyX2lkOmNudHJfbmF0aXZl" + ) + table = AsyncMock() + table.find_unique.return_value = None + prisma_client = SimpleNamespace( + db=SimpleNamespace(litellm_managedobjecttable=table) + ) + monkeypatch.setattr( + ownership, + "_get_prisma_client", + AsyncMock(return_value=prisma_client), + ) + auth = UserAPIKeyAuth(team_id="team-1") + + await ownership.record_container_owners_from_responses_response( + response={ + "output": [ + { + "type": "code_interpreter_call", + "container_id": encoded_container_id, + } + ], + "_hidden_params": {"custom_llm_provider": "azure"}, + }, + user_api_key_dict=auth, + ) + + original_id, provider = await ownership.assert_user_can_access_container( + container_id=encoded_container_id, + user_api_key_dict=auth, + custom_llm_provider="azure", + ) + assert original_id == "cntr_native" + assert provider == "azure" + + +@pytest.mark.asyncio +async def test_should_record_container_ownership_after_streaming_responses_finish( + monkeypatch, +): + """Streaming /v1/responses calls return through the + ``select_data_generator`` branch and never reach the non-streaming + container-ownership tail. The wrapper must read + ``completed_response`` off the upstream iterator once iteration + finishes and write the row, otherwise code-interpreter containers + created during the stream stay unregistered and follow-up file API + calls 403. + """ + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + + encoded_container_id = ( + "cntr_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOmF6dXJlO21vZGVsX2lkOmR" + "lZi0xMjM7Y29udGFpbmVyX2lkOmNudHJfbmF0aXZl" + ) + response_body = SimpleNamespace( + output=[ + SimpleNamespace( + type="code_interpreter_call", + container_id=encoded_container_id, + code_interpreter_call=None, + ) + ] + ) + stream_response = SimpleNamespace( + completed_response=SimpleNamespace(response=response_body), + _hidden_params={"custom_llm_provider": "azure"}, + ) + + async def fake_sse_generator(): + yield "data: chunk-1\n\n" + yield "data: chunk-2\n\n" + + table = AsyncMock() + table.find_unique.return_value = None + prisma_client = SimpleNamespace( + db=SimpleNamespace(litellm_managedobjecttable=table) + ) + monkeypatch.setattr( + ownership, + "_get_prisma_client", + AsyncMock(return_value=prisma_client), + ) + auth = UserAPIKeyAuth(team_id="team-1") + + wrapped = ( + ProxyBaseLLMRequestProcessing._wrap_responses_stream_for_container_ownership( + original_stream_response=stream_response, + wrapped_generator=fake_sse_generator(), + user_api_key_dict=auth, + ) + ) + + chunks = [chunk async for chunk in wrapped] + assert chunks == ["data: chunk-1\n\n", "data: chunk-2\n\n"] + + table.create.assert_awaited_once() + created_data = table.create.await_args.kwargs["data"] + assert created_data["created_by"] == "team:team-1" + assert created_data["unified_object_id"] == encoded_container_id + + +@pytest.mark.asyncio +async def test_streaming_ownership_wrap_no_op_when_stream_did_not_complete( + monkeypatch, +): + """If the stream errored before ``response.completed``, + ``completed_response`` is ``None`` — we must skip the ownership + write rather than crash the response generator.""" + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + + stream_response = SimpleNamespace(completed_response=None) + + async def fake_sse_generator(): + yield "data: chunk-1\n\n" + + record = AsyncMock() + monkeypatch.setattr( + ownership, + "record_container_owners_from_responses_response", + record, + ) + + wrapped = ( + ProxyBaseLLMRequestProcessing._wrap_responses_stream_for_container_ownership( + original_stream_response=stream_response, + wrapped_generator=fake_sse_generator(), + user_api_key_dict=UserAPIKeyAuth(user_id="user-1"), + ) + ) + chunks = [chunk async for chunk in wrapped] + + assert chunks == ["data: chunk-1\n\n"] + record.assert_not_awaited() diff --git a/tests/test_litellm/integrations/datadog/test_datadog_cost_management.py b/tests/test_litellm/integrations/datadog/test_datadog_cost_management.py index be2084969a5..cb786d9c292 100644 --- a/tests/test_litellm/integrations/datadog/test_datadog_cost_management.py +++ b/tests/test_litellm/integrations/datadog/test_datadog_cost_management.py @@ -3,7 +3,7 @@ import time from unittest.mock import AsyncMock import pytest -from httpx import Response +from httpx import Request, Response from litellm.integrations.datadog.datadog_cost_management import ( DatadogCostManagementLogger, @@ -167,3 +167,230 @@ async def test_async_send_batch(clean_env): content = json.loads(call_args[1]["content"]) assert content[0]["ProviderName"] == "openai" assert content[0]["BilledCost"] == 0.01 + + +_PUT_REQUEST = Request("PUT", "https://api.test.datadoghq.com/api/v2/cost/custom_costs") + + +@pytest.mark.asyncio +async def test_async_send_batch_clears_queue_on_success(clean_env): + """Bug 1 regression: log_queue must be empty after a successful upload.""" + logger = DatadogCostManagementLogger() + logger.async_client = AsyncMock() + logger.async_client.put.return_value = Response( + 202, json={"status": "ok"}, request=_PUT_REQUEST + ) + logger.log_queue = [ + StandardLoggingPayload( + custom_llm_provider="openai", + model="gpt-4", + response_cost=0.01, + startTime=time.time(), + ) + ] + await logger.async_send_batch() + assert logger.log_queue == [] + + +@pytest.mark.asyncio +async def test_async_send_batch_preserves_events_added_during_upload(clean_env): + """Events appended while the upload is in flight survive (land on the cleared queue).""" + logger = DatadogCostManagementLogger() + + later_event = StandardLoggingPayload( + custom_llm_provider="anthropic", + model="claude-3", + response_cost=0.02, + startTime=time.time(), + ) + + async def slow_put(*args, **kwargs): + logger.log_queue.append(later_event) + return Response(202, json={"status": "ok"}, request=_PUT_REQUEST) + + logger.async_client = AsyncMock() + logger.async_client.put.side_effect = slow_put + logger.log_queue = [ + StandardLoggingPayload( + custom_llm_provider="openai", + model="gpt-4", + response_cost=0.01, + startTime=time.time(), + ) + ] + await logger.async_send_batch() + assert logger.log_queue == [later_event] + + +@pytest.mark.asyncio +async def test_async_send_batch_requeues_on_upload_failure(clean_env): + """Failed upload requeues the original batch (no data loss).""" + logger = DatadogCostManagementLogger() + logger.async_client = AsyncMock() + logger.async_client.put.side_effect = Exception("boom") + original = StandardLoggingPayload( + custom_llm_provider="openai", + model="gpt-4", + response_cost=0.01, + startTime=time.time(), + ) + logger.log_queue = [original] + await logger.async_send_batch() + assert logger.log_queue == [original] + + +@pytest.mark.asyncio +async def test_extract_tags_emits_canonical_focus_dimensions(clean_env): + """provider, model, model_id always emitted regardless of cost_tag_keys.""" + logger = DatadogCostManagementLogger() + log = StandardLoggingPayload( + custom_llm_provider="openai", + model="gpt-4o", + model_id="router-id-123", + response_cost=0.01, + startTime=time.time(), + ) + tags = logger._extract_tags(log) + assert tags["provider"] == "openai" + assert tags["model"] == "gpt-4o" + assert tags["model_id"] == "router-id-123" + + +@pytest.mark.asyncio +async def test_extract_tags_allowlist_filters_request_tags(clean_env): + """Only request_tags whose key is in cost_tag_keys reach the Tags dict.""" + logger = DatadogCostManagementLogger(cost_tag_keys=["capability", "tier"]) + log = StandardLoggingPayload( + custom_llm_provider="openai", + model="gpt-4", + response_cost=0.01, + startTime=time.time(), + request_tags=["capability:chat", "tier:gold", "secret:disallowed"], + ) + tags = logger._extract_tags(log) + assert tags["capability"] == "chat" + assert tags["tier"] == "gold" + assert "secret" not in tags + + +@pytest.mark.asyncio +async def test_extract_tags_allowlist_filters_metadata(clean_env): + """Only metadata keys in cost_tag_keys flow through; others (and dict/list values) are dropped.""" + logger = DatadogCostManagementLogger(cost_tag_keys=["capability", "owner"]) + log = StandardLoggingPayload( + custom_llm_provider="openai", + model="gpt-4", + response_cost=0.01, + startTime=time.time(), + metadata={ + "capability": "chat", + "owner": "team-x", + "secret_field": "sensitive", + "nested_obj": {"a": 1}, + }, + ) + tags = logger._extract_tags(log) + assert tags["capability"] == "chat" + assert tags["owner"] == "team-x" + assert "secret_field" not in tags + assert "nested_obj" not in tags + + +@pytest.mark.asyncio +async def test_extract_tags_empty_allowlist_default(clean_env): + """With no cost_tag_keys, request_tags and arbitrary metadata.* do NOT leak into Tags.""" + logger = DatadogCostManagementLogger() + log = StandardLoggingPayload( + custom_llm_provider="openai", + model="gpt-4", + response_cost=0.01, + startTime=time.time(), + request_tags=["capability:chat"], + metadata={"capability": "chat", "user_api_key_alias": "alice"}, + ) + tags = logger._extract_tags(log) + assert "capability" not in tags + # Backwards-compat keys still flow: + assert tags["user"] == "alice" + + +@pytest.mark.asyncio +async def test_extract_tags_nested_metadata_allowlisted(clean_env): + """spend_logs_metadata and requester_metadata get spread one level under the allowlist.""" + logger = DatadogCostManagementLogger(cost_tag_keys=["env", "platform"]) + log = StandardLoggingPayload( + custom_llm_provider="openai", + model="gpt-4", + response_cost=0.01, + startTime=time.time(), + metadata={ + "spend_logs_metadata": {"platform": "web", "ignored": "x"}, + "requester_metadata": {"env": "prod"}, + }, + ) + tags = logger._extract_tags(log) + assert tags["platform"] == "web" + # "env" is a reserved trusted dimension — requester_metadata.env must NOT + # overwrite the value sourced from get_datadog_env(). + assert tags["env"] != "prod" + assert "ignored" not in tags + + +@pytest.mark.asyncio +async def test_extract_tags_allowlist_cannot_override_reserved_dimensions(clean_env): + """ + Reserved tag keys (env, service, host, pod_name, provider, model, model_id, + team, user, model_group) must not be overwritten by user-controlled + request_tags or metadata, even when listed in cost_tag_keys. + """ + reserved = [ + "env", + "service", + "host", + "pod_name", + "provider", + "model", + "model_id", + "team", + "user", + "model_group", + ] + logger = DatadogCostManagementLogger(cost_tag_keys=reserved) + + metadata_attack = {k: f"attacker-meta-{k}" for k in reserved} + metadata_attack["user_api_key_alias"] = "trusted-user" + metadata_attack["user_api_key_team_alias"] = "trusted-team" + metadata_attack["model_group"] = "trusted-group" + metadata_attack["spend_logs_metadata"] = { + k: f"attacker-spend-{k}" for k in reserved + } + metadata_attack["requester_metadata"] = {k: f"attacker-req-{k}" for k in reserved} + + log = StandardLoggingPayload( + custom_llm_provider="openai", + model="gpt-4", + model_id="router-id-123", + response_cost=0.01, + startTime=time.time(), + request_tags=[f"{k}:attacker-rt-{k}" for k in reserved], + metadata=metadata_attack, + ) + + tags = logger._extract_tags(log) + + # Canonical FOCUS dims keep their trusted (top-level payload) values. + assert tags["provider"] == "openai" + assert tags["model"] == "gpt-4" + assert tags["model_id"] == "router-id-123" + + # Backwards-compat trusted dims keep their proxy-controlled metadata values. + assert tags["user"] == "trusted-user" + assert tags["team"] == "trusted-team" + assert tags["model_group"] == "trusted-group" + + # No reserved key carries an attacker-supplied prefix from any path. + for k in reserved: + assert not tags[k].startswith("attacker-"), ( + f"reserved key {k!r} was overwritten by user-controlled input: " + f"{tags[k]!r}" + ) diff --git a/tests/test_litellm/integrations/test_galileo.py b/tests/test_litellm/integrations/test_galileo.py new file mode 100644 index 00000000000..aab220f46be --- /dev/null +++ b/tests/test_litellm/integrations/test_galileo.py @@ -0,0 +1,397 @@ +import os +import sys +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +sys.path.insert(0, os.path.abspath("../..")) + +from litellm.integrations.galileo import GalileoObserve +from litellm.types.utils import ( + Choices, + EmbeddingResponse, + ImageObject, + ImageResponse, + Message, + ModelResponse, + TextCompletionResponse, +) + + +@pytest.fixture +def galileo_v2_env(monkeypatch): + monkeypatch.setenv("GALILEO_API_KEY", "test-api-key") + monkeypatch.setenv("GALILEO_PROJECT_ID", "86ff8ebe-a297-4134-b167-748bdd8d2c20") + monkeypatch.setenv("GALILEO_LOG_STREAM_ID", "76c4ea50-8aa3-4771-a0d7-8567b112210f") + monkeypatch.setenv("GALILEO_BASE_URL", "https://api.galileo.ai") + + +@pytest.mark.asyncio +async def test_galileo_v2_ingest_url_and_headers(galileo_v2_env): + logger = GalileoObserve() + logger.in_memory_records = [ + { + "latency_ms": 100, + "status_code": 200, + "input_text": "hi", + "output_text": "hello", + "node_type": "acompletion", + "model": "gpt-5.2", + "num_input_tokens": 1, + "num_output_tokens": 2, + "created_at": "2026-05-25T12:00:00", + } + ] + + url, payload = logger._get_ingest_request() + assert ( + url + == "https://api.galileo.ai/v2/projects/86ff8ebe-a297-4134-b167-748bdd8d2c20/spans" + ) + assert payload["log_stream_id"] == "76c4ea50-8aa3-4771-a0d7-8567b112210f" + assert payload["spans"][0]["type"] == "llm" + assert payload["spans"][0]["output"]["content"] == "hello" + + assert await logger._ensure_headers() is True + assert logger.headers["Galileo-API-Key"] == "test-api-key" + + +def test_galileo_v2_span_preserves_message_roles(galileo_v2_env): + record = { + "latency_ms": 1, + "status_code": 200, + "input_text": "fallback", + "output_text": "ok", + "node_type": "acompletion", + "model": "gpt-5.2", + "num_input_tokens": 0, + "num_output_tokens": 0, + "created_at": "2026-05-25T12:00:00", + "messages": [ + {"role": "system", "content": "be helpful"}, + {"role": "user", "content": "hello"}, + ], + } + span = GalileoObserve._record_to_v2_span(record) + assert span["input"] == [ + {"role": "system", "content": "be helpful"}, + {"role": "user", "content": "hello"}, + ] + + +def test_galileo_output_text_from_model_response(galileo_v2_env): + logger = GalileoObserve() + response = ModelResponse( + choices=[ + Choices( + message=Message( + content="assistant reply", + role="assistant", + annotations=[], + ) + ) + ] + ) + + output = logger.get_output_str_from_response(response, {"call_type": "acompletion"}) + assert output == "assistant reply" + + +@pytest.mark.asyncio +async def test_galileo_flush_swallows_http_errors(galileo_v2_env): + logger = GalileoObserve() + logger.in_memory_records = [ + { + "latency_ms": 1, + "status_code": 200, + "input_text": "a", + "output_text": "b", + "node_type": "acompletion", + "model": "gpt-5.2", + "num_input_tokens": 0, + "num_output_tokens": 0, + "created_at": "2026-05-25T12:00:00", + } + ] + + with patch.object( + logger.async_httpx_handler, "post", new_callable=AsyncMock + ) as mock_post: + mock_post.side_effect = Exception("404 Not Found") + await logger.flush_in_memory_records() + + assert len(logger.in_memory_records) == 1 + + +@pytest.mark.asyncio +async def test_galileo_flush_clears_records_on_201(galileo_v2_env): + logger = GalileoObserve() + logger.in_memory_records = [ + { + "latency_ms": 1, + "status_code": 200, + "input_text": "a", + "output_text": "b", + "node_type": "acompletion", + "model": "gpt-5.2", + "num_input_tokens": 0, + "num_output_tokens": 0, + "created_at": "2026-05-25T12:00:00", + } + ] + + mock_response = AsyncMock() + mock_response.is_success = True + mock_response.status_code = 201 + + with patch.object( + logger.async_httpx_handler, "post", new_callable=AsyncMock + ) as mock_post: + mock_post.return_value = mock_response + await logger.flush_in_memory_records() + + assert logger.in_memory_records == [] + + +def test_galileo_normalize_base_url_none(monkeypatch): + monkeypatch.delenv("GALILEO_API_KEY", raising=False) + monkeypatch.delenv("GALILEO_BASE_URL", raising=False) + monkeypatch.delenv("GALILEO_PROJECT_ID", raising=False) + logger = GalileoObserve() + assert logger.base_url is None + assert logger._normalize_base_url(None) is None + assert logger._normalize_base_url("https://x.example/") == "https://x.example" + + +def test_galileo_is_configured_branches(monkeypatch): + monkeypatch.delenv("GALILEO_API_KEY", raising=False) + monkeypatch.delenv("GALILEO_BASE_URL", raising=False) + monkeypatch.delenv("GALILEO_PROJECT_ID", raising=False) + monkeypatch.delenv("GALILEO_USERNAME", raising=False) + monkeypatch.delenv("GALILEO_PASSWORD", raising=False) + + no_env = GalileoObserve() + assert no_env._is_configured() is False + + monkeypatch.setenv("GALILEO_API_KEY", "k") + monkeypatch.setenv("GALILEO_PROJECT_ID", "p") + v2 = GalileoObserve() + assert v2._is_configured() is True + + monkeypatch.delenv("GALILEO_API_KEY", raising=False) + monkeypatch.setenv("GALILEO_USERNAME", "u") + monkeypatch.setenv("GALILEO_PASSWORD", "pw") + monkeypatch.setenv("GALILEO_BASE_URL", "https://galileo.example") + legacy = GalileoObserve() + assert legacy._is_configured() is True + + monkeypatch.delenv("GALILEO_PASSWORD", raising=False) + no_pw = GalileoObserve() + assert no_pw._is_configured() is False + + +def test_galileo_input_messages_fallbacks(): + assert GalileoObserve._galileo_input_messages(None, "hi") == [ + {"role": "user", "content": "hi"} + ] + assert GalileoObserve._galileo_input_messages( + ["not-a-dict", {"content": "no role"}], "fallback" + ) == [{"role": "user", "content": "fallback"}] + + +def test_galileo_record_to_v2_span_with_tags_and_offset(): + span = GalileoObserve._record_to_v2_span( + { + "latency_ms": 5, + "status_code": 200, + "input_text": "in", + "output_text": "out", + "node_type": "acompletion", + "model": "gpt-5.2", + "num_input_tokens": 1, + "num_output_tokens": 2, + "created_at": "2026-05-25T12:00:00", + "tags": ["t1"], + } + ) + assert span["tags"] == ["t1"] + assert span["created_at"].endswith("Z") + + offset = GalileoObserve._record_to_v2_span( + {"created_at": "2026-05-25T12:00:00-05:00"} + ) + assert offset["created_at"] == "2026-05-25T12:00:00-05:00" + + +def test_galileo_get_output_str_variants(galileo_v2_env): + logger = GalileoObserve() + assert logger.get_output_str_from_response(None, {}) is None + assert ( + logger.get_output_str_from_response( + EmbeddingResponse(), {"call_type": "embedding"} + ) + is None + ) + + text_resp = TextCompletionResponse() + text_resp.choices = [MagicMock(text="text-completion-output")] + assert ( + logger.get_output_str_from_response(text_resp, {"call_type": "text_completion"}) + == "text-completion-output" + ) + + image_resp = ImageResponse(data=[ImageObject(url="https://x/y.png")]) + assert "y.png" in logger.get_output_str_from_response(image_resp, {}) + + assert logger.get_output_str_from_response("not-a-supported-type", {}) is None + + +def test_galileo_get_ingest_request_unconfigured(monkeypatch): + monkeypatch.delenv("GALILEO_API_KEY", raising=False) + monkeypatch.delenv("GALILEO_BASE_URL", raising=False) + monkeypatch.delenv("GALILEO_PROJECT_ID", raising=False) + logger = GalileoObserve() + assert logger._get_ingest_request() is None + + +def test_galileo_get_ingest_request_legacy(monkeypatch): + monkeypatch.delenv("GALILEO_API_KEY", raising=False) + monkeypatch.setenv("GALILEO_USERNAME", "u") + monkeypatch.setenv("GALILEO_PASSWORD", "pw") + monkeypatch.setenv("GALILEO_BASE_URL", "https://galileo.example/") + monkeypatch.setenv("GALILEO_PROJECT_ID", "proj") + logger = GalileoObserve() + logger.in_memory_records = [{"foo": "bar"}] + url, payload = logger._get_ingest_request() + assert url == "https://galileo.example/projects/proj/observe/ingest" + assert payload == {"records": [{"foo": "bar"}]} + + +@pytest.mark.asyncio +async def test_galileo_ensure_headers_v2_missing_key(monkeypatch): + monkeypatch.delenv("GALILEO_API_KEY", raising=False) + monkeypatch.setenv("GALILEO_PROJECT_ID", "p") + monkeypatch.setenv("GALILEO_BASE_URL", "https://x") + logger = GalileoObserve() + logger.use_v2_api = True + logger.api_key = None + assert await logger._ensure_headers() is False + + +@pytest.mark.asyncio +async def test_galileo_ensure_headers_cached(galileo_v2_env): + logger = GalileoObserve() + logger.headers = {"Galileo-API-Key": "already-set"} + assert await logger._ensure_headers() is True + + +@pytest.mark.asyncio +async def test_galileo_ensure_headers_legacy_login(monkeypatch): + monkeypatch.delenv("GALILEO_API_KEY", raising=False) + monkeypatch.setenv("GALILEO_USERNAME", "u") + monkeypatch.setenv("GALILEO_PASSWORD", "pw") + monkeypatch.setenv("GALILEO_BASE_URL", "https://galileo.example") + monkeypatch.setenv("GALILEO_PROJECT_ID", "p") + logger = GalileoObserve() + + login_resp = MagicMock() + login_resp.raise_for_status = MagicMock() + login_resp.json = MagicMock(return_value={"access_token": "tok"}) + + with patch.object( + logger.async_httpx_handler, "post", new_callable=AsyncMock + ) as mock_post: + mock_post.return_value = login_resp + assert await logger._ensure_headers() is True + + assert logger.headers["Authorization"] == "Bearer tok" + + +@pytest.mark.asyncio +async def test_galileo_ensure_headers_legacy_login_failure(monkeypatch): + monkeypatch.delenv("GALILEO_API_KEY", raising=False) + monkeypatch.setenv("GALILEO_USERNAME", "u") + monkeypatch.setenv("GALILEO_PASSWORD", "pw") + monkeypatch.setenv("GALILEO_BASE_URL", "https://galileo.example") + monkeypatch.setenv("GALILEO_PROJECT_ID", "p") + logger = GalileoObserve() + + with patch.object( + logger.async_httpx_handler, "post", new_callable=AsyncMock + ) as mock_post: + mock_post.side_effect = Exception("boom") + assert await logger._ensure_headers() is False + + +@pytest.mark.asyncio +async def test_galileo_flush_noop_when_unconfigured(monkeypatch): + monkeypatch.delenv("GALILEO_API_KEY", raising=False) + monkeypatch.delenv("GALILEO_BASE_URL", raising=False) + monkeypatch.delenv("GALILEO_PROJECT_ID", raising=False) + logger = GalileoObserve() + logger.in_memory_records = [{"foo": "bar"}] + await logger.flush_in_memory_records() + assert logger.in_memory_records == [{"foo": "bar"}] + + +@pytest.mark.asyncio +async def test_galileo_flush_resets_headers_on_401(monkeypatch): + monkeypatch.delenv("GALILEO_API_KEY", raising=False) + monkeypatch.setenv("GALILEO_USERNAME", "u") + monkeypatch.setenv("GALILEO_PASSWORD", "pw") + monkeypatch.setenv("GALILEO_BASE_URL", "https://galileo.example") + monkeypatch.setenv("GALILEO_PROJECT_ID", "p") + logger = GalileoObserve() + logger.headers = {"Authorization": "Bearer stale"} + logger.in_memory_records = [{"records": "x"}] + + mock_response = MagicMock() + mock_response.is_success = False + mock_response.status_code = 401 + mock_response.text = "unauthorized" + + with patch.object( + logger.async_httpx_handler, "post", new_callable=AsyncMock + ) as mock_post: + mock_post.return_value = mock_response + await logger.flush_in_memory_records() + + assert logger.headers is None + assert logger.in_memory_records == [{"records": "x"}] + + +@pytest.mark.asyncio +async def test_galileo_async_log_success_appends_and_flushes(galileo_v2_env): + import datetime + + logger = GalileoObserve() + response = ModelResponse( + choices=[ + Choices(message=Message(content="reply", role="assistant", annotations=[])) + ], + usage={"prompt_tokens": 1, "completion_tokens": 2}, + ) + + flushed_url: dict = {} + mock_response = MagicMock() + mock_response.is_success = True + mock_response.status_code = 200 + + async def fake_post(**kwargs): + flushed_url["url"] = kwargs.get("url") + return mock_response + + with patch.object(logger.async_httpx_handler, "post", side_effect=fake_post): + await logger.async_log_success_event( + kwargs={ + "call_type": "acompletion", + "model": "gpt", + "messages": [{"role": "user", "content": "hi"}], + }, + response_obj=response, + start_time=datetime.datetime(2026, 5, 25, 12, 0, 0), + end_time=datetime.datetime(2026, 5, 25, 12, 0, 1), + ) + + assert "/v2/projects/" in flushed_url["url"] + assert logger.in_memory_records == [] diff --git a/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py b/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py index acf8e4d2a14..7913efe8294 100644 --- a/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py +++ b/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py @@ -476,6 +476,50 @@ async def test_transcription_captured_in_backend_to_client(): assert logging_obj.model_call_details["messages"] == streaming.input_messages +@pytest.mark.asyncio +async def test_client_ack_caches_setup_to_prevent_duplicate_session_update_setup(): + websocket = MagicMock() + backend_ws = MagicMock() + logging_obj = MagicMock() + logging_obj.pre_call = MagicMock() + + # Two session.update messages arrive before setupComplete round-trip. + websocket.receive_text = AsyncMock( + side_effect=[ + json.dumps({"type": "session.update", "session": {"tools": []}}), + json.dumps({"type": "session.update", "session": {"tools": []}}), + Exception("client done"), + ] + ) + + provider_config = MagicMock() + + def _transform(message: str, model: str, session_configuration_request=None): + if session_configuration_request is None: + return [json.dumps({"setup": {"model": "models/gemini-2.5-flash"}})] + return [] + + provider_config.transform_realtime_request = MagicMock(side_effect=_transform) + + backend_ws.send = AsyncMock() + + streaming = RealTimeStreaming( + websocket=websocket, + backend_ws=backend_ws, + logging_obj=logging_obj, + provider_config=provider_config, + model="gemini-2.5-flash", + ) + + await streaming.client_ack_messages() + + # Setup should be forwarded exactly once even with repeated session.update. + assert backend_ws.send.await_count == 1 + assert streaming.session_configuration_request is not None + sent_payload = json.loads(backend_ws.send.await_args_list[0].args[0]) + assert "setup" in sent_payload + + def test_collect_session_tools_from_session_update(): """ Test that tools from session.update events are collected. @@ -879,6 +923,169 @@ async def test_realtime_text_input_guardrail_blocks_and_returns_error(): litellm.callbacks = [] # cleanup +@pytest.mark.asyncio +async def test_realtime_function_call_output_guardrail_blocks_and_returns_error(): + """ + Test that a client-supplied function_call_output whose content triggers a + guardrail is blocked: it is not forwarded to the backend, and an error + event is sent to the client. + """ + from fastapi import HTTPException + + import litellm + from litellm.integrations.custom_guardrail import CustomGuardrail + from litellm.types.guardrails import GuardrailEventHooks + + class BlockingGuardrail(CustomGuardrail): + async def apply_guardrail( + self, inputs, request_data, input_type, logging_obj=None + ): + texts = inputs.get("texts", []) + for text in texts: + if "@" in text: + raise HTTPException( + status_code=403, + detail={"error": "email address detected"}, + ) + return inputs + + guardrail = BlockingGuardrail( + guardrail_name="email-blocker", + event_hook=GuardrailEventHooks.pre_call, + default_on=True, + ) + litellm.callbacks = [guardrail] + + client_ws = MagicMock() + client_ws.send_text = AsyncMock() + + backend_ws = MagicMock() + backend_ws.send = AsyncMock() + backend_ws.recv = AsyncMock(side_effect=ConnectionClosed(None, None)) + + logging_obj = MagicMock() + logging_obj.pre_call = MagicMock() + + streaming = RealTimeStreaming(client_ws, backend_ws, logging_obj) + + item_create_msg = json.dumps( + { + "type": "conversation.item.create", + "item": { + "type": "function_call_output", + "call_id": "call_123", + "output": "Tool says: my email is test@example.com", + }, + } + ) + + client_ws.receive_text = AsyncMock( + side_effect=[ + item_create_msg, + Exception("connection closed"), + ] + ) + + await streaming.client_ack_messages() + + sent_texts = [json.loads(c.args[0]) for c in client_ws.send_text.call_args_list] + error_events = [e for e in sent_texts if e.get("type") == "error"] + assert len(error_events) == 1, f"Expected one error event, got: {sent_texts}" + assert error_events[0]["error"]["type"] == "guardrail_violation" + + sent_to_backend = [c.args[0] for c in backend_ws.send.call_args_list if c.args] + forwarded_tool_outputs = [ + json.loads(m) + for m in sent_to_backend + if isinstance(m, str) + and json.loads(m).get("type") == "conversation.item.create" + and json.loads(m).get("item", {}).get("type") == "function_call_output" + ] + # A sanitized placeholder must reach the backend so providers that pair + # every toolCall with a toolResponse (Gemini/Vertex Live) exit their + # pending-tool-call state instead of stalling. The placeholder must NOT + # contain any of the blocked content. + assert len(forwarded_tool_outputs) == 1, ( + f"Sanitized function_call_output should be forwarded, got: " + f"{forwarded_tool_outputs}" + ) + sanitized_item = forwarded_tool_outputs[0]["item"] + assert sanitized_item["call_id"] == "call_123" + assert "test@example.com" not in sanitized_item["output"] + + litellm.callbacks = [] # cleanup + + +@pytest.mark.asyncio +async def test_realtime_function_call_output_guardrail_allows_clean_output(): + """ + Test that a clean function_call_output passes through and reaches the backend + when guardrails are configured. + """ + import litellm + from litellm.integrations.custom_guardrail import CustomGuardrail + from litellm.types.guardrails import GuardrailEventHooks + + class BlockingGuardrail(CustomGuardrail): + async def apply_guardrail( + self, inputs, request_data, input_type, logging_obj=None + ): + return inputs + + guardrail = BlockingGuardrail( + guardrail_name="noop", + event_hook=GuardrailEventHooks.pre_call, + default_on=True, + ) + litellm.callbacks = [guardrail] + + client_ws = MagicMock() + client_ws.send_text = AsyncMock() + + backend_ws = MagicMock() + backend_ws.send = AsyncMock() + backend_ws.recv = AsyncMock(side_effect=ConnectionClosed(None, None)) + + logging_obj = MagicMock() + logging_obj.pre_call = MagicMock() + + streaming = RealTimeStreaming(client_ws, backend_ws, logging_obj) + + item_create_msg = json.dumps( + { + "type": "conversation.item.create", + "item": { + "type": "function_call_output", + "call_id": "call_456", + "output": '{"temperature": 72, "unit": "F"}', + }, + } + ) + + client_ws.receive_text = AsyncMock( + side_effect=[ + item_create_msg, + Exception("connection closed"), + ] + ) + + await streaming.client_ack_messages() + + sent_to_backend = [c.args[0] for c in backend_ws.send.call_args_list if c.args] + forwarded = [ + json.loads(m) + for m in sent_to_backend + if isinstance(m, str) + and json.loads(m).get("type") == "conversation.item.create" + and json.loads(m).get("item", {}).get("type") == "function_call_output" + ] + assert ( + len(forwarded) == 1 + ), f"Clean function_call_output should be forwarded, got: {forwarded}" + + litellm.callbacks = [] # cleanup + + @pytest.mark.asyncio async def test_realtime_text_input_guardrail_uses_pre_call_mode(): """ @@ -1160,3 +1367,406 @@ async def test_on_violation_end_session_closes_on_first_fail(): assert streaming._violation_count == 1 litellm.callbacks = [] # cleanup + + +@pytest.mark.asyncio +async def test_provider_path_suppresses_duplicate_session_created_after_synthetic(): + client_ws = MagicMock() + client_ws.send_text = AsyncMock() + + backend_ws = MagicMock() + backend_ws.recv = AsyncMock( + side_effect=[b'{"setupComplete": {}}', ConnectionClosed(None, None)] + ) + backend_ws.send = AsyncMock() + + provider_config = MagicMock() + provider_config.transform_realtime_response = MagicMock( + return_value={ + "response": [ + { + "type": "session.created", + "event_id": "event_1", + "session": {"id": "sess_1", "modalities": ["audio"]}, + } + ], + "current_output_item_id": None, + "current_response_id": None, + "current_delta_chunks": [], + "current_conversation_id": None, + "current_item_chunks": [], + "current_delta_type": None, + "session_configuration_request": None, + } + ) + + logging_obj = MagicMock() + logging_obj.litellm_trace_id = "trace_1" + logging_obj.async_success_handler = AsyncMock() + logging_obj.success_handler = MagicMock() + + streaming = RealTimeStreaming( + websocket=client_ws, + backend_ws=backend_ws, + logging_obj=logging_obj, + provider_config=provider_config, + model="gemini-2.5-flash", + ) + # Simulate synthetic session.created already sent by llm_http_handler. + streaming._session_created_sent_to_client = True + + await streaming.backend_to_client_send_messages() + + sent_payloads = [json.loads(c.args[0]) for c in client_ws.send_text.call_args_list] + assert not any( + payload.get("type") == "session.created" for payload in sent_payloads + ), f"Expected duplicate session.created to be suppressed, got: {sent_payloads}" + + +@pytest.mark.asyncio +async def test_duplicate_session_created_still_triggers_guardrail_turn_detection_update(): + client_ws = MagicMock() + client_ws.send_text = AsyncMock() + + backend_ws = MagicMock() + backend_ws.recv = AsyncMock( + side_effect=[b'{"setupComplete": {}}', ConnectionClosed(None, None)] + ) + backend_ws.send = AsyncMock() + + provider_config = MagicMock() + provider_config.transform_realtime_response = MagicMock( + return_value={ + "response": [ + { + "type": "session.created", + "event_id": "event_1", + "session": {"id": "sess_1", "modalities": ["audio"]}, + } + ], + "current_output_item_id": None, + "current_response_id": None, + "current_delta_chunks": [], + "current_conversation_id": None, + "current_item_chunks": [], + "current_delta_type": None, + "session_configuration_request": None, + } + ) + + logging_obj = MagicMock() + logging_obj.litellm_trace_id = "trace_1" + logging_obj.async_success_handler = AsyncMock() + logging_obj.success_handler = MagicMock() + + streaming = RealTimeStreaming( + websocket=client_ws, + backend_ws=backend_ws, + logging_obj=logging_obj, + provider_config=provider_config, + model="gemini-2.5-flash", + ) + # Synthetic session.created already sent by llm_http_handler. + streaming._session_created_sent_to_client = True + streaming._has_audio_transcription_guardrails = MagicMock(return_value=True) # type: ignore[method-assign] + streaming._send_to_backend = AsyncMock() # type: ignore[method-assign] + + await streaming.backend_to_client_send_messages() + + # Duplicate session.created should still cause the one-time guardrail + # turn_detection update to be sent to backend. + assert streaming._send_to_backend.await_count == 1 + sent_update = json.loads(streaming._send_to_backend.await_args_list[0].args[0]) + assert sent_update["type"] == "session.update" + injected_session = sent_update["session"] + assert injected_session["type"] == "realtime" + assert ( + injected_session["audio"]["input"]["turn_detection"]["create_response"] is False + ) + + +@pytest.mark.asyncio +async def test_guardrail_update_respects_idempotency_flag(): + """Verify guardrail turn-detection update uses idempotency flag correctly.""" + client_ws = AsyncMock() + backend_ws = MagicMock() + backend_ws.send = AsyncMock() + + logging_obj = MagicMock() + logging_obj.litellm_trace_id = "trace_1" + logging_obj.async_success_handler = AsyncMock() + logging_obj.success_handler = MagicMock() + + provider_config = MagicMock() + provider_config.transform_realtime_request = MagicMock( + side_effect=lambda msg, model, session_config: [msg] + ) + + streaming = RealTimeStreaming( + websocket=client_ws, + backend_ws=backend_ws, + logging_obj=logging_obj, + provider_config=provider_config, + model="gemini-2.5-flash", + ) + streaming._has_audio_transcription_guardrails = MagicMock(return_value=True) # type: ignore[method-assign] + + # First call should send the update + assert streaming._guardrail_turn_detection_update_sent is False + await streaming._maybe_send_guardrail_turn_detection_update() + assert streaming._guardrail_turn_detection_update_sent is True + assert backend_ws.send.await_count == 1 + + # Second call should be a no-op (idempotent) + await streaming._maybe_send_guardrail_turn_detection_update() + assert backend_ws.send.await_count == 1 # Still 1, not 2 + + +@pytest.mark.asyncio +async def test_guardrail_turn_detection_injected_into_first_session_update_deferred_mode(): + """Verify turn_detection is injected into first session.update in deferred mode.""" + client_ws = AsyncMock() + client_ws.receive_text = AsyncMock( + side_effect=[ + json.dumps( + { + "type": "session.update", + "session": { + "modalities": ["text", "audio"], + "tools": [{"type": "function", "name": "get_weather"}], + }, + } + ), + ConnectionClosed(None, None), + ] + ) + backend_ws = MagicMock() + backend_ws.send = AsyncMock() + + logging_obj = MagicMock() + logging_obj.litellm_trace_id = "trace_1" + logging_obj.async_success_handler = AsyncMock() + logging_obj.success_handler = MagicMock() + + provider_config = MagicMock() + transformed_messages = [] + + def mock_transform(msg, model, session_config): + transformed_messages.append((msg, session_config)) + return [msg] # Pass through for simplicity + + provider_config.transform_realtime_request = MagicMock(side_effect=mock_transform) + + streaming = RealTimeStreaming( + websocket=client_ws, + backend_ws=backend_ws, + logging_obj=logging_obj, + provider_config=provider_config, + model="gemini-2.5-flash", + ) + streaming._has_audio_transcription_guardrails = MagicMock(return_value=True) # type: ignore[method-assign] + + # Simulate first session.update in deferred mode + await streaming.client_ack_messages() + + # Verify turn_detection was injected into the session.update. The + # injection runs before the GA remap, so the create_response flag ends + # up nested under audio.input.turn_detection in the GA-shaped payload. + assert len(transformed_messages) == 1 + transformed_msg, session_config = transformed_messages[0] + msg_obj = json.loads(transformed_msg) + assert msg_obj["type"] == "session.update" + session_obj = msg_obj["session"] + injected_turn_detection = session_obj.get("turn_detection") or session_obj.get( + "audio", {} + ).get("input", {}).get("turn_detection") + assert injected_turn_detection is not None + assert injected_turn_detection["create_response"] is False + assert streaming._guardrail_turn_detection_update_sent is True + + +@pytest.mark.asyncio +@pytest.mark.parametrize("existing_turn_detection", [None, "auto", 42, ["server_vad"]]) +async def test_guardrail_turn_detection_injection_tolerates_non_dict_value( + existing_turn_detection, +): + """Client-supplied non-dict turn_detection must not crash client_ack_messages.""" + client_ws = AsyncMock() + client_ws.receive_text = AsyncMock( + side_effect=[ + json.dumps( + { + "type": "session.update", + "session": { + "modalities": ["text", "audio"], + "turn_detection": existing_turn_detection, + }, + } + ), + ConnectionClosed(None, None), + ] + ) + backend_ws = MagicMock() + backend_ws.send = AsyncMock() + + logging_obj = MagicMock() + logging_obj.litellm_trace_id = "trace_1" + logging_obj.async_success_handler = AsyncMock() + logging_obj.success_handler = MagicMock() + + provider_config = MagicMock() + transformed_messages = [] + + def mock_transform(msg, model, session_config): + transformed_messages.append((msg, session_config)) + return [msg] + + provider_config.transform_realtime_request = MagicMock(side_effect=mock_transform) + + streaming = RealTimeStreaming( + websocket=client_ws, + backend_ws=backend_ws, + logging_obj=logging_obj, + provider_config=provider_config, + model="gemini-2.5-flash", + ) + streaming._has_audio_transcription_guardrails = MagicMock(return_value=True) # type: ignore[method-assign] + + await streaming.client_ack_messages() + + assert len(transformed_messages) == 1 + transformed_msg, _ = transformed_messages[0] + msg_obj = json.loads(transformed_msg) + session_obj = msg_obj["session"] + injected_turn_detection = session_obj.get("turn_detection") or session_obj.get( + "audio", {} + ).get("input", {}).get("turn_detection") + assert isinstance(injected_turn_detection, dict) + assert injected_turn_detection["create_response"] is False + assert streaming._guardrail_turn_detection_update_sent is True + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "client_session", + [ + {"turn_detection": {"type": "server_vad", "create_response": True}}, + { + "audio": { + "input": { + "turn_detection": {"type": "server_vad", "create_response": True} + } + } + }, + ], +) +async def test_subsequent_session_update_cannot_reenable_vad_when_guardrails_active( + client_session, +): + """A subsequent client session.update must not be allowed to flip + ``create_response`` back to True once audio transcription guardrails have + disabled VAD auto-response. Covers both the flat beta shape and the + nested GA ``audio.input.turn_detection`` shape. + """ + client_ws = AsyncMock() + client_ws.receive_text = AsyncMock( + side_effect=[ + json.dumps({"type": "session.update", "session": client_session}), + ConnectionClosed(None, None), + ] + ) + backend_ws = MagicMock() + backend_ws.send = AsyncMock() + + logging_obj = MagicMock() + logging_obj.litellm_trace_id = "trace_1" + logging_obj.async_success_handler = AsyncMock() + logging_obj.success_handler = MagicMock() + + provider_config = MagicMock() + transformed_messages = [] + + def mock_transform(msg, model, session_config): + transformed_messages.append((msg, session_config)) + return [msg] + + provider_config.transform_realtime_request = MagicMock(side_effect=mock_transform) + + streaming = RealTimeStreaming( + websocket=client_ws, + backend_ws=backend_ws, + logging_obj=logging_obj, + provider_config=provider_config, + model="gemini-2.5-flash", + ) + streaming._has_audio_transcription_guardrails = MagicMock(return_value=True) # type: ignore[method-assign] + # Simulate that initial setup + guardrail disable have already happened. + streaming.session_configuration_request = json.dumps({"setup": {"model": "x"}}) + streaming._guardrail_turn_detection_update_sent = True + + await streaming.client_ack_messages() + + assert len(transformed_messages) == 1 + forwarded_msg, _ = transformed_messages[0] + msg_obj = json.loads(forwarded_msg) + session_obj = msg_obj["session"] + forwarded_turn_detection = session_obj.get("turn_detection") or session_obj.get( + "audio", {} + ).get("input", {}).get("turn_detection") + assert isinstance(forwarded_turn_detection, dict) + assert forwarded_turn_detection["create_response"] is False + + +@pytest.mark.asyncio +async def test_follow_up_setup_updates_cached_session_configuration_request(): + """A follow-up setup produced by a subsequent session.update must replace + the cached ``session_configuration_request`` so downstream readers + (e.g. modality lookup in ``response.created``) see the latest config.""" + client_ws = AsyncMock() + client_ws.receive_text = AsyncMock( + side_effect=[ + json.dumps({"type": "session.update", "session": {"tools": []}}), + ConnectionClosed(None, None), + ] + ) + backend_ws = MagicMock() + backend_ws.send = AsyncMock() + + logging_obj = MagicMock() + logging_obj.async_success_handler = AsyncMock() + logging_obj.success_handler = MagicMock() + + provider_config = MagicMock() + follow_up_setup = json.dumps( + { + "setup": { + "model": "models/gemini-2.5-flash", + "generationConfig": {"responseModalities": ["TEXT"]}, + "tools": [{"function_declarations": []}], + } + } + ) + provider_config.transform_realtime_request = MagicMock( + return_value=[follow_up_setup] + ) + + streaming = RealTimeStreaming( + websocket=client_ws, + backend_ws=backend_ws, + logging_obj=logging_obj, + provider_config=provider_config, + model="gemini-2.5-flash", + ) + # Simulate that the original auto-setup was already cached. + streaming.session_configuration_request = json.dumps( + { + "setup": { + "model": "models/gemini-2.5-flash", + "generationConfig": {"responseModalities": ["AUDIO"]}, + } + } + ) + + await streaming.client_ack_messages() + + assert streaming.session_configuration_request == follow_up_setup diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_overhead.py b/tests/test_litellm/litellm_core_utils/test_streaming_overhead.py new file mode 100644 index 00000000000..8fb0659ab5a --- /dev/null +++ b/tests/test_litellm/litellm_core_utils/test_streaming_overhead.py @@ -0,0 +1,508 @@ +""" +Tests for CustomStreamWrapper per-chunk behavior across Anthropic, +Bedrock Invoke, and Bedrock Converse: text passthrough, usage stripping, +hidden_params propagation, finish_reason, sync/async parity, and the +per-stream caches (_GCHUNK_FIELDS, _post_streaming_hooks). +""" + +import asyncio +import time +from typing import List, Optional +from unittest.mock import MagicMock, patch + +import litellm +from litellm.litellm_core_utils.streaming_handler import ( + CustomStreamWrapper, + _GCHUNK_FIELDS, + generic_chunk_has_all_required_fields, +) +from litellm.types.utils import ( + Delta, + GenericStreamingChunk as GChunk, + ModelResponseStream, + StreamingChoices, + Usage, +) + +# --------------------------------------------------------------------------- +# Shared helpers +# --------------------------------------------------------------------------- + + +def _make_logging_obj(provider: str = "anthropic") -> MagicMock: + logging_obj = MagicMock() + logging_obj.model_call_details = { + "custom_llm_provider": provider, + "litellm_params": {}, + } + logging_obj.call_type = "completion" + logging_obj.stream_options = None + logging_obj.messages = [{"role": "user", "content": "hi"}] + logging_obj.completion_start_time = None + logging_obj._llm_caching_handler = None + return logging_obj + + +def _make_generic_chunk( + text: str, + is_finished: bool = False, + finish_reason: str = "", + usage: Optional[dict] = None, +) -> GChunk: + return GChunk( + text=text, + is_finished=is_finished, + finish_reason=finish_reason, + usage=usage, + index=0, + tool_use=None, + ) + + +def _make_bedrock_converse_chunk( + text: str = "", + finish_reason: str = "", + usage: Optional[Usage] = None, +) -> ModelResponseStream: + """Simulate what AWSEventStreamDecoder.converse_chunk_parser returns.""" + return ModelResponseStream( + choices=[ + StreamingChoices( + finish_reason=finish_reason or None, + index=0, + delta=Delta(content=text, role="assistant"), + ) + ], + id="msg-test", + model="anthropic.claude-3-5-sonnet", + usage=usage, + ) + + +async def _async_iter(chunks: list): + """Wrap a list as a proper async iterator for use in __anext__ async branch.""" + for chunk in chunks: + yield chunk + + +def _make_wrapper( + chunks: list, + provider: str = "anthropic", + async_stream: bool = False, +) -> CustomStreamWrapper: + logging_obj = _make_logging_obj(provider) + stream = _async_iter(chunks) if async_stream else iter(chunks) + wrapper = CustomStreamWrapper( + completion_stream=stream, + model="claude-3-5-sonnet", + logging_obj=logging_obj, + custom_llm_provider=provider, + ) + return wrapper + + +def _drain_sync(wrapper: CustomStreamWrapper) -> List[ModelResponseStream]: + results = [] + for chunk in wrapper: + results.append(chunk) + return results + + +async def _drain_async(wrapper: CustomStreamWrapper) -> List[ModelResponseStream]: + results = [] + async for chunk in wrapper: + results.append(chunk) + return results + + +# --------------------------------------------------------------------------- +# 1. Module-level _GCHUNK_FIELDS constant +# --------------------------------------------------------------------------- + + +def test_gchunk_fields_is_frozenset(): + """_GCHUNK_FIELDS must be a frozenset built from GChunk.__annotations__.""" + assert isinstance(_GCHUNK_FIELDS, frozenset) + assert _GCHUNK_FIELDS == frozenset(GChunk.__annotations__) + + +def test_generic_chunk_has_all_required_fields_uses_module_constant(monkeypatch): + """generic_chunk_has_all_required_fields must use _GCHUNK_FIELDS, not __annotations__. + + The check semantics: every key in `chunk` must be a known GChunk field. + This identifies GChunk-shaped dicts (all keys are valid GChunk fields). + """ + valid_chunk = _make_generic_chunk("hello") + assert generic_chunk_has_all_required_fields(valid_chunk) is True + + # A dict with an extra unknown key should return False — the unknown key + # is not a GChunk field, so the chunk is not a pure GChunk. + extra_key_chunk = dict(valid_chunk) + extra_key_chunk["unknown_extra_key"] = "value" + assert generic_chunk_has_all_required_fields(extra_key_chunk) is False + + # A dict with only known GChunk fields but fewer keys still passes because + # all its keys are valid (subset of GChunk fields). + partial_chunk = {"text": "hi", "is_finished": False} + assert generic_chunk_has_all_required_fields(partial_chunk) is True + + +# --------------------------------------------------------------------------- +# 2. Cached model name and provider at init time +# --------------------------------------------------------------------------- + + +def test_cached_model_name_simple(): + """For non-openai providers the cached model name must match the model arg.""" + wrapper = _make_wrapper([], provider="anthropic") + assert wrapper._cached_model_name == "claude-3-5-sonnet" + assert wrapper._cached_logging_llm_provider == "anthropic" + + +def test_cached_model_name_openai_prefix(): + """For openai provider when logging provider differs, model name is prefixed.""" + logging_obj = _make_logging_obj(provider="azure") + wrapper = CustomStreamWrapper( + completion_stream=iter([]), + model="gpt-4o", + logging_obj=logging_obj, + custom_llm_provider="openai", + ) + assert wrapper._cached_model_name == "azure/gpt-4o" + assert wrapper._cached_logging_llm_provider == "azure" + + +def test_base_hidden_params_precomputed(): + """_base_hidden_params must be pre-built from _hidden_params at init.""" + wrapper = _make_wrapper([], provider="anthropic") + assert "response_cost" in wrapper._base_hidden_params + assert wrapper._base_hidden_params["response_cost"] is None + # Must include all keys from _hidden_params + for k in wrapper._hidden_params: + assert k in wrapper._base_hidden_params + + +# --------------------------------------------------------------------------- +# 3. Sync path: model_dump() is NOT called on non-usage chunks +# --------------------------------------------------------------------------- + + +def test_sync_path_no_model_dump_on_text_chunks(): + """ + The sync __next__ must NOT call model_dump() on chunks that have no usage. + + ModelResponseStream declares `usage` as a field, so a `hasattr` check + would always succeed and trigger the model_dump()+recreate path on every + chunk. The wrapper must check `is not None` instead. + """ + chunks = [ + _make_generic_chunk("Hello"), + _make_generic_chunk(" world"), + _make_generic_chunk("", is_finished=True, finish_reason="stop"), + ] + wrapper = _make_wrapper(chunks) + + model_dump_call_count = 0 + original_model_dump = ModelResponseStream.model_dump + + def counting_model_dump(self, **kwargs): + nonlocal model_dump_call_count + model_dump_call_count += 1 + return original_model_dump(self, **kwargs) + + with patch.object(ModelResponseStream, "model_dump", counting_model_dump): + results = _drain_sync(wrapper) + + text_chunks = [r for r in results if r.choices and r.choices[0].delta.content] + assert len(text_chunks) >= 2, "Expected at least 2 text chunks" + assert model_dump_call_count <= 1, ( + f"model_dump() called {model_dump_call_count} times — " + "usage check is firing on every chunk" + ) + + +# --------------------------------------------------------------------------- +# 4. Sync path: usage chunk is stripped from body but preserved in hidden_params +# --------------------------------------------------------------------------- + + +def test_sync_path_usage_stripped_from_body_preserved_in_hidden_params(): + """Usage data must be removed from the returned chunk but added to _hidden_params.""" + usage_dict = {"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30} + chunks = [ + _make_generic_chunk("Hello"), + _make_generic_chunk( + "", is_finished=True, finish_reason="stop", usage=usage_dict + ), + ] + wrapper = _make_wrapper(chunks) + results = _drain_sync(wrapper) + + # The usage chunk must be returned (not silently dropped) + finish_chunks = [ + r for r in results if r.choices and r.choices[0].finish_reason == "stop" + ] + assert finish_chunks, "Finish-reason chunk was not returned" + + # The final chunk must carry usage in _hidden_params + final = results[-1] + assert "usage" in final._hidden_params, "usage missing from _hidden_params" + hidden_usage = final._hidden_params["usage"] + assert hidden_usage is not None + + +# --------------------------------------------------------------------------- +# 5. Async path: usage chunk is stripped from body but preserved in hidden_params +# --------------------------------------------------------------------------- + + +def test_async_path_usage_stripped_from_body_preserved_in_hidden_params(): + """Async path mirrors sync path for usage handling.""" + usage_dict = {"prompt_tokens": 5, "completion_tokens": 15, "total_tokens": 20} + chunks = [ + _make_generic_chunk("Hi"), + _make_generic_chunk( + "", is_finished=True, finish_reason="stop", usage=usage_dict + ), + ] + + async def _run(): + # async_stream=True forces the real async-for branch of __anext__ + wrapper = _make_wrapper(chunks, async_stream=True) + return await _drain_async(wrapper) + + results = asyncio.run(_run()) + final = results[-1] + assert "usage" in final._hidden_params + assert final._hidden_params["usage"] is not None + + +# --------------------------------------------------------------------------- +# 6. Bedrock Converse: ModelResponseStream chunks pass through correctly +# --------------------------------------------------------------------------- + + +def test_bedrock_converse_text_chunks_pass_through(): + """ + Bedrock Converse returns ModelResponseStream objects directly. + They should pass through chunk_creator and appear in output unchanged. + """ + chunks = [ + _make_bedrock_converse_chunk("Hello"), + _make_bedrock_converse_chunk(" world"), + _make_bedrock_converse_chunk("", finish_reason="end_turn"), + ] + wrapper = _make_wrapper(chunks, provider="bedrock") + results = _drain_sync(wrapper) + + texts = [ + r.choices[0].delta.content + for r in results + if r.choices and r.choices[0].delta.content + ] + assert "Hello" in texts or any("Hello" in (t or "") for t in texts) + + +def test_bedrock_converse_usage_chunk_stripped_and_in_hidden_params(): + """Usage in a Bedrock Converse ModelResponseStream chunk is handled correctly.""" + usage = Usage(prompt_tokens=8, completion_tokens=12, total_tokens=20) + chunks = [ + _make_bedrock_converse_chunk("Hi"), + _make_bedrock_converse_chunk("", finish_reason="end_turn", usage=usage), + ] + wrapper = _make_wrapper(chunks, provider="bedrock") + results = _drain_sync(wrapper) + + final = results[-1] + assert "usage" in final._hidden_params + assert final._hidden_params["usage"] is not None + + +# --------------------------------------------------------------------------- +# 7. Anthropic generic chunk (GChunk) path +# --------------------------------------------------------------------------- + + +def test_anthropic_generic_chunks_text_pass_through(): + """GChunk text chunks must arrive in the output with correct content.""" + chunks = [ + _make_generic_chunk("The"), + _make_generic_chunk(" answer"), + _make_generic_chunk("", is_finished=True, finish_reason="stop"), + ] + wrapper = _make_wrapper(chunks, provider="anthropic") + results = _drain_sync(wrapper) + + texts = [ + r.choices[0].delta.content + for r in results + if r.choices and r.choices[0].delta.content + ] + assert len(texts) >= 2 + + +def test_anthropic_finish_reason_propagated(): + """finish_reason must be set on the final streaming chunk.""" + chunks = [ + _make_generic_chunk("Hi"), + _make_generic_chunk("", is_finished=True, finish_reason="stop"), + ] + wrapper = _make_wrapper(chunks, provider="anthropic") + results = _drain_sync(wrapper) + + finish_reasons = [ + r.choices[0].finish_reason + for r in results + if r.choices and r.choices[0].finish_reason + ] + assert "stop" in finish_reasons + + +# --------------------------------------------------------------------------- +# 8. Callback caching: _post_streaming_hooks resolved once per stream +# --------------------------------------------------------------------------- + + +def test_post_streaming_hooks_cached_after_first_call(): + """ + _post_streaming_hooks must be None before the first hook call and a list after. + The same list object must be reused on subsequent calls (not re-built). + """ + wrapper = _make_wrapper([], provider="anthropic") + assert wrapper._post_streaming_hooks is None, "Must be None before first call" + + async def _run(): + # Simulate hook resolution with an empty callback list + with patch.object(litellm, "callbacks", []): + await wrapper._call_post_streaming_deployment_hook( + MagicMock(spec=ModelResponseStream) + ) + first_list = wrapper._post_streaming_hooks + assert isinstance(first_list, list) + + # Second call must reuse the same list object + with patch.object(litellm, "callbacks", []): + await wrapper._call_post_streaming_deployment_hook( + MagicMock(spec=ModelResponseStream) + ) + assert ( + wrapper._post_streaming_hooks is first_list + ), "_post_streaming_hooks was rebuilt on second call — caching broken" + + asyncio.run(_run()) + + +def test_post_streaming_hooks_filters_correctly(): + """ + Only CustomLogger instances must be included; plain callables are excluded. + + Note: CustomLogger's base class already defines + async_post_call_streaming_deployment_hook, so ALL CustomLogger subclasses + pass the hasattr() check regardless of whether they override the method. + The filter therefore keeps any CustomLogger instance and drops anything else. + """ + from litellm.integrations.custom_logger import CustomLogger + + class MyLogger(CustomLogger): + pass + + plain_callable = MagicMock() + + wrapper = _make_wrapper([], provider="anthropic") + + async def _run(): + with patch.object(litellm, "callbacks", [MyLogger(), plain_callable]): + await wrapper._call_post_streaming_deployment_hook( + MagicMock(spec=ModelResponseStream) + ) + + # plain_callable must be excluded; MyLogger (CustomLogger subclass) included + assert len(wrapper._post_streaming_hooks) == 1 + assert isinstance(wrapper._post_streaming_hooks[0], MyLogger) + + asyncio.run(_run()) + + +# --------------------------------------------------------------------------- +# 9. model_response_creator: hidden_params built correctly +# --------------------------------------------------------------------------- + + +def test_model_response_creator_hidden_params_no_chunk(): + """model_response_creator() with no args must include all _base_hidden_params.""" + wrapper = _make_wrapper([], provider="anthropic") + response = wrapper.model_response_creator() + + assert response._hidden_params.get("response_cost") is None + assert response._hidden_params.get("custom_llm_provider") == "anthropic" + assert "created_at" in response._hidden_params + + +def test_model_response_creator_hidden_params_caller_merged(): + """When hidden_params are passed by caller, they must be included in result.""" + wrapper = _make_wrapper([], provider="anthropic") + caller_params = {"some_key": "some_value"} + response = wrapper.model_response_creator(hidden_params=caller_params) + + assert response._hidden_params.get("some_key") == "some_value" + assert response._hidden_params.get("response_cost") is None + + +def test_model_response_creator_stream_key_stripped(): + """The 'stream' key must be removed from chunk before constructing ModelResponseStream.""" + wrapper = _make_wrapper([], provider="anthropic") + chunk = {"stream": True, "choices": []} + # Should not raise even if 'stream' would be an invalid ModelResponseStream field + response = wrapper.model_response_creator(chunk=chunk) + assert response is not None + + +# --------------------------------------------------------------------------- +# 10. Per-chunk overhead regression: sync path must not regress +# --------------------------------------------------------------------------- + + +def test_sync_streaming_overhead_not_regressed(): + """ + Micro-benchmark: the sync hot path must process 200 text chunks in < 2 s. + + This test acts as a canary for gross per-chunk overhead regressions. + It is intentionally generous (2 s) to avoid flakiness on slow CI runners. + """ + n_chunks = 200 + chunks = [_make_generic_chunk(f"token-{i}") for i in range(n_chunks)] + chunks.append(_make_generic_chunk("", is_finished=True, finish_reason="stop")) + + wrapper = _make_wrapper(chunks, provider="anthropic") + + start = time.monotonic() + results = _drain_sync(wrapper) + elapsed = time.monotonic() - start + + assert len(results) > 0, "No chunks returned" + assert elapsed < 2.0, ( + f"Sync streaming of {n_chunks} chunks took {elapsed:.3f}s — " + "per-chunk overhead regression detected" + ) + + +def test_async_streaming_overhead_not_regressed(): + """ + Micro-benchmark for the async path: 200 text chunks in < 2 s. + """ + n_chunks = 200 + chunks = [_make_generic_chunk(f"token-{i}") for i in range(n_chunks)] + chunks.append(_make_generic_chunk("", is_finished=True, finish_reason="stop")) + + async def _run(): + wrapper = _make_wrapper(chunks, provider="anthropic") + start = time.monotonic() + results = await _drain_async(wrapper) + return results, time.monotonic() - start + + results, elapsed = asyncio.run(_run()) + assert len(results) > 0 + assert elapsed < 2.0, ( + f"Async streaming of {n_chunks} chunks took {elapsed:.3f}s — " + "per-chunk overhead regression detected" + ) diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py index 7d9e4768303..687c5a2e733 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -1622,6 +1622,29 @@ def test_effort_output_config_preservation(): assert result["output_config"]["effort"] == "medium" +def test_output_config_format_preservation_and_beta_header(): + """Test that output_config.format is preserved and treated as structured output.""" + config = AnthropicConfig() + output_format = { + "type": "json_schema", + "schema": {"type": "object", "properties": {"answer": {"type": "string"}}}, + } + optional_params = {"output_config": {"format": output_format, "effort": "xhigh"}} + + result = config.transform_request( + model="claude-opus-4-7", + messages=[{"role": "user", "content": "Test"}], + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + headers = config.update_headers_with_optional_anthropic_beta({}, optional_params) + + assert result["output_config"]["format"] == output_format + assert result["output_config"]["effort"] == "xhigh" + assert "structured-outputs-2025-11-13" in headers["anthropic-beta"] + + def test_effort_beta_header_injection(): """Test that effort beta header is automatically added when output_config is detected.""" from litellm.llms.anthropic.common_utils import AnthropicModelInfo @@ -1648,7 +1671,7 @@ def test_effort_validation(): messages = [{"role": "user", "content": "Test"}] - # Valid values should work + # Valid values should work (xhigh is Opus 4.7+ only, not 4.5) for effort in ["high", "medium", "low"]: optional_params = {"output_config": {"effort": effort}} result = config.transform_request( @@ -2513,14 +2536,14 @@ def test_reasoning_effort_accepts_dict_shape_for_adaptive_model(reasoning_effort ) # thinking must be set (adaptive for 4.6+) - assert "thinking" in result, ( - f"thinking missing for reasoning_effort={reasoning_effort_value!r}" - ) + assert ( + "thinking" in result + ), f"thinking missing for reasoning_effort={reasoning_effort_value!r}" assert result["thinking"]["type"] == "adaptive" # output_config must carry the mapped effort - assert "output_config" in result, ( - f"output_config missing for reasoning_effort={reasoning_effort_value!r}" - ) + assert ( + "output_config" in result + ), f"output_config missing for reasoning_effort={reasoning_effort_value!r}" assert result["output_config"]["effort"] == "low" @@ -2532,7 +2555,9 @@ def test_reasoning_effort_accepts_dict_shape_for_adaptive_model(reasoning_effort {"effort": "low", "summary": "concise"}, ], ) -def test_reasoning_effort_accepts_dict_shape_for_non_adaptive_model(reasoning_effort_value): +def test_reasoning_effort_accepts_dict_shape_for_non_adaptive_model( + reasoning_effort_value, +): """ Non-adaptive (pre-4.6) branch: dict-shape reasoning_effort must still map to ``thinking.type='enabled'`` + ``budget_tokens``. ``output_config`` must @@ -2547,9 +2572,9 @@ def test_reasoning_effort_accepts_dict_shape_for_non_adaptive_model(reasoning_ef drop_params=False, ) - assert "thinking" in result, ( - f"thinking missing for reasoning_effort={reasoning_effort_value!r}" - ) + assert ( + "thinking" in result + ), f"thinking missing for reasoning_effort={reasoning_effort_value!r}" assert result["thinking"]["type"] == "enabled" assert "budget_tokens" in result["thinking"] assert result["thinking"]["budget_tokens"] > 0 @@ -2582,12 +2607,12 @@ def test_reasoning_effort_unparseable_dict_is_dropped(bad_value): model="claude-sonnet-4-6-20260219", drop_params=False, ) - assert "thinking" not in result, ( - f"thinking should not be set for bad value {bad_value!r}" - ) - assert "output_config" not in result, ( - f"output_config should not be set for bad value {bad_value!r}" - ) + assert ( + "thinking" not in result + ), f"thinking should not be set for bad value {bad_value!r}" + assert ( + "output_config" not in result + ), f"output_config should not be set for bad value {bad_value!r}" @pytest.mark.parametrize( diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_messages_structured_outputs.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_messages_structured_outputs.py index 3c81bfaa0f9..e6d5c6f4ee1 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_messages_structured_outputs.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_messages_structured_outputs.py @@ -48,6 +48,39 @@ def test_output_format_supported_and_transforms_correctly(): assert "structured-outputs-2025-11-13" in headers["anthropic-beta"] +def test_output_config_format_supported_and_transforms_correctly(): + """Test that output_config.format is preserved and adds the structured-output beta.""" + config = AnthropicMessagesConfig() + + supported_params = config.get_supported_anthropic_messages_params("claude-opus-4-7") + assert "output_config" in supported_params + + output_format = { + "type": "json_schema", + "schema": {"type": "object", "properties": {"result": {"type": "string"}}}, + } + optional_params = { + "max_tokens": 1024, + "output_config": {"format": output_format, "effort": "xhigh"}, + } + headers = {} + + result = config.transform_anthropic_messages_request( + model="claude-opus-4-7", + messages=[{"role": "user", "content": "test"}], + anthropic_messages_optional_request_params=optional_params.copy(), + litellm_params={}, + headers=headers, + ) + + headers = config._update_headers_with_anthropic_beta(headers, optional_params) + + assert result["output_config"]["format"] == output_format + assert result["output_config"]["effort"] == "xhigh" + assert "anthropic-beta" in headers + assert "structured-outputs-2025-11-13" in headers["anthropic-beta"] + + def test_output_format_works_with_bedrock_and_azure(): """Test that output_format works with Bedrock and Azure Foundry models.""" config = AnthropicMessagesConfig() diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_reasoning_effort_translation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_reasoning_effort_translation.py index 83716b8c8d3..54bf0c4ac0f 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_reasoning_effort_translation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_reasoning_effort_translation.py @@ -6,6 +6,9 @@ from litellm.llms.anthropic.common_utils import AnthropicError from litellm.llms.anthropic.experimental_pass_through.messages.transformation import ( AnthropicMessagesConfig, ) +from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import ( + AmazonAnthropicClaudeMessagesConfig, +) @pytest.mark.parametrize( @@ -102,7 +105,6 @@ def test_invalid_reasoning_effort_raises_400(bad_effort): "model,bad_effort", [ ("claude-opus-4-6", "xhigh"), - ("bedrock/invoke/us.anthropic.claude-opus-4-6-v1", "xhigh"), ("claude-sonnet-4-6", "xhigh"), ], ) @@ -123,6 +125,56 @@ def test_reasoning_effort_unsupported_tier_raises_400_messages(model, bad_effort assert "not supported by this model" in str(exc_info.value) +@pytest.mark.parametrize( + "model,effort,expected_effort", + [ + ("invoke/us.anthropic.claude-opus-4-6-v1", "xhigh", "max"), + ("invoke/us.anthropic.claude-opus-4-6-v1", "max", "max"), + ("invoke/us.anthropic.claude-opus-4-6-v1", "high", "high"), + ("invoke/us.anthropic.claude-opus-4-7", "xhigh", "xhigh"), + ], +) +def test_bedrock_invoke_messages_clamps_effort_to_ceiling( + model, effort, expected_effort +): + """Bedrock Invoke /v1/messages degrades effort to the model's ceiling. + + Claude Code "goal mode" sends ``xhigh``; Opus 4.6 must clamp to ``max`` + instead of raising, while Opus 4.7 (ceiling ``xhigh``) keeps ``xhigh``. + """ + config = AmazonAnthropicClaudeMessagesConfig() + optional_params = {"max_tokens": 1024, "reasoning_effort": effort} + + result = config.transform_anthropic_messages_request( + model=model, + messages=[{"role": "user", "content": "Hello"}], + anthropic_messages_optional_request_params=optional_params, + litellm_params={}, + headers={}, + ) + + assert result["output_config"]["effort"] == expected_effort + assert result["thinking"]["type"] == "adaptive" + + +def test_bedrock_invoke_messages_rejects_xhigh_without_ceiling(): + """Sonnet 4.6 on Bedrock has no effort ceiling, so xhigh is still rejected.""" + config = AmazonAnthropicClaudeMessagesConfig() + optional_params = {"max_tokens": 1024, "reasoning_effort": "xhigh"} + + with pytest.raises(AnthropicError) as exc_info: + config.transform_anthropic_messages_request( + model="invoke/us.anthropic.claude-sonnet-4-6", + messages=[{"role": "user", "content": "Hello"}], + anthropic_messages_optional_request_params=optional_params, + litellm_params={}, + headers={}, + ) + + assert exc_info.value.status_code == 400 + assert "not supported by this model" in str(exc_info.value) + + @pytest.mark.parametrize( "model", [ diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/test_reasoning_effort_fields.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/test_reasoning_effort_fields.py index d42d109f21b..08fef8c6a24 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/test_reasoning_effort_fields.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/test_reasoning_effort_fields.py @@ -63,7 +63,13 @@ class TestGetModelInfoReasoningEffortFields: class TestModelRegistryReasoningEffortFields: """Verify specific models have the expected reasoning effort capability - values in the JSON registry file.""" + values in the JSON registry file. + + Claude models intentionally OMIT ``supports_minimal_reasoning_effort``: + ``minimal`` is not a real Anthropic effort level (the API accepts only + low/medium/high/xhigh/max), so LiteLLM degrades ``minimal`` to ``low`` + regardless of the flag. These tests guard against the flag being + re-added to the Claude fleet.""" @pytest.fixture(autouse=True) def _load_registry(self): @@ -77,41 +83,41 @@ class TestModelRegistryReasoningEffortFields: entry = self.registry["claude-opus-4-6"] assert entry.get("supports_max_reasoning_effort") is True - def test_opus_4_7_supports_minimal(self): + def test_opus_4_7_omits_minimal(self): entry = self.registry["claude-opus-4-7"] - assert entry.get("supports_minimal_reasoning_effort") is True + assert "supports_minimal_reasoning_effort" not in entry - def test_opus_4_6_supports_minimal(self): + def test_opus_4_6_omits_minimal(self): entry = self.registry["claude-opus-4-6"] - assert entry.get("supports_minimal_reasoning_effort") is True + assert "supports_minimal_reasoning_effort" not in entry - def test_sonnet_4_6_supports_minimal(self): + def test_sonnet_4_6_omits_minimal(self): entry = self.registry["anthropic.claude-sonnet-4-6"] - assert entry.get("supports_minimal_reasoning_effort") is True + assert "supports_minimal_reasoning_effort" not in entry def test_bedrock_opus_4_7_supports_max(self): entry = self.registry["anthropic.claude-opus-4-7"] assert entry.get("supports_max_reasoning_effort") is True - assert entry.get("supports_minimal_reasoning_effort") is True + assert "supports_minimal_reasoning_effort" not in entry def test_vertex_opus_4_7_supports_max(self): entry = self.registry["vertex_ai/claude-opus-4-7"] assert entry.get("supports_max_reasoning_effort") is True - assert entry.get("supports_minimal_reasoning_effort") is True + assert "supports_minimal_reasoning_effort" not in entry def test_vertex_opus_4_6_supports_max(self): entry = self.registry["vertex_ai/claude-opus-4-6"] assert entry.get("supports_max_reasoning_effort") is True - assert entry.get("supports_minimal_reasoning_effort") is True + assert "supports_minimal_reasoning_effort" not in entry - def test_azure_ai_opus_4_6_supports_minimal(self): + def test_azure_ai_opus_4_6_omits_minimal(self): entry = self.registry["azure_ai/claude-opus-4-6"] - assert entry.get("supports_minimal_reasoning_effort") is True + assert "supports_minimal_reasoning_effort" not in entry def test_azure_ai_opus_4_7_supports_max(self): entry = self.registry["azure_ai/claude-opus-4-7"] assert entry.get("supports_max_reasoning_effort") is True - assert entry.get("supports_minimal_reasoning_effort") is True + assert "supports_minimal_reasoning_effort" not in entry # --------------------------------------------------------------------------- diff --git a/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py b/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py index b2e254901f4..4c4c0e17a38 100644 --- a/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py @@ -430,6 +430,61 @@ def test_output_config_forwarded_for_bedrock_chat_invoke_request(): assert result["max_tokens"] == 100 +def test_output_config_format_converted_for_bedrock_chat_invoke_request(): + """Bedrock Invoke chat path consumes ``output_config.format`` before forwarding.""" + config = AmazonAnthropicClaudeConfig() + schema = { + "type": "object", + "properties": {"answer": {"type": "string"}}, + } + + result = config.transform_request( + model="anthropic.claude-opus-4-7", + messages=[{"role": "user", "content": "test"}], + optional_params={ + "max_tokens": 100, + "output_config": { + "effort": "xhigh", + "format": {"type": "json_schema", "schema": schema}, + }, + }, + litellm_params={}, + headers={}, + ) + + assert result.get("output_config") == {"effort": "xhigh"} + last_content = result["messages"][0]["content"] + assert json.loads(last_content[-1]["text"]) == schema + + +@pytest.mark.parametrize( + "model,expected_effort", + [ + ("anthropic.claude-opus-4-5-20251101-v1:0", "high"), + ("anthropic.claude-opus-4-6-v1", "max"), + ("anthropic.claude-opus-4-7", "xhigh"), + ], +) +def test_output_config_effort_normalized_for_bedrock_chat_invoke_request( + model, expected_effort +): + """Bedrock Invoke chat path accepts ``xhigh`` and forwards the provider-safe effort.""" + config = AmazonAnthropicClaudeConfig() + + result = config.transform_request( + model=model, + messages=[{"role": "user", "content": "test"}], + optional_params={ + "max_tokens": 100, + "output_config": {"effort": "xhigh"}, + }, + litellm_params={}, + headers={}, + ) + + assert result.get("output_config") == {"effort": expected_effort} + + def test_bedrock_chat_invoke_checks_output_config_support_with_bedrock_provider(): config = AmazonAnthropicClaudeConfig() messages = [{"role": "user", "content": "test"}] diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index 5f2ed3dc00f..c8e72b7ac5b 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -318,6 +318,7 @@ def test_reasoning_effort_none_omits_thinking_for_anthropic_converse(model): ("bedrock/converse/us.anthropic.claude-opus-4-7", "high", "high"), ("bedrock/converse/us.anthropic.claude-opus-4-7", "xhigh", "xhigh"), ("bedrock/converse/us.anthropic.claude-opus-4-7", "max", "max"), + ("bedrock/converse/us.anthropic.claude-opus-4-6-v1", "xhigh", "max"), ("bedrock/converse/us.anthropic.claude-opus-4-6-v1", "max", "max"), ("bedrock/converse/us.anthropic.claude-sonnet-4-6", "high", "high"), ("bedrock/converse/us.anthropic.claude-sonnet-4-6", "minimal", "low"), @@ -369,6 +370,132 @@ def test_output_config_effort_forwarded_into_additional_request_fields(model): assert additional.get("output_config") == {"effort": "high"} +def test_output_config_format_translated_to_native_output_config_converse(): + """``output_config.format`` becomes Bedrock ``outputConfig`` and is not forwarded raw.""" + config = AmazonConverseConfig() + schema = { + "type": "object", + "properties": {"answer": {"type": "string"}}, + } + + result = config._transform_request( + model="bedrock/converse/us.anthropic.claude-opus-4-7", + messages=[{"role": "user", "content": "hi"}], + optional_params={ + "maxTokens": 256, + "thinking": {"type": "adaptive"}, + "output_config": { + "effort": "xhigh", + "format": {"type": "json_schema", "schema": schema}, + }, + }, + litellm_params={}, + headers={}, + ) + + additional = result.get("additionalModelRequestFields", {}) + assert additional.get("output_config") == {"effort": "xhigh"} + assert "format" not in additional["output_config"] + assert result["outputConfig"]["textFormat"]["type"] == "json_schema" + parsed_schema = json.loads( + result["outputConfig"]["textFormat"]["structure"]["jsonSchema"]["schema"] + ) + assert parsed_schema == {**schema, "additionalProperties": False} + + +def test_output_config_format_dropped_on_unsupported_converse_model_warns(caplog): + """When Converse model lacks native structured-output support, the silently + dropped ``output_config.format`` must surface as a warning so callers can + diagnose plain-text responses.""" + from unittest.mock import patch + + config = AmazonConverseConfig() + schema = { + "type": "object", + "properties": {"answer": {"type": "string"}}, + } + + with patch.object( + AmazonConverseConfig, + "_supports_native_structured_outputs", + return_value=False, + ): + with caplog.at_level("WARNING"): + result = config._transform_request( + model="bedrock/converse/us.anthropic.claude-3-haiku-20240307-v1:0", + messages=[{"role": "user", "content": "hi"}], + optional_params={ + "maxTokens": 256, + "output_config": { + "format": {"type": "json_schema", "schema": schema}, + }, + }, + litellm_params={}, + headers={}, + ) + + assert "outputConfig" not in result + assert any( + "dropping `output_config.format`" in record.getMessage() + for record in caplog.records + ) + + +def test_output_config_normalized_marker_does_not_leak_into_optional_params(): + """The internal ``_output_config_normalized`` marker set by + ``_handle_reasoning_effort_parameter`` must be consumed during request + preparation so it does not linger on the caller's ``optional_params``.""" + config = AmazonConverseConfig() + + optional_params = config.map_openai_params( + non_default_params={"reasoning_effort": "xhigh"}, + optional_params={}, + model="bedrock/converse/us.anthropic.claude-opus-4-6-v1", + drop_params=False, + ) + assert optional_params.get("_output_config_normalized") is True + + config._transform_request( + model="bedrock/converse/us.anthropic.claude-opus-4-6-v1", + messages=[{"role": "user", "content": "hi"}], + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + assert "_output_config_normalized" not in optional_params + + +@pytest.mark.parametrize( + "model,expected_effort", + [ + ("bedrock/converse/us.anthropic.claude-opus-4-5-20251101-v1:0", "high"), + ("bedrock/converse/us.anthropic.claude-opus-4-6-v1", "max"), + ("bedrock/converse/us.anthropic.claude-opus-4-7", "xhigh"), + ], +) +def test_output_config_effort_normalized_for_bedrock_converse_opus( + model, expected_effort +): + """Bedrock Converse accepts ``xhigh`` and forwards the provider-safe effort.""" + config = AmazonConverseConfig() + + result = config._transform_request( + model=model, + messages=[{"role": "user", "content": "hi"}], + optional_params={ + "maxTokens": 256, + "thinking": {"type": "adaptive"}, + "output_config": {"effort": "xhigh"}, + }, + litellm_params={}, + headers={}, + ) + + additional = result.get("additionalModelRequestFields", {}) + assert additional.get("output_config") == {"effort": expected_effort} + + @pytest.mark.parametrize( "effort", ["disabled", "invalid", ""], diff --git a/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py b/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py index 2e315a535f0..c92a9905229 100644 --- a/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py +++ b/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py @@ -767,6 +767,163 @@ def test_bedrock_messages_forwards_output_config_with_output_format(): assert "output_format" not in result +def test_bedrock_messages_converts_output_config_format_to_inline_schema(): + """``output_config.format`` is consumed so Bedrock does not see an unknown nested key.""" + from unittest.mock import patch + + from litellm.types.router import GenericLiteLLMParams + + cfg = AmazonAnthropicClaudeMessagesConfig() + messages = [{"role": "user", "content": [{"type": "text", "text": "Hello"}]}] + schema = { + "type": "object", + "properties": {"answer": {"type": "string"}}, + } + optional_params = { + "max_tokens": 4096, + "output_config": { + "effort": "xhigh", + "format": {"type": "json_schema", "schema": schema}, + }, + } + + with patch( + "litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation._supports_factory", + return_value=True, + ): + result = cfg.transform_anthropic_messages_request( + model="anthropic.claude-opus-4-7", + messages=messages, + anthropic_messages_optional_request_params=optional_params, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert result.get("output_config") == {"effort": "xhigh"} + assert "output_format" not in result + last_content = result["messages"][0]["content"] + assert json.loads(last_content[-1]["text"]) == schema + + +@pytest.mark.parametrize( + "model,expected_effort", + [ + ("anthropic.claude-opus-4-5-20251101-v1:0", "high"), + ("anthropic.claude-opus-4-6-v1", "max"), + ("anthropic.claude-opus-4-7", "xhigh"), + ], +) +def test_bedrock_messages_normalizes_output_config_effort_for_opus( + model, expected_effort +): + """Bedrock /v1/messages accepts ``xhigh`` and forwards the provider-safe effort.""" + from unittest.mock import patch + + from litellm.types.router import GenericLiteLLMParams + + cfg = AmazonAnthropicClaudeMessagesConfig() + + with patch( + "litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation._supports_factory", + return_value=True, + ): + result = cfg.transform_anthropic_messages_request( + model=model, + messages=[{"role": "user", "content": [{"type": "text", "text": "Hello"}]}], + anthropic_messages_optional_request_params={ + "max_tokens": 4096, + "output_config": {"effort": "xhigh"}, + }, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert result.get("output_config") == {"effort": expected_effort} + + +def test_bedrock_messages_does_not_mutate_callers_messages_when_embedding_schema(): + """Inline-schema embedding must not mutate the caller's ``messages`` list, + message dicts, or content list.""" + from unittest.mock import patch + + from litellm.types.router import GenericLiteLLMParams + + cfg = AmazonAnthropicClaudeMessagesConfig() + caller_content = [{"type": "text", "text": "Hello"}] + caller_message = {"role": "user", "content": caller_content} + caller_messages = [caller_message] + schema = {"type": "object", "properties": {"answer": {"type": "string"}}} + optional_params = { + "max_tokens": 4096, + "output_config": { + "effort": "xhigh", + "format": {"type": "json_schema", "schema": schema}, + }, + } + + with patch( + "litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation._supports_factory", + return_value=True, + ): + result = cfg.transform_anthropic_messages_request( + model="anthropic.claude-opus-4-7", + messages=caller_messages, + anthropic_messages_optional_request_params=optional_params, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert caller_messages == [ + {"role": "user", "content": [{"type": "text", "text": "Hello"}]} + ] + assert caller_message == { + "role": "user", + "content": [{"type": "text", "text": "Hello"}], + } + assert caller_content == [{"type": "text", "text": "Hello"}] + last_content = result["messages"][-1]["content"] + assert json.loads(last_content[-1]["text"]) == schema + + +def test_bedrock_messages_does_not_mutate_callers_output_config(): + """`pop_bedrock_invoke_output_config_format` / effort normalization must not + leak into the caller's ``optional_params`` dict.""" + from unittest.mock import patch + + from litellm.types.router import GenericLiteLLMParams + + cfg = AmazonAnthropicClaudeMessagesConfig() + schema = { + "type": "object", + "properties": {"answer": {"type": "string"}}, + } + caller_output_config = { + "effort": "xhigh", + "format": {"type": "json_schema", "schema": schema}, + } + optional_params = { + "max_tokens": 4096, + "output_config": caller_output_config, + } + + with patch( + "litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation._supports_factory", + return_value=True, + ): + cfg.transform_anthropic_messages_request( + model="anthropic.claude-opus-4-5-20251101-v1:0", + messages=[{"role": "user", "content": [{"type": "text", "text": "Hello"}]}], + anthropic_messages_optional_request_params=optional_params, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert caller_output_config == { + "effort": "xhigh", + "format": {"type": "json_schema", "schema": schema}, + } + + def test_bedrock_messages_strips_output_config_with_output_format(): """ When both output_config and output_format are present, output_format @@ -1071,9 +1228,7 @@ def test_bedrock_messages_preserves_compact_context_management_and_adds_beta(): messages = [{"role": "user", "content": [{"type": "text", "text": "Hi"}]}] optional_params = { "max_tokens": 4096, - "context_management": { - "edits": [{"type": "compact_20260112"}] - }, + "context_management": {"edits": [{"type": "compact_20260112"}]}, } result = cfg.transform_anthropic_messages_request( @@ -1084,9 +1239,7 @@ def test_bedrock_messages_preserves_compact_context_management_and_adds_beta(): headers={}, ) - assert result.get("context_management") == { - "edits": [{"type": "compact_20260112"}] - } + assert result.get("context_management") == {"edits": [{"type": "compact_20260112"}]} assert "compact-2026-01-12" in result.get("anthropic_beta", []) assert result["max_tokens"] == 4096 @@ -1118,9 +1271,7 @@ def test_bedrock_messages_filters_unsupported_context_management_edits(): headers={}, ) - assert result.get("context_management") == { - "edits": [{"type": "compact_20260112"}] - } + assert result.get("context_management") == {"edits": [{"type": "compact_20260112"}]} assert "compact-2026-01-12" in result.get("anthropic_beta", []) diff --git a/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py b/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py index c39fb427a01..6298eeb25e9 100644 --- a/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py +++ b/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py @@ -1,9 +1,7 @@ -import json import os import sys import pytest -from fastapi.testclient import TestClient sys.path.insert( 0, os.path.abspath("../../../..") @@ -12,7 +10,6 @@ sys.path.insert( from litellm.llms.bedrock.common_utils import BedrockModelInfo - # --------------------------------------------------------------------------- # # get_bedrock_response_stream_shape lazy-load tests # # --------------------------------------------------------------------------- # @@ -24,8 +21,10 @@ def _reset_bedrock_response_stream_shape_cache(): import litellm.llms.bedrock.common_utils as mod mod.get_bedrock_response_stream_shape.cache_clear() + mod._get_local_model_cost_map.cache_clear() yield mod.get_bedrock_response_stream_shape.cache_clear() + mod._get_local_model_cost_map.cache_clear() def test_bedrock_response_stream_shape_lazy_loads_once(): @@ -222,3 +221,45 @@ def test_context_window_suffix_stripped_for_cost_lookup(): get_bedrock_base_model("anthropic.claude-3-5-sonnet-20241022-v2:0:51k") == "anthropic.claude-3-5-sonnet-20241022-v2:0" ) + + +def test_output_config_effort_normalization_uses_model_info_ceiling(monkeypatch): + import litellm.llms.bedrock.common_utils as mod + + calls = [] + + def fake_get_model_info(model, custom_llm_provider=None): + calls.append((model, custom_llm_provider)) + return {"bedrock_output_config_effort_ceiling": "max"} + + monkeypatch.setattr(mod, "_get_model_info", fake_get_model_info) + output_config = {"effort": "xhigh"} + + mod.normalize_bedrock_opus_output_config_effort( + model="custom-bedrock-alias-without-opus-pattern", + output_config=output_config, + ) + + assert output_config == {"effort": "max"} + assert calls == [("custom-bedrock-alias-without-opus-pattern", "bedrock")] + + +@pytest.mark.parametrize( + "model,expected_ceiling", + [ + ("anthropic.claude-opus-4-5-20251101-v1:0", "high"), + ("anthropic.claude-opus-4-6-v1", "max"), + ("anthropic.claude-opus-4-7", "xhigh"), + ("us.anthropic.claude-opus-4-5-20251101-v1:0", "high"), + ("us.anthropic.claude-opus-4-6-v1", "max"), + ("us.anthropic.claude-opus-4-7", "xhigh"), + ], +) +def test_bundled_bedrock_opus_model_info_declares_output_config_effort_ceiling( + model, expected_ceiling +): + from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap + + model_info = GetModelCostMap.load_local_model_cost_map()[model] + + assert model_info["bedrock_output_config_effort_ceiling"] == expected_ceiling diff --git a/tests/test_litellm/llms/gemini/realtime/test_gemini_realtime_transformation.py b/tests/test_litellm/llms/gemini/realtime/test_gemini_realtime_transformation.py index cc0a32d2ce6..cc0adc4277c 100644 --- a/tests/test_litellm/llms/gemini/realtime/test_gemini_realtime_transformation.py +++ b/tests/test_litellm/llms/gemini/realtime/test_gemini_realtime_transformation.py @@ -20,8 +20,10 @@ def test_gemini_realtime_transformation_session_created(): assert config is not None session_configuration_request = { - "model": "gemini-1.5-flash", - "generationConfig": {"responseModalities": ["TEXT"]}, + "setup": { + "model": "gemini-1.5-flash", + "generationConfig": {"responseModalities": ["TEXT"]}, + } } session_configuration_request_str = json.dumps(session_configuration_request) session_created_message = {"setupComplete": {}} @@ -45,8 +47,54 @@ def test_gemini_realtime_transformation_session_created(): }, ) - print(transformed_message) - assert transformed_message["response"][0]["type"] == "session.created" + session_created = transformed_message["response"][0] + assert session_created["type"] == "session.created" + # Verify the setup-wrapped configuration reaches the modality lookup so + # the synthetic session.created reflects the cached responseModalities. + assert session_created["session"]["modalities"] == ["text"] + + +def test_session_created_does_not_overwrite_session_configuration_request(): + config = GeminiRealtimeConfig() + + session_configuration_request_str = json.dumps( + { + "setup": { + "model": "models/gemini-2.5-flash-native-audio", + "generationConfig": {"responseModalities": ["AUDIO"]}, + } + } + ) + setup_complete_message = {"setupComplete": {}} + + logging_obj = MagicMock() + logging_obj.litellm_trace_id = "trace_123" + + transformed = config.transform_realtime_response( + json.dumps(setup_complete_message), + "gemini-2.5-flash-native-audio", + logging_obj, + realtime_response_transform_input={ + "session_configuration_request": session_configuration_request_str, + "current_output_item_id": None, + "current_response_id": None, + "current_conversation_id": None, + "current_delta_chunks": [], + "current_item_chunks": [], + "current_delta_type": None, + }, + ) + + # Must keep original setup payload (with "setup"), not overwrite with session.created event. + assert ( + transformed["session_configuration_request"] + == session_configuration_request_str + ) + + # Also verify emitted session.created reflects audio modality from setup payload. + session_created = transformed["response"][0] + assert session_created["type"] == "session.created" + assert "audio" in session_created["session"]["modalities"] def test_gemini_realtime_transformation_content_delta(): @@ -54,8 +102,10 @@ def test_gemini_realtime_transformation_content_delta(): assert config is not None session_configuration_request = { - "model": "gemini-1.5-flash", - "generationConfig": {"responseModalities": ["TEXT"]}, + "setup": { + "model": "gemini-1.5-flash", + "generationConfig": {"responseModalities": ["TEXT"]}, + } } session_configuration_request_str = json.dumps(session_configuration_request) session_created_message = { @@ -147,8 +197,10 @@ def test_gemini_realtime_transformation_audio_delta(): assert config is not None session_configuration_request = { - "model": "gemini-1.5-flash", - "generationConfig": {"responseModalities": ["AUDIO"]}, + "setup": { + "model": "gemini-1.5-flash", + "generationConfig": {"responseModalities": ["AUDIO"]}, + } } session_configuration_request_str = json.dumps(session_configuration_request) @@ -196,8 +248,10 @@ def test_gemini_realtime_transformation_generation_complete(): assert config is not None session_configuration_request = { - "model": "gemini-1.5-flash", - "generationConfig": {"responseModalities": ["AUDIO"]}, + "setup": { + "model": "gemini-1.5-flash", + "generationConfig": {"responseModalities": ["AUDIO"]}, + } } session_configuration_request_str = json.dumps(session_configuration_request) @@ -225,9 +279,9 @@ def test_gemini_realtime_transformation_generation_complete(): contains_audio_done_event = False for response in responses: if response["type"] == OpenAIRealtimeEventTypes.RESPONSE_AUDIO_DONE.value: - contains_audio_delta = True + contains_audio_done_event = True break - assert contains_audio_delta, "Expected audio delta event" + assert contains_audio_done_event, "Expected audio done event" def test_gemini_3_1_flash_live_preview_model_cost_map_entry(): @@ -242,3 +296,1211 @@ def test_gemini_3_1_flash_live_preview_model_cost_map_entry(): assert info.get("max_output_tokens") == 65536 assert "video" in info.get("supported_modalities", []) assert info.get("supports_function_calling") is True + + +def test_gemini_realtime_tool_call_transformation(): + """Test transformation of Gemini toolCall to OpenAI function_call_arguments.done format.""" + config = GeminiRealtimeConfig() + + # Gemini toolCall message format + gemini_tool_call = { + "toolCall": { + "functionCalls": [ + { + "id": "call_123", + "name": "get_weather", + "args": {"location": "San Francisco", "unit": "fahrenheit"}, + } + ] + } + } + + gemini_tool_call_str = json.dumps(gemini_tool_call) + logging_obj = MagicMock() + logging_obj.litellm_trace_id = "test-trace-123" + + # Transform the toolCall message + result = config.transform_realtime_response( + gemini_tool_call_str, + "gemini-2.5-flash", + logging_obj, + realtime_response_transform_input={ + "session_configuration_request": None, + "current_output_item_id": "item_123", + "current_response_id": "resp_123", + "current_conversation_id": None, + "current_delta_chunks": [], + "current_item_chunks": [], + "current_delta_type": None, + }, + ) + + print("Tool call transformation result:", json.dumps(result, indent=2)) + + # Verify the transformation + responses = result["response"] + assert len(responses) > 0, "Expected at least one response event" + + # Find the function_call_arguments.done event + function_call_event = None + for event in responses: + if event.get("type") == "response.function_call_arguments.done": + function_call_event = event + break + + assert ( + function_call_event is not None + ), "Expected function_call_arguments.done event" + assert function_call_event["call_id"] == "call_123" + assert function_call_event["name"] == "get_weather" + assert function_call_event["response_id"] == "resp_123" + assert function_call_event["item_id"] == "item_123_tool_0" + assert function_call_event["output_index"] == 0 + + # Verify arguments are properly serialized as JSON string + args = json.loads(function_call_event["arguments"]) + assert args["location"] == "San Francisco" + assert args["unit"] == "fahrenheit" + + +def test_gemini_realtime_session_update_with_tools(): + """Test transformation of OpenAI session.update with tools to Gemini setup format.""" + config = GeminiRealtimeConfig() + + # OpenAI format session update with tools + session_update = { + "type": "session.update", + "session": { + "instructions": "You are a helpful assistant with weather tools.", + "temperature": 0.7, + "max_response_output_tokens": 1024, + "modalities": ["audio"], + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather for a location.", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city name", + }, + "unit": { + "type": "string", + "enum": ["fahrenheit", "celsius"], + }, + }, + "required": ["location"], + }, + }, + } + ], + }, + } + + # Transform to Gemini format (first session.update, so setup should be sent) + messages = config.transform_realtime_request( + json.dumps(session_update), + "gemini-2.5-flash", + session_configuration_request=None, + ) + + assert len(messages) == 1, "Expected one setup message" + + gemini_setup = json.loads(messages[0]) + assert "setup" in gemini_setup + + setup_config = gemini_setup["setup"] + + # Verify tools are at top level, not in generationConfig + assert "tools" in setup_config + assert "tools" not in setup_config.get("generationConfig", {}) + + # Verify tool structure matches Gemini format + tools = setup_config["tools"] + assert len(tools) == 1 + assert "function_declarations" in tools[0] + + function_decl = tools[0]["function_declarations"][0] + assert function_decl["name"] == "get_weather" + assert "Get the current weather" in function_decl["description"] + assert "parameters" in function_decl + + +def test_gemini_session_update_defaults_to_audio_modality(): + config = GeminiRealtimeConfig() + + session_update = { + "type": "session.update", + "session": { + "instructions": "You are a helpful assistant.", + # No modalities on purpose + }, + } + + messages = config.transform_realtime_request( + json.dumps(session_update), + "gemini-2.5-flash", + session_configuration_request=None, + ) + + assert len(messages) == 1 + setup_payload = json.loads(messages[0])["setup"] + assert setup_payload["generationConfig"]["responseModalities"] == ["AUDIO"] + + +def test_gemini_requires_session_configuration_feature_flag(monkeypatch): + config = GeminiRealtimeConfig() + + # Default behavior remains backwards-compatible (auto setup on connect) + monkeypatch.setattr(litellm, "gemini_live_defer_setup", False, raising=False) + assert config.requires_session_configuration() is True + + # Opt-in behavior: defer setup until client sends session.update + monkeypatch.setattr(litellm, "gemini_live_defer_setup", True, raising=False) + assert config.requires_session_configuration() is False + + +def test_gemini_realtime_function_call_output_transformation(): + """Test transformation of OpenAI function_call_output to Gemini toolResponse format. + + Exercises the full production round-trip: a Gemini toolCall arrives first + and populates the call_id -> name mapping, then the OpenAI + function_call_output is transformed and must carry the function name back + to Gemini in functionResponses. + """ + config = GeminiRealtimeConfig() + + # Receive a toolCall from Gemini first to populate the call_id -> name mapping. + logging_obj = MagicMock() + logging_obj.litellm_trace_id = "trace_func_output" + config.transform_realtime_response( + json.dumps( + { + "toolCall": { + "functionCalls": [ + { + "id": "call_123", + "name": "get_weather", + "args": {"location": "San Francisco"}, + } + ] + } + } + ), + "gemini-2.5-flash", + logging_obj, + realtime_response_transform_input={ + "session_configuration_request": None, + "current_output_item_id": None, + "current_response_id": None, + "current_conversation_id": None, + "current_delta_chunks": [], + "current_item_chunks": [], + "current_delta_type": None, + }, + ) + assert config._tool_call_id_to_name.get("call_123") == "get_weather" + + # OpenAI format function call output + function_output = { + "type": "conversation.item.create", + "item": { + "type": "function_call_output", + "call_id": "call_123", + "output": json.dumps( + { + "location": "San Francisco", + "temperature": 72, + "unit": "fahrenheit", + "conditions": "sunny", + } + ), + }, + } + + # Transform to Gemini format + messages = config.transform_realtime_request( + json.dumps(function_output), + "gemini-2.5-flash", + session_configuration_request="existing", + ) + + assert len(messages) == 1, "Expected one toolResponse message" + + gemini_response = json.loads(messages[0]) + assert "toolResponse" in gemini_response + + tool_response = gemini_response["toolResponse"] + assert "functionResponses" in tool_response + assert len(tool_response["functionResponses"]) == 1 + + func_response = tool_response["functionResponses"][0] + assert func_response["id"] == "call_123" + assert func_response["name"] == "get_weather" + assert "response" in func_response + assert func_response["response"]["temperature"] == 72 + assert func_response["response"]["conditions"] == "sunny" + + # A retry of the same function_call_output (e.g. a client SDK that + # re-sends the result) must still produce a functionResponses payload + # carrying ``name`` — the call_id → name mapping must not be evicted + # after the first lookup. + retry_messages = config.transform_realtime_request( + json.dumps(function_output), + "gemini-2.5-flash", + session_configuration_request="existing", + ) + retry_response = json.loads(retry_messages[0])["toolResponse"]["functionResponses"][ + 0 + ] + assert retry_response["name"] == "get_weather" + + +def test_gemini_realtime_user_text_transformation(): + """Test transformation of OpenAI user message to Gemini clientContent format.""" + config = GeminiRealtimeConfig() + + # OpenAI format user message + user_message = { + "type": "conversation.item.create", + "item": { + "type": "message", + "role": "user", + "content": [ + {"type": "input_text", "text": "What's the weather in London?"} + ], + }, + } + + # Transform to Gemini format + messages = config.transform_realtime_request( + json.dumps(user_message), + "gemini-2.5-flash", + session_configuration_request="existing", + ) + + assert len(messages) == 1, "Expected one clientContent message" + + gemini_message = json.loads(messages[0]) + assert "clientContent" in gemini_message + + client_content = gemini_message["clientContent"] + assert "turns" in client_content + assert len(client_content["turns"]) == 1 + + turn = client_content["turns"][0] + assert turn["role"] == "user" + assert len(turn["parts"]) == 1 + assert turn["parts"][0]["text"] == "What's the weather in London?" + assert client_content["turnComplete"] is True + + +def test_return_new_content_delta_events_without_session_config_does_not_error(): + config = GeminiRealtimeConfig() + + events = config.return_new_content_delta_events( + response_id="resp_1", + output_item_id="item_1", + conversation_id="conv_1", + delta_type="text", + session_configuration_request=None, + ) + + assert len(events) >= 1 + assert events[0]["type"] == "response.created" + + +def test_gemini_realtime_multi_tool_calls_have_unique_item_ids(): + config = GeminiRealtimeConfig() + logging_obj = MagicMock() + logging_obj.litellm_trace_id = "test-trace-123" + + gemini_tool_call = { + "toolCall": { + "functionCalls": [ + { + "id": "call_1", + "name": "get_weather", + "args": {"location": "SF"}, + }, + { + "id": "call_2", + "name": "get_weather", + "args": {"location": "NYC"}, + }, + ] + } + } + + result = config.transform_realtime_response( + json.dumps(gemini_tool_call), + "gemini-2.5-flash", + logging_obj, + realtime_response_transform_input={ + "session_configuration_request": None, + "current_output_item_id": "item_123", + "current_response_id": "resp_123", + "current_conversation_id": None, + "current_delta_chunks": [], + "current_item_chunks": [], + "current_delta_type": None, + }, + ) + + responses = [ + ev + for ev in result["response"] + if ev.get("type") == "response.function_call_arguments.done" + ] + assert len(responses) == 2 + assert responses[0]["response_id"] == "resp_123" + assert responses[1]["response_id"] == "resp_123" + assert responses[0]["item_id"] == "item_123_tool_0" + assert responses[1]["item_id"] == "item_123_tool_1" + assert responses[0]["item_id"] != responses[1]["item_id"] + assert responses[0]["output_index"] == 0 + assert responses[1]["output_index"] == 1 + + +def test_gemini_session_update_includes_input_audio_transcription_default(): + """Verify _handle_session_update includes inputAudioTranscription default.""" + config = GeminiRealtimeConfig() + session_update = { + "type": "session.update", + "session": { + "modalities": ["text", "audio"], + "tools": [ + { + "type": "function", + "name": "get_weather", + "description": "Get weather", + "parameters": { + "type": "object", + "properties": {"location": {"type": "string"}}, + }, + } + ], + }, + } + + result = config.transform_realtime_request( + json.dumps(session_update), + "gemini-2.5-flash", + session_configuration_request=None, + ) + + assert len(result) == 1 + setup = json.loads(result[0]) + assert "setup" in setup + assert "inputAudioTranscription" in setup["setup"] + assert setup["setup"]["inputAudioTranscription"] == {} + + +def test_gemini_tool_call_emits_response_created_preamble(): + """Verify response.created is emitted before tool call events when response_id is None.""" + config = GeminiRealtimeConfig() + logging_obj = MagicMock() + logging_obj.litellm_trace_id = "trace_123" + + gemini_tool_call = { + "toolCall": { + "functionCalls": [ + { + "id": "call_123", + "name": "get_weather", + "args": {"location": "San Francisco", "unit": "fahrenheit"}, + } + ] + } + } + + # Transform with current_response_id=None to trigger preamble emission + result = config.transform_realtime_response( + json.dumps(gemini_tool_call), + "gemini-2.5-flash", + logging_obj, + realtime_response_transform_input={ + "session_configuration_request": None, + "current_output_item_id": None, + "current_response_id": None, + "current_conversation_id": None, + "current_delta_chunks": [], + "current_item_chunks": [], + "current_delta_type": None, + }, + ) + + responses = result["response"] + # Should have: response.created, output_item.added, function_call_arguments.delta, function_call_arguments.done, output_item.done, conversation.item.created, response.done + assert len(responses) >= 7 + assert responses[0]["type"] == "response.created" + assert "response" in responses[0] + assert responses[0]["response"]["status"] == "in_progress" + # response.created on the tool-call path mirrors the audio/text preamble: + # modalities/temperature/max_output_tokens are present so spec-compliant + # clients see consistent response metadata regardless of payload type. + assert "modalities" in responses[0]["response"] + assert "temperature" in responses[0]["response"] + assert "max_output_tokens" in responses[0]["response"] + assert responses[1]["type"] == "response.output_item.added" + assert responses[1]["item"]["type"] == "function_call" + assert responses[1]["item"]["status"] == "in_progress" + assert responses[2]["type"] == "response.function_call_arguments.delta" + assert responses[2]["call_id"] == "call_123" + assert responses[2]["delta"] == responses[3]["arguments"] + assert responses[3]["type"] == "response.function_call_arguments.done" + assert responses[4]["type"] == "response.output_item.done" + assert responses[4]["item"]["type"] == "function_call" + assert responses[4]["item"]["status"] == "completed" + assert responses[5]["type"] == "conversation.item.created" + assert responses[5]["item"]["type"] == "function_call" + assert responses[5]["item"]["status"] == "completed" + assert responses[6]["type"] == "response.done" + assert responses[6]["response"]["status"] == "completed" + assert len(responses[6]["response"]["output"]) == 1 + assert responses[6]["response"]["output"][0]["type"] == "function_call" + assert result["current_output_item_id"] is None + assert result["current_response_id"] is None + + +def test_gemini_tool_call_resets_ids_for_post_tool_model_turn(): + """After tool-call response.done, a subsequent modelTurn must emit response.created.""" + config = GeminiRealtimeConfig() + logging_obj = MagicMock() + logging_obj.litellm_trace_id = "trace_123" + + session_configuration_request = json.dumps( + { + "setup": { + "model": "gemini-1.5-flash", + "generationConfig": {"responseModalities": ["TEXT"]}, + } + } + ) + + tool_result = config.transform_realtime_response( + json.dumps( + { + "toolCall": { + "functionCalls": [ + { + "id": "call_123", + "name": "get_weather", + "args": {"location": "San Francisco"}, + } + ] + } + } + ), + "gemini-2.5-flash", + logging_obj, + realtime_response_transform_input={ + "session_configuration_request": session_configuration_request, + "current_output_item_id": None, + "current_response_id": None, + "current_conversation_id": None, + "current_delta_chunks": [], + "current_item_chunks": [], + "current_delta_type": None, + }, + ) + + tool_response_id = tool_result["response"][0]["response"]["id"] + assert tool_result["current_output_item_id"] is None + assert tool_result["current_response_id"] is None + + post_tool_result = config.transform_realtime_response( + json.dumps( + { + "serverContent": { + "modelTurn": {"parts": [{"text": "The weather is sunny."}]} + } + } + ), + "gemini-2.5-flash", + logging_obj, + realtime_response_transform_input={ + "session_configuration_request": session_configuration_request, + "current_output_item_id": tool_result["current_output_item_id"], + "current_response_id": tool_result["current_response_id"], + "current_conversation_id": tool_result["current_conversation_id"], + "current_delta_chunks": tool_result["current_delta_chunks"], + "current_item_chunks": tool_result["current_item_chunks"], + "current_delta_type": tool_result["current_delta_type"], + }, + ) + + post_tool_events = post_tool_result["response"] + assert post_tool_events[0]["type"] == "response.created" + assert post_tool_events[0]["response"]["id"] != tool_response_id + assert ( + post_tool_result["current_response_id"] == post_tool_events[0]["response"]["id"] + ) + + +def test_gemini_empty_tool_call_does_not_crash_websocket(): + """A toolCall payload with no functionCalls must not raise the + 'Unknown message type' guard — that would terminate the WebSocket session + on what is at worst a benign no-op from Gemini.""" + config = GeminiRealtimeConfig() + logging_obj = MagicMock() + logging_obj.litellm_trace_id = "trace_empty_tool_call" + + result = config.transform_realtime_response( + json.dumps({"toolCall": {"functionCalls": []}}), + "gemini-2.5-flash", + logging_obj, + realtime_response_transform_input={ + "session_configuration_request": None, + "current_output_item_id": None, + "current_response_id": None, + "current_conversation_id": None, + "current_delta_chunks": [], + "current_item_chunks": [], + "current_delta_type": None, + }, + ) + + assert result["response"] == [] + assert result["current_response_id"] is None + assert result["current_output_item_id"] is None + + +def test_gemini_empty_tool_call_with_sibling_usage_metadata_does_not_crash(): + """A toolCall with empty functionCalls alongside a sibling key (e.g. + ``usageMetadata``) must still be handled as a benign no-op: the empty + toolCall is consumed and the metadata sibling is skipped, without + raising ``Unknown message type``.""" + config = GeminiRealtimeConfig() + logging_obj = MagicMock() + logging_obj.litellm_trace_id = "trace_empty_tool_call_with_sibling" + + result = config.transform_realtime_response( + json.dumps( + { + "toolCall": {"functionCalls": []}, + "usageMetadata": {"totalTokenCount": 7}, + } + ), + "gemini-2.5-flash", + logging_obj, + realtime_response_transform_input={ + "session_configuration_request": None, + "current_output_item_id": "item_existing", + "current_response_id": "resp_existing", + "current_conversation_id": "conv_existing", + "current_delta_chunks": [], + "current_item_chunks": [], + "current_delta_type": None, + }, + ) + + assert result["response"] == [] + # In-flight response IDs must survive the benign no-op. + assert result["current_response_id"] == "resp_existing" + assert result["current_output_item_id"] == "item_existing" + + +def test_gemini_tool_call_response_done_includes_usage_from_sibling_metadata(): + """A ``toolCall`` frame with a sibling ``usageMetadata`` must propagate the + real token counts onto the emitted ``response.done`` so spend/budget + accounting records tokens consumed by the tool-call turn — otherwise an + authenticated client can repeatedly drive tool calls with zero spend.""" + config = GeminiRealtimeConfig() + logging_obj = MagicMock() + logging_obj.litellm_trace_id = "trace_tool_call_usage" + + result = config.transform_realtime_response( + json.dumps( + { + "toolCall": { + "functionCalls": [ + { + "id": "call_usage", + "name": "get_weather", + "args": {"location": "NYC"}, + } + ] + }, + "usageMetadata": { + "promptTokenCount": 17, + "responseTokenCount": 4, + "totalTokenCount": 21, + }, + } + ), + "gemini-2.5-flash", + logging_obj, + realtime_response_transform_input={ + "session_configuration_request": None, + "current_output_item_id": None, + "current_response_id": None, + "current_conversation_id": None, + "current_delta_chunks": [], + "current_item_chunks": [], + "current_delta_type": None, + }, + ) + + response_done = next( + ev for ev in result["response"] if ev.get("type") == "response.done" + ) + usage = response_done["response"]["usage"] + assert usage["input_tokens"] == 17 + assert usage["output_tokens"] == 4 + assert usage["total_tokens"] == 21 + + +def test_gemini_tool_call_response_done_falls_back_to_empty_usage(): + """Without sibling ``usageMetadata`` the tool-call ``response.done`` still + carries a valid empty usage block so OpenAI-compatible clients (which + expect ``usage`` on every ``response.done``) don't break.""" + config = GeminiRealtimeConfig() + logging_obj = MagicMock() + logging_obj.litellm_trace_id = "trace_tool_call_no_usage" + + result = config.transform_realtime_response( + json.dumps( + { + "toolCall": { + "functionCalls": [ + { + "id": "call_no_usage", + "name": "get_weather", + "args": {"location": "NYC"}, + } + ] + } + } + ), + "gemini-2.5-flash", + logging_obj, + realtime_response_transform_input={ + "session_configuration_request": None, + "current_output_item_id": None, + "current_response_id": None, + "current_conversation_id": None, + "current_delta_chunks": [], + "current_item_chunks": [], + "current_delta_type": None, + }, + ) + + response_done = next( + ev for ev in result["response"] if ev.get("type") == "response.done" + ) + usage = response_done["response"]["usage"] + assert usage["input_tokens"] == 0 + assert usage["output_tokens"] == 0 + assert usage["total_tokens"] == 0 + + +def test_gemini_function_call_output_includes_name(): + """Verify function_call_output includes name field from stored mapping.""" + config = GeminiRealtimeConfig() + + # First, receive a toolCall from Gemini (this stores the call_id → name mapping) + gemini_tool_call = { + "toolCall": { + "functionCalls": [ + { + "id": "call_123", + "name": "get_weather", + "args": {"location": "San Francisco"}, + } + ] + } + } + + logging_obj = MagicMock() + logging_obj.litellm_trace_id = "trace_123" + + config.transform_realtime_response( + json.dumps(gemini_tool_call), + "gemini-2.5-flash", + logging_obj, + realtime_response_transform_input={ + "session_configuration_request": None, + "current_output_item_id": None, + "current_response_id": None, + "current_conversation_id": None, + "current_delta_chunks": [], + "current_item_chunks": [], + "current_delta_type": None, + }, + ) + + # Verify mapping was stored + assert "call_123" in config._tool_call_id_to_name + assert config._tool_call_id_to_name["call_123"] == "get_weather" + + # Now send a function_call_output back (this should include the name) + function_output = { + "type": "conversation.item.create", + "item": { + "type": "function_call_output", + "call_id": "call_123", + "output": json.dumps({"result": "72 degrees"}), + }, + } + + result = config.transform_realtime_request( + json.dumps(function_output), + "gemini-2.5-flash", + session_configuration_request="{}", + ) + + assert len(result) == 1 + tool_response = json.loads(result[0]) + assert "toolResponse" in tool_response + assert "functionResponses" in tool_response["toolResponse"] + assert len(tool_response["toolResponse"]["functionResponses"]) == 1 + + function_response = tool_response["toolResponse"]["functionResponses"][0] + assert function_response["id"] == "call_123" + assert function_response["name"] == "get_weather" # ✅ Name is included + assert "response" in function_response + + +def test_gemini_subsequent_session_update_forwards_tools_merged_with_original_setup(): + """A client session.update sent after the auto-setup must forward tools/ + instructions as a follow-up setup, merged with the original setup so we + don't drop the pre-existing config (model, generationConfig, etc.).""" + config = GeminiRealtimeConfig() + + original_setup = { + "setup": { + "model": "models/gemini-2.5-flash-native-audio", + "generationConfig": {"responseModalities": ["AUDIO"]}, + "inputAudioTranscription": {}, + "systemInstruction": {"role": "user", "parts": [{"text": "original"}]}, + } + } + + session_update = { + "type": "session.update", + "session": { + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather.", + "parameters": { + "type": "object", + "properties": {"location": {"type": "string"}}, + "required": ["location"], + }, + }, + } + ], + "instructions": "Be concise.", + }, + } + + messages = config.transform_realtime_request( + json.dumps(session_update), + "gemini-2.5-flash-native-audio", + session_configuration_request=json.dumps(original_setup), + ) + + assert len(messages) == 1 + follow_up = json.loads(messages[0])["setup"] + assert "tools" in follow_up + assert follow_up["tools"][0]["function_declarations"][0]["name"] == "get_weather" + # systemInstruction overwritten by client's instructions + assert follow_up["systemInstruction"]["parts"][0]["text"] == "Be concise." + # Original generationConfig / model / inputAudioTranscription preserved + assert follow_up["generationConfig"]["responseModalities"] == ["AUDIO"] + assert follow_up["model"] == "models/gemini-2.5-flash-native-audio" + assert follow_up["inputAudioTranscription"] == {} + + +def test_gemini_subsequent_session_update_with_turn_detection_only_preserves_original_tools(): + """A subsequent session.update carrying only turn_detection (the + guardrail-injected disable) must keep the original tools/generationConfig.""" + config = GeminiRealtimeConfig() + + original_setup = { + "setup": { + "model": "models/gemini-2.5-flash-native-audio", + "generationConfig": {"responseModalities": ["AUDIO"]}, + "inputAudioTranscription": {}, + "tools": [ + { + "function_declarations": [ + {"name": "lookup", "description": "x", "parameters": {}} + ] + } + ], + } + } + + session_update = { + "type": "session.update", + "session": {"turn_detection": {"create_response": False}}, + } + + messages = config.transform_realtime_request( + json.dumps(session_update), + "gemini-2.5-flash-native-audio", + session_configuration_request=json.dumps(original_setup), + ) + + assert len(messages) == 1 + follow_up = json.loads(messages[0])["setup"] + assert follow_up["tools"] == original_setup["setup"]["tools"] + assert ( + follow_up["realtimeInputConfig"]["automaticActivityDetection"]["disabled"] + is True + ) + + +def test_gemini_follow_up_session_update_preserves_response_modalities_on_partial_generation_config(): + """A follow-up session.update that only sets `temperature` (or any other + generationConfig sub-field) must not wipe `responseModalities` from the + original setup.""" + config = GeminiRealtimeConfig() + + original_setup = { + "setup": { + "model": "models/gemini-2.5-flash-native-audio", + "generationConfig": { + "responseModalities": ["AUDIO"], + "maxOutputTokens": 2048, + }, + "inputAudioTranscription": {}, + } + } + + session_update = { + "type": "session.update", + "session": {"temperature": 0.7}, + } + + messages = config.transform_realtime_request( + json.dumps(session_update), + "gemini-2.5-flash-native-audio", + session_configuration_request=json.dumps(original_setup), + ) + + follow_up = json.loads(messages[0])["setup"] + assert follow_up["generationConfig"]["responseModalities"] == ["AUDIO"] + assert follow_up["generationConfig"]["maxOutputTokens"] == 2048 + assert follow_up["generationConfig"]["temperature"] == 0.7 + + +def test_gemini_subsequent_session_update_preserves_automatic_activity_detection_subfields(): + config = GeminiRealtimeConfig() + + original_setup = { + "setup": { + "model": "models/gemini-2.5-flash-native-audio", + "generationConfig": {"responseModalities": ["AUDIO"]}, + "realtimeInputConfig": { + "automaticActivityDetection": { + "disabled": False, + "silenceDurationMs": 500, + "prefixPaddingMs": 100, + } + }, + } + } + + session_update = { + "type": "session.update", + "session": {"turn_detection": {"create_response": False}}, + } + + messages = config.transform_realtime_request( + json.dumps(session_update), + "gemini-2.5-flash-native-audio", + session_configuration_request=json.dumps(original_setup), + ) + + automatic_activity_detection = json.loads(messages[0])["setup"][ + "realtimeInputConfig" + ]["automaticActivityDetection"] + assert automatic_activity_detection["disabled"] is True + assert automatic_activity_detection["silenceDurationMs"] == 500 + assert automatic_activity_detection["prefixPaddingMs"] == 100 + + +def test_gemini_tool_call_id_to_name_evicts_oldest_when_capped(): + """The call_id → name LRU must evict the oldest entry once the cap is + reached so long sessions with many tool calls don't grow unboundedly, + while keeping recently-seen call_ids resolvable for retried + function_call_output messages.""" + config = GeminiRealtimeConfig() + logging_obj = MagicMock() + logging_obj.litellm_trace_id = "trace_lru" + + config._TOOL_CALL_ID_TO_NAME_MAX = 4 + + for idx in range(8): + config.transform_realtime_response( + json.dumps( + { + "toolCall": { + "functionCalls": [ + { + "id": f"call_{idx}", + "name": f"fn_{idx}", + "args": {}, + } + ] + } + } + ), + "gemini-2.5-flash", + logging_obj, + realtime_response_transform_input={ + "session_configuration_request": None, + "current_output_item_id": None, + "current_response_id": None, + "current_conversation_id": None, + "current_delta_chunks": [], + "current_item_chunks": [], + "current_delta_type": None, + }, + ) + + assert len(config._tool_call_id_to_name) == 4 + # Most recent 4 retained; oldest 4 evicted. + assert list(config._tool_call_id_to_name) == [ + "call_4", + "call_5", + "call_6", + "call_7", + ] + + +def test_gemini_standalone_usage_metadata_does_not_crash_websocket(): + """A Gemini frame containing only sibling metadata (e.g. a standalone + ``usageMetadata`` block emitted between turns) must not trip the + ``Unknown message type`` guard — that would terminate the WebSocket + session on a benign no-op frame.""" + config = GeminiRealtimeConfig() + logging_obj = MagicMock() + logging_obj.litellm_trace_id = "trace_usage_only" + + result = config.transform_realtime_response( + json.dumps( + { + "usageMetadata": { + "promptTokenCount": 12, + "responseTokenCount": 34, + "totalTokenCount": 46, + } + } + ), + "gemini-2.5-flash", + logging_obj, + realtime_response_transform_input={ + "session_configuration_request": None, + "current_output_item_id": "item_existing", + "current_response_id": "resp_existing", + "current_conversation_id": "conv_existing", + "current_delta_chunks": [], + "current_item_chunks": [], + "current_delta_type": None, + }, + ) + + assert result["response"] == [] + # State must be returned unchanged so subsequent frames continue the + # in-flight response correctly. + assert result["current_output_item_id"] == "item_existing" + assert result["current_response_id"] == "resp_existing" + assert result["current_conversation_id"] == "conv_existing" + + +def test_gemini_standalone_usage_metadata_is_attributed_to_next_tool_call_response_done(): + """A standalone ``usageMetadata`` frame emitted between turns must not + silently drop the consumed tokens. The next tool-call ``response.done`` + must carry those token counts so an authenticated client cannot drive + tool-call turns whose token usage is recorded as zero, bypassing + spend/budget accounting.""" + config = GeminiRealtimeConfig() + logging_obj = MagicMock() + logging_obj.litellm_trace_id = "trace_standalone_usage_then_tool_call" + + standalone_result = config.transform_realtime_response( + json.dumps( + { + "usageMetadata": { + "promptTokenCount": 31, + "responseTokenCount": 9, + "totalTokenCount": 40, + } + } + ), + "gemini-2.5-flash", + logging_obj, + realtime_response_transform_input={ + "session_configuration_request": None, + "current_output_item_id": None, + "current_response_id": None, + "current_conversation_id": None, + "current_delta_chunks": [], + "current_item_chunks": [], + "current_delta_type": None, + }, + ) + assert standalone_result["response"] == [] + + tool_call_result = config.transform_realtime_response( + json.dumps( + { + "toolCall": { + "functionCalls": [ + { + "id": "call_buffered", + "name": "get_weather", + "args": {"location": "NYC"}, + } + ] + } + } + ), + "gemini-2.5-flash", + logging_obj, + realtime_response_transform_input={ + "session_configuration_request": None, + "current_output_item_id": None, + "current_response_id": None, + "current_conversation_id": None, + "current_delta_chunks": [], + "current_item_chunks": [], + "current_delta_type": None, + }, + ) + + response_done = next( + ev for ev in tool_call_result["response"] if ev.get("type") == "response.done" + ) + usage = response_done["response"]["usage"] + assert usage["input_tokens"] == 31 + assert usage["output_tokens"] == 9 + assert usage["total_tokens"] == 40 + # Buffer must be cleared after attribution so a subsequent tool-call + # turn without its own usage does not double-count the previous frame. + assert config._pending_usage_metadata is None + + +def test_gemini_standalone_usage_metadata_is_attributed_to_next_response_done(): + """A standalone ``usageMetadata`` frame must also flow into the normal + (non-tool-call) ``response.done`` path so audio/text turns whose usage + arrives in a separate frame are still billed correctly.""" + config = GeminiRealtimeConfig() + logging_obj = MagicMock() + logging_obj.litellm_trace_id = "trace_standalone_usage_then_turn_complete" + + config.transform_realtime_response( + json.dumps( + { + "usageMetadata": { + "promptTokenCount": 5, + "responseTokenCount": 11, + "totalTokenCount": 16, + } + } + ), + "gemini-2.5-flash", + logging_obj, + realtime_response_transform_input={ + "session_configuration_request": None, + "current_output_item_id": None, + "current_response_id": None, + "current_conversation_id": None, + "current_delta_chunks": [], + "current_item_chunks": [], + "current_delta_type": None, + }, + ) + + turn_complete_result = config.transform_realtime_response( + json.dumps({"serverContent": {"turnComplete": True}}), + "gemini-2.5-flash", + logging_obj, + realtime_response_transform_input={ + "session_configuration_request": None, + "current_output_item_id": None, + "current_response_id": None, + "current_conversation_id": None, + "current_delta_chunks": [], + "current_item_chunks": [], + "current_delta_type": None, + }, + ) + + response_done = next( + ev + for ev in turn_complete_result["response"] + if ev.get("type") == "response.done" + ) + usage = response_done["response"]["usage"] + assert usage["input_tokens"] == 5 + assert usage["output_tokens"] == 11 + assert usage["total_tokens"] == 16 + assert config._pending_usage_metadata is None + + +def test_gemini_in_frame_usage_metadata_clears_pending_buffer(): + """When ``usageMetadata`` arrives in the same frame as the closing + ``toolCall`` / ``turnComplete``, the in-frame counts are authoritative + and any buffered standalone metadata must be discarded so a later + turn's ``response.done`` does not double-count tokens.""" + config = GeminiRealtimeConfig() + config._pending_usage_metadata = { + "promptTokenCount": 99, + "responseTokenCount": 99, + "totalTokenCount": 198, + } + logging_obj = MagicMock() + logging_obj.litellm_trace_id = "trace_in_frame_clears_buffer" + + result = config.transform_realtime_response( + json.dumps( + { + "toolCall": { + "functionCalls": [ + { + "id": "call_in_frame", + "name": "get_weather", + "args": {"location": "NYC"}, + } + ] + }, + "usageMetadata": { + "promptTokenCount": 3, + "responseTokenCount": 2, + "totalTokenCount": 5, + }, + } + ), + "gemini-2.5-flash", + logging_obj, + realtime_response_transform_input={ + "session_configuration_request": None, + "current_output_item_id": None, + "current_response_id": None, + "current_conversation_id": None, + "current_delta_chunks": [], + "current_item_chunks": [], + "current_delta_type": None, + }, + ) + + response_done = next( + ev for ev in result["response"] if ev.get("type") == "response.done" + ) + usage = response_done["response"]["usage"] + assert usage["input_tokens"] == 3 + assert usage["output_tokens"] == 2 + assert usage["total_tokens"] == 5 + assert config._pending_usage_metadata is None diff --git a/tests/test_litellm/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py b/tests/test_litellm/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py index 1baaf912568..0ad614099de 100644 --- a/tests/test_litellm/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py @@ -19,6 +19,7 @@ import websockets.exceptions # registers websockets.exceptions on the websocket sys.path.insert(0, os.path.abspath("../../../../..")) +import litellm from litellm.llms.vertex_ai.realtime.transformation import VertexAIRealtimeConfig # --------------------------------------------------------------------------- @@ -82,6 +83,85 @@ def test_session_configuration_request_model_format(): ) +def test_vertex_requires_session_configuration_feature_flag(monkeypatch): + cfg = VertexAIRealtimeConfig( + access_token="tok", project="my-proj", location="us-central1" + ) + + # Default remains backwards-compatible (auto setup on connect) + monkeypatch.setattr(litellm, "gemini_live_defer_setup", False, raising=False) + assert cfg.requires_session_configuration() is True + + # Opt-in deferred setup for tool-injection flow + monkeypatch.setattr(litellm, "gemini_live_defer_setup", True, raising=False) + assert cfg.requires_session_configuration() is False + + +def test_vertex_session_update_defaults_to_audio_modality(): + cfg = VertexAIRealtimeConfig( + access_token="tok", project="my-proj", location="us-central1" + ) + + session_update = { + "type": "session.update", + "session": { + "instructions": "You are a helpful assistant.", + # No modalities provided on purpose + }, + } + + messages = cfg.transform_realtime_request( + json.dumps(session_update), + "gemini-live-2.5-flash-native-audio", + session_configuration_request=None, + ) + assert len(messages) == 1 + setup_payload = json.loads(messages[0])["setup"] + assert setup_payload["generationConfig"]["responseModalities"] == ["AUDIO"] + + +def test_vertex_session_update_normalizes_ga_remapped_fields(): + """GA-format clients send ``output_modalities`` and nested + ``audio.input.transcription`` / ``audio.input.turn_detection``. These must + be normalised back to the flat beta keys before ``map_openai_params`` + runs so client preferences aren't silently dropped. + """ + cfg = VertexAIRealtimeConfig( + access_token="tok", project="my-proj", location="us-central1" + ) + + session_update = { + "type": "session.update", + "session": { + "instructions": "Be concise.", + "output_modalities": ["text"], + "audio": { + "input": { + "transcription": {}, + "turn_detection": {"silence_duration_ms": 1500}, + }, + }, + }, + } + + messages = cfg.transform_realtime_request( + json.dumps(session_update), + "gemini-live-2.5-flash-native-audio", + session_configuration_request=None, + ) + assert len(messages) == 1 + setup_payload = json.loads(messages[0])["setup"] + + assert setup_payload["generationConfig"]["responseModalities"] == ["TEXT"] + assert setup_payload["inputAudioTranscription"] == {} + assert ( + setup_payload["realtimeInputConfig"]["automaticActivityDetection"][ + "silenceDurationMs" + ] + == 1500 + ) + + # --------------------------------------------------------------------------- # Round-trip test: text-in / text-out via RealTimeStreaming # --------------------------------------------------------------------------- @@ -208,3 +288,61 @@ async def test_vertex_realtime_text_in_text_out(): # response.done should have been forwarded done_msgs = [m for m in sent_to_client if '"response.done"' in m] assert done_msgs, "Expected response.done to be sent to client" + + +def test_vertex_warns_when_dropping_guardrail_turn_detection_update(caplog): + """A subsequent session.update carrying the guardrail's + ``create_response: False`` cannot be forwarded as a follow-up setup on + Vertex AI (1007). Surface a warning so operators know the auto-response + suppression is being silently dropped.""" + import logging + + cfg = VertexAIRealtimeConfig( + access_token="tok", project="my-proj", location="us-central1" + ) + + session_update = { + "type": "session.update", + "session": {"turn_detection": {"create_response": False}}, + } + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + result = cfg.transform_realtime_request( + json.dumps(session_update), + "gemini-live-2.5-flash-native-audio", + session_configuration_request=json.dumps({"setup": {"model": "x"}}), + ) + + assert result == [] + assert any( + "Vertex AI Realtime" in record.message + and "create_response=False" in record.message + for record in caplog.records + ) + + +def test_vertex_does_not_warn_when_dropping_non_guardrail_session_update(caplog): + """A subsequent session.update without ``create_response: False`` is a + routine drop and should stay at debug level (no warning).""" + import logging + + cfg = VertexAIRealtimeConfig( + access_token="tok", project="my-proj", location="us-central1" + ) + + session_update = { + "type": "session.update", + "session": {"instructions": "Be concise."}, + } + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + cfg.transform_realtime_request( + json.dumps(session_update), + "gemini-live-2.5-flash-native-audio", + session_configuration_request=json.dumps({"setup": {"model": "x"}}), + ) + + assert not any( + "Vertex AI Realtime" in record.message and "session.update" in record.message + for record in caplog.records + ) diff --git a/tests/test_litellm/llms/vertex_ai/videos/test_vertex_video_transformation.py b/tests/test_litellm/llms/vertex_ai/videos/test_vertex_video_transformation.py index 70583cfe61b..55197d3165c 100644 --- a/tests/test_litellm/llms/vertex_ai/videos/test_vertex_video_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/videos/test_vertex_video_transformation.py @@ -456,6 +456,127 @@ class TestVertexAIVideoConfig: raw_response=mock_response, logging_obj=self.mock_logging_obj ) + def test_get_video_edit_prefetch_params(self): + """Test that prefetch params returns the fetchPredictOperation URL and body.""" + operation_name = "projects/test-project/locations/us-central1/publishers/google/models/veo-3.1-generate-001/operations/op-123" + api_base = "https://us-central1-aiplatform.googleapis.com/v1/projects/test-project/locations/us-central1/publishers/google/models" + + fetch_url, fetch_body = self.config.get_video_edit_prefetch_params( + video_id=operation_name, + api_base=api_base, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert "fetchPredictOperation" in fetch_url + assert "veo-3.1-generate-001" in fetch_url + assert fetch_body == {"operationName": operation_name} + + def test_transform_video_edit_request_with_bytes(self): + """Test video edit request builds predictLongRunning body from pre-fetched bytes.""" + operation_name = "projects/test-project/locations/us-central1/publishers/google/models/veo-3.1-generate-001/operations/op-123" + api_base = "https://us-central1-aiplatform.googleapis.com/v1/projects/test-project/locations/us-central1/publishers/google/models" + fake_bytes = base64.b64encode(b"fake_video").decode() + + prefetched = { + "done": True, + "response": { + "videos": [{"bytesBase64Encoded": fake_bytes, "mimeType": "video/mp4"}] + }, + } + + url, data = self.config.transform_video_edit_request( + prompt="Make it brighter", + video_id=operation_name, + api_base=api_base, + litellm_params=GenericLiteLLMParams(), + headers={"Authorization": "Bearer token"}, + prefetched_source_data=prefetched, + ) + + assert url.endswith(":predictLongRunning") + assert "veo-3.1-generate-001" in url + instance = data["instances"][0] + assert instance["prompt"] == "Make it brighter" + assert instance["video"]["bytesBase64Encoded"] == fake_bytes + assert instance["video"]["mimeType"] == "video/mp4" + + def test_transform_video_edit_request_with_gcs_uri(self): + """Test that gcsUri is used when present in source video.""" + operation_name = "projects/test-project/locations/us-central1/publishers/google/models/veo-3.1-generate-001/operations/op-456" + api_base = "https://us-central1-aiplatform.googleapis.com/v1/projects/test-project/locations/us-central1/publishers/google/models" + + prefetched = { + "done": True, + "response": { + "videos": [{"gcsUri": "gs://bucket/video.mp4", "mimeType": "video/mp4"}] + }, + } + + _, data = self.config.transform_video_edit_request( + prompt="Make it darker", + video_id=operation_name, + api_base=api_base, + litellm_params=GenericLiteLLMParams(), + headers={}, + prefetched_source_data=prefetched, + ) + + assert data["instances"][0]["video"] == {"gcsUri": "gs://bucket/video.mp4"} + + def test_transform_video_edit_request_source_not_done_raises(self): + """Test that editing an in-progress video raises a clear error.""" + operation_name = "projects/test-project/locations/us-central1/publishers/google/models/veo-3.1-generate-001/operations/op-789" + api_base = "https://us-central1-aiplatform.googleapis.com/v1/projects/test-project/locations/us-central1/publishers/google/models" + + with pytest.raises(ValueError, match="not complete yet"): + self.config.transform_video_edit_request( + prompt="Make it brighter", + video_id=operation_name, + api_base=api_base, + litellm_params=GenericLiteLLMParams(), + headers={}, + prefetched_source_data={"done": False}, + ) + + def test_transform_video_edit_response(self): + """Test that edit response returns a processing VideoObject with encoded ID.""" + operation_name = "projects/test-project/locations/us-central1/publishers/google/models/veo-3.1-generate-001/operations/new-op-123" + mock_response = Mock(spec=httpx.Response) + mock_response.json.return_value = {"name": operation_name} + + video_obj = self.config.transform_video_edit_response( + raw_response=mock_response, + logging_obj=self.mock_logging_obj, + custom_llm_provider="vertex_ai", + ) + + assert isinstance(video_obj, VideoObject) + assert video_obj.status == "processing" + assert video_obj.id + assert video_obj.model == "veo-3.1-generate-001" + + def test_transform_video_edit_response_includes_usage_for_cost(self): + """Edit responses include duration/resolution usage for spend accounting.""" + operation_name = "projects/test-project/locations/us-central1/publishers/google/models/veo-3.1-generate-001/operations/new-op-123" + mock_response = Mock(spec=httpx.Response) + mock_response.json.return_value = {"name": operation_name} + request_data = { + "instances": [{"prompt": "Make it brighter", "video": {}}], + "parameters": {"durationSeconds": 8, "resolution": "1080p"}, + } + + video_obj = self.config.transform_video_edit_response( + raw_response=mock_response, + logging_obj=self.mock_logging_obj, + custom_llm_provider="vertex_ai", + request_data=request_data, + ) + + assert video_obj.usage is not None + assert video_obj.usage["duration_seconds"] == 8.0 + assert video_obj.usage["video_resolution"] == "1080p" + def test_transform_video_remix_request_not_supported(self): """Test that video remix raises NotImplementedError.""" with pytest.raises(NotImplementedError, match="Video remix is not supported"): diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py b/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py index 54a36eac2a1..1499f7e474f 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py @@ -2444,13 +2444,14 @@ async def test_get_team_object_permission_with_core_auth_auto_loading(): @pytest.mark.asyncio async def test_get_allowed_mcp_servers_for_team_uses_helper(): """ - Test that _get_allowed_mcp_servers_for_team properly uses _get_team_object_permission - helper which handles both loaded and unloaded object_permission cases. + Test that _get_allowed_mcp_servers_for_team resolves both legacy + object_permission fields (mcp_servers, mcp_access_groups) and the unified + team.access_group_ids → access_mcp_server_ids path. """ from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - from litellm.proxy._types import LiteLLM_ObjectPermissionTable + from litellm.proxy._types import LiteLLM_ObjectPermissionTable, LiteLLM_TeamTable from litellm.types.mcp import MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -2464,53 +2465,51 @@ async def test_get_allowed_mcp_servers_for_team_uses_helper(): transport=MCPTransport.http, ) try: - # Create mock object permission with servers and access groups mock_object_permission = LiteLLM_ObjectPermissionTable( object_permission_id="perm-789", mcp_servers=["direct-server1", "direct-server2"], mcp_access_groups=["dev-group"], vector_stores=[], ) + mock_team = LiteLLM_TeamTable( + team_id="team-789", + access_group_ids=[], + object_permission_id="perm-789", + ) + mock_team.object_permission = mock_object_permission - # Create mock user auth mock_user_auth = UserAPIKeyAuth( api_key="test-key", user_id="test-user", team_id="team-789", ) - # Mock the helper methods - with patch.object( - MCPRequestHandler, "_get_team_object_permission" - ) as mock_get_team_perm: - with patch.object( - MCPRequestHandler, "_get_mcp_servers_from_access_groups" - ) as mock_get_access_group_servers: - # Configure mocks - mock_get_team_perm.return_value = mock_object_permission - mock_get_access_group_servers.return_value = [ - "group-server1", - "group-server2", - ] + with ( + patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), + patch( + "litellm.proxy.auth.auth_checks.get_team_object", + new_callable=AsyncMock, + return_value=mock_team, + ), + patch.object( + MCPRequestHandler, + "_get_mcp_servers_from_access_groups", + new_callable=AsyncMock, + return_value=["group-server1", "group-server2"], + ) as mock_get_access_group_servers, + ): + result = await MCPRequestHandler._get_allowed_mcp_servers_for_team( + mock_user_auth + ) - # Call the method - result = await MCPRequestHandler._get_allowed_mcp_servers_for_team( - mock_user_auth - ) + assert set(result) == { + "direct-server1", + "direct-server2", + "group-server1", + "group-server2", + } - # Assert the result contains both direct and access group servers - assert set(result) == { - "direct-server1", - "direct-server2", - "group-server1", - "group-server2", - } - - # Verify _get_team_object_permission was called (the helper we fixed) - mock_get_team_perm.assert_called_once_with(mock_user_auth) - - # Verify access groups were resolved - mock_get_access_group_servers.assert_called_once_with(["dev-group"]) + mock_get_access_group_servers.assert_called_once_with(["dev-group"]) finally: for sid in ("direct-server1", "direct-server2"): global_mcp_server_manager.registry.pop(sid, None) @@ -2520,32 +2519,36 @@ async def test_get_allowed_mcp_servers_for_team_uses_helper(): async def test_get_allowed_mcp_servers_for_team_with_no_object_permission(): """ Test that _get_allowed_mcp_servers_for_team returns empty list when - team has no object_permission. + the team has no object_permission and no access_group_ids. """ - # Create mock user auth + from litellm.proxy._types import LiteLLM_TeamTable + + mock_team = LiteLLM_TeamTable( + team_id="team-no-perm", + access_group_ids=[], + object_permission_id=None, + ) + mock_user_auth = UserAPIKeyAuth( api_key="test-key", user_id="test-user", team_id="team-no-perm", ) - # Mock the helper to return None (no object permission) - with patch.object( - MCPRequestHandler, "_get_team_object_permission" - ) as mock_get_team_perm: - mock_get_team_perm.return_value = None - - # Call the method + with ( + patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), + patch( + "litellm.proxy.auth.auth_checks.get_team_object", + new_callable=AsyncMock, + return_value=mock_team, + ), + ): result = await MCPRequestHandler._get_allowed_mcp_servers_for_team( mock_user_auth ) - # Assert empty list is returned assert result == [] - # Verify the helper was called - mock_get_team_perm.assert_called_once_with(mock_user_auth) - @pytest.mark.asyncio async def test_get_allowed_mcp_servers_for_team_without_user_auth_returns_empty(): @@ -3456,3 +3459,185 @@ async def test_get_allowed_mcp_servers_no_union_when_no_authorized_extras(): # key ∩ team = {} (no overlap), extras = [] → final = [] result = await MCPRequestHandler.get_allowed_mcp_servers(auth) assert result == [] + + +# --------------------------------------------------------------------------- +# Issue #27657: team unified access_group_ids resolve to MCP servers +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_team_access_group_ids_resolve_to_mcp_servers(): + """A virtual key with empty access_group_ids inherits MCP servers from + its team's access_group_ids (mirror of the model-side resolution). + + Reproduction of https://github.com/BerriAI/litellm/issues/27657: + the runtime used to ignore team.access_group_ids when computing the + MCP scope, so virtual keys saw empty server lists even when their + team had an MCP-granting access group attached. + """ + from litellm.proxy._types import LiteLLM_TeamTable + + mock_team = LiteLLM_TeamTable( + team_id="team-a", + access_group_ids=["mcp-premium"], + object_permission_id=None, + ) + + auth = UserAPIKeyAuth( + token="test-token-hash", + api_key="sk-test", + team_id="team-a", + access_group_ids=[], + ) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), + patch( + "litellm.proxy.auth.auth_checks.get_team_object", + new_callable=AsyncMock, + return_value=mock_team, + ), + patch( + "litellm.proxy.auth.auth_checks._get_mcp_server_ids_from_access_groups", + new_callable=AsyncMock, + return_value=["srv-stripe"], + ) as mock_resolver, + ): + result = await MCPRequestHandler._get_allowed_mcp_servers_for_team(auth) + + assert result == ["srv-stripe"] + mock_resolver.assert_called_once() + assert mock_resolver.call_args.kwargs["access_group_ids"] == ["mcp-premium"] + + +@pytest.mark.asyncio +async def test_team_access_group_ids_union_with_object_permission(): + """When both legacy object_permission and unified team.access_group_ids + grant MCP servers, the final list is their union.""" + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + from litellm.proxy._types import LiteLLM_ObjectPermissionTable, LiteLLM_TeamTable + from litellm.types.mcp import MCPTransport + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + for sid in ("srv-direct",): + global_mcp_server_manager.registry[sid] = MCPServer( + server_id=sid, + name=sid, + server_name=sid, + url=f"https://{sid}.example.com", + transport=MCPTransport.http, + ) + try: + mock_object_permission = LiteLLM_ObjectPermissionTable( + object_permission_id="perm-1", + mcp_servers=["srv-direct"], + mcp_access_groups=[], + vector_stores=[], + ) + mock_team = LiteLLM_TeamTable( + team_id="team-a", + access_group_ids=["mcp-premium"], + object_permission_id="perm-1", + ) + mock_team.object_permission = mock_object_permission + + auth = UserAPIKeyAuth( + token="test-token-hash", + api_key="sk-test", + team_id="team-a", + ) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), + patch( + "litellm.proxy.auth.auth_checks.get_team_object", + new_callable=AsyncMock, + return_value=mock_team, + ), + patch( + "litellm.proxy.auth.auth_checks._get_mcp_server_ids_from_access_groups", + new_callable=AsyncMock, + return_value=["srv-stripe"], + ), + ): + result = await MCPRequestHandler._get_allowed_mcp_servers_for_team(auth) + + assert set(result) == {"srv-direct", "srv-stripe"} + finally: + global_mcp_server_manager.registry.pop("srv-direct", None) + + +@pytest.mark.asyncio +async def test_team_access_group_ids_empty_returns_no_extras(): + """Empty team.access_group_ids → resolver called with [], short-circuits + without DB access, no extras added.""" + from litellm.proxy._types import LiteLLM_TeamTable + + mock_team = LiteLLM_TeamTable( + team_id="team-a", + access_group_ids=[], + object_permission_id=None, + ) + + auth = UserAPIKeyAuth( + token="test-token-hash", + api_key="sk-test", + team_id="team-a", + ) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), + patch( + "litellm.proxy.auth.auth_checks.get_team_object", + new_callable=AsyncMock, + return_value=mock_team, + ), + patch( + "litellm.proxy.auth.auth_checks._get_mcp_server_ids_from_access_groups", + new_callable=AsyncMock, + return_value=[], + ) as mock_resolver, + ): + result = await MCPRequestHandler._get_allowed_mcp_servers_for_team(auth) + + assert result == [] + mock_resolver.assert_called_once() + assert mock_resolver.call_args.kwargs["access_group_ids"] == [] + + +@pytest.mark.asyncio +async def test_get_allowed_mcp_servers_includes_team_access_group_extras_end_to_end(): + """End-to-end: virtual key has nothing of its own, team has an MCP + access group → key sees the granted server through get_allowed_mcp_servers.""" + auth = UserAPIKeyAuth( + token="test-token", + api_key="sk-test", + team_id="team-a", + access_group_ids=[], + ) + + with ( + patch.object( + MCPRequestHandler, + "_get_allowed_mcp_servers_for_key", + new_callable=AsyncMock, + return_value=[], + ), + patch.object( + MCPRequestHandler, + "_get_allowed_mcp_servers_for_team", + new_callable=AsyncMock, + return_value=["srv-stripe"], + ), + patch.object( + MCPRequestHandler, + "_get_key_access_group_mcp_server_extras", + new_callable=AsyncMock, + return_value=[], + ), + ): + result = await MCPRequestHandler.get_allowed_mcp_servers(auth) + assert result == ["srv-stripe"] diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_jwt_mcp_enforcement.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_jwt_mcp_enforcement.py index c2e42d2f592..d8e4a342e52 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_jwt_mcp_enforcement.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_jwt_mcp_enforcement.py @@ -462,6 +462,9 @@ async def test_e2e_jwt_team_mcp_key_intersection(monkeypatch): monkeypatch.setattr( "litellm.proxy.auth.handle_jwt.get_team_object", mock_get_team_object ) + monkeypatch.setattr( + "litellm.proxy.auth.auth_checks.get_team_object", mock_get_team_object + ) jwt_handler = JWTHandler() jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth(team_ids_jwt_field="groups") @@ -495,28 +498,25 @@ async def test_e2e_jwt_team_mcp_key_intersection(monkeypatch): object_permission=key_object_permission, # Key has its own permissions ) - # Mock the helper methods to return our test data - with patch.object( - MCPRequestHandler, "_get_team_object_permission" - ) as mock_team_perm: - mock_team_perm.return_value = team_object_permission + with ( + patch.object( + MCPRequestHandler, + "_get_key_object_permission", + return_value=key_object_permission, + ), + patch.object( + MCPRequestHandler, + "_get_mcp_servers_from_access_groups", + new_callable=AsyncMock, + return_value=[], + ), + ): + allowed_servers = await MCPRequestHandler.get_allowed_mcp_servers( + user_api_key_auth + ) - with patch.object( - MCPRequestHandler, "_get_key_object_permission" - ) as mock_key_perm: - mock_key_perm.return_value = key_object_permission - - with patch.object( - MCPRequestHandler, "_get_mcp_servers_from_access_groups" - ) as mock_access_groups: - mock_access_groups.return_value = [] - - allowed_servers = await MCPRequestHandler.get_allowed_mcp_servers( - user_api_key_auth - ) - - # Should be intersection: only server-2 is in both - expected = ["server-2"] - assert sorted(allowed_servers) == sorted( - expected - ), f"Expected intersection {expected}, got {allowed_servers}" + # Should be intersection: only server-2 is in both + expected = ["server-2"] + assert sorted(allowed_servers) == sorted( + expected + ), f"Expected intersection {expected}, got {allowed_servers}" diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_jwt_mcp_simple.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_jwt_mcp_simple.py index 2ae575b6d99..052231b562a 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_jwt_mcp_simple.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_jwt_mcp_simple.py @@ -41,37 +41,44 @@ async def test_simple_jwt_mcp_permissions_enforced(): object_permission_id="perm-123", mcp_servers=team_mcp_servers, ) + team_obj = LiteLLM_TeamTable( + team_id="my-team", + access_group_ids=[], + object_permission_id="perm-123", + ) + team_obj.object_permission = team_object_permission - # 3. Mock the team permission lookup - with patch.object( - MCPRequestHandler, "_get_team_object_permission", new_callable=AsyncMock - ) as mock_team_perm: - mock_team_perm.return_value = team_object_permission + # 3. Mock the team object lookup (object_permission attached) and prisma_client + with ( + patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), + patch( + "litellm.proxy.auth.auth_checks.get_team_object", + new_callable=AsyncMock, + return_value=team_obj, + ) as mock_get_team, + patch.object( + MCPRequestHandler, + "_get_key_object_permission", + new_callable=AsyncMock, + return_value=None, + ), + patch.object( + MCPRequestHandler, + "_get_mcp_servers_from_access_groups", + new_callable=AsyncMock, + return_value=[], + ), + ): + # 4. Call get_allowed_mcp_servers - this is what MCP routes use + allowed = await MCPRequestHandler.get_allowed_mcp_servers(user_auth) - # Mock key permissions (empty - user has no key-level MCP permissions) - with patch.object( - MCPRequestHandler, "_get_key_object_permission", new_callable=AsyncMock - ) as mock_key_perm: - mock_key_perm.return_value = None + # 5. Verify only team's MCP servers are returned + assert sorted(allowed) == sorted( + team_mcp_servers + ), f"Expected {team_mcp_servers}, got {allowed}" - # Mock access groups (empty) - with patch.object( - MCPRequestHandler, - "_get_mcp_servers_from_access_groups", - new_callable=AsyncMock, - ) as mock_access_groups: - mock_access_groups.return_value = [] - - # 4. Call get_allowed_mcp_servers - this is what MCP routes use - allowed = await MCPRequestHandler.get_allowed_mcp_servers(user_auth) - - # 5. Verify only team's MCP servers are returned - assert sorted(allowed) == sorted( - team_mcp_servers - ), f"Expected {team_mcp_servers}, got {allowed}" - - # Verify team permission was looked up - mock_team_perm.assert_called_once_with(user_auth) + # Verify team was looked up + mock_get_team.assert_called() @pytest.mark.asyncio @@ -120,25 +127,33 @@ async def test_simple_jwt_team_id_required_for_mcp_permissions(): object_permission_id="perm-1", mcp_servers=team_mcp_servers, ) + team_obj = LiteLLM_TeamTable( + team_id="team-abc", + access_group_ids=[], + object_permission_id="perm-1", + ) + team_obj.object_permission = team_perm - with patch.object( - MCPRequestHandler, "_get_team_object_permission", new_callable=AsyncMock - ) as mock_perm: - mock_perm.return_value = team_perm - - with patch.object( + with ( + patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), + patch( + "litellm.proxy.auth.auth_checks.get_team_object", + new_callable=AsyncMock, + return_value=team_obj, + ) as mock_get_team, + patch.object( MCPRequestHandler, "_get_mcp_servers_from_access_groups", new_callable=AsyncMock, - ) as mock_groups: - mock_groups.return_value = [] + return_value=[], + ), + ): + result = await MCPRequestHandler._get_allowed_mcp_servers_for_team( + user_with_team + ) - result = await MCPRequestHandler._get_allowed_mcp_servers_for_team( - user_with_team - ) - - assert sorted(result) == sorted(team_mcp_servers) - mock_perm.assert_called_once() # Permission WAS checked + assert sorted(result) == sorted(team_mcp_servers) + mock_get_team.assert_called() # Team WAS looked up # Case 2: team_id is None -> team permissions NOT checked user_without_team = UserAPIKeyAuth( diff --git a/tests/test_litellm/proxy/auth/test_auth_checks.py b/tests/test_litellm/proxy/auth/test_auth_checks.py index 155bc198c98..52e7a0ff373 100644 --- a/tests/test_litellm/proxy/auth/test_auth_checks.py +++ b/tests/test_litellm/proxy/auth/test_auth_checks.py @@ -1625,6 +1625,50 @@ async def test_reject_clientside_metadata_tags_non_llm_route(): assert result is True +@pytest.mark.asyncio +async def test_reject_clientside_metadata_tags_allows_key_tags_without_client_tags(): + """Key metadata.tags are injected after the reject check; requests without + client metadata.tags must not be blocked when reject_clientside_metadata_tags is on.""" + from fastapi import Request + + from litellm.proxy.auth.auth_checks import common_checks + + request_body = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "test"}], + } + + general_settings = {"reject_clientside_metadata_tags": True} + mock_request = MagicMock(spec=Request) + valid_token = UserAPIKeyAuth( + token="test-token", + models=["gpt-3.5-turbo"], + metadata={"tags": ["engineering"]}, + ) + + with patch( + "litellm.proxy.auth.auth_checks.get_tag_objects_batch", + new_callable=AsyncMock, + return_value={}, + ): + result = await common_checks( + request_body=request_body, + team_object=None, + user_object=None, + end_user_object=None, + global_proxy_spend=None, + general_settings=general_settings, + route="/chat/completions", + llm_router=None, + proxy_logging_obj=MagicMock(), + valid_token=valid_token, + request=mock_request, + ) + + assert result is True + assert request_body["metadata"]["tags"] == ["engineering"] + + @pytest.mark.asyncio async def test_virtual_key_soft_budget_check_with_user_obj(): """Test _virtual_key_soft_budget_check includes user_email when user_obj is provided""" diff --git a/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py b/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py index 0d7becd3e2e..ce8f0802ae1 100644 --- a/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py +++ b/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py @@ -1149,6 +1149,13 @@ async def test_apply_guardrail_not_found(mocker): "litellm.proxy.guardrails.guardrail_endpoints.GUARDRAIL_REGISTRY", mock_registry ) + mock_proxy_logging = mocker.Mock() + mock_proxy_logging.post_call_failure_hook = AsyncMock() + mocker.patch("litellm.proxy.proxy_server.proxy_logging_obj", mock_proxy_logging) + mocker.patch("litellm.proxy.proxy_server.general_settings", {}) + mocker.patch("litellm.proxy.proxy_server.proxy_config", mocker.Mock()) + mocker.patch("litellm.proxy.proxy_server.version", "test") + # Create request request = ApplyGuardrailRequest( guardrail_name="non-existent-guardrail", text="Test input text" @@ -1159,7 +1166,11 @@ async def test_apply_guardrail_not_found(mocker): # Call endpoint and expect ProxyException with pytest.raises(ProxyException) as exc_info: - await apply_guardrail(request=request, user_api_key_dict=mock_user_auth) + await apply_guardrail( + fastapi_request=mocker.Mock(), + request=request, + user_api_key_dict=mock_user_auth, + ) # Verify error details assert str(exc_info.value.code) == "404" @@ -1186,6 +1197,25 @@ async def test_apply_guardrail_execution_error(mocker): "litellm.proxy.guardrails.guardrail_endpoints.GUARDRAIL_REGISTRY", mock_registry ) + mock_logging_obj = mocker.Mock() + mock_logging_obj.async_failure_handler = AsyncMock() + mock_logging_obj.model_call_details = {} + mock_processor = mocker.Mock() + mock_processor.common_processing_pre_call_logic = AsyncMock( + return_value=({"guardrail_name": "test-guardrail"}, mock_logging_obj) + ) + mocker.patch( + "litellm.proxy.common_request_processing.ProxyBaseLLMRequestProcessing", + return_value=mock_processor, + ) + mock_proxy_logging = mocker.Mock() + mock_proxy_logging.post_call_failure_hook = AsyncMock() + mocker.patch("litellm.proxy.proxy_server.proxy_logging_obj", mock_proxy_logging) + mocker.patch("litellm.proxy.proxy_server.general_settings", {}) + mocker.patch("litellm.proxy.proxy_server.proxy_config", mocker.Mock()) + mocker.patch("litellm.proxy.proxy_server.version", "test") + mocker.patch("litellm.litellm_core_utils.thread_pool_executor.executor") + # Create request request = ApplyGuardrailRequest( guardrail_name="test-guardrail", text="Test input text with forbidden content" @@ -1196,12 +1226,70 @@ async def test_apply_guardrail_execution_error(mocker): # Call endpoint and expect ProxyException with pytest.raises(ProxyException) as exc_info: - await apply_guardrail(request=request, user_api_key_dict=mock_user_auth) + await apply_guardrail( + fastapi_request=mocker.Mock(), + request=request, + user_api_key_dict=mock_user_auth, + ) # Verify error is properly handled assert "Bedrock guardrail failed" in str(exc_info.value.message) +@pytest.mark.asyncio +async def test_apply_guardrail_invokes_logging_pipeline(mocker): + mock_guardrail = mocker.Mock() + mock_guardrail.apply_guardrail = AsyncMock(return_value={"texts": ["masked"]}) + + mock_registry = mocker.Mock() + mock_registry.get_initialized_guardrail_callback.return_value = mock_guardrail + mocker.patch( + "litellm.proxy.guardrails.guardrail_endpoints.GUARDRAIL_REGISTRY", mock_registry + ) + + mock_logging_obj = mocker.Mock() + mock_logging_obj.async_success_handler = AsyncMock() + mock_logging_obj.model_call_details = {} + mock_processor = mocker.Mock() + mock_processor.common_processing_pre_call_logic = AsyncMock( + return_value=({"guardrail_name": "test-guardrail"}, mock_logging_obj) + ) + mocker.patch( + "litellm.proxy.common_request_processing.ProxyBaseLLMRequestProcessing", + return_value=mock_processor, + ) + + mock_proxy_logging = mocker.Mock() + mock_proxy_logging.post_call_success_hook = AsyncMock() + mocker.patch("litellm.proxy.proxy_server.proxy_logging_obj", mock_proxy_logging) + mocker.patch("litellm.proxy.proxy_server.general_settings", {}) + mocker.patch("litellm.proxy.proxy_server.proxy_config", mocker.Mock()) + mocker.patch("litellm.proxy.proxy_server.version", "test") + mock_executor = mocker.Mock() + mocker.patch( + "litellm.litellm_core_utils.thread_pool_executor.executor", mock_executor + ) + + request = ApplyGuardrailRequest( + guardrail_name="test-guardrail", text="hello@example.com" + ) + response = await apply_guardrail( + fastapi_request=mocker.Mock(), + request=request, + user_api_key_dict=UserAPIKeyAuth(), + ) + + assert response.response_text == "masked" + mock_processor.common_processing_pre_call_logic.assert_awaited_once() + mock_proxy_logging.post_call_success_hook.assert_awaited_once() + mock_logging_obj.async_success_handler.assert_awaited_once() + assert mock_logging_obj.call_type == "pass_through_endpoint" + mock_executor.submit.assert_called_once() + assert mock_logging_obj.async_success_handler.await_args.kwargs["result"] == { + "response": {"response_text": "masked"} + } + + @pytest.mark.asyncio async def test_get_guardrail_info_endpoint_config_guardrail(mocker): """ diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py index 13bb39c35c9..d580f1f7703 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py @@ -2832,6 +2832,7 @@ async def test_list_team_v2_security_check_non_admin_user_own_teams(): ] mock_db.litellm_teamtable.find_many = AsyncMock(return_value=mock_teams) mock_db.litellm_teamtable.count = AsyncMock(return_value=2) + mock_db.litellm_verificationtoken.group_by = AsyncMock(return_value=[]) with patch( "litellm.proxy.management_endpoints.team_endpoints.get_user_object", @@ -2888,6 +2889,7 @@ async def test_list_team_v2_security_check_admin_user(): ] mock_db.litellm_teamtable.find_many = AsyncMock(return_value=mock_teams) mock_db.litellm_teamtable.count = AsyncMock(return_value=2) + mock_db.litellm_verificationtoken.group_by = AsyncMock(return_value=[]) # Should NOT raise an exception result = await list_team_v2( @@ -3036,6 +3038,7 @@ async def test_list_team_v2_org_admin_sees_org_teams(): } mock_db.litellm_teamtable.find_many = AsyncMock(return_value=[mock_team]) mock_db.litellm_teamtable.count = AsyncMock(return_value=1) + mock_db.litellm_verificationtoken.group_by = AsyncMock(return_value=[]) result = await list_team_v2( http_request=mock_request, @@ -3211,6 +3214,7 @@ async def test_list_team_v2_org_admin_with_user_id_returns_user_teams(): } mock_db.litellm_teamtable.find_many = AsyncMock(return_value=[mock_team]) mock_db.litellm_teamtable.count = AsyncMock(return_value=1) + mock_db.litellm_verificationtoken.group_by = AsyncMock(return_value=[]) result = await list_team_v2( http_request=mock_request, @@ -3390,6 +3394,163 @@ async def test_list_team_v2_search_composes_with_user_id_filter(): assert where["team_id"] == {"in": ["team_a", "team_b"]} +@pytest.mark.asyncio +async def test_list_team_v2_populates_keys_count(): + """ + Test that list_team_v2 returns a keys_count per team derived from a single + batched group_by against LiteLLM_VerificationToken. + """ + from unittest.mock import AsyncMock, Mock, patch + + from fastapi import Request + + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.management_endpoints.team_endpoints import list_team_v2 + + mock_request = Mock(spec=Request) + mock_user_api_key_dict_admin = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, + user_id="admin_user_123", + ) + + with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma_client: + mock_db = Mock() + mock_prisma_client.db = mock_db + + team_a = Mock() + team_a.team_id = "team_a" + team_a.model_dump = lambda: { + "team_id": "team_a", + "team_alias": "Team A", + "members_with_roles": [{"user_id": "u1", "role": "user"}], + } + team_b = Mock() + team_b.team_id = "team_b" + team_b.model_dump = lambda: { + "team_id": "team_b", + "team_alias": "Team B", + "members_with_roles": [], + } + + mock_db.litellm_teamtable.find_many = AsyncMock(return_value=[team_a, team_b]) + mock_db.litellm_teamtable.count = AsyncMock(return_value=2) + mock_db.litellm_verificationtoken.group_by = AsyncMock( + return_value=[ + {"team_id": "team_a", "_count": {"team_id": 3}}, + # team_b intentionally absent → expect 0 + ] + ) + + result = await list_team_v2( + http_request=mock_request, + user_id=None, + user_api_key_dict=mock_user_api_key_dict_admin, + page=1, + page_size=10, + status=None, + ) + + assert result["total"] == 2 + by_id = {t.team_id: t for t in result["teams"]} + assert by_id["team_a"].keys_count == 3 + assert by_id["team_b"].keys_count == 0 + + # The aggregate is one batched query, filtered by the page's team IDs. + group_by_kwargs = mock_db.litellm_verificationtoken.group_by.call_args.kwargs + assert group_by_kwargs["by"] == ["team_id"] + assert group_by_kwargs["where"] == {"team_id": {"in": ["team_a", "team_b"]}} + assert group_by_kwargs["count"] == {"team_id": True} + + +@pytest.mark.asyncio +async def test_list_team_v2_keys_count_skipped_for_empty_page(): + """ + When the page has no teams, the keys-count group_by must not be issued. + """ + from unittest.mock import AsyncMock, Mock, patch + + from fastapi import Request + + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.management_endpoints.team_endpoints import list_team_v2 + + mock_request = Mock(spec=Request) + mock_user_api_key_dict_admin = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, + user_id="admin_user_123", + ) + + with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma_client: + mock_db = Mock() + mock_prisma_client.db = mock_db + + mock_db.litellm_teamtable.find_many = AsyncMock(return_value=[]) + mock_db.litellm_teamtable.count = AsyncMock(return_value=0) + mock_db.litellm_verificationtoken.group_by = AsyncMock(return_value=[]) + + result = await list_team_v2( + http_request=mock_request, + user_id=None, + user_api_key_dict=mock_user_api_key_dict_admin, + page=1, + page_size=10, + status=None, + ) + + assert result["total"] == 0 + assert result["teams"] == [] + mock_db.litellm_verificationtoken.group_by.assert_not_called() + + +@pytest.mark.asyncio +async def test_list_team_v2_keys_count_skipped_for_deleted_status(): + """ + The deleted-table branch returns LiteLLM_DeletedTeamTable items, which do + not carry keys_count — group_by must not be issued. + """ + from unittest.mock import AsyncMock, Mock, patch + + from fastapi import Request + + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.management_endpoints.team_endpoints import list_team_v2 + + mock_request = Mock(spec=Request) + mock_user_api_key_dict_admin = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, + user_id="admin_user_123", + ) + + with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma_client: + mock_db = Mock() + mock_prisma_client.db = mock_db + + mock_deleted = Mock() + mock_deleted.team_id = "team_d" + mock_deleted.model_dump = lambda: { + "team_id": "team_d", + "team_alias": "Deleted Team", + } + + mock_db.litellm_deletedteamtable.find_many = AsyncMock( + return_value=[mock_deleted] + ) + mock_db.litellm_deletedteamtable.count = AsyncMock(return_value=1) + mock_db.litellm_verificationtoken.group_by = AsyncMock(return_value=[]) + + result = await list_team_v2( + http_request=mock_request, + user_id=None, + user_api_key_dict=mock_user_api_key_dict_admin, + page=1, + page_size=10, + status="deleted", + ) + + assert result["total"] == 1 + mock_db.litellm_verificationtoken.group_by.assert_not_called() + + @pytest.mark.asyncio async def test_team_member_delete_cleans_membership(mock_db_client, mock_admin_auth): """ diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index fc9813ba530..f336c632546 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -515,6 +515,59 @@ async def test_add_litellm_data_to_request_body_snapshot_excludes_secret_fields( ) +@pytest.mark.asyncio +async def test_add_litellm_data_to_request_body_snapshot_excludes_proxy_server_request(): + """Regression: the body snapshot used to include the proxy_server_request + key itself, producing the path + ``proxy_server_request.body.proxy_server_request.body == body``. Custom + loggers and audit consumers must not see the self-referencing structure + (independent of redaction — fires on every successful call). + """ + from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request + + request_mock = MagicMock(spec=Request) + request_mock.url.path = "/v1/chat/completions" + request_mock.url = MagicMock() + request_mock.url.__str__.return_value = "http://localhost/v1/chat/completions" + request_mock.method = "POST" + request_mock.query_params = {} + request_mock.headers = {"Content-Type": "application/json"} + request_mock.client = MagicMock() + request_mock.client.host = "127.0.0.1" + + data = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "hello"}], + } + + user_api_key_dict = UserAPIKeyAuth( + api_key="hashed-key", + user_id="test-user", + metadata={}, + team_metadata={}, + spend=0.0, + max_budget=100.0, + model_max_budget={}, + team_spend=0.0, + team_max_budget=200.0, + ) + + updated = await add_litellm_data_to_request( + data=data, + request=request_mock, + user_api_key_dict=user_api_key_dict, + proxy_config=MagicMock(), + general_settings={}, + version="test-version", + ) + + snapshot_body = updated["proxy_server_request"]["body"] + assert "proxy_server_request" not in snapshot_body, ( + "proxy_server_request must be excluded from its own body snapshot " + "to prevent the body from self-referencing" + ) + + @pytest.mark.asyncio async def test_add_litellm_data_to_request_strips_string_encoded_admin_injection(): """Regression: metadata arriving as a JSON string (multipart/form-data or @@ -4182,6 +4235,209 @@ class TestApplyClientTagPolicyPreAuth: assert exc_info.value.max_budget == 0.10 +class TestApplyKeyTagsPreAuth: + def test_merges_key_tags_into_metadata(self): + data = {"model": "gpt-3.5-turbo"} + user_api_key_dict = UserAPIKeyAuth( + api_key="hashed-key", + metadata={"tags": ["engineering", "production"]}, + team_metadata={}, + ) + + LiteLLMProxyRequestSetup.apply_key_tags_pre_auth( + request_data=data, + user_api_key_dict=user_api_key_dict, + ) + + assert data["metadata"]["tags"] == ["engineering", "production"] + + def test_unions_key_tags_with_existing_request_tags(self): + data = { + "model": "gpt-3.5-turbo", + "metadata": {"tags": ["request-tag"]}, + } + user_api_key_dict = UserAPIKeyAuth( + api_key="hashed-key", + metadata={"tags": ["key-tag", "request-tag"]}, + team_metadata={}, + ) + + LiteLLMProxyRequestSetup.apply_key_tags_pre_auth( + request_data=data, + user_api_key_dict=user_api_key_dict, + ) + + # request-tag deduplicated; key-tag appended + assert data["metadata"]["tags"] == ["request-tag", "key-tag"] + + def test_no_key_tags_no_mutation(self): + data = {"model": "gpt-3.5-turbo"} + user_api_key_dict = UserAPIKeyAuth( + api_key="hashed-key", + metadata={}, + team_metadata={}, + ) + + LiteLLMProxyRequestSetup.apply_key_tags_pre_auth( + request_data=data, + user_api_key_dict=user_api_key_dict, + ) + + assert "metadata" not in data or "tags" not in data.get("metadata", {}) + + def test_empty_key_metadata_no_mutation(self): + data = {"model": "gpt-3.5-turbo"} + user_api_key_dict = UserAPIKeyAuth( + api_key="hashed-key", + metadata={}, + team_metadata={}, + ) + + LiteLLMProxyRequestSetup.apply_key_tags_pre_auth( + request_data=data, + user_api_key_dict=user_api_key_dict, + ) + + assert "metadata" not in data + + def test_uses_litellm_metadata_when_present(self): + data = { + "model": "gpt-3.5-turbo", + "litellm_metadata": {"foo": "bar"}, + } + user_api_key_dict = UserAPIKeyAuth( + api_key="hashed-key", + metadata={"tags": ["key-tag"]}, + team_metadata={}, + ) + + LiteLLMProxyRequestSetup.apply_key_tags_pre_auth( + request_data=data, + user_api_key_dict=user_api_key_dict, + ) + + assert data["litellm_metadata"]["tags"] == ["key-tag"] + assert "tags" not in data.get("metadata", {}) + + def test_string_metadata_parsed_before_merge(self): + data = { + "model": "gpt-3.5-turbo", + "metadata": '{"tags": ["existing"]}', + } + user_api_key_dict = UserAPIKeyAuth( + api_key="hashed-key", + metadata={"tags": ["key-tag"]}, + team_metadata={}, + ) + + LiteLLMProxyRequestSetup.apply_key_tags_pre_auth( + request_data=data, + user_api_key_dict=user_api_key_dict, + ) + + assert isinstance(data["metadata"], dict) + assert data["metadata"]["tags"] == ["existing", "key-tag"] + + @pytest.mark.asyncio + async def test_key_tags_visible_to_tag_max_budget_check(self): + from litellm.proxy._types import LiteLLM_BudgetTable, LiteLLM_TagTable + from litellm.proxy.auth.auth_checks import _tag_max_budget_check + from litellm.proxy.utils import ProxyLogging + + data = {"model": "gpt-3.5-turbo"} + user_api_key_dict = UserAPIKeyAuth( + api_key="hashed-key", + metadata={"tags": ["engineering"]}, + team_metadata={}, + ) + + LiteLLMProxyRequestSetup.apply_key_tags_pre_auth( + request_data=data, + user_api_key_dict=user_api_key_dict, + ) + + tag_object = LiteLLM_TagTable( + tag_name="engineering", + spend=0.0, + litellm_budget_table=LiteLLM_BudgetTable(max_budget=0.10), + ) + + async def mock_get_current_spend(counter_key, fallback_spend): + if counter_key == "spend:tag:engineering": + return 0.50 + return fallback_spend + + with ( + patch( + "litellm.proxy.proxy_server.get_current_spend", + mock_get_current_spend, + ), + patch( + "litellm.proxy.auth.auth_checks.get_tag_objects_batch", + new_callable=AsyncMock, + return_value={"engineering": tag_object}, + ), + ): + with pytest.raises(litellm.BudgetExceededError) as exc_info: + await _tag_max_budget_check( + request_body=data, + prisma_client=MagicMock(), + user_api_key_cache=MagicMock(), + proxy_logging_obj=ProxyLogging(user_api_key_cache=None), + valid_token=UserAPIKeyAuth(token="test-token"), + ) + assert exc_info.value.current_cost == 0.50 + assert exc_info.value.max_budget == 0.10 + + @pytest.mark.asyncio + async def test_key_tags_within_budget_passes_check(self): + from litellm.proxy._types import LiteLLM_BudgetTable, LiteLLM_TagTable + from litellm.proxy.auth.auth_checks import _tag_max_budget_check + from litellm.proxy.utils import ProxyLogging + + data = {"model": "gpt-3.5-turbo"} + user_api_key_dict = UserAPIKeyAuth( + api_key="hashed-key", + metadata={"tags": ["engineering"]}, + team_metadata={}, + ) + + LiteLLMProxyRequestSetup.apply_key_tags_pre_auth( + request_data=data, + user_api_key_dict=user_api_key_dict, + ) + + tag_object = LiteLLM_TagTable( + tag_name="engineering", + spend=0.05, + litellm_budget_table=LiteLLM_BudgetTable(max_budget=0.10), + ) + + async def mock_get_current_spend(counter_key, fallback_spend): + if counter_key == "spend:tag:engineering": + return 0.05 + return fallback_spend + + with ( + patch( + "litellm.proxy.proxy_server.get_current_spend", + mock_get_current_spend, + ), + patch( + "litellm.proxy.auth.auth_checks.get_tag_objects_batch", + new_callable=AsyncMock, + return_value={"engineering": tag_object}, + ), + ): + await _tag_max_budget_check( + request_body=data, + prisma_client=MagicMock(), + user_api_key_cache=MagicMock(), + proxy_logging_obj=ProxyLogging(user_api_key_cache=None), + valid_token=UserAPIKeyAuth(token="test-token"), + ) + + # ============================================================================ # Tests for #27516: provider hint resolution from deployment when the # user-facing model name has no provider prefix. diff --git a/tests/test_litellm/test_claude_haiku_4_5_config.py b/tests/test_litellm/test_claude_haiku_4_5_config.py index 7ed8197fa87..8755e5d156f 100644 --- a/tests/test_litellm/test_claude_haiku_4_5_config.py +++ b/tests/test_litellm/test_claude_haiku_4_5_config.py @@ -42,11 +42,6 @@ def test_bedrock_haiku_4_5_configuration(): model_info.get("supports_vision") is True ), f"{model} should support vision" - # Verify tool use system prompt tokens - assert ( - model_info.get("tool_use_system_prompt_tokens") == 346 - ), f"{model} should have tool_use_system_prompt_tokens set to 346" - # Verify core capabilities assert model_info.get("supports_computer_use") is True assert model_info.get("supports_function_calling") is True @@ -96,7 +91,6 @@ def test_bedrock_haiku_4_5_matches_sonnet_capabilities(): "supports_pdf_input", "supports_assistant_prefill", "supports_reasoning", - "tool_use_system_prompt_tokens", ] for capability in shared_capabilities: diff --git a/tests/test_litellm/test_claude_opus_4_6_config.py b/tests/test_litellm/test_claude_opus_4_6_config.py index 654ef1b9771..d946d1b41af 100644 --- a/tests/test_litellm/test_claude_opus_4_6_config.py +++ b/tests/test_litellm/test_claude_opus_4_6_config.py @@ -82,31 +82,26 @@ def test_opus_4_6_model_pricing_and_capabilities(): "claude-opus-4-6": { "provider": "anthropic", "has_long_context_pricing": False, - "tool_use_system_prompt_tokens": 346, "max_input_tokens": 1000000, }, "claude-opus-4-6-20260205": { "provider": "anthropic", "has_long_context_pricing": False, - "tool_use_system_prompt_tokens": 346, "max_input_tokens": 1000000, }, "anthropic.claude-opus-4-6-v1": { "provider": "bedrock_converse", "has_long_context_pricing": False, - "tool_use_system_prompt_tokens": 346, "max_input_tokens": 1000000, }, "vertex_ai/claude-opus-4-6": { "provider": "vertex_ai-anthropic_models", "has_long_context_pricing": False, - "tool_use_system_prompt_tokens": 346, "max_input_tokens": 1000000, }, "azure_ai/claude-opus-4-6": { "provider": "azure_ai", "has_long_context_pricing": False, - "tool_use_system_prompt_tokens": 159, "max_input_tokens": 200000, }, } @@ -143,10 +138,6 @@ def test_opus_4_6_model_pricing_and_capabilities(): assert info["supports_reasoning"] is True assert info["supports_tool_choice"] is True assert info["supports_vision"] is True - assert ( - info["tool_use_system_prompt_tokens"] - == config["tool_use_system_prompt_tokens"] - ) def test_opus_4_6_bedrock_regional_model_pricing(): @@ -191,7 +182,6 @@ def test_opus_4_6_bedrock_regional_model_pricing(): assert info["max_output_tokens"] == 128000 assert info["max_tokens"] == 128000 assert info["supports_assistant_prefill"] is False - assert info["tool_use_system_prompt_tokens"] == 346 assert "input_cost_per_token_above_200k_tokens" not in info assert "output_cost_per_token_above_200k_tokens" not in info assert "cache_creation_input_token_cost_above_200k_tokens" not in info @@ -220,7 +210,6 @@ def test_opus_4_6_alias_and_dated_metadata_match(): "cache_creation_input_token_cost_above_1hr", "cache_read_input_token_cost", "supports_assistant_prefill", - "tool_use_system_prompt_tokens", ] for key in keys_to_match: assert alias[key] == dated[key], f"Mismatch for {key}" diff --git a/tests/test_litellm/test_claude_opus_4_8_config.py b/tests/test_litellm/test_claude_opus_4_8_config.py new file mode 100644 index 00000000000..0ea4026e165 --- /dev/null +++ b/tests/test_litellm/test_claude_opus_4_8_config.py @@ -0,0 +1,184 @@ +""" +Validate Claude Opus 4.8 model configuration entries. + +Regression coverage for the wildcard-routing failure where a bare model name +(``claude-opus-4-8``) could not match an ``anthropic/*`` deployment because +LiteLLM could not infer its provider — the model was simply missing from the +model cost map, so ``get_llm_provider`` raised and the router returned +"no healthy deployments for this model". The fix is the cost-map entries added +for Anthropic, Bedrock, Vertex AI, and Azure AI; those entries are what populate +``litellm.anthropic_models`` at import time, which is what the bare-name lookup +in ``get_llm_provider`` consumes. +""" + +import json +import os + +import pytest + +import litellm +from litellm.constants import BEDROCK_CONVERSE_MODELS +from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap + +REPO_ROOT = os.path.join(os.path.dirname(__file__), "../..") + + +def _load_root_cost_map() -> dict: + json_path = os.path.join(REPO_ROOT, "model_prices_and_context_window.json") + with open(json_path) as f: + return json.load(f) + + +@pytest.fixture +def local_model_cost_map(monkeypatch): + """Force the bundled backup cost map so assertions don't depend on the + network-fetched ``main`` copy (which lags this branch until merge).""" + original_model_cost = litellm.model_cost + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.get_model_info.cache_clear() + try: + yield + finally: + litellm.model_cost = original_model_cost + litellm.get_model_info.cache_clear() + + +def test_opus_4_8_model_pricing_and_capabilities(): + model_data = _load_root_cost_map() + + expected_models = { + "claude-opus-4-8": { + "provider": "anthropic", + "max_input_tokens": 1000000, + }, + "anthropic.claude-opus-4-8": { + "provider": "bedrock_converse", + "max_input_tokens": 1000000, + }, + "vertex_ai/claude-opus-4-8": { + "provider": "vertex_ai-anthropic_models", + "max_input_tokens": 1000000, + }, + # Microsoft Foundry / Azure caps Opus 4.8 at a 200k context window. + "azure_ai/claude-opus-4-8": { + "provider": "azure_ai", + "max_input_tokens": 200000, + }, + } + + for model_name, config in expected_models.items(): + assert model_name in model_data, f"Missing model entry: {model_name}" + info = model_data[model_name] + + assert info["litellm_provider"] == config["provider"] + assert info["mode"] == "chat" + assert info["max_input_tokens"] == config["max_input_tokens"] + assert info["max_output_tokens"] == 128000 + assert info["max_tokens"] == 128000 + + # Base pricing matches Opus 4.7: $5 / $25 per MTok, with the standard + # 1.25x cache-write and 0.1x cache-read multipliers. + assert info["input_cost_per_token"] == 5e-06 + assert info["output_cost_per_token"] == 2.5e-05 + assert info["cache_creation_input_token_cost"] == 6.25e-06 + assert info["cache_read_input_token_cost"] == 5e-07 + + # Opus 4.x flagships are flat-rate across the full context window. + assert "input_cost_per_token_above_200k_tokens" not in info + assert "output_cost_per_token_above_200k_tokens" not in info + + assert info["supports_assistant_prefill"] is False + assert info["supports_function_calling"] is True + assert info["supports_prompt_caching"] is True + assert info["supports_reasoning"] is True + assert info["supports_tool_choice"] is True + assert info["supports_vision"] is True + + +def test_opus_4_8_bedrock_regional_model_pricing(): + model_data = _load_root_cost_map() + + # Global endpoints use base pricing; regional endpoints carry a 10% premium. + expected_models = { + "global.anthropic.claude-opus-4-8": { + "input_cost_per_token": 5e-06, + "output_cost_per_token": 2.5e-05, + "cache_creation_input_token_cost": 6.25e-06, + "cache_read_input_token_cost": 5e-07, + }, + "us.anthropic.claude-opus-4-8": { + "input_cost_per_token": 5.5e-06, + "output_cost_per_token": 2.75e-05, + "cache_creation_input_token_cost": 6.875e-06, + "cache_read_input_token_cost": 5.5e-07, + }, + "eu.anthropic.claude-opus-4-8": { + "input_cost_per_token": 5.5e-06, + "output_cost_per_token": 2.75e-05, + "cache_creation_input_token_cost": 6.875e-06, + "cache_read_input_token_cost": 5.5e-07, + }, + "au.anthropic.claude-opus-4-8": { + "input_cost_per_token": 5.5e-06, + "output_cost_per_token": 2.75e-05, + "cache_creation_input_token_cost": 6.875e-06, + "cache_read_input_token_cost": 5.5e-07, + }, + } + + for model_name, expected in expected_models.items(): + assert model_name in model_data, f"Missing model entry: {model_name}" + info = model_data[model_name] + assert info["litellm_provider"] == "bedrock_converse" + assert info["max_input_tokens"] == 1000000 + assert info["max_output_tokens"] == 128000 + assert info["bedrock_output_config_effort_ceiling"] == "xhigh" + for key, value in expected.items(): + assert info[key] == value + + +def test_opus_4_8_fast_mode_multiplier(): + """Opus 4.8 dropped fast-mode pricing to 2x base ($10/$50 per MTok); + Opus 4.7 was 6x ($30/$150).""" + model_data = _load_root_cost_map() + entry = model_data["claude-opus-4-8"]["provider_specific_entry"] + assert entry["us"] == 1.1 + assert entry["fast"] == 2.0 + + +def test_opus_4_8_present_in_bundled_backup(): + """The bundled backup is the runtime fallback (and what tests load with + ``LITELLM_LOCAL_MODEL_COST_MAP=True``) — it must carry the same entries as + the root cost map, otherwise the model resolves on one path but not the + other.""" + backup = GetModelCostMap.load_local_model_cost_map() + for model_name in ( + "claude-opus-4-8", + "anthropic.claude-opus-4-8", + "global.anthropic.claude-opus-4-8", + "us.anthropic.claude-opus-4-8", + "eu.anthropic.claude-opus-4-8", + "au.anthropic.claude-opus-4-8", + "vertex_ai/claude-opus-4-8", + "vertex_ai/claude-opus-4-8@default", + "azure_ai/claude-opus-4-8", + ): + assert model_name in backup, f"Missing from backup cost map: {model_name}" + + +def test_opus_4_8_registered_for_bedrock_converse(): + assert "anthropic.claude-opus-4-8" in BEDROCK_CONVERSE_MODELS + + +def test_opus_4_8_provider_resolves_via_model_info(local_model_cost_map): + """Regression: ``claude-opus-4-8`` must resolve to provider ``anthropic``. + + Before the cost-map entry existed, the model was unknown to LiteLLM, so it + could not be tied to the ``anthropic`` provider and an ``anthropic/*`` + wildcard deployment would not match it. + """ + info = litellm.get_model_info(model="claude-opus-4-8") + assert info["litellm_provider"] == "anthropic" + assert info["max_input_tokens"] == 1000000 + assert info["max_output_tokens"] == 128000 diff --git a/tests/test_litellm/test_claude_sonnet_4_6_config.py b/tests/test_litellm/test_claude_sonnet_4_6_config.py index 434ef9bdeb1..27023d4ee6d 100644 --- a/tests/test_litellm/test_claude_sonnet_4_6_config.py +++ b/tests/test_litellm/test_claude_sonnet_4_6_config.py @@ -50,7 +50,6 @@ def test_bedrock_sonnet_4_6_region_prefixes(): assert model_info.get("supports_pdf_input") is True assert model_info.get("supports_assistant_prefill") is True assert model_info.get("supports_reasoning") is True - assert model_info.get("tool_use_system_prompt_tokens") == 346 def test_bedrock_sonnet_4_6_jp_matches_other_regional_pricing(): diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index e646c75eda0..6a78653ec99 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -859,8 +859,11 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "supports_adaptive_thinking": {"type": "boolean"}, "supports_service_tier": {"type": "boolean"}, "supports_preset": {"type": "boolean"}, - "supports_output_config": {"type": "boolean"}, - "tool_use_system_prompt_tokens": {"type": "number"}, + "supports_output_config": {"type": "boolean"}, + "bedrock_output_config_effort_ceiling": { + "type": "string", + "enum": ["low", "medium", "high", "max", "xhigh"], + }, "tpm": {"type": "number"}, "provider_specific_entry": {"type": "object"}, "supported_endpoints": { diff --git a/tests/test_litellm/test_video_generation.py b/tests/test_litellm/test_video_generation.py index b0eb2438b95..3d0472ef96e 100644 --- a/tests/test_litellm/test_video_generation.py +++ b/tests/test_litellm/test_video_generation.py @@ -398,6 +398,34 @@ class TestVideoGeneration: ) assert abs(cost - 0.8) < 0.001 + def test_completion_cost_video_edit_uses_video_calculator(self): + """video_edit is charged via the same video cost path as create_video.""" + from litellm.cost_calculator import completion_cost + + mock_response = MagicMock() + mock_response.usage = MagicMock() + mock_response.usage.duration_seconds = 10.0 + type(mock_response)._hidden_params = {} + + mock_logging_obj = MagicMock() + mock_logging_obj.litellm_params = { + "metadata": { + "model_info": { + "output_cost_per_video_per_second": 0.05, + } + } + } + + cost = completion_cost( + completion_response=mock_response, + model="vertex_ai/veo-3.1-generate-001", + call_type="video_edit", + custom_llm_provider="vertex_ai", + custom_pricing=True, + litellm_logging_obj=mock_logging_obj, + ) + assert cost == 0.5 + def test_video_generation_with_files(self): """Test video generation with file uploads.""" config = OpenAIVideoConfig() diff --git a/ui/litellm-dashboard/CLAUDE.md b/ui/litellm-dashboard/CLAUDE.md new file mode 100644 index 00000000000..3d43019c749 --- /dev/null +++ b/ui/litellm-dashboard/CLAUDE.md @@ -0,0 +1 @@ +Never put LiteLLM tokens or API keys in `localStorage`. `localStorage` survives browser close. Prefer `httpOnly` cookies, or `sessionStorage` at most, understanding that any web storage is readable by injected scripts (XSS), and only httpOnly cookies are not diff --git a/ui/litellm-dashboard/e2e_tests/fixtures/seed.sql b/ui/litellm-dashboard/e2e_tests/fixtures/seed.sql index 91312e66ce0..5e5313240e6 100644 --- a/ui/litellm-dashboard/e2e_tests/fixtures/seed.sql +++ b/ui/litellm-dashboard/e2e_tests/fixtures/seed.sql @@ -33,6 +33,7 @@ VALUES ('e2e-internal-viewer', 'viewer@test.local', 'internal_user_viewer', '{"e2e-team-crud"}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), ('e2e-team-admin', 'teamadmin@test.local', 'internal_user', '{"e2e-team-crud","e2e-team-delete"}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), ('e2e-invitable-user', 'invitable@test.local', 'internal_user', '{}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), + ('e2e-invitable-by-team-admin', 'invitable-team@test.local', 'internal_user', '{}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), ('e2e-removable-member', 'removable@test.local', 'internal_user', '{"e2e-team-crud"}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'); -- 5. Teams (members_with_roles is required JSON) diff --git a/ui/litellm-dashboard/e2e_tests/serverRootPath.config.ts b/ui/litellm-dashboard/e2e_tests/serverRootPath.config.ts new file mode 100644 index 00000000000..83831f82da0 --- /dev/null +++ b/ui/litellm-dashboard/e2e_tests/serverRootPath.config.ts @@ -0,0 +1,32 @@ +import { defineConfig, devices } from "@playwright/test"; + +// Minimal config for the SERVER_ROOT_PATH redirect spec. Deliberately does NOT +// reuse the main e2e config because: +// - globalSetup logs in via http://localhost:4000/ui/login, which 404s when +// the proxy is mounted under a non-root path. +// - The redirect spec must run against a clean, unauthenticated session, so +// no storage state should be loaded. +export default defineConfig({ + testDir: "./tests/login", + testMatch: ["serverRootPathRedirect.spec.ts"], + fullyParallel: false, + forbidOnly: !!process.env.CI, + retries: process.env.CI ? 2 : 0, + workers: 1, + reporter: "list", + use: { + trace: "on-first-retry", + actionTimeout: 15 * 1000, + navigationTimeout: 30 * 1000, + }, + projects: [ + { + name: "chromium", + use: { ...devices["Desktop Chrome"] }, + }, + ], + timeout: 60 * 1000, + expect: { + timeout: 10 * 1000, + }, +}); diff --git a/ui/litellm-dashboard/e2e_tests/tests/login/login.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/login/login.spec.ts index 5d4b2508444..994d211cc18 100644 --- a/ui/litellm-dashboard/e2e_tests/tests/login/login.spec.ts +++ b/ui/litellm-dashboard/e2e_tests/tests/login/login.spec.ts @@ -10,4 +10,21 @@ test("user can log in", async ({ page }) => { await expect(loginButton).toBeEnabled(); await loginButton.click(); await expect(page.getByText("Virtual Keys")).toBeVisible(); + + // Match the navbar account button by its stable aria-label (UserDropdown.tsx + // emits "Account menu — — signed in as "). Earlier this used + // `hasText: /^User$/`, which never matched the rendered button (text is + // displayName = "Account" for the master-key admin), so the trigger evaluate + // would time out in CI. + const userTrigger = page.locator('button[aria-label^="Account menu"]').first(); + await userTrigger.click(); + + // Filter by the popupRender wrapper class to disambiguate from other + // ant-dropdown popups. + const popup = page.locator(".ant-dropdown:visible").filter({ + has: page.locator(".bg-white.rounded-lg.shadow-lg"), + }).first(); + await expect(popup).toBeVisible({ timeout: 5_000 }); + await expect(popup.getByText("Admin", { exact: true })).toBeVisible({ timeout: 5_000 }); + await expect(popup.getByText("default_user_id", { exact: true })).toBeVisible({ timeout: 5_000 }); }); diff --git a/ui/litellm-dashboard/e2e_tests/tests/login/serverRootPathRedirect.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/login/serverRootPathRedirect.spec.ts new file mode 100644 index 00000000000..62a3e913184 --- /dev/null +++ b/ui/litellm-dashboard/e2e_tests/tests/login/serverRootPathRedirect.spec.ts @@ -0,0 +1,32 @@ +import { expect, test } from "@playwright/test"; + +// Driven by the SERVER_ROOT_PATH env var injected by the workflow; the container +// is booted with the same value, so the asset paths and the runtime config it +// serves at /litellm/.well-known/litellm-ui-config will both reflect it. +const ROOT_PATH = process.env.SERVER_ROOT_PATH ?? ""; + +test.skip(!ROOT_PATH, "Requires SERVER_ROOT_PATH env var"); + +// Contract: an unauthenticated visit must redirect to a login URL that preserves +// the SERVER_ROOT_PATH prefix. The redirect URL is built client-side from +// `proxyBaseUrl`, which is populated by an async fetch of the runtime UI config. +// If the redirect fires before that fetch resolves, the URL is missing the +// prefix and the user lands on a 404. To make the race deterministic across +// runners, the config endpoint is intentionally delayed. +test("unauth redirect preserves SERVER_ROOT_PATH prefix", async ({ page }) => { + // Matches both `/litellm/.well-known/litellm-ui-config` and + // `${SERVER_ROOT_PATH}/.well-known/litellm-ui-config` (the proxy rewrites the + // bundle at boot when a root path is set). + await page.route("**/.well-known/litellm-ui-config", async (route) => { + await new Promise((resolve) => setTimeout(resolve, 500)); + await route.continue(); + }); + + await page.context().clearCookies(); + + await page.goto(`http://localhost:4000${ROOT_PATH}/ui/?page=virtual-keys`); + + await page.waitForURL((url) => url.pathname.endsWith("/ui/login"), { timeout: 15_000 }); + + expect(page.url()).toContain(`${ROOT_PATH}/ui/login`); +}); diff --git a/ui/litellm-dashboard/e2e_tests/tests/mcp/mcpServers.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/mcp/mcpServers.spec.ts new file mode 100644 index 00000000000..f953a82daaa --- /dev/null +++ b/ui/litellm-dashboard/e2e_tests/tests/mcp/mcpServers.spec.ts @@ -0,0 +1,62 @@ +import { test, expect } from "@playwright/test"; +import { ADMIN_STORAGE_PATH } from "../../constants"; +import { navigateToPage } from "../../helpers/navigation"; +import { Page } from "../../fixtures/pages"; + +// Coverage scope: only the happy-path Streamable HTTP + None auth create flow. +// See E2E_COVERAGE.md (#29 row) for the full list of uncovered MCP surfaces +// — SSE / stdio / OpenAPI transports, API Key / Bearer / OAuth2 / Basic / Token +// / AWS SigV4 auth, edit/delete, BYOK credentials, tool list/call (needs a real +// or mocked MCP server in the e2e fixture stack), and access-group permissions. +test.describe("MCP Servers", () => { + test.use({ storageState: ADMIN_STORAGE_PATH }); + + test("Add a custom MCP server via the discovery → custom form", async ({ page }) => { + await navigateToPage(page, Page.McpServers); + + // Open the discovery modal, then drop into the custom-server form + await page.getByRole("button", { name: /Add New MCP Server/i }).click(); + const discovery = page.locator(".ant-modal:visible").filter({ hasText: "Add MCP Server" }); + await expect(discovery).toBeVisible({ timeout: 5_000 }); + await discovery.getByRole("button", { name: /Custom Server/i }).click(); + + const formModal = page.locator(".ant-modal:visible").filter({ hasText: "MCP Server Name" }); + await expect(formModal).toBeVisible({ timeout: 5_000 }); + + // Name — no spaces or hyphens per validateMCPServerName + const uniqueName = `e2e_mcp_${Date.now()}`; + await formModal.locator('input[id="server_name"]').fill(uniqueName); + + // Transport: Streamable HTTP — the only value the proxy actually accepts is "http" + const transportField = formModal.locator(".ant-form-item", { hasText: "Transport Type" }); + await transportField.locator(".ant-select").click(); + await page.locator(".ant-select-dropdown:visible").getByText("Streamable HTTP").click(); + + // URL — use a fake URL; the form just persists it, it doesn't have to be reachable + await formModal.locator('input[id="url"]').fill("https://e2e-fake-mcp.test.local/mcp"); + + // Authentication: None + // The auth_type Form.Item has no label prop (create_mcp_server.tsx:795), so + // it can't be anchored by label text. Scope via the enclosing Collapse + // panel ("Authentication") instead — that anchor is stable even if the + // placeholder copy changes. + const authSection = formModal.locator(".ant-collapse-item", { hasText: /^Authentication/ }); + const authField = authSection.locator(".ant-form-item").first(); + await authField.locator(".ant-select").click(); + await page.locator(".ant-select-dropdown:visible").getByText("None", { exact: true }).click(); + + // Submit + await formModal.getByRole("button", { name: /^Add MCP Server$/ }).click(); + + // No teardown needed — the e2e runner spins up a fresh DB per invocation. + + // Success toast and the new row in the table. Scope the row lookup to + // the MCP servers table so the form modal's `server_name` input — which + // still holds the timestamped value during its close animation — can't + // satisfy the assertion before the server actually lands in the list. + await expect(page.getByText("MCP Server created successfully").first()) + .toBeVisible({ timeout: 15_000 }); + await expect(page.locator("table tbody").getByText(uniqueName).first()) + .toBeVisible({ timeout: 10_000 }); + }); +}); diff --git a/ui/litellm-dashboard/e2e_tests/tests/modelHub/modelHub.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/modelHub/modelHub.spec.ts new file mode 100644 index 00000000000..ada4dfb735e --- /dev/null +++ b/ui/litellm-dashboard/e2e_tests/tests/modelHub/modelHub.spec.ts @@ -0,0 +1,79 @@ +import { test, expect } from "@playwright/test"; +import { ADMIN_STORAGE_PATH } from "../../constants"; +import { navigateToPage, dismissFeedbackPopup } from "../../helpers/navigation"; +import { Page } from "../../fixtures/pages"; + +test.describe("AI Hub (internal admin view)", () => { + test.use({ storageState: ADMIN_STORAGE_PATH }); + + test("Make models public via the multi-step modal", async ({ page }) => { + await navigateToPage(page, Page.ModelHubTable); + + // Open the "Select Models to Make Public" modal + await page.getByRole("button", { name: /Select Models to Make Public/i }).click(); + + const modal = page.locator(".ant-modal:visible").filter({ hasText: "Make Models Public" }); + await expect(modal).toBeVisible({ timeout: 5_000 }); + + // Guard: the "Select All (N)" label only shows a count when filteredData + // has at least one row. Asserting N>=1 here turns a missing-seed-data + // failure into an immediate diagnostic rather than a downstream timeout + // on the disabled-Next button or the success toast. + await expect(modal.getByText(/Select All \(\d+\)/)).toBeVisible({ timeout: 5_000 }); + + // Step 1: pick the seeded models via "Select All" + await modal.getByText(/Select All/i).click(); + + // Move to confirm step + await modal.getByRole("button", { name: "Next" }).click(); + await expect(modal.getByText("Confirm Making Models Public")).toBeVisible({ timeout: 5_000 }); + + // Submit + await modal.getByRole("button", { name: "Make Public" }).click(); + + await expect(page.getByText(/Successfully made .* model group\(s\) public/i).first()) + .toBeVisible({ timeout: 15_000 }); + }); + + test("AI Hub tab list renders Model Hub, Agent Hub, MCP Hub and Skill Hub", async ({ page }) => { + await navigateToPage(page, Page.ModelHubTable); + + // The tab strip lives in the main view; check each tab is present and clickable. + // (The "Claude Code Plugin Marketplace" tab from the manual-QA checklist was + // renamed to "Skill Hub" — verify the current label here so the test stays + // in sync with the UI.) + // + // Note: unlike the public /ui/model_hub_table view (test below), the admin + // ModelHubTable renders all four tabs unconditionally — there are no `&&` + // guards around Agent Hub or MCP Hub in the source + // (ModelHubTable.tsx ~L436-439). Asserting all four here is intentional: + // this pins the manual-QA contract that the AI Hub tab strip exposes + // exactly these labels regardless of seeded agent/MCP data. + for (const tabName of ["Model Hub", "Agent Hub", "MCP Hub", "Skill Hub"]) { + const tab = page.getByRole("tab", { name: tabName }); + await expect(tab, `${tabName} tab should be present`).toBeVisible({ timeout: 5_000 }); + await tab.click(); + } + }); +}); + +test.describe("Public model hub (/ui/model_hub_table)", () => { + // No storageState — the public page is reached anonymously with a `key` query param. + + test("Public model_hub_table loads and renders the Model Hub tab", async ({ page }) => { + // The page expects the proxy key as the `key` query param. Use the master + // key the e2e runner already exports — this matches what the AI Hub copy + // button hands out. + const masterKey = process.env.LITELLM_MASTER_KEY || "sk-1234"; + await page.goto(`/ui/model_hub_table?key=${masterKey}`); + + // Dismiss the feedback popup before asserting on the tab, so a popup + // race can't briefly mask the tab while we're evaluating visibility. + await dismissFeedbackPopup(page); + + // Page loads (no auth redirect) and the Model Hub tab is always present. + // Agent Hub and MCP Hub tabs are conditionally rendered only when public + // agents/MCP servers exist, so we don't assert on them in a fresh CI run. + await expect(page.getByRole("tab", { name: "Model Hub" })).toBeVisible({ timeout: 10_000 }); + }); +}); diff --git a/ui/litellm-dashboard/e2e_tests/tests/modelsPage/addModel.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/modelsPage/addModel.spec.ts index c3bd8489027..bb53fb7a23b 100644 --- a/ui/litellm-dashboard/e2e_tests/tests/modelsPage/addModel.spec.ts +++ b/ui/litellm-dashboard/e2e_tests/tests/modelsPage/addModel.spec.ts @@ -1,5 +1,5 @@ import { test, expect } from "@playwright/test"; -import { ADMIN_STORAGE_PATH, E2E_TEAM_CRUD_ID } from "../../constants"; +import { ADMIN_STORAGE_PATH, E2E_TEAM_CRUD_ALIAS, E2E_TEAM_CRUD_ID } from "../../constants"; import { Role, users } from "../../fixtures/users"; import { navigateToPage } from "../../helpers/navigation"; import { Page } from "../../fixtures/pages"; @@ -150,6 +150,111 @@ test.describe("Add Model", () => { await expect(tableBody.getByText("claude-haiku-4-5").first()).toBeVisible({ timeout: 15_000 }); }); + test("Add team-only model via Team-BYOK toggle and verify it appears with the team", async ({ page, request }) => { + // The Team-BYOK switch is gated on `premiumUser` — without a license set + // for the proxy under test, the toggle is disabled and this manual-QA + // step cannot be exercised. + test.skip( + !process.env.LITELLM_LICENSE, + "LITELLM_LICENSE not set in test env — Team-BYOK switch is disabled", + ); + + // Make the test idempotent across retries and local reruns: delete any + // Cohere model already scoped to the e2e team before we start, and again + // after we finish. The sibling "Add wildcard route" test creates a + // team-less Cohere wildcard, so we only target rows that have BOTH the + // cohere/* model_name AND team_id == e2e-team-crud. + const masterKey = users[Role.ProxyAdmin].password; + const auth = { Authorization: `Bearer ${masterKey}` }; + const deleteTeamScopedCohereModels = async () => { + const res = await request.get("/v2/model/info", { headers: auth }); + if (!res.ok()) return; + const body = await res.json(); + const matches: Array<{ id: string }> = (body?.data ?? []).filter((m: any) => + typeof m?.model_name === "string" && + m.model_name.startsWith("cohere") && + m?.model_info?.team_id === E2E_TEAM_CRUD_ID, + ); + for (const m of matches) { + await request.post("/model/delete", { headers: auth, data: { id: m.id } }); + } + }; + await deleteTeamScopedCohereModels(); + + try { + await navigateToPage(page, Page.Models); + await page.getByRole("tab", { name: "Add Model" }).click(); + + await selectProvider(page, "Cohere"); + + const modelDropdown = page.locator(".ant-select-selection-overflow").first(); + await modelDropdown.click(); + const wildcardOption = page.getByTitle(/All .* Models \(Wildcard\)/); + await wildcardOption.click(); + await page.keyboard.press("Escape"); + + const apiKeyInput = page.locator('input[type="password"]').first(); + await apiKeyInput.fill("sk-any-key-for-team-byok-test"); + + // Flip the Team-BYOK switch on (Form.Item label "Team-BYOK Model") + const teamByokRow = page.locator(".ant-form-item", { hasText: "Team-BYOK Model" }); + await teamByokRow.getByRole("switch").click(); + + // The Team dropdown appears underneath once the switch is on. TeamDropdown + // renders its Select.Option children with custom / markup, so + // the popup items don't carry role="option" — match by text content, + // scoped to the visible dropdown so a stale tag elsewhere in the form + // can't satisfy it. + const teamDropdown = page.getByTestId("team-dropdown"); + await expect(teamDropdown).toBeVisible({ timeout: 5_000 }); + await teamDropdown.click(); + const teamOption = page.locator(".ant-select-dropdown:visible") + .getByText(E2E_TEAM_CRUD_ID) + .first(); + await expect(teamOption).toBeVisible({ timeout: 5_000 }); + await teamOption.click(); + + await page.getByRole("button", { name: "Add Model" }).last().click(); + + // Scope the success toast to antd's notification container so a stale + // success message from an earlier test in the same context can't satisfy + // the assertion. + await expect(page.locator(".ant-notification").getByText("created successfully").last()) + .toBeVisible({ timeout: 15_000 }); + + // Verify the model is now in All Models with the team_id attached. The + // Models table renders team-scoped models with the team id in the row. + await page.getByRole("tab", { name: "All Models" }).click(); + await page.waitForLoadState("networkidle"); + // Match the sibling tests in this file — networkidle fires before the + // table finishes re-rendering, so give it the same 2s settle before + // searching. + await page.waitForTimeout(2000); + + await page.locator('input[placeholder="Search model names..."]').fill("cohere"); + await page.waitForTimeout(1000); + + // Confirm the search returned at least one result — gives a clear + // failure message when the table is empty instead of timing out on a + // row assertion. + await expect(page.getByTestId("models-results-count")).toHaveText( + /Showing \d+ - \d+ of \d+ results/, + { timeout: 15_000 }, + ); + + // Stronger than "alias appears somewhere in tbody" — pin the assertion + // to a single row that has BOTH the cohere model_name AND the seeded + // team alias, so a stale cohere row from "Add wildcard route" (no team) + // can't satisfy the check. + const teamCohereRow = page.locator("table tbody tr") + .filter({ hasText: "cohere/" }) + .filter({ hasText: E2E_TEAM_CRUD_ALIAS }); + await expect(teamCohereRow).toHaveCount(1, { timeout: 15_000 }); + } finally { + await deleteTeamScopedCohereModels(); + } + }); + test("Add wildcard route and verify it appears in All Models", async ({ page }) => { await navigateToPage(page, Page.Models); await page.getByRole("tab", { name: "Add Model" }).click(); diff --git a/ui/litellm-dashboard/e2e_tests/tests/proxy-admin/teams.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/proxy-admin/teams.spec.ts index a1864b22a43..6f6e8373390 100644 --- a/ui/litellm-dashboard/e2e_tests/tests/proxy-admin/teams.spec.ts +++ b/ui/litellm-dashboard/e2e_tests/tests/proxy-admin/teams.spec.ts @@ -131,4 +131,49 @@ test.describe("Proxy Admin - Teams", () => { await expect(page.getByText(/updated|success/i).first()).toBeVisible({ timeout: 10_000 }); }); + + test("Edit team model selection", async ({ page, request }) => { + // Restore the seeded models via API in case a prior run (or a CI retry) + // left this team mutated — the assertion below requires fake-anthropic-claude + // to be present. + const masterKey = process.env.LITELLM_MASTER_KEY || "sk-1234"; + const seededModels = ["fake-openai-gpt-4", "fake-anthropic-claude"]; + const restore = async () => { + const res = await request.post("http://localhost:4000/team/update", { + headers: { Authorization: `Bearer ${masterKey}` }, + data: { team_id: E2E_TEAM_CRUD_ID, models: seededModels }, + }); + expect(res.ok(), `restore failed: ${res.status()} ${await res.text()}`).toBeTruthy(); + }; + await restore(); + + try { + await navigateToPage(page, Page.Teams); + await dismissFeedbackPopup(page); + + await clickTeamId(page, E2E_TEAM_CRUD_ID); + + await page.getByRole("tab", { name: "Settings" }).click(); + await page.getByRole("button", { name: "Edit Settings" }).click(); + + // Remove the anthropic tag — other tests against this team use "All Team + // Models" so they pick up whatever remains. + const modelsSelect = page.locator("[data-testid='models-select']"); + await expect(modelsSelect).toBeVisible({ timeout: 10_000 }); + + const anthropicTag = modelsSelect + .locator(".ant-select-selection-item") + .filter({ hasText: "fake-anthropic-claude" }); + await expect(anthropicTag).toBeVisible({ timeout: 5_000 }); + await anthropicTag.locator(".ant-select-selection-item-remove").click(); + + await page.getByRole("button", { name: "Save Changes" }).click(); + + await expect(page.getByText(/Team settings updated|updated successfully/i).first()) + .toBeVisible({ timeout: 10_000 }); + } finally { + // Leave the team in its seeded state for any subsequent test or rerun. + await restore(); + } + }); }); diff --git a/ui/litellm-dashboard/e2e_tests/tests/settings/routerSettings.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/settings/routerSettings.spec.ts new file mode 100644 index 00000000000..8dd5571f7af --- /dev/null +++ b/ui/litellm-dashboard/e2e_tests/tests/settings/routerSettings.spec.ts @@ -0,0 +1,102 @@ +import { test, expect } from "@playwright/test"; +import { ADMIN_STORAGE_PATH } from "../../constants"; +import { navigateToPage } from "../../helpers/navigation"; +import { Page } from "../../fixtures/pages"; +import { Role, users } from "../../fixtures/users"; + +const PRIMARY = "fake-openai-gpt-4"; +const FALLBACK = "fake-anthropic-claude"; + +/** + * Wipe any fallbacks for the primary model so the test is idempotent across + * retries and local reruns (the proxy persists router_settings to the DB). + */ +async function clearFallbackForPrimary(request: import("@playwright/test").APIRequestContext) { + const masterKey = users[Role.ProxyAdmin].password; + const auth = { Authorization: `Bearer ${masterKey}` }; + + const current = await request.get("http://localhost:4000/get/config/callbacks", { headers: auth }); + if (!current.ok()) return; + const body = await current.json(); + const router = body?.router_settings ?? {}; + const existing: Array> = Array.isArray(router.fallbacks) ? router.fallbacks : []; + const next = existing.filter((entry) => !(entry && PRIMARY in entry)); + if (next.length === existing.length) return; + + await request.post("http://localhost:4000/config/update", { + headers: auth, + data: { router_settings: { ...router, fallbacks: next } }, + }); +} + +test.describe("Router Settings - Fallbacks", () => { + test.use({ storageState: ADMIN_STORAGE_PATH }); + + test.beforeEach(async ({ request }) => { + await clearFallbackForPrimary(request); + }); + + test.afterEach(async ({ request }) => { + await clearFallbackForPrimary(request); + }); + + test("Add a fallback and verify it appears in the table", async ({ page }) => { + await navigateToPage(page, Page.RouterSettings); + + // Four tabs: Loadbalancing / Routing Groups / Fallbacks / General — click Fallbacks + await page.getByRole("tab", { name: "Fallbacks" }).click(); + + // The model options come from /model_group/info, which AddFallbacks + // fires only after the modal mounts. Wait for that response so the + // dropdown is populated before we try to pick from it — without this + // the test races on CI (local SLOWMO masks the gap). + const modelsLoaded = page.waitForResponse( + (res) => res.url().includes("/model_group/info") && res.status() === 200, + { timeout: 15_000 }, + ); + await page.getByRole("button", { name: /Add Fallbacks/i }).click(); + await modelsLoaded; + + const modal = page.locator(".ant-modal:visible"); + await expect(modal).toBeVisible({ timeout: 5_000 }); + + // FallbackGroupConfig.tsx renders both selects with `showSearch`. The + // most stable interaction is: click to open + focus, type the model name to + // narrow the listbox to a single highlighted option, then press Enter. + // Verify each selection landed by watching the dialog's own state transition + // (the tab title updates to the picked primary; the fallback chain list + // populates) rather than by asserting on the dropdown popup, which sits in + // a custom getPopupContainer and is awkward to scope reliably. + const primarySelect = modal.locator(".ant-select").filter({ hasText: "Select primary model" }); + await primarySelect.click(); + await page.keyboard.type(PRIMARY); + await page.keyboard.press("Enter"); + await expect(modal.getByRole("tab", { name: PRIMARY })).toBeVisible({ timeout: 10_000 }); + + const fallbackSelect = modal.locator(".ant-select").filter({ hasText: "Select fallback models" }); + await fallbackSelect.click(); + await page.keyboard.type(FALLBACK); + await page.keyboard.press("Enter"); + await page.keyboard.press("Escape"); + // The Fallback Chain helper text reads "(N/10 used)"; once it ticks to 1 the + // selection has been recorded. + await expect(modal.getByText("(1/10 used)")).toBeVisible({ timeout: 10_000 }); + + // Save + await modal.getByRole("button", { name: /Save All Configurations/i }).click(); + + // Success toast + await expect(page.getByText(/fallback configuration\(s\) added successfully/i).first()) + .toBeVisible({ timeout: 10_000 }); + + // Modal closes, and a single row contains BOTH the primary and the fallback + // model — stronger than asserting each name appears somewhere in tbody, + // which could be satisfied by leftover rows from prior runs. + await expect(modal).not.toBeVisible({ timeout: 5_000 }); + + const newRow = page.locator("table tbody tr") + .filter({ hasText: PRIMARY }) + .filter({ hasText: FALLBACK }); + await expect(newRow).toHaveCount(1, { timeout: 10_000 }); + }); +}); diff --git a/ui/litellm-dashboard/e2e_tests/tests/team-admin/teamAdmin.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/team-admin/teamAdmin.spec.ts new file mode 100644 index 00000000000..1612e6929bd --- /dev/null +++ b/ui/litellm-dashboard/e2e_tests/tests/team-admin/teamAdmin.spec.ts @@ -0,0 +1,117 @@ +import { test, expect } from "@playwright/test"; +import { + E2E_INTERNAL_USER_KEY_ALIAS, + E2E_TEAM_CRUD_ALIAS, + E2E_TEAM_CRUD_ID, + TEAM_ADMIN_STORAGE_PATH, +} from "../../constants"; +import { Page } from "../../fixtures/pages"; +import { navigateToPage, dismissFeedbackPopup } from "../../helpers/navigation"; + +async function clickTeamId(page: import("@playwright/test").Page, teamId: string) { + const cell = page.locator("td").filter({ hasText: teamId }).first(); + await expect(cell).toBeVisible({ timeout: 10_000 }); + await cell.click(); + await expect(page.getByText("Back to Teams")).toBeVisible({ timeout: 10_000 }); +} + +test.describe("Team Admin", () => { + test.use({ storageState: TEAM_ADMIN_STORAGE_PATH }); + + test("Team admin can see all team keys including internal user keys", async ({ page }) => { + // Step from the manual-QA checklist: navigate into the team info page, + // open the Virtual Keys tab, and confirm a key belonging to another + // team member (the seeded internal user) is visible. + await navigateToPage(page, Page.Teams); + await dismissFeedbackPopup(page); + + await clickTeamId(page, E2E_TEAM_CRUD_ID); + + await page.getByRole("tab", { name: "Virtual Keys" }).click(); + await expect(page.getByText(E2E_INTERNAL_USER_KEY_ALIAS).first()) + .toBeVisible({ timeout: 10_000 }); + + // And from the global Virtual Keys page, the same key should be visible. + await navigateToPage(page, Page.ApiKeys); + await expect(page.getByText(E2E_INTERNAL_USER_KEY_ALIAS).first()) + .toBeVisible({ timeout: 10_000 }); + }); + + test("Team admin can add a member to their team", async ({ page }) => { + await navigateToPage(page, Page.Teams); + await dismissFeedbackPopup(page); + + await clickTeamId(page, E2E_TEAM_CRUD_ID); + + await page.getByRole("tab", { name: "Members" }).click(); + await page.getByRole("button", { name: /Add Member/i }).click(); + + const modal = page.locator(".ant-modal:visible"); + await expect(modal).toBeVisible({ timeout: 5_000 }); + + // Use a dedicated invitee user so this doesn't race with the proxy-admin + // "Invite a user" test that adds invitable@test.local to the same team. + await modal.locator(".ant-select").first().click(); + await page.keyboard.type("invitable-team@test.local"); + + const emailOption = page.getByRole("option", { name: "invitable-team@test.local" }).first(); + await expect(emailOption).toBeAttached({ timeout: 10_000 }); + await page.keyboard.press("Enter"); + + await modal.getByRole("button", { name: /Add Member/i }).click(); + + await expect(page.getByText("Team member added successfully").first()) + .toBeVisible({ timeout: 10_000 }); + }); + + test("Team admin can remove a member from their team", async ({ page }) => { + await navigateToPage(page, Page.Teams); + await dismissFeedbackPopup(page); + + await clickTeamId(page, E2E_TEAM_CRUD_ID); + + await page.getByRole("tab", { name: "Members" }).click(); + + // Seeded members appear in the roster by user_id (members_with_roles has no + // email), so match the row on the user_id rather than the email. + const row = page.locator("tr", { hasText: "e2e-removable-member" }).first(); + await expect(row).toBeVisible({ timeout: 10_000 }); + await row.getByTestId("delete-member").click(); + + const modal = page.locator(".ant-modal:visible"); + await expect(modal).toBeVisible({ timeout: 5_000 }); + await modal.getByRole("button", { name: /^Delete$/ }).click(); + + await expect(page.getByText("Team member removed successfully").first()) + .toBeVisible({ timeout: 10_000 }); + }); + + test("Team admin can create a team key with All Team Models", async ({ page }) => { + await navigateToPage(page, Page.ApiKeys); + await dismissFeedbackPopup(page); + + await page.getByRole("button", { name: /Create New Key/i }).click(); + await expect(page.getByText("Key Ownership")).toBeVisible({ timeout: 10_000 }); + + const keyName = `e2e-team-admin-key-${Date.now()}`; + await page.getByTestId("base-input").fill(keyName); + + // Team selector — same locator pattern as the proxy-admin keys test. + const teamSelect = page.locator(".ant-select", { hasText: "Search or select a team" }); + await teamSelect.click(); + await page.keyboard.type(E2E_TEAM_CRUD_ALIAS); + await page.locator(".ant-select-dropdown:visible").getByText(E2E_TEAM_CRUD_ALIAS).first().click(); + + // Models — pick "All Team Models" + await page.locator(".ant-select-selection-overflow").click(); + await page.locator(".ant-select-dropdown:visible").getByText("All Team Models").click(); + await page.keyboard.press("Escape"); + + await page.getByRole("button", { name: "Create Key", exact: true }).click(); + + await expect(page.getByText("Save your Key")).toBeVisible({ timeout: 10_000 }); + await page.keyboard.press("Escape"); + + await expect(page.getByText(keyName)).toBeVisible({ timeout: 10_000 }); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/experimental/api-playground/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/experimental/api-playground/page.tsx deleted file mode 100644 index 0948b7626db..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/experimental/api-playground/page.tsx +++ /dev/null @@ -1,12 +0,0 @@ -"use client"; - -import TransformRequestPanel from "@/components/transform_request"; -import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; - -const APIPlaygroundPage = () => { - const { accessToken } = useAuthorized(); - - return ; -}; - -export default APIPlaygroundPage; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/experimental/budgets/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/experimental/budgets/page.tsx deleted file mode 100644 index e49bd342c05..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/experimental/budgets/page.tsx +++ /dev/null @@ -1,12 +0,0 @@ -"use client"; - -import BudgetPanel from "@/components/budgets/budget_panel"; -import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; - -const BudgetsPage = () => { - const { accessToken } = useAuthorized(); - - return ; -}; - -export default BudgetsPage; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/experimental/caching/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/experimental/caching/page.tsx deleted file mode 100644 index 6dcbcdc697c..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/experimental/caching/page.tsx +++ /dev/null @@ -1,20 +0,0 @@ -"use client"; - -import CacheDashboard from "@/components/cache_dashboard"; -import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; - -const CachingPage = () => { - const { token, accessToken, userRole, userId, premiumUser } = useAuthorized(); - - return ( - - ); -}; - -export default CachingPage; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/experimental/claude-code-plugins/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/experimental/claude-code-plugins/page.tsx deleted file mode 100644 index c92c39639c6..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/experimental/claude-code-plugins/page.tsx +++ /dev/null @@ -1,17 +0,0 @@ -"use client"; - -import ClaudeCodePluginsPanel from "@/components/claude_code_plugins"; -import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; - -const ClaudeCodePluginsPage = () => { - const { accessToken, userRole } = useAuthorized(); - - return ( - - ); -}; - -export default ClaudeCodePluginsPage; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/experimental/old-usage/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/experimental/old-usage/page.tsx deleted file mode 100644 index 9521f4f69f1..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/experimental/old-usage/page.tsx +++ /dev/null @@ -1,23 +0,0 @@ -"use client"; - -import Usage from "@/components/usage"; -import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; -import { useState } from "react"; - -const OldUsagePage = () => { - const { accessToken, token, userRole, userId, premiumUser } = useAuthorized(); - const [keys, setKeys] = useState([]); - - return ( - - ); -}; - -export default OldUsagePage; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/experimental/prompts/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/experimental/prompts/page.tsx deleted file mode 100644 index 0836a03b7e7..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/experimental/prompts/page.tsx +++ /dev/null @@ -1,12 +0,0 @@ -"use client"; - -import PromptsPanel from "@/components/prompts"; -import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; - -const PromptsPage = () => { - const { accessToken } = useAuthorized(); - - return ; -}; - -export default PromptsPage; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/experimental/tag-management/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/experimental/tag-management/page.tsx deleted file mode 100644 index 0e686387b34..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/experimental/tag-management/page.tsx +++ /dev/null @@ -1,12 +0,0 @@ -"use client"; - -import TagManagement from "@/components/tag_management"; -import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; - -const TagManagementPage = () => { - const { accessToken, userId, userRole } = useAuthorized(); - - return ; -}; - -export default TagManagementPage; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/page.tsx deleted file mode 100644 index 50cee215eb9..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/page.tsx +++ /dev/null @@ -1,12 +0,0 @@ -"use client"; - -import GuardrailsPanel from "@/components/guardrails"; -import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; - -const GuardrailsPage = () => { - const { accessToken } = useAuthorized(); - - return ; -}; - -export default GuardrailsPage; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/logs/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/logs/page.tsx deleted file mode 100644 index 43ce427131b..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/logs/page.tsx +++ /dev/null @@ -1,20 +0,0 @@ -"use client"; - -import SpendLogsTable from "@/components/view_logs"; -import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; - -const LogsPage = () => { - const { accessToken, token, userRole, userId, premiumUser } = useAuthorized(); - - return ( - - ); -}; - -export default LogsPage; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/model-hub/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/model-hub/page.tsx deleted file mode 100644 index c37a935976b..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/model-hub/page.tsx +++ /dev/null @@ -1,12 +0,0 @@ -"use client"; - -import ModelHubTable from "@/components/AIHub/ModelHubTable"; -import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; - -const ModelHubPage = () => { - const { accessToken, premiumUser, userRole } = useAuthorized(); - - return ; -}; - -export default ModelHubPage; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/policies/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/policies/page.tsx deleted file mode 100644 index c1f6ec51d78..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/policies/page.tsx +++ /dev/null @@ -1,17 +0,0 @@ -"use client"; - -import PoliciesPanel from "@/components/policies"; -import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; - -const PoliciesPage = () => { - const { accessToken, userRole } = useAuthorized(); - - return ( - - ); -}; - -export default PoliciesPage; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/settings/admin-settings/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/settings/admin-settings/page.tsx deleted file mode 100644 index 8dae33afe7e..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/settings/admin-settings/page.tsx +++ /dev/null @@ -1,13 +0,0 @@ -"use client"; - -import AdminPanel from "@/components/AdminPanel"; - -const AdminSettings = () => { - - return ( - - ); -}; - -export default AdminSettings; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/settings/logging-and-alerts/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/settings/logging-and-alerts/page.tsx deleted file mode 100644 index b13e3c42f9e..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/settings/logging-and-alerts/page.tsx +++ /dev/null @@ -1,12 +0,0 @@ -"use client"; - -import Settings from "@/components/settings"; -import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; - -const LoggingAndAlertsPage = () => { - const { accessToken, userRole, userId, premiumUser } = useAuthorized(); - - return ; -}; - -export default LoggingAndAlertsPage; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/settings/router-settings/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/settings/router-settings/page.tsx deleted file mode 100644 index 2b5463cd81f..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/settings/router-settings/page.tsx +++ /dev/null @@ -1,12 +0,0 @@ -"use client"; - -import GeneralSettings from "@/components/general_settings"; -import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; - -const RouterSettingsPage = () => { - const { accessToken, userRole, userId } = useAuthorized(); - - return ; -}; - -export default RouterSettingsPage; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/settings/ui-theme/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/settings/ui-theme/page.tsx deleted file mode 100644 index c6826cf11df..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/settings/ui-theme/page.tsx +++ /dev/null @@ -1,12 +0,0 @@ -"use client"; - -import UIThemeSettings from "@/components/ui_theme_settings"; -import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; - -const UIThemePage = () => { - const { userId, userRole, accessToken } = useAuthorized(); - - return ; -}; - -export default UIThemePage; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/skills/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/skills/page.tsx deleted file mode 100644 index 47d2331bed8..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/skills/page.tsx +++ /dev/null @@ -1,17 +0,0 @@ -"use client"; - -import ClaudeCodePluginsPanel from "@/components/claude_code_plugins"; -import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; - -const SkillsPage = () => { - const { accessToken, userRole } = useAuthorized(); - - return ( - - ); -}; - -export default SkillsPage; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/teams/TeamsView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/teams/TeamsView.tsx deleted file mode 100644 index 94a0e03304e..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/teams/TeamsView.tsx +++ /dev/null @@ -1,370 +0,0 @@ -import React, { useState, useEffect } from "react"; -import { useQueryClient } from "@tanstack/react-query"; -import { organizationKeys } from "@/app/(dashboard)/hooks/organizations/useOrganizations"; -import { teamDeleteCall, Organization } from "@/components/networking"; -import { fetchTeams } from "@/components/common_components/fetch_teams"; -import { Form } from "antd"; -import TeamInfoView from "@/components/team/TeamInfo"; -import TeamSSOSettings from "@/components/TeamSSOSettings"; -import { isAdminRole } from "@/utils/roles"; -import { Card, Button, Col, Text, Grid, TabPanel } from "@tremor/react"; -import AvailableTeamsPanel from "@/components/team/available_teams"; -import type { KeyResponse, Team } from "@/components/key_team_helpers/key_list"; - -import { Member, v2TeamListCall } from "@/components/networking"; -import { updateExistingKeys } from "@/utils/dataUtils"; -import TeamsHeaderTabs from "@/app/(dashboard)/teams/components/TeamsHeaderTabs"; -import TeamsFilters from "@/app/(dashboard)/teams/components/TeamsFilters"; -import useFetchTeams from "@/app/(dashboard)/teams/hooks/useFetchTeams"; -import TeamsTable from "@/app/(dashboard)/teams/components/TeamsTable/TeamsTable"; -import DeleteTeamModal from "@/app/(dashboard)/teams/components/modals/DeleteTeamModal"; -import CreateTeamModal from "@/app/(dashboard)/teams/components/modals/CreateTeamModal"; - -interface TeamProps { - teams: Team[] | null; - accessToken: string | null; - setTeams: React.Dispatch>; - userID: string | null; - userRole: string | null; - organizations: Organization[] | null; - premiumUser?: boolean; -} - -interface FilterState { - team_id: string; - team_alias: string; - organization_id: string; - sort_by: string; - sort_order: "asc" | "desc"; -} - -interface TeamInfo { - members_with_roles: Member[]; -} - -interface PerTeamInfo { - keys: KeyResponse[]; - team_info: TeamInfo; -} - -const TeamsView: React.FC = ({ - teams, - accessToken, - setTeams, - userID, - userRole, - organizations, - premiumUser = false, -}) => { - const queryClient = useQueryClient(); - const [currentOrg, setCurrentOrg] = useState(null); - const [showFilters, setShowFilters] = useState(false); - const [filters, setFilters] = useState({ - team_id: "", - team_alias: "", - organization_id: "", - sort_by: "created_at", - sort_order: "desc", - }); - - const [form] = Form.useForm(); - const [memberForm] = Form.useForm(); - - const [selectedTeamId, setSelectedTeamId] = useState(null); - const [editTeam, setEditTeam] = useState(false); - - const [isTeamModalVisible, setIsTeamModalVisible] = useState(false); - const [isAddMemberModalVisible, setIsAddMemberModalVisible] = useState(false); - const [isEditMemberModalVisible, setIsEditMemberModalVisible] = useState(false); - const [userModels, setUserModels] = useState([]); - const [isDeleteModalOpen, setIsDeleteModalOpen] = useState(false); - const [teamToDelete, setTeamToDelete] = useState(null); - const [perTeamInfo, setPerTeamInfo] = useState>({}); - - const [loggingSettings, setLoggingSettings] = useState([]); - const [modelAliases, setModelAliases] = useState<{ [key: string]: string }>({}); - const { lastRefreshed, onRefreshClick: handleRefreshClick } = useFetchTeams({ currentOrg, setTeams }); - - useEffect(() => { - const fetchTeamInfo = () => { - if (!teams) return; - - const newPerTeamInfo = teams.reduce( - (acc, team) => { - acc[team.team_id] = { - keys: team.keys || [], - team_info: { - members_with_roles: team.members_with_roles || [], - }, - }; - return acc; - }, - {} as Record, - ); - - setPerTeamInfo(newPerTeamInfo); - }; - - fetchTeamInfo(); - }, [teams]); - - const handleOk = () => { - setIsTeamModalVisible(false); - form.resetFields(); - setLoggingSettings([]); - setModelAliases({}); - }; - - const handleMemberOk = () => { - setIsAddMemberModalVisible(false); - setIsEditMemberModalVisible(false); - memberForm.resetFields(); - }; - - const handleCancel = () => { - setIsTeamModalVisible(false); - form.resetFields(); - setLoggingSettings([]); - setModelAliases({}); - }; - - const handleDelete = async (team_id: string) => { - // Set the team to delete and open the confirmation modal - setTeamToDelete(team_id); - setIsDeleteModalOpen(true); - }; - - const confirmDelete = async () => { - if (teamToDelete == null || teams == null || accessToken == null) { - return; - } - - try { - await teamDeleteCall(accessToken, teamToDelete); - queryClient.invalidateQueries({ queryKey: organizationKeys.all }); - // Successfully completed the deletion. Update the state to trigger a rerender. - fetchTeams(accessToken, userID, userRole, currentOrg, setTeams); - } catch (error) { - console.error("Error deleting the team:", error); - // Handle any error situations, such as displaying an error message to the user. - } - - // Close the confirmation modal and reset the teamToDelete - setIsDeleteModalOpen(false); - setTeamToDelete(null); - }; - - const cancelDelete = () => { - // Close the confirmation modal and reset the teamToDelete - setIsDeleteModalOpen(false); - setTeamToDelete(null); - }; - - const is_team_admin = (team: any) => { - if (team == null || team.members_with_roles == null) { - return false; - } - for (let i = 0; i < team.members_with_roles.length; i++) { - let member = team.members_with_roles[i]; - if (member.user_id == userID && member.role == "admin") { - return true; - } - } - return false; - }; - - const handleFilterChange = (key: keyof FilterState, value: string) => { - const newFilters = { ...filters, [key]: value }; - setFilters(newFilters); - // Call teamListCall with the new filters - if (accessToken) { - v2TeamListCall( - accessToken, - newFilters.organization_id || null, - null, - newFilters.team_id || null, - newFilters.team_alias || null, - ) - .then((response) => { - if (response && response.teams) { - setTeams(response.teams); - } - }) - .catch((error) => { - console.error("Error fetching teams:", error); - }); - } - }; - - const handleSortChange = (sortBy: string, sortOrder: "asc" | "desc") => { - const newFilters = { - ...filters, - sort_by: sortBy, - sort_order: sortOrder, - }; - setFilters(newFilters); - // Call teamListCall with the new sort parameters - if (accessToken) { - v2TeamListCall( - accessToken, - filters.organization_id || null, - null, - filters.team_id || null, - filters.team_alias || null, - ) - .then((response) => { - if (response && response.teams) { - setTeams(response.teams); - } - }) - .catch((error) => { - console.error("Error fetching teams:", error); - }); - } - }; - - const handleFilterReset = () => { - setFilters({ - team_id: "", - team_alias: "", - organization_id: "", - sort_by: "created_at", - sort_order: "desc", - }); - // Reset teams list - if (accessToken) { - v2TeamListCall(accessToken, null, userID || null, null, null) - .then((response) => { - if (response && response.teams) { - setTeams(response.teams); - } - }) - .catch((error) => { - console.error("Error fetching teams:", error); - }); - } - }; - - return ( -

- - - {(userRole == "Admin" || userRole == "Org Admin") && ( - - )} - {selectedTeamId ? ( - { - setTeams((teams) => { - if (teams == null) { - return teams; - } - const updated = teams.map((team) => { - if (data.team_id === team.team_id) { - return updateExistingKeys(team, data); - } - return team; - }); - // Minimal fix: refresh the full team list after an update - if (accessToken) { - fetchTeams(accessToken, userID, userRole, currentOrg, setTeams); - } - return updated; - }); - }} - onClose={() => { - setSelectedTeamId(null); - setEditTeam(false); - }} - accessToken={accessToken} - is_team_admin={is_team_admin(teams?.find((team) => team.team_id === selectedTeamId))} - is_proxy_admin={userRole == "Admin"} - is_org_admin={(() => { - const team = teams?.find((t) => t.team_id === selectedTeamId); - if (!team?.organization_id || !organizations || !userID) return false; - const org = organizations.find((o) => o.organization_id === team.organization_id); - return org?.members?.some((m: any) => m.user_id === userID && m.user_role === "org_admin") ?? false; - })()} - userModels={userModels} - editTeam={editTeam} - premiumUser={premiumUser} - /> - ) : ( - - - - Click on “Team ID” to view team details and manage team members. - - - - -
-
- -
-
- - {isDeleteModalOpen && ( - - )} - - - - - - - - {isAdminRole(userRole || "") && ( - - - - )} - - )} - {(userRole == "Admin" || userRole == "Org Admin") && ( - - )} - - -
- ); -}; - -export default TeamsView; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/teams/components/TeamsFilters.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/teams/components/TeamsFilters.test.tsx deleted file mode 100644 index 9a818c27624..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/teams/components/TeamsFilters.test.tsx +++ /dev/null @@ -1,151 +0,0 @@ -import { render, screen, within } from "@testing-library/react"; -import userEvent from "@testing-library/user-event"; -import React from "react"; -import { describe, expect, it, vi } from "vitest"; -import { Organization } from "@/components/networking"; -import TeamsFilters from "./TeamsFilters"; - -type FilterState = { - team_id: string; - team_alias: string; - organization_id: string; - sort_by: string; - sort_order: "asc" | "desc"; -}; - -const emptyFilters: FilterState = { - team_alias: "", - team_id: "", - organization_id: "", - sort_by: "", - sort_order: "asc", -}; - -const mockOrganizations: Organization[] = [ - { organization_id: "org-1", organization_alias: "Acme Corp" } as Organization, - { organization_id: "org-2", organization_alias: "Globex" } as Organization, -]; - -const renderFilters = (overrides: Partial[0]> = {}) => { - const defaults = { - filters: emptyFilters, - organizations: mockOrganizations, - showFilters: false, - onToggleFilters: vi.fn(), - onChange: vi.fn(), - onReset: vi.fn(), - }; - return render(); -}; - -describe("TeamsFilters", () => { - it("should render the team name search input, Filters button, and Reset Filters button", () => { - renderFilters(); - - expect(screen.getByPlaceholderText("Search by Team Name...")).toBeInTheDocument(); - expect(screen.getByRole("button", { name: /^filters$/i })).toBeInTheDocument(); - expect(screen.getByRole("button", { name: /reset filters/i })).toBeInTheDocument(); - }); - - it("should reflect the current team_alias filter value in the search input", () => { - renderFilters({ filters: { ...emptyFilters, team_alias: "Platform" } }); - - expect(screen.getByPlaceholderText("Search by Team Name...")).toHaveValue("Platform"); - }); - - it("should call onChange with 'team_alias' key when the search input changes", async () => { - const user = userEvent.setup(); - const onChange = vi.fn(); - renderFilters({ onChange }); - - await user.type(screen.getByPlaceholderText("Search by Team Name..."), "Dev"); - - expect(onChange).toHaveBeenCalledWith("team_alias", expect.stringContaining("D")); - }); - - it("should call onToggleFilters with the inverted boolean when the Filters button is clicked", async () => { - const user = userEvent.setup(); - const onToggleFilters = vi.fn(); - renderFilters({ showFilters: false, onToggleFilters }); - - await user.click(screen.getByRole("button", { name: /^filters$/i })); - - expect(onToggleFilters).toHaveBeenCalledWith(true); - }); - - it("should call onToggleFilters(false) when filters are currently expanded", async () => { - const user = userEvent.setup(); - const onToggleFilters = vi.fn(); - renderFilters({ showFilters: true, onToggleFilters }); - - await user.click(screen.getByRole("button", { name: /^filters$/i })); - - expect(onToggleFilters).toHaveBeenCalledWith(false); - }); - - it("should call onReset when the Reset Filters button is clicked", async () => { - const user = userEvent.setup(); - const onReset = vi.fn(); - renderFilters({ onReset }); - - await user.click(screen.getByRole("button", { name: /reset filters/i })); - - expect(onReset).toHaveBeenCalledTimes(1); - }); - - it("should not show the Team ID input when showFilters is false", () => { - renderFilters({ showFilters: false }); - - expect(screen.queryByPlaceholderText("Enter Team ID")).not.toBeInTheDocument(); - }); - - it("should show the Team ID input when showFilters is true", () => { - renderFilters({ showFilters: true }); - - expect(screen.getByPlaceholderText("Enter Team ID")).toBeInTheDocument(); - }); - - it("should call onChange with 'team_id' key when the Team ID input changes", async () => { - const user = userEvent.setup(); - const onChange = vi.fn(); - renderFilters({ showFilters: true, onChange }); - - await user.type(screen.getByPlaceholderText("Enter Team ID"), "abc"); - - expect(onChange).toHaveBeenCalledWith("team_id", expect.stringContaining("a")); - }); - - it("should reflect the current team_id filter value in the Team ID input", () => { - renderFilters({ showFilters: true, filters: { ...emptyFilters, team_id: "team-xyz" } }); - - expect(screen.getByPlaceholderText("Enter Team ID")).toHaveValue("team-xyz"); - }); - - it("should show the active filter indicator on the Filters button when team_alias is set", () => { - renderFilters({ filters: { ...emptyFilters, team_alias: "Platform" } }); - - const filtersButton = screen.getByRole("button", { name: /^filters$/i }); - expect(within(filtersButton).getByTestId("active-filter-indicator")).toBeInTheDocument(); - }); - - it("should show the active filter indicator on the Filters button when team_id is set", () => { - renderFilters({ filters: { ...emptyFilters, team_id: "team-123" } }); - - const filtersButton = screen.getByRole("button", { name: /^filters$/i }); - expect(within(filtersButton).getByTestId("active-filter-indicator")).toBeInTheDocument(); - }); - - it("should show the active filter indicator on the Filters button when organization_id is set", () => { - renderFilters({ filters: { ...emptyFilters, organization_id: "org-1" } }); - - const filtersButton = screen.getByRole("button", { name: /^filters$/i }); - expect(within(filtersButton).getByTestId("active-filter-indicator")).toBeInTheDocument(); - }); - - it("should not show the active filter indicator when all filters are empty", () => { - renderFilters({ filters: emptyFilters }); - - const filtersButton = screen.getByRole("button", { name: /^filters$/i }); - expect(within(filtersButton).queryByTestId("active-filter-indicator")).not.toBeInTheDocument(); - }); -}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/teams/components/TeamsFilters.tsx b/ui/litellm-dashboard/src/app/(dashboard)/teams/components/TeamsFilters.tsx deleted file mode 100644 index 04c65ffe268..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/teams/components/TeamsFilters.tsx +++ /dev/null @@ -1,141 +0,0 @@ -import { Select, SelectItem } from "@tremor/react"; -import React from "react"; -import { Organization } from "@/components/networking"; - -interface TeamsFiltersProps { - filters: FilterState; - organizations: Organization[] | null; - showFilters: boolean; - onToggleFilters: (toggle: boolean) => void; - onChange: (key: K, value: FilterState[K]) => void; - onReset: () => void; -} - -type FilterState = { - team_id: string; - team_alias: string; - organization_id: string; - sort_by: string; - sort_order: "asc" | "desc"; -}; - -const TeamsFilters = ({ - filters, - organizations, - showFilters, - onToggleFilters, - onChange, - onReset, -}: TeamsFiltersProps) => { - return ( -
- {/* Search and Filter Controls */} -
- {/* Team Alias Search */} -
- onChange("team_alias", e.target.value)} - /> - - - -
- - {/* Filter Button */} - - - {/* Reset Filters Button */} - -
- - {/* Additional Filters */} - {showFilters && ( -
- {/* Team ID Search */} -
- onChange("team_id", e.target.value)} - /> - - - -
- - {/* Organization Dropdown */} -
- -
-
- )} -
- ); -}; - -export default TeamsFilters; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/teams/components/TeamsHeaderTabs.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/teams/components/TeamsHeaderTabs.test.tsx deleted file mode 100644 index 50a7f10f047..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/teams/components/TeamsHeaderTabs.test.tsx +++ /dev/null @@ -1,54 +0,0 @@ -import { render, screen } from "@testing-library/react"; -import React from "react"; -import { describe, expect, it, vi } from "vitest"; -import TeamsHeaderTabs from "./TeamsHeaderTabs"; - -vi.mock("@tremor/react", () => ({ - TabGroup: ({ children, ...props }: any) =>
{children}
, - TabList: ({ children, ...props }: any) =>
{children}
, - Tab: ({ children, ...props }: any) => , - TabPanels: ({ children, ...props }: any) =>
{children}
, - Text: ({ children, ...props }: any) => {children}, - Icon: ({ onClick, ...props }: any) => -
-
- {keyCount > 0 && ( -
-
- -
-
-

- Warning: This team has {keyCount} associated key{keyCount > 1 ? "s" : ""}. -

-

- Deleting the team will also delete all associated keys. This action is irreversible. -

-
-
- )} -

- Are you sure you want to force delete this team and all its keys? -

-
- - setDeleteConfirmInput(e.target.value)} - placeholder="Enter team name exactly" - className="w-full px-4 py-3 border border-gray-300 rounded-md focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-blue-500 text-base" - autoFocus - /> -
-
- -
- - -
- - - ); -}; - -export default DeleteTeamModal; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/teams/hooks/useFetchTeams.ts b/ui/litellm-dashboard/src/app/(dashboard)/teams/hooks/useFetchTeams.ts deleted file mode 100644 index c02787896f9..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/teams/hooks/useFetchTeams.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { useCallback, useEffect, useState } from "react"; -import { fetchTeams } from "@/components/common_components/fetch_teams"; -import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; -import { Organization, Team } from "@/components/networking"; - -interface useFetchTeamsProps { - currentOrg: Organization | null; - setTeams: (teams: Team[] | null) => void; -} - -const useFetchTeams = ({ currentOrg, setTeams }: useFetchTeamsProps) => { - const [lastRefreshed, setLastRefreshed] = useState(""); - const { accessToken, userId, userRole } = useAuthorized(); - - const onRefreshClick = useCallback(() => { - const currentDate = new Date(); - setLastRefreshed(currentDate.toLocaleString()); - }, []); - - useEffect(() => { - if (accessToken) { - fetchTeams(accessToken, userId, userRole, currentOrg, setTeams).then(); - } - onRefreshClick(); - }, [accessToken, currentOrg, lastRefreshed, onRefreshClick, setTeams, userId, userRole]); - - return { lastRefreshed, setLastRefreshed, onRefreshClick }; -}; - -export default useFetchTeams; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/teams/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/teams/page.tsx deleted file mode 100644 index 041c50dd32a..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/teams/page.tsx +++ /dev/null @@ -1,31 +0,0 @@ -"use client"; - -import TeamsView from "@/app/(dashboard)/teams/TeamsView"; -import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; -import useTeams from "@/app/(dashboard)/hooks/useTeams"; -import { useEffect, useState } from "react"; -import { Organization } from "@/components/networking"; -import { fetchOrganizations } from "@/components/organizations"; - -const TeamsPage = () => { - const { accessToken, userId, userRole } = useAuthorized(); - const { teams, setTeams } = useTeams(); - const [organizations, setOrganizations] = useState([]); - - useEffect(() => { - fetchOrganizations(accessToken, setOrganizations).then(() => {}); - }, [accessToken]); - - return ( - - ); -}; - -export default TeamsPage; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/test-key/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/test-key/page.tsx deleted file mode 100644 index 0f984686c40..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/test-key/page.tsx +++ /dev/null @@ -1,45 +0,0 @@ -"use client"; - -import ChatUI from "@/components/playground/chat_ui/ChatUI"; -import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; -import { useState, useEffect } from "react"; -import { fetchProxySettings } from "@/utils/proxyUtils"; - -interface ProxySettings { - PROXY_BASE_URL?: string; - LITELLM_UI_API_DOC_BASE_URL?: string | null; -} - -const TestKeyPage = () => { - const { token, accessToken, userRole, userId, disabledPersonalKeyCreation } = useAuthorized(); - const [proxySettings, setProxySettings] = useState(undefined); - - useEffect(() => { - const initializeProxySettings = async () => { - if (accessToken) { - const settings = await fetchProxySettings(accessToken); - if (settings) { - setProxySettings({ - PROXY_BASE_URL: settings.PROXY_BASE_URL || undefined, - LITELLM_UI_API_DOC_BASE_URL: settings.LITELLM_UI_API_DOC_BASE_URL, - }); - } - } - }; - - initializeProxySettings(); - }, [accessToken]); - - return ( - - ); -}; - -export default TestKeyPage; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/tools/mcp-servers/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/tools/mcp-servers/page.tsx deleted file mode 100644 index 9b94de6c9f2..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/tools/mcp-servers/page.tsx +++ /dev/null @@ -1,12 +0,0 @@ -"use client"; - -import { MCPServers } from "@/components/mcp_tools"; -import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; - -const MCPServersPage = () => { - const { accessToken, userRole, userId } = useAuthorized(); - - return ; -}; - -export default MCPServersPage; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/tools/vector-stores/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/tools/vector-stores/page.tsx deleted file mode 100644 index 8516a0faa1a..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/tools/vector-stores/page.tsx +++ /dev/null @@ -1,12 +0,0 @@ -"use client"; - -import VectorStoreManagement from "@/components/vector_store_management"; -import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; - -const VectorStoresPage = () => { - const { accessToken, userId, userRole } = useAuthorized(); - - return ; -}; - -export default VectorStoresPage; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/page.tsx deleted file mode 100644 index 477c1163ce7..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/page.tsx +++ /dev/null @@ -1,14 +0,0 @@ -"use client"; - -import UsagePageView from "@/components/UsagePage/components/UsagePageView"; -import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; -import useTeams from "@/app/(dashboard)/hooks/useTeams"; - -const UsagePage = () => { - const { accessToken, userRole, userId, premiumUser } = useAuthorized(); - const { teams } = useTeams(); - - return ; -}; - -export default UsagePage; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/users/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/users/page.tsx deleted file mode 100644 index 9874dd48865..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/users/page.tsx +++ /dev/null @@ -1,53 +0,0 @@ -"use client"; - -import ViewUserDashboard from "@/components/view_users"; -import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; -import useTeams from "@/app/(dashboard)/hooks/useTeams"; -import { useOrganizations } from "@/app/(dashboard)/hooks/organizations/useOrganizations"; -import { isProxyAdminRole } from "@/utils/roles"; -import { useState, useMemo } from "react"; -import { Organization } from "@/components/networking"; - -const UsersPage = () => { - const { accessToken, userRole, userId, token } = useAuthorized(); - const [keys, setKeys] = useState([]); - - const { teams } = useTeams(); - const { data: organizations, isLoading: isOrgsLoading } = useOrganizations(); - - // Three states: - // - undefined: org data still loading (non-proxy-admin) — query should wait - // - null: proxy admin or no org filtering needed — query runs unfiltered - // - Array<{organization_id, organization_alias}>: org admin orgs — query runs filtered - const orgAdminOrgIds = useMemo((): Array<{organization_id: string, organization_alias: string}> | null | undefined => { - if (!userId || !userRole) return null; - // Proxy admins see all users — no org filtering - if (isProxyAdminRole(userRole)) return null; - - // Still loading org data — signal "not ready yet" - if (isOrgsLoading || !organizations) return undefined; - - const adminOrgs = organizations - .filter((org: Organization) => - org.members?.some((member) => member.user_id === userId && member.user_role === "org_admin") - ) - .map((org: Organization) => ({ organization_id: org.organization_id, organization_alias: org.organization_alias })); - - return adminOrgs.length > 0 ? adminOrgs : null; - }, [userId, organizations, userRole, isOrgsLoading]); - - return ( - - ); -}; - -export default UsersPage; diff --git a/ui/litellm-dashboard/src/app/layout.tsx b/ui/litellm-dashboard/src/app/layout.tsx index a4ed17cde39..f79b7eb7028 100644 --- a/ui/litellm-dashboard/src/app/layout.tsx +++ b/ui/litellm-dashboard/src/app/layout.tsx @@ -3,6 +3,7 @@ import { Inter } from "next/font/google"; import "./globals.css"; import AntdGlobalProvider from "@/contexts/AntdGlobalProvider"; +import { AuthProvider } from "@/contexts/AuthContext"; import ReactQueryProvider from "@/contexts/ReactQueryProvider"; const inter = Inter({ subsets: ["latin"] }); @@ -22,7 +23,9 @@ export default function RootLayout({ - {children} + + {children} + diff --git a/ui/litellm-dashboard/src/app/page.tsx b/ui/litellm-dashboard/src/app/page.tsx index 57618185968..ce12967c911 100644 --- a/ui/litellm-dashboard/src/app/page.tsx +++ b/ui/litellm-dashboard/src/app/page.tsx @@ -20,7 +20,7 @@ import { Team } from "@/components/key_team_helpers/key_list"; import { MCPServers } from "@/components/mcp_tools"; import ModelHubTable from "@/components/AIHub/ModelHubTable"; import Navbar from "@/components/navbar"; -import { getUiConfig, Organization, proxyBaseUrl, setGlobalLitellmHeaderName, getInProductNudgesCall } from "@/components/networking"; +import { Organization, proxyBaseUrl, getInProductNudgesCall } from "@/components/networking"; import NewUsagePage from "@/components/UsagePage/components/UsagePageView"; import OldTeams from "@/components/OldTeams"; import { fetchUserModels, CreateKeyPrefillData } from "@/components/organisms/create_key_button"; @@ -45,24 +45,14 @@ import WorkflowRuns from "@/components/workflow_runs"; import SpendLogsTable from "@/components/view_logs"; import ViewUserDashboard from "@/components/view_users"; import { ThemeProvider } from "@/contexts/ThemeContext"; -import { clearTokenCookies, getCookie } from "@/utils/cookieUtils"; -import { isJwtExpired } from "@/utils/jwtUtils"; +import { useAuth } from "@/contexts/AuthContext"; import { buildLoginUrlWithReturn, consumeReturnUrl, isValidReturnUrl, normalizeUrlForCompare, storeReturnUrl } from "@/utils/returnUrlUtils"; -import { formatUserRole, isAdminRole } from "@/utils/roles"; +import { isAdminRole } from "@/utils/roles"; import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; -import { jwtDecode } from "jwt-decode"; import { useRouter, useSearchParams } from "next/navigation"; import { Suspense, useEffect, useMemo, useRef, useState } from "react"; import { ConfigProvider, theme } from "antd"; -function deleteCookie(name: string, path = "/") { - // Best-effort client-side clear (works for non-HttpOnly cookies without Domain) - document.cookie = `${name}=; Max-Age=0; Path=${path}`; - if (name === "token") { - clearTokenCookies(); - } -} - interface ProxySettings { PROXY_BASE_URL: string; PROXY_LOGOUT_URL: string; @@ -77,10 +67,18 @@ interface ProxySettings { const LEGACY_REDIRECTS: Record = {}; function CreateKeyPageContent() { - const [userRole, setUserRole] = useState(""); - const [premiumUser, setPremiumUser] = useState(false); - const [disabledPersonalKeyCreation, setDisabledPersonalKeyCreation] = useState(false); - const [userEmail, setUserEmail] = useState(null); + const { + authLoading, + token, + userID, + userRole, + userEmail, + accessToken, + premiumUser, + setUserRole, + setUserEmail, + } = useAuth(); + const [teams, setTeams] = useState(null); const [keys, setKeys] = useState([]); const [organizations, setOrganizations] = useState([]); @@ -90,14 +88,10 @@ function CreateKeyPageContent() { PROXY_LOGOUT_URL: "", }); - const [showSSOBanner, setShowSSOBanner] = useState(true); const router = useRouter(); const searchParams = useSearchParams()!; const [modelData, setModelData] = useState({ data: [] }); - const [token, setToken] = useState(null); const [createClicked, setCreateClicked] = useState(false); - const [authLoading, setAuthLoading] = useState(true); - const [userID, setUserID] = useState(null); // Survey state - always show by default const [showSurveyPrompt, setShowSurveyPrompt] = useState(true); @@ -185,7 +179,6 @@ function CreateKeyPageContent() { setPage(newPage); }; - const [accessToken, setAccessToken] = useState(null); const [sidebarCollapsed, setSidebarCollapsed] = useState(false); // Track if we've already attempted a return URL redirect to prevent race conditions @@ -201,38 +194,6 @@ function CreateKeyPageContent() { }; const redirectToLogin = authLoading === false && token === null && invitation_id === null; - useEffect(() => { - let cancelled = false; - - (async () => { - try { - await getUiConfig(); // ensures proxyBaseUrl etc. are ready - } catch { - // proceed regardless; we still need to decide auth state - } - - if (cancelled) return; - - const raw = getCookie("token"); - const valid = raw && !isJwtExpired(raw) ? raw : null; - - // If token exists but is invalid/expired, clear it so downstream code - // doesn't keep trying to use it and cause redirect spasms. - if (raw && !valid) { - deleteCookie("token", "/"); - } - - if (!cancelled) { - setToken(valid); - setAuthLoading(false); - } - })(); - - return () => { - cancelled = true; - }; - }, []); - useEffect(() => { if (redirectToLogin) { // Store the current URL so we can redirect back after login @@ -291,62 +252,6 @@ function CreateKeyPageContent() { } }, [token]); - useEffect(() => { - if (!token) { - return; - } - - // Defensive: re-check expiry in case cookie changed after mount - if (isJwtExpired(token)) { - deleteCookie("token", "/"); - setToken(null); - return; - } - - let decoded: any = null; - try { - decoded = jwtDecode(token); - } catch { - // Malformed token → treat as unauthenticated - deleteCookie("token", "/"); - setToken(null); - return; - } - - if (decoded) { - // set accessToken - setAccessToken(decoded.key); - - setDisabledPersonalKeyCreation(decoded.disabled_non_admin_personal_key_creation); - - // check if userRole is defined - if (decoded.user_role) { - const formattedUserRole = formatUserRole(decoded.user_role); - setUserRole(formattedUserRole); - } - - if (decoded.user_email) { - setUserEmail(decoded.user_email); - } - - if (decoded.login_method) { - setShowSSOBanner(decoded.login_method == "username_password" ? true : false); - } - - if (decoded.premium_user) { - setPremiumUser(decoded.premium_user); - } - - if (decoded.auth_header_name) { - setGlobalLitellmHeaderName(decoded.auth_header_name); - } - - if (decoded.user_id) { - setUserID(decoded.user_id); - } - } - }, [token]); - useEffect(() => { if (accessToken && userID && userRole) { fetchUserModels(userID, userRole, accessToken, setUserModels); diff --git a/ui/litellm-dashboard/src/components/OldTeams.test.tsx b/ui/litellm-dashboard/src/components/OldTeams.test.tsx index 4b89820bad6..b8707c1a338 100644 --- a/ui/litellm-dashboard/src/components/OldTeams.test.tsx +++ b/ui/litellm-dashboard/src/components/OldTeams.test.tsx @@ -1018,3 +1018,83 @@ describe("OldTeams - organization alias display", () => { }); }); }); + +describe("OldTeams - Resources column keys badge", () => { + beforeEach(() => { + vi.clearAllMocks(); + mockUseOrganizations.mockReturnValue({ data: [] }); + }); + + it("renders keys_count from the v2 payload in the Resources badge", async () => { + const { container } = renderWithQueryClient( + , + ); + + await waitFor(() => { + expect(screen.getByText("Team With Keys")).toBeInTheDocument(); + }); + const cyanTag = container.querySelector(".ant-tag-cyan"); + expect(cyanTag).not.toBeNull(); + expect(cyanTag?.textContent).toContain("3"); + }); + + it("falls back to keys.length when keys_count is absent", async () => { + const { container } = renderWithQueryClient( + , + ); + + await waitFor(() => { + expect(screen.getByText("Legacy Team")).toBeInTheDocument(); + }); + const cyanTag = container.querySelector(".ant-tag-cyan"); + expect(cyanTag).not.toBeNull(); + expect(cyanTag?.textContent).toContain("2"); + }); +}); diff --git a/ui/litellm-dashboard/src/components/OldTeams.tsx b/ui/litellm-dashboard/src/components/OldTeams.tsx index da00ad911b0..8f9e5a75c20 100644 --- a/ui/litellm-dashboard/src/components/OldTeams.tsx +++ b/ui/litellm-dashboard/src/components/OldTeams.tsx @@ -105,6 +105,7 @@ interface TeamInfo { interface PerTeamInfo { keys: KeyResponse[]; + keys_count: number; team_info: TeamInfo; } @@ -364,6 +365,7 @@ const Teams: React.FC = ({ (acc, team) => { acc[team.team_id] = { keys: team.keys || [], + keys_count: team.keys_count ?? team.keys?.length ?? 0, team_info: { members_with_roles: team.members_with_roles || [], }, @@ -745,7 +747,7 @@ const Teams: React.FC = ({ render: (_: unknown, record: Team) => { const memberCount = perTeamInfo?.[record.team_id]?.team_info?.members_with_roles?.length ?? 0; const modelCount = record.models?.length ?? 0; - const keyCount = perTeamInfo?.[record.team_id]?.keys?.length ?? 0; + const keyCount = perTeamInfo?.[record.team_id]?.keys_count ?? 0; return ( @@ -977,17 +979,23 @@ const Teams: React.FC = ({ { + const deleteKeyCount = + teamToDelete?.keys_count ?? teamToDelete?.keys?.length ?? 0; + return deleteKeyCount === 0 ? undefined - : `Warning: This team has ${teamToDelete?.keys?.length} keys associated with it. Deleting the team will also delete all associated keys. This action is irreversible.` - } + : `Warning: This team has ${deleteKeyCount} keys associated with it. Deleting the team will also delete all associated keys. This action is irreversible.`; + })()} message="Are you sure you want to delete this team and all its keys? This action cannot be undone." resourceInformationTitle="Team Information" resourceInformation={[ { label: "Team ID", value: teamToDelete?.team_id, code: true }, { label: "Team Name", value: teamToDelete?.team_alias }, - { label: "Keys", value: teamToDelete?.keys?.length }, + { + label: "Keys", + value: + teamToDelete?.keys_count ?? teamToDelete?.keys?.length ?? 0, + }, { label: "Members", value: teamToDelete?.members_with_roles?.length }, ]} requiredConfirmation={teamToDelete?.team_alias} diff --git a/ui/litellm-dashboard/src/components/key_team_helpers/key_list.tsx b/ui/litellm-dashboard/src/components/key_team_helpers/key_list.tsx index 6b3c65aaf7b..60568da48ed 100644 --- a/ui/litellm-dashboard/src/components/key_team_helpers/key_list.tsx +++ b/ui/litellm-dashboard/src/components/key_team_helpers/key_list.tsx @@ -13,6 +13,7 @@ export interface Team { organization_id: string; created_at: string; keys: KeyResponse[]; + keys_count?: number; members_with_roles: Member[]; spend: number; access_group_ids?: string[]; diff --git a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx index 7d3f077dafc..b590a7dc043 100644 --- a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx +++ b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx @@ -439,6 +439,12 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp // Update the formValues with the final metadata formValues.metadata = JSON.stringify(metadata); + // disable_global_guardrails is premium-gated server-side; only send it when enabled + // so non-premium key creation isn't blocked by that gate. + if (!formValues.disable_global_guardrails) { + delete formValues.disable_global_guardrails; + } + // Transform allowed_vector_store_ids and allowed_mcp_servers_and_groups into object_permission format if (formValues.allowed_vector_store_ids && formValues.allowed_vector_store_ids.length > 0) { formValues.object_permission = { diff --git a/ui/litellm-dashboard/src/components/templates/key_info_view.tsx b/ui/litellm-dashboard/src/components/templates/key_info_view.tsx index 60b7e31d478..2de40925b1c 100644 --- a/ui/litellm-dashboard/src/components/templates/key_info_view.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_info_view.tsx @@ -34,8 +34,15 @@ interface KeyInfoViewProps { backButtonText?: string; } -// Must stay in sync with LiteLLM_ManagementEndpoint_MetadataFields_Premium -// in litellm/proxy/_types.py — limited to fields the key-edit form submits. +// Premium fields (from LiteLLM_ManagementEndpoint_MetadataFields_Premium in +// litellm/proxy/_types.py) that the key-edit form submits as arrays/strings, where +// "empty" means "unset". The loop below drops them when they're empty-and-were-empty +// so a non-premium edit of unrelated fields doesn't trip the server's premium gate. +// +// Boolean premium fields (e.g. disable_global_guardrails) do NOT belong here: false is +// a real value, not "empty", so isEmptyValue(false) is false and the loop would never +// drop it — we'd resend false on every edit and trip the gate. Booleans get their own +// "send only when changed" guard instead (see disable_global_guardrails below). const PREMIUM_METADATA_FIELDS = [ "policies", "guardrails", @@ -174,6 +181,15 @@ export default function KeyInfoView({ } } + // disable_global_guardrails is premium-gated server-side; only send it when it + // changed so a non-premium edit of unrelated fields isn't blocked by that gate. + const previousDisableGlobalGuardrails = Boolean( + (currentKeyData.metadata as Record | undefined)?.disable_global_guardrails, + ); + if (Boolean(formValues.disable_global_guardrails) === previousDisableGlobalGuardrails) { + delete formValues.disable_global_guardrails; + } + // Handle max budget empty string formValues.max_budget = mapEmptyStringToNull(formValues.max_budget); diff --git a/ui/litellm-dashboard/src/contexts/AuthContext.tsx b/ui/litellm-dashboard/src/contexts/AuthContext.tsx new file mode 100644 index 00000000000..3693d858952 --- /dev/null +++ b/ui/litellm-dashboard/src/contexts/AuthContext.tsx @@ -0,0 +1,157 @@ +"use client"; + +import React, { createContext, useContext, useEffect, useState } from "react"; +import { jwtDecode } from "jwt-decode"; +import { clearTokenCookies, getCookie } from "@/utils/cookieUtils"; +import { isJwtExpired } from "@/utils/jwtUtils"; +import { formatUserRole } from "@/utils/roles"; +import { getUiConfig, setGlobalLitellmHeaderName } from "@/components/networking"; + +function deleteCookie(name: string, path = "/") { + document.cookie = `${name}=; Max-Age=0; Path=${path}`; + if (name === "token") { + clearTokenCookies(); + } +} + +type AuthContextValue = { + authLoading: boolean; + token: string | null; + userID: string | null; + userRole: string; + userEmail: string | null; + accessToken: string | null; + premiumUser: boolean; + disabledPersonalKeyCreation: boolean; + showSSOBanner: boolean; + + setToken: React.Dispatch>; + setUserID: React.Dispatch>; + setUserRole: React.Dispatch>; + setUserEmail: React.Dispatch>; + setAccessToken: React.Dispatch>; + setPremiumUser: React.Dispatch>; + setShowSSOBanner: React.Dispatch>; +}; + +const AuthContext = createContext(null); + +export function AuthProvider({ children }: { children: React.ReactNode }) { + const [authLoading, setAuthLoading] = useState(true); + const [token, setToken] = useState(null); + const [userID, setUserID] = useState(null); + const [userRole, setUserRole] = useState(""); + const [userEmail, setUserEmail] = useState(null); + const [accessToken, setAccessToken] = useState(null); + const [premiumUser, setPremiumUser] = useState(false); + const [disabledPersonalKeyCreation, setDisabledPersonalKeyCreation] = useState(false); + const [showSSOBanner, setShowSSOBanner] = useState(true); + + // Load runtime UI config (populates proxyBaseUrl etc.) before clearing + // authLoading, so any consumer that builds proxy-rooted URLs from authLoading=false + // (e.g. the unauthenticated login redirect) sees the resolved value rather than + // the module-init default. Then read the cookie and validate JWT expiry. + useEffect(() => { + let cancelled = false; + + (async () => { + try { + await getUiConfig(); + } catch { + // proceed regardless; auth state must still be resolved + } + + if (cancelled) return; + + const raw = getCookie("token"); + const valid = raw && !isJwtExpired(raw) ? raw : null; + + // Clear expired/invalid token so downstream code doesn't keep trying to use it. + if (raw && !valid) { + deleteCookie("token", "/"); + } + + setToken(valid); + setAuthLoading(false); + })(); + + return () => { + cancelled = true; + }; + }, []); + + // Decode JWT and populate derived auth state whenever the token changes. + useEffect(() => { + if (!token) { + return; + } + + if (isJwtExpired(token)) { + deleteCookie("token", "/"); + setToken(null); + return; + } + + let decoded: { [k: string]: any } | null = null; + try { + decoded = jwtDecode(token); + } catch { + deleteCookie("token", "/"); + setToken(null); + return; + } + + if (!decoded) return; + + setAccessToken(decoded.key); + setDisabledPersonalKeyCreation(decoded.disabled_non_admin_personal_key_creation); + + if (decoded.user_role) { + setUserRole(formatUserRole(decoded.user_role)); + } + if (decoded.user_email) { + setUserEmail(decoded.user_email); + } + if (decoded.login_method) { + setShowSSOBanner(decoded.login_method === "username_password"); + } + if (decoded.premium_user) { + setPremiumUser(decoded.premium_user); + } + if (decoded.auth_header_name) { + setGlobalLitellmHeaderName(decoded.auth_header_name); + } + if (decoded.user_id) { + setUserID(decoded.user_id); + } + }, [token]); + + const value: AuthContextValue = { + authLoading, + token, + userID, + userRole, + userEmail, + accessToken, + premiumUser, + disabledPersonalKeyCreation, + showSSOBanner, + setToken, + setUserID, + setUserRole, + setUserEmail, + setAccessToken, + setPremiumUser, + setShowSSOBanner, + }; + + return {children}; +} + +export function useAuth(): AuthContextValue { + const ctx = useContext(AuthContext); + if (!ctx) { + throw new Error("useAuth must be used within an AuthProvider"); + } + return ctx; +} diff --git a/ui/litellm-dashboard/tests/CreateKeyPage.expiredToken.test.tsx b/ui/litellm-dashboard/tests/CreateKeyPage.expiredToken.test.tsx index 1d725572a33..3b9232861b9 100644 --- a/ui/litellm-dashboard/tests/CreateKeyPage.expiredToken.test.tsx +++ b/ui/litellm-dashboard/tests/CreateKeyPage.expiredToken.test.tsx @@ -147,6 +147,18 @@ vi.mock("@/lib/cva.config", () => ({ })); import CreateKeyPage from "@/app/page"; +import { AuthProvider } from "@/contexts/AuthContext"; + +// The page consumes auth state via useAuth(). Wrap it so the hook resolves +// against a real provider — the provider's effects (cookie read, JWT decode, +// redirect-on-expired) are what these tests exercise. +function PageUnderTest() { + return ( + + + + ); +} /** ---------------------------- * Helpers @@ -207,7 +219,7 @@ describe("CreateKeyPage auth behavior", () => { const cookieSetSpy = vi.spyOn(document, "cookie", "set"); // Act - render(); + render(); // Assert: we eventually redirect to SSO login with return URL (single replace, not assign/href) await waitFor(() => { @@ -243,7 +255,7 @@ describe("CreateKeyPage auth behavior", () => { }); // Act - render(); + render(); // Assert: no redirect await waitFor(() => { @@ -286,7 +298,7 @@ describe("CreateKeyPage auth behavior", () => { // Return URL has the same params in a different order consumeReturnUrlMock.mockReturnValue("http://localhost/ui?a=1&b=2"); - render(); + render(); await waitFor(() => { expect(window.location.replace).not.toHaveBeenCalled(); diff --git a/uv.lock b/uv.lock index fe3e0e037cf..080cd89df6b 100644 --- a/uv.lock +++ b/uv.lock @@ -9,7 +9,7 @@ resolution-markers = [ ] [options] -exclude-newer = "2026-05-19T01:14:41.559325863Z" +exclude-newer = "2026-05-26T01:39:13.630743Z" exclude-newer-span = "P3D" [manifest] @@ -1465,17 +1465,18 @@ wheels = [ [[package]] name = "fastapi" -version = "0.124.4" +version = "0.136.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "annotated-doc" }, { name = "pydantic" }, { name = "starlette" }, { name = "typing-extensions" }, + { name = "typing-inspection" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/cd/21/ade3ff6745a82ea8ad88552b4139d27941549e4f19125879f848ac8f3c3d/fastapi-0.124.4.tar.gz", hash = "sha256:0e9422e8d6b797515f33f500309f6e1c98ee4e85563ba0f2debb282df6343763", size = 378460, upload-time = "2025-12-12T15:00:43.891Z" } +sdist = { url = "https://files.pythonhosted.org/packages/81/2d/ff8d91d7b564d464629a0fd50a4489c97fcb836ac230bf3a7269232a9b1f/fastapi-0.136.3.tar.gz", hash = "sha256:e487fae93ad408e6f47641ee4dfe389864fd7bec92e547ea8498fc13f43e83ab", size = 396410, upload-time = "2026-05-23T18:53:15.192Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/3e/57/aa70121b5008f44031be645a61a7c4abc24e0e888ad3fc8fda916f4d188e/fastapi-0.124.4-py3-none-any.whl", hash = "sha256:6d1e703698443ccb89e50abe4893f3c84d9d6689c0cf1ca4fad6d3c15cf69f15", size = 113281, upload-time = "2025-12-12T15:00:42.44Z" }, + { url = "https://files.pythonhosted.org/packages/e0/82/45359b62a067409bd929ae8a56b8ed13e5a8c8a61194b3c236920999ab83/fastapi-0.136.3-py3-none-any.whl", hash = "sha256:3d2a69bdf04b7e9f3afa292c3bc7a98816bbfafa10bc9b45f3f3700d2f761620", size = 117481, upload-time = "2026-05-23T18:53:16.924Z" }, ] [[package]] @@ -2082,77 +2083,71 @@ grpc = [ [[package]] name = "granian" -version = "2.5.7" +version = "2.7.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/db/b1/100c5add0409559ddbbecca5835c17217b7a2e026eff999bfa359a630686/granian-2.5.7.tar.gz", hash = "sha256:4702a7bcc736454803426bd2c4e7a374739ae1e4b11d27bcdc49b691d316fa0c", size = 112206, upload-time = "2025-11-05T12:18:29.258Z" } +sdist = { url = "https://files.pythonhosted.org/packages/db/0c/27aa25280b6c1f323312e83088304da8a7f3e5c1e568d3a560365ec6fa67/granian-2.7.4.tar.gz", hash = "sha256:1dc0530d7ae6b0ae43aafafe771ac0b8c38af68bbd71ab355828817faf13aac1", size = 128212, upload-time = "2026-04-23T11:55:55.275Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e1/6f/7719fc97aa081915024939f0d35fdae57dfd3d7214f7ef4a7fa664abbbc3/granian-2.5.7-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:7d84a254e9c88da874ba349f7892278a871acc391ab6af21cc32f58d27cd50a9", size = 2854526, upload-time = "2025-11-05T12:15:29.721Z" }, - { url = "https://files.pythonhosted.org/packages/9f/cd/af33b780602f962c282ba3341131f7ee3b224a6c856a9fb11a017750a48f/granian-2.5.7-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8857d5a6ed94ea64d6b92d1d5fa8f7c1676bbecd71e6ca3d71fcd7118448af1d", size = 2537151, upload-time = "2025-11-05T12:15:31.659Z" }, - { url = "https://files.pythonhosted.org/packages/6d/58/1a0d529d3d3ddc11b2b292b8f2a7566812d8691de7b1fc8ea5c8f36fd81a/granian-2.5.7-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9914dfc93f04a53a92d8cfdb059c11d620ff83e9326a99880491a9c5bc5940ef", size = 3017277, upload-time = "2025-11-05T12:15:33.42Z" }, - { url = "https://files.pythonhosted.org/packages/a4/78/2a3c198ee379392d9998e4ff0cfd9ffa95b2d2c683bd15a7266a09325d43/granian-2.5.7-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:24c972fe009ca3a08fd7fb182e07fcb16bffe49c87b1c3489a6986c9e9248dc1", size = 2859098, upload-time = "2025-11-05T12:15:35.15Z" }, - { url = "https://files.pythonhosted.org/packages/6e/44/7b9fba226083170e9ba221b23ab29d7ffcb761b1ef2b6ed6dac2081bc7fe/granian-2.5.7-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:034df207e62f104d39db479b693e03072c7eb8e202493cdf58948ff83e753cca", size = 3119567, upload-time = "2025-11-05T12:15:36.674Z" }, - { url = "https://files.pythonhosted.org/packages/ff/76/f1e348991c031a50d30d3ab0625fec3b7e811092cdb0d1e996885abf1605/granian-2.5.7-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:0719052a27caca73bf4000ccdb0339a9d6705e7a4b6613b9fa88ba27c72ba659", size = 2901389, upload-time = "2025-11-05T12:15:39.557Z" }, - { url = "https://files.pythonhosted.org/packages/f0/69/71b3d7d90d56fda5617fd98838ac481756ad64f76c1fc1b5e21c43a51f15/granian-2.5.7-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:be5b9224ec2583ea3b6ca90788b7f59253b6e07fcf817d14c205e6611faaf2be", size = 2989856, upload-time = "2025-11-05T12:15:41.001Z" }, - { url = "https://files.pythonhosted.org/packages/74/42/603db3d0ede778adc979c6acc1eaafa5c670c795f5e0e14feb07772ed197/granian-2.5.7-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:ff246af31840369a1d06030f4d291c6a93841f68ee1f836036bce6625ae73b30", size = 3147378, upload-time = "2025-11-05T12:15:42.432Z" }, - { url = "https://files.pythonhosted.org/packages/35/b5/cc557e30ba23c2934c33935768dd0233ef7a10b1e8c81dbbc63d5e2562b5/granian-2.5.7-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:cf79375e37a63217f9c1dc4ad15200bc5a89860b321ca30d8a5086a6ea1202e4", size = 3210930, upload-time = "2025-11-05T12:15:45.263Z" }, - { url = "https://files.pythonhosted.org/packages/c3/67/ba90520cafcd13b5c76d147d713556b9eef877ca001f9ccf44d5443738b6/granian-2.5.7-cp310-cp310-win_amd64.whl", hash = "sha256:b4269a390054c0f71d9ce9d7c75ce2da0c59e78cb522016eb2f5a506c3eb6573", size = 2176887, upload-time = "2025-11-05T12:15:46.615Z" }, - { url = "https://files.pythonhosted.org/packages/61/21/da3ade91b49ae99146daac6426701cc25b2c5f1413b6c8cb1cc048877036/granian-2.5.7-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:7aa90dcda1fbf03604e229465380138954d9c000eca2947a94dcfbd765414d32", size = 2854652, upload-time = "2025-11-05T12:15:48.342Z" }, - { url = "https://files.pythonhosted.org/packages/76/67/a6fa402ca5ebddebec5d46dacf646ce073872e5251915a725f6abf2a23bb/granian-2.5.7-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:da4f27323be1188f9e325711016ee108840e14a5971bb4b4d15b65b2d1b00a2d", size = 2537539, upload-time = "2025-11-05T12:15:50.136Z" }, - { url = "https://files.pythonhosted.org/packages/f9/70/accb5afd83ef785bd9e32067a13547c51cb0139076a8f2857d6d436773df/granian-2.5.7-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8ca5b7028b6ebafce30419ddb6ee7fbfb236fdd0da89427811324ddd38c7d314", size = 3017554, upload-time = "2025-11-05T12:15:52.962Z" }, - { url = "https://files.pythonhosted.org/packages/74/45/98356af5f36af2b6b47a91fef0d326c275e508bf4bcf0c08bd35ed314db8/granian-2.5.7-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b83e95b18be5dfa92296bc8acfeb353488123399c90cc5f0eccf451e88bc4caf", size = 2859127, upload-time = "2025-11-05T12:15:54.49Z" }, - { url = "https://files.pythonhosted.org/packages/27/7a/04d3ec13b197509c40340ec80414fbbc2b0913f6e1a18c3987cc608c8571/granian-2.5.7-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9aad9e920441232a7b8ad33bef7f04aae986e0e386ab7f13312477c3ea2c85df", size = 3119494, upload-time = "2025-11-05T12:15:56.324Z" }, - { url = "https://files.pythonhosted.org/packages/b9/5d/1a82a596725824f6e76b8f7b853ceb464cd0334b2b8143c278aa46f23b6d/granian-2.5.7-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:777d35961d5139d203cf54d872ad5979b171e6496a471a5bcb8032f4471bdec6", size = 2901511, upload-time = "2025-11-05T12:15:58.7Z" }, - { url = "https://files.pythonhosted.org/packages/94/45/b53d6d7df5cd35c3b8bb329f5ee1c7b31ead7a61a6f2046f6562028d7e1b/granian-2.5.7-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ae72c7ba1e8f35d3021dafb2ba6c4ef89f93f877218f8c6ed1cb672145cd81ad", size = 2989828, upload-time = "2025-11-05T12:16:00.341Z" }, - { url = "https://files.pythonhosted.org/packages/7f/80/bb57b0fa24fcd518cd64442249459bd214ab1ec5f32590fd30389944261c/granian-2.5.7-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:3764d87edd3fddaf557dce32be396a2a56dfc5b9ad2989b1f98952983ae4a21c", size = 3147694, upload-time = "2025-11-05T12:16:01.826Z" }, - { url = "https://files.pythonhosted.org/packages/7f/00/f8747aaf8dcd488e4462db89f7273dd9ae702fd17a58d72193b48eff0470/granian-2.5.7-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f5e21bbf1daebb0219253576cac4e5edc8fa8356ad85d66577c4f3ea2d5c6e3c", size = 3211169, upload-time = "2025-11-05T12:16:03.308Z" }, - { url = "https://files.pythonhosted.org/packages/1f/69/8593d539898a870692cad447d22c2c4cc34566ad9070040ca216db6ac184/granian-2.5.7-cp311-cp311-win_amd64.whl", hash = "sha256:d210dd98852825c8a49036a6ec23cdfaa7689d1cb12ddc651c6466b412047349", size = 2176921, upload-time = "2025-11-05T12:16:04.63Z" }, - { url = "https://files.pythonhosted.org/packages/b5/cf/f76d05e950f76924ffb6c5212561be4dd93fa569518869cc1233a0c77613/granian-2.5.7-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:41e3a293ac23c76d18628d1bd8376ce3230fb3afe3cf71126b8885e8da4e40c4", size = 2850787, upload-time = "2025-11-05T12:16:06.028Z" }, - { url = "https://files.pythonhosted.org/packages/3f/d7/6972aa8c38d26b4cf9f35bcc9b7d3a26a3aa930e612d5913d8f4181331a1/granian-2.5.7-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8b345b539bcbe6dedf8a9323b0c960530cb1fb2cfb887139e6ae9513b6c04d8c", size = 2529552, upload-time = "2025-11-05T12:16:07.389Z" }, - { url = "https://files.pythonhosted.org/packages/56/b4/cd5958b6af674a32296a0fef73fb499c2bf2874025062323f5dbc838f4fc/granian-2.5.7-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:12e4d7ba8e3223e2bf974860a59c29b06fa805a98ad4304be4e77180d3a28f55", size = 3009131, upload-time = "2025-11-05T12:16:08.759Z" }, - { url = "https://files.pythonhosted.org/packages/7a/69/f3828de736c2802fd7fcac0bb1a0387b3332d432f0eeacb8116094926f06/granian-2.5.7-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e727d3518f038b64cb0352b34f43b387aafe5eb12b6c4b57ef598b811e40d4ed", size = 2852544, upload-time = "2025-11-05T12:16:10.22Z" }, - { url = "https://files.pythonhosted.org/packages/6f/c3/b8c65cf86d473b6e99e6d985c678cb192c9b9776a966a2f4b009696bb650/granian-2.5.7-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:59fe2b352a828a2b04bcfd105e623d66786f217759d2d6245651a7b81e4ac294", size = 3131904, upload-time = "2025-11-05T12:16:13.249Z" }, - { url = "https://files.pythonhosted.org/packages/df/7e/b60421bddf187ab2a46682423e4a94b2b22a6ddff6842bf9ca2194e62ac2/granian-2.5.7-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:ec5fb593c2d436a323e711010e79718e6d5d1491d0d660fb7c9d97f7e5900830", size = 2908851, upload-time = "2025-11-05T12:16:15.305Z" }, - { url = "https://files.pythonhosted.org/packages/2f/cf/3f2426e19dc955a74dc94a5a47c4170e68acb060c541ac080f71a9d55d5d/granian-2.5.7-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:48fbc25f3717d01e11547afe0e9cdf9d7c41c9f316b9623a40c22ea6b2128d36", size = 2993270, upload-time = "2025-11-05T12:16:17.133Z" }, - { url = "https://files.pythonhosted.org/packages/40/2e/67e1e05ee0d503cc6e9fe53b03f69eb2f267a589d7b40873d120c417385f/granian-2.5.7-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:770935fec3374b814d21c01508c0697842d7c3750731a8ea129738b537ac594c", size = 3134662, upload-time = "2025-11-05T12:16:18.598Z" }, - { url = "https://files.pythonhosted.org/packages/17/d5/9d3242bbd911434c4f3d4f14c48e73774a8ddb591e0f975eaeeaef1d5081/granian-2.5.7-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:5db2600c92f74da74f624d2fdb01afe9e9365b50bd4e695a78e54961dc132f1b", size = 3220446, upload-time = "2025-11-05T12:16:20.598Z" }, - { url = "https://files.pythonhosted.org/packages/10/27/b2baa0443a42d8eb59f3dfbe8186e8c80a090655584af4611f22f1592d7a/granian-2.5.7-cp312-cp312-win_amd64.whl", hash = "sha256:bc368bdeb21646a965adf9f43dd2f4a770647e50318ba1b7cf387d4916ed7e69", size = 2179465, upload-time = "2025-11-05T12:16:22.031Z" }, - { url = "https://files.pythonhosted.org/packages/54/ec/bf1b7eefe824630d1d3ae9a8af397d823f2339d3adec71e9ee49d667409c/granian-2.5.7-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:fafb9c17def635bb0a5e20e145601598a6767b879bc2501663dbb45a57d1bc2e", size = 2850581, upload-time = "2025-11-05T12:16:23.516Z" }, - { url = "https://files.pythonhosted.org/packages/28/f7/5172daf1968c3a2337c51c50f4a3013aaab564d012d3a79e8390cc66403b/granian-2.5.7-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:9616a197eba637d59242661be8a46127c3f79f7c9bbfa44c0ea8c8c790a11d5e", size = 2529452, upload-time = "2025-11-05T12:16:25.088Z" }, - { url = "https://files.pythonhosted.org/packages/92/10/4344ccacc3f8dea973d630306491de43fbd4a0248e3f7cc9ff09ed5cc524/granian-2.5.7-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cfd7a09d5eb00a271ec79e3e0bbf069aa62ce376b64825bdeacb668d2b2a4041", size = 3008798, upload-time = "2025-11-05T12:16:26.584Z" }, - { url = "https://files.pythonhosted.org/packages/5e/33/638cf8c7f23ab905d3f6a371b5f87d03fd611678424223a0f1d0f7766cc7/granian-2.5.7-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1438a82264690fce6e82de66a95c77f5b0a5c33b93269eb85fc69ce0112c12d5", size = 2852309, upload-time = "2025-11-05T12:16:28.064Z" }, - { url = "https://files.pythonhosted.org/packages/18/42/6ec25d37ffc1f08679e6b325e9f9ac199ba5def948904c9205cd34fbfe6b/granian-2.5.7-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b3573121da77aac1af64cf90a88f29b2daecbf92458beec187421a382039f366", size = 3131335, upload-time = "2025-11-05T12:16:29.588Z" }, - { url = "https://files.pythonhosted.org/packages/b0/1e/db85dac58d84d3e50e427fe5b60b4f8e8a561d9784971fa3b2879198ad88/granian-2.5.7-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:34cdb82024efbcc9de01c7505213be17e4ba5e7a3acabe74ecd93ba31de7673e", size = 2908705, upload-time = "2025-11-05T12:16:31.049Z" }, - { url = "https://files.pythonhosted.org/packages/d9/25/a38fd12e1661bbd8535203a8b61240feac7b6b96726bff4de23b0078ab9f/granian-2.5.7-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:572451e94de69df228e4314cb91a50dee1565c4a53d33ffac5936c6ec9c5aba2", size = 2993118, upload-time = "2025-11-05T12:16:32.767Z" }, - { url = "https://files.pythonhosted.org/packages/d3/cd/852913a0fc30efc24495453c0f973dd74ef13aa0561afb352afa4b6ecbc2/granian-2.5.7-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:6e1679a4b102511b483774397134d244108851ae7a1e8bef09a8ef927ab4d370", size = 3134260, upload-time = "2025-11-05T12:16:34.552Z" }, - { url = "https://files.pythonhosted.org/packages/60/64/0dff100ce1e43c700918b39656cc000b1163c144eac3a12563a5f692dcd1/granian-2.5.7-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:285be70dcf3c70121afec03e691596db94bd786f9bebc229e9e0319686857d82", size = 3219987, upload-time = "2025-11-05T12:16:36.43Z" }, - { url = "https://files.pythonhosted.org/packages/19/ab/e66cf9bf57800dd7c2a2a4b8f23124603fce561a65a176f4cf3794a85b92/granian-2.5.7-cp313-cp313-win_amd64.whl", hash = "sha256:1273c9b1d38d19bcdd550a9a846d07112e541cfa1f99be04fbb926f2a003df3d", size = 2179201, upload-time = "2025-11-05T12:16:37.869Z" }, - { url = "https://files.pythonhosted.org/packages/da/0e/feca4a20e7b9e7de0e58103278c6581ebf3d5c1b972ed1c2dcfd25741f15/granian-2.5.7-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:75b9798bc13baa76e35165e5a778cd58a7258d5a2112ed6ef84ef84874244856", size = 2776744, upload-time = "2025-11-05T12:16:41.969Z" }, - { url = "https://files.pythonhosted.org/packages/f7/fe/65ca38ba9b9f4805495d96ed7b774dfd300f7c944f088db39c676c16501e/granian-2.5.7-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4cb8247728680ca308b7dc41a6d27582b78e15e902377e89000711f1126524dd", size = 2465942, upload-time = "2025-11-05T12:16:43.762Z" }, - { url = "https://files.pythonhosted.org/packages/75/d1/b9dea32fbafabe5c7b049fb0209149a37c6b8468c698d066448cbe88dc85/granian-2.5.7-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64348b83f1ad2f7a29df7932dc518ad669cb61a08a9cde02ca8ede8e9b110506", size = 3015413, upload-time = "2025-11-05T12:16:45.265Z" }, - { url = "https://files.pythonhosted.org/packages/cb/9e/d29485ab18896e4d911e33b006af7a9b7098316a78938d6b7455c523fea5/granian-2.5.7-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:e2292d4a4661c79d471fa0ff6fe640018c923b6a6dd1bb5383b368b3d5ec2a0c", size = 2783371, upload-time = "2025-11-05T12:16:46.762Z" }, - { url = "https://files.pythonhosted.org/packages/41/cd/58c67dc191caeecbbb15ee39d433136dd064c13778b4551661bd902b5a78/granian-2.5.7-cp313-cp313t-musllinux_1_1_aarch64.whl", hash = "sha256:45903d2f2f88a9cd4a7d0b8ec329db1fb2d9e15bf38153087a3b217b9cdb0046", size = 2979946, upload-time = "2025-11-05T12:16:48.255Z" }, - { url = "https://files.pythonhosted.org/packages/16/0b/04e4977df3ef7607a8b6625caed7cac107a049120d2452c33392d4544875/granian-2.5.7-cp313-cp313t-musllinux_1_1_armv7l.whl", hash = "sha256:106e8988e42e527c18b763be5faae7e8f602caac6cb93657793638fc9ab41c98", size = 3123177, upload-time = "2025-11-05T12:16:49.724Z" }, - { url = "https://files.pythonhosted.org/packages/c7/89/4e10e18fc107e5929143a06d9257646963cf5621c928b3d2774e5a85652a/granian-2.5.7-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:711632e602c4ea08b827bf6095c2c6fbe6005c7a05f142ae2b4d9e1d45cefbd9", size = 3211773, upload-time = "2025-11-05T12:16:51.438Z" }, - { url = "https://files.pythonhosted.org/packages/57/81/94e416056d8b4b1cd09cc8065a1e240b0af99f21301c209571530cd83dd0/granian-2.5.7-cp313-cp313t-win_amd64.whl", hash = "sha256:1c571733aa0fdb6755be9ffb3cd728ef965ae565ba896e407d6019bad929d7bb", size = 2174154, upload-time = "2025-11-05T12:16:53.411Z" }, - { url = "https://files.pythonhosted.org/packages/0e/25/2a4112983df5ce0ec8407121ad72c17d27ebfad57085749b8e4164d69e63/granian-2.5.7-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:cdae1c86357bfe895ffd0065c0403913bc008f752e2f77ab363d4e3b4276009b", size = 2838744, upload-time = "2025-11-05T12:17:45.904Z" }, - { url = "https://files.pythonhosted.org/packages/d7/0a/eb0c5b71355e8f99b89dc335f16cd5108763c554e96a2aae5e7162ef4997/granian-2.5.7-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:bc1d8aaf5bfc5fc9f8f590a42e9f88a43d19ad71f670c6969fa791b52ce1f5ec", size = 2538706, upload-time = "2025-11-05T12:17:47.471Z" }, - { url = "https://files.pythonhosted.org/packages/f2/9c/4c592c5a813a921033a37a0f003278b1f772a6c9abd16f821bcb119151f0/granian-2.5.7-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:288b62c19aea5b162d27e229469b6307a78cb272aa8fcc296dbfca9fbbda4d8f", size = 3117369, upload-time = "2025-11-05T12:17:49.172Z" }, - { url = "https://files.pythonhosted.org/packages/f1/35/96af9f0995a7c45f0cd31261ab6284e5d6028afa17c6fcfe757cccb0afb5/granian-2.5.7-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:66c3d2619dc5e845d658cf3ed4f7370f83d5323a85ff8338e7c7a27d9a333841", size = 2904972, upload-time = "2025-11-05T12:17:50.863Z" }, - { url = "https://files.pythonhosted.org/packages/fc/93/45c253983c2001f534ba2c7bc1e53718fc8cecf196b1e1a0469d5874ae54/granian-2.5.7-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:323e35d5d5054d2568fc824798471e7d33314f47aebd556c4fbf4894e539347d", size = 2991986, upload-time = "2025-11-05T12:17:52.602Z" }, - { url = "https://files.pythonhosted.org/packages/25/77/c03e60c7bed386ab16cf15b317dea7f95dde5095af6e17cbd657cd82c21b/granian-2.5.7-pp310-pypy310_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:026ef2588a2b991b250768bf47538fd5fd864549535f885239b6908b214299c4", size = 3163649, upload-time = "2025-11-05T12:17:54.402Z" }, - { url = "https://files.pythonhosted.org/packages/5e/c9/2bce3db4e3da8d3a697c363c8f699b71f05b7f7a0458e1ba345eaea53fcd/granian-2.5.7-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:4717a62c0a1b79372c495b99ade18bfc3c4a365242bf75770c96a4767a9bcf66", size = 3201886, upload-time = "2025-11-05T12:17:56.553Z" }, - { url = "https://files.pythonhosted.org/packages/78/66/997ebfd8cc4a0640befb970bc846a76437d1f0b55dff179e69f29fa4615b/granian-2.5.7-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:4b57ae0a2e1dbc7a248e3c08440b490b3f247e7e4f997faa72e82f5a89d0ea4c", size = 2175219, upload-time = "2025-11-05T12:17:58.126Z" }, - { url = "https://files.pythonhosted.org/packages/16/0f/da2588ac78254a4d0be90a6f733d0bb7dd1edb78a10d9e59fa9837687e94/granian-2.5.7-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:bee545c9b9e38eabcdd675e3fec1a2112b8193dc864739952b9de8131433a31c", size = 2838886, upload-time = "2025-11-05T12:17:59.809Z" }, - { url = "https://files.pythonhosted.org/packages/7d/34/75def8343534e9d48362c43c3cbd06242a2d7804fbfbc824c8aa9fb75a30/granian-2.5.7-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:73c76c0f1ee46506224e92df193b4d271ea89f0d82cd69301784ca85bc1db515", size = 2538597, upload-time = "2025-11-05T12:18:01.496Z" }, - { url = "https://files.pythonhosted.org/packages/c3/5d/d828d97aad050cfc5b18a0163b532c289a35ad214e31f5a129695b2b4cae/granian-2.5.7-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:68879c27aed972f647a8e8ef37f9046f71d7507dc9b3ceffa97d2fbffe6a16c8", size = 3117570, upload-time = "2025-11-05T12:18:03.818Z" }, - { url = "https://files.pythonhosted.org/packages/2d/57/b8380f3d6b6dcdcd454d720cf11dbecb0e2071a870f44eb834011f14b573/granian-2.5.7-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:ea9cbdfbd750813866dcc9c020018e5f20a57a4e3a83bd049ccc1f6da0559b75", size = 2905089, upload-time = "2025-11-05T12:18:05.567Z" }, - { url = "https://files.pythonhosted.org/packages/0b/e9/04a7c3b83650afc4a4ad82b67e6306d99f80ac1a6aacb3a8ba182f7359d6/granian-2.5.7-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:d142ff5ee6027515370e56f95d179ec3e81bd265d5b4958de2b19adcdf34887d", size = 2991867, upload-time = "2025-11-05T12:18:07.223Z" }, - { url = "https://files.pythonhosted.org/packages/2b/bf/a1cdbff73cbac4fddf817d06c13ce6cdc75c22d6da1b257e3563fea4c3c5/granian-2.5.7-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:222f0fb1688a62ca23cb3da974cefa69e7fdc40fd548d1ae87a953225e1d1cbb", size = 3164141, upload-time = "2025-11-05T12:18:09.267Z" }, - { url = "https://files.pythonhosted.org/packages/c8/cc/35c6a55ac2c211e86a9f0c728eb81b6ad19f05a3055d79c6f11a1b71f5d5/granian-2.5.7-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:40494c6cda1ad881ae07efbb2dc4a1ca8f12d5c6cf28d1ab8b0f2db13826617b", size = 3201599, upload-time = "2025-11-05T12:18:10.962Z" }, - { url = "https://files.pythonhosted.org/packages/f3/0a/5a95a3889532bc5a5f652cdc78dae8ffa16d4228b4d35256a98be89e33ef/granian-2.5.7-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:c3942d08af2c8b67d0ef569b6c567284433ebf09b4af3ea68388abb7caccad2b", size = 2175240, upload-time = "2025-11-05T12:18:12.956Z" }, + { url = "https://files.pythonhosted.org/packages/1c/23/ccaa1c786aa528a672d6f25ce236156aeb06a63931109aa6f2d4d3c8a350/granian-2.7.4-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:1c2a13c5c119e34369f984d8414edb8ba3793d7c78c37bb795942648dda3eca1", size = 6354293, upload-time = "2026-04-23T11:53:32.922Z" }, + { url = "https://files.pythonhosted.org/packages/91/b7/2b0b0e2dc07cb17febbbf3b349c96f425a53d913ba76278091f821adfc22/granian-2.7.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:91963c4928a355d772f14075057ff721423bce70612a619edc2daf04dd258577", size = 6050464, upload-time = "2026-04-23T11:53:34.912Z" }, + { url = "https://files.pythonhosted.org/packages/a3/9c/1feb488abb85446ac7d12d05a93788a3a6b42810d64d5806ebc309c65fc9/granian-2.7.4-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9d068796cb7e8e0b7a4c8d51077701e37104a39cd103c655a5c232ad561fb07c", size = 6883201, upload-time = "2026-04-23T11:53:36.73Z" }, + { url = "https://files.pythonhosted.org/packages/c8/ac/176b47bb48689fe3575ccbf372d67dd52fb0390686f4d8b7174ab2538ce5/granian-2.7.4-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c932f5c292b643019c4dd410a352789dbb8cb2cb41ec5b373779a87375de398a", size = 6135933, upload-time = "2026-04-23T11:53:38.552Z" }, + { url = "https://files.pythonhosted.org/packages/25/5d/4d894789683bd074bbe0361df7e3e524a2b763e44d771e326a16d4ea014b/granian-2.7.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b7a8f411408b0b65a07460e39cb53178e30a15ff5f0c77ed6aa31e1106590ea9", size = 6783971, upload-time = "2026-04-23T11:53:40.602Z" }, + { url = "https://files.pythonhosted.org/packages/88/e3/fe781d55306f1542b3db0fac13b70b39846b5e9a400fda9a3a2bf04a9af6/granian-2.7.4-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:6b7ab6a1a0c0d77ec1dd1145b7c8f3da5251ec7926c005da22f7415bf1b217a7", size = 6906647, upload-time = "2026-04-23T11:53:42.352Z" }, + { url = "https://files.pythonhosted.org/packages/76/11/c10af6940d8bb73cfb4eaa6781eb3f5c7880e1b69d9d87edc63292171538/granian-2.7.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:6036316f781f7ad1412d7aa10b49c5a25e69fae3f67ed766b0923ebb43aa5118", size = 6930262, upload-time = "2026-04-23T11:53:43.855Z" }, + { url = "https://files.pythonhosted.org/packages/f4/3d/5d129792626ac990d3b14d484736b8eafa202966f1a541c5b665c30fc880/granian-2.7.4-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:acef581d94270a22763fba192fc8cef0df77dac125080ca27e6e847a5e59cd07", size = 7050428, upload-time = "2026-04-23T11:53:45.385Z" }, + { url = "https://files.pythonhosted.org/packages/4c/e0/9a9d9f9d0e68277d49829d53746b2e8b3761e7b79a666fd9a4f445587321/granian-2.7.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:187a85fe36561c74a1db94b858175824c3154ebe6d0aa61c97124427f5c5a5fa", size = 6899482, upload-time = "2026-04-23T11:53:47.372Z" }, + { url = "https://files.pythonhosted.org/packages/19/de/04de408275968d4bff8e6fed9d6abf406beccac27412a08c8daaab7c9534/granian-2.7.4-cp310-cp310-win_amd64.whl", hash = "sha256:c73c6099206288c903a305d975064fbb51f9d0c78d06c914b23dde56165105c9", size = 3995374, upload-time = "2026-04-23T11:53:48.751Z" }, + { url = "https://files.pythonhosted.org/packages/df/00/a7db7e3627992c59927f57d5447638be515e683e2c8037ab7845250270d2/granian-2.7.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:455c51baf51dd0c3d22004fc04f9afb0662cb84ab2b75b48e5d6bb8b3e4e3548", size = 6353285, upload-time = "2026-04-23T11:53:50.113Z" }, + { url = "https://files.pythonhosted.org/packages/b2/23/337ab1a0929cb0cfbdedc06879cff62d6c08cb725fa2d4e139c7e305fed3/granian-2.7.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f2c54f3fe69790aa4b685372bcc8f382a8e9ba570b8ea4cb476e3b240a5a5a7c", size = 6050711, upload-time = "2026-04-23T11:53:51.887Z" }, + { url = "https://files.pythonhosted.org/packages/28/ba/3605834adaf5dc9ac3701b817bc9d42c73c89fb67815c7c87c7f64a9b6e1/granian-2.7.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f9549c44b325fe51ee4fc57308761f5178add4d531f1cc333b4a1eedf4a5b7af", size = 6882199, upload-time = "2026-04-23T11:53:53.298Z" }, + { url = "https://files.pythonhosted.org/packages/2e/1e/f11c9773dbf07ed326efc26a771b39ce97f7ee25608c16d69248db2da8da/granian-2.7.4-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0910390ea8f893cc4c3f38a28c923a321609358cf46d31aa7df5c3d3e58e8337", size = 6135800, upload-time = "2026-04-23T11:53:55.186Z" }, + { url = "https://files.pythonhosted.org/packages/f1/96/ca238b4f5d813643264abace48ca630efb1ab6d10409bd9e2c05c1d1ef12/granian-2.7.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b0de44552990b3dacb87ea3f37ebbcce67881712c0b0db500013821b14df7e4e", size = 6784786, upload-time = "2026-04-23T11:53:56.706Z" }, + { url = "https://files.pythonhosted.org/packages/ae/85/2feeffc37fe3c8a0f3e932393bcc99c8972984fe95907b34b380284caf1c/granian-2.7.4-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:5c9c6d51a675d9b7084244e63157899dd1afe6f1a5ab014015bc86afd4871df5", size = 6906836, upload-time = "2026-04-23T11:53:58.991Z" }, + { url = "https://files.pythonhosted.org/packages/ae/b6/faed26e3abd741e1d261defc0e7e3b2ecb9a2189c557e829bb28c3281456/granian-2.7.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:6be8c6ebbc53efea03284aef87de9b7367df3c9433f7df3b46c1edceaaa9d840", size = 6929985, upload-time = "2026-04-23T11:54:00.846Z" }, + { url = "https://files.pythonhosted.org/packages/25/5c/9f7a836177b5e94ad15da49046254e5b837e1d2c3d03981111d4af9a9d2b/granian-2.7.4-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:c10e056a6e76da640adb35f88d41ba40ae44065c5e04d4bc35f47c19a7f83a99", size = 7050668, upload-time = "2026-04-23T11:54:02.466Z" }, + { url = "https://files.pythonhosted.org/packages/08/88/19c01761d639b5e2c2eb2f12ff064c6765f32fa7f129c9f48162cdca0668/granian-2.7.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0e60a3153456f8922ca73d3a427cc3bb594c021f70ec08ecded6581efe25f48c", size = 6899480, upload-time = "2026-04-23T11:54:03.948Z" }, + { url = "https://files.pythonhosted.org/packages/04/ec/9022f3c2728fcd99f9374ce633e254b201f4bf76fbe60031bbc137f1cf8a/granian-2.7.4-cp311-cp311-win_amd64.whl", hash = "sha256:e9cafbf391d16ea8b8a2e9f88501783fac8da75eb948620899062a17929c4a84", size = 3996087, upload-time = "2026-04-23T11:54:05.771Z" }, + { url = "https://files.pythonhosted.org/packages/8d/d9/148024fd3a8bd974bb5c68a0cb48d15df7763fd1364bf090ccc2d423028a/granian-2.7.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:2c2f40aaecf2ba3d8232e55181c8f6db7bc68d9112a419ab8d5f9e2f33f631f5", size = 6374067, upload-time = "2026-04-23T11:54:07.293Z" }, + { url = "https://files.pythonhosted.org/packages/fc/bb/c53b61a7cb67d33677d96913438eca3d79de1b1b7173a361fcdf2753ade7/granian-2.7.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a8111d5e74b27721e0fdda3edba7c154d44c41b469466857ca3c51b088e3846b", size = 6046338, upload-time = "2026-04-23T11:54:08.684Z" }, + { url = "https://files.pythonhosted.org/packages/29/8d/5c9dc91b9c9a05bf6ed0b795d30f4bb8f290d61502779a89ed2fd75f9fb6/granian-2.7.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:74adbb6c1920dbf4271b824135639318b2a20ff5e33bc35639a8e2928a777234", size = 7000585, upload-time = "2026-04-23T11:54:10.451Z" }, + { url = "https://files.pythonhosted.org/packages/d9/7c/c770593b24a472ab5265a44546f56079757efbf89f8e8b2229a8443e453b/granian-2.7.4-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0b778d356b61e0389c823016ad2be50a634b80d3d28a33922f7ac39553e828ad", size = 6255544, upload-time = "2026-04-23T11:54:12.484Z" }, + { url = "https://files.pythonhosted.org/packages/15/46/796147587edb494a330294cb001cf68520ad8296a7da91d80ec672ac8615/granian-2.7.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3607b091c4ef225ee99150f3b02cb827de8d677b52fc75f0b28893244f7bab27", size = 6875124, upload-time = "2026-04-23T11:54:13.967Z" }, + { url = "https://files.pythonhosted.org/packages/c5/25/b867f624886e11053e7a6235244de26fd864a136e65d12295e728b3e5005/granian-2.7.4-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:3d3cf4fe3cafd9b874d8b749c66c790cbf2b4225f2a7d9fb284c51b77a8e938d", size = 6982394, upload-time = "2026-04-23T11:54:15.733Z" }, + { url = "https://files.pythonhosted.org/packages/4f/e1/5746bfe202bd2f6a1506346463ce52dd015c2b5d03d07a53ecf0fddefa3f/granian-2.7.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:846c9cbfea8684ab13d21d66855ad06dc077fb95b5590e7f5040e79994d6429d", size = 6991457, upload-time = "2026-04-23T11:54:17.325Z" }, + { url = "https://files.pythonhosted.org/packages/e0/45/fc6992839d367b6ae8fa8d88b5e70ec293162c3a2e0e6b90fc426f228df2/granian-2.7.4-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:d34d97cfe4a7805ecb5b1b1684f3f197bb4baf019d2a9f18e34fd1d697a03a7f", size = 7148499, upload-time = "2026-04-23T11:54:19.234Z" }, + { url = "https://files.pythonhosted.org/packages/fe/12/16ffd64a1213858d4cf824767b398758be807dd1a6df5a303dc76994b6d6/granian-2.7.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:f11336e4bcd8ef5c5143b075b5260e37e8431eb36d68564cc39416ca526c797f", size = 7006829, upload-time = "2026-04-23T11:54:20.804Z" }, + { url = "https://files.pythonhosted.org/packages/95/9a/f2fcda200f8739ddf25be72591b7a28897be0ffd952a76ec655e5f877144/granian-2.7.4-cp312-cp312-win_amd64.whl", hash = "sha256:9e0a4370773ec4a0e92a55a33fc700b60003e335480e5c7fe941f4bc3dda2e18", size = 4026771, upload-time = "2026-04-23T11:54:22.36Z" }, + { url = "https://files.pythonhosted.org/packages/0a/0f/fa7c63afedcb214edb96703cade360d946d5f1ca59ddb0b3d8e04587fb45/granian-2.7.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:d11da4a4527ba8dc28b5533d5e3241d8d9212e593195d27c6e72c8a422010af5", size = 6373513, upload-time = "2026-04-23T11:54:24.246Z" }, + { url = "https://files.pythonhosted.org/packages/be/39/3088ce32d940f7982102ea3bdc230090e34ac56dc0bce04f2d03b56ea435/granian-2.7.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:057a3db87e93eca1a11255dd13b45b5dd83f798a750fd87f02e14d54db5741b6", size = 6045232, upload-time = "2026-04-23T11:54:25.708Z" }, + { url = "https://files.pythonhosted.org/packages/ac/61/588f6b5397ea4f5bd9fc8de4b8cc092c555b8d95371c03d149b3bc419277/granian-2.7.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bb63d64c686799cea850c0c328d21adf75e323991a20be04923afc729432d2b5", size = 7001059, upload-time = "2026-04-23T11:54:27.532Z" }, + { url = "https://files.pythonhosted.org/packages/58/63/2affbcecfe96f940744c2086ea3793935d5f6898207590a579c92fc8588f/granian-2.7.4-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f406648c47569e983f0c58bd0853bac30a2bcdc6227428255ee5cc65a8ee62b6", size = 6255487, upload-time = "2026-04-23T11:54:29.397Z" }, + { url = "https://files.pythonhosted.org/packages/87/ac/31f7155a467020e7640e91af15ca3a70b0e7da210de42e3d3344e5eba8d0/granian-2.7.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bd56306eed06e293f4848c5ea997e1d019d1ad13b8252dde1f0bc773aca85ef", size = 6875068, upload-time = "2026-04-23T11:54:31.128Z" }, + { url = "https://files.pythonhosted.org/packages/99/22/402cc903e5c4e82bd363177392d4e1dcab8b27c1f7006c5316c37c597056/granian-2.7.4-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:732639e612e6b6e8d481f399f367e8c9bbb6f0e1b7b0aa74db340c574ee3dd98", size = 6982487, upload-time = "2026-04-23T11:54:32.704Z" }, + { url = "https://files.pythonhosted.org/packages/d3/92/3878f977bda82fc3a66fc7e95a54366a7b82edd53e6c9fdb3ec053693280/granian-2.7.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:47b8fdbfb369d52bb3fb884514a6a3a7e4d8e81c65fd26e5232985f2b46ebe0f", size = 6990683, upload-time = "2026-04-23T11:54:34.301Z" }, + { url = "https://files.pythonhosted.org/packages/ce/b3/a1239f3bc4e9034e07cb32403e6a6d26db01bba1c244dd654f6a76bf2612/granian-2.7.4-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:b679086082bfd7c1aa8c248ef673b715616a4ce58eec6fbeef8b83b30ac84283", size = 7148570, upload-time = "2026-04-23T11:54:36.494Z" }, + { url = "https://files.pythonhosted.org/packages/89/3c/fef781ea7356b21f671615dd0d53adc00fad81031a9ea506f80d1f46a43d/granian-2.7.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:a29191e949a99ffae2807abb7a864f7493f7a744e4fe2ddd2b5cd8db9b71378d", size = 7006976, upload-time = "2026-04-23T11:54:38.135Z" }, + { url = "https://files.pythonhosted.org/packages/56/54/ae2979fc45c06fbb37f595ee10eb6b138b6056202163b8e274d140d3f87b/granian-2.7.4-cp313-cp313-win_amd64.whl", hash = "sha256:07d26325cc69371ea2dc9d3a9cd0cc851c1c8e3dce40aca90e8c204547b5ba7e", size = 4027044, upload-time = "2026-04-23T11:54:39.957Z" }, + { url = "https://files.pythonhosted.org/packages/21/51/10344430e495bfa128dccc114957b33e712e971f91668788c08fe791df73/granian-2.7.4-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:4e093fe9511387313ad7ec9a76b0c78397cc584ef3dff47d46c336c5aee9cd8d", size = 6249290, upload-time = "2026-04-23T11:54:41.738Z" }, + { url = "https://files.pythonhosted.org/packages/ec/46/c7eda2e71a89a13e174598649f721c63ed3d908c0904b62621e8a433af0f/granian-2.7.4-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:227889f821526b8b60c5edf31b01fc987c4193bb0fc198c0998e0841e0cb719c", size = 5901799, upload-time = "2026-04-23T11:54:43.708Z" }, + { url = "https://files.pythonhosted.org/packages/72/d8/79e51f9f794389a9d6cab3d7c6b834b87d65fba72a43784eb5d2664a57a6/granian-2.7.4-cp313-cp313t-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:2b28d4aec5a9f2758a48da1897649a01b70ee1c00f2c4649db574527a3d00943", size = 6037594, upload-time = "2026-04-23T11:54:45.595Z" }, + { url = "https://files.pythonhosted.org/packages/ce/d8/835873a407279435fa0c8e8ac52392d3ba5c9a652bb15c0036aa07d9c302/granian-2.7.4-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f708fea5024a40e0dfba1c17c1c4b09e02e00ac0ac9ac1e345b409f0c11b71e5", size = 6966672, upload-time = "2026-04-23T11:54:47.242Z" }, + { url = "https://files.pythonhosted.org/packages/92/5f/21eacdda27c38e4194de5f9bef36c4045058daf6d58533fadb7c54c70573/granian-2.7.4-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f7006dfe9852cded794bc60008a168faf4dc2ecc18f1d74b5fde545685b699ec", size = 6563668, upload-time = "2026-04-23T11:54:49.751Z" }, + { url = "https://files.pythonhosted.org/packages/bd/06/9b19956d75277df44ee380e873a86b9890c431f2e2bcde32b3ba341f0efa/granian-2.7.4-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:77103af44034e30505fb5577b8214b0ad39cd6cbdc854ff980d4755faf93adaa", size = 6664285, upload-time = "2026-04-23T11:54:51.502Z" }, + { url = "https://files.pythonhosted.org/packages/85/33/740e0c9478be49c0778c4ea1773357680980e10e84b59bc19664033996dc/granian-2.7.4-cp313-cp313t-musllinux_1_1_aarch64.whl", hash = "sha256:b23194e1e0652297086224212605edb4998442511637e732d6009506277f8ff9", size = 6820367, upload-time = "2026-04-23T11:54:53.506Z" }, + { url = "https://files.pythonhosted.org/packages/c2/ad/3453fc1212268a01fee957122f2b1699af0efe50eca07ac570e11d1be12b/granian-2.7.4-cp313-cp313t-musllinux_1_1_armv7l.whl", hash = "sha256:f62941a4ffa1f1c2c5750cfc0b0ad96aa85d63b016125289779eef8888f5340d", size = 7132366, upload-time = "2026-04-23T11:54:55.123Z" }, + { url = "https://files.pythonhosted.org/packages/6c/ca/8479e4d2a02f210ce68b5dc73c77953ec1dfd3769bf725d06e6ec420d502/granian-2.7.4-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:ea6f97d2ade676f1bf49b79088fa4b5640b8b9804b7470218486df3d4be50046", size = 6842094, upload-time = "2026-04-23T11:54:56.665Z" }, + { url = "https://files.pythonhosted.org/packages/0d/96/71f95c73220726aee3e908b3ad2745c4c44fbfba508cb5ed615a9d4d367f/granian-2.7.4-cp313-cp313t-win_amd64.whl", hash = "sha256:759140ceef02ef72e57a184461927d72bcc2ddd3664c3cbbf4def7516f818041", size = 3974523, upload-time = "2026-04-23T11:54:58.541Z" }, + { url = "https://files.pythonhosted.org/packages/51/18/577637bb861ab688db8eb5d698ad700133818debd7ae6f58c0574c43f70e/granian-2.7.4-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:ce50300cf876f418ba0545f6e8c56d8c75038fc503add0fd1b58d9a3057d95ea", size = 6363314, upload-time = "2026-04-23T11:55:39.837Z" }, + { url = "https://files.pythonhosted.org/packages/48/41/11a6219baa10270f1a6a2a101cfa372e5d55a46a839a43b49a8d087fac09/granian-2.7.4-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:eb7f727f14d7d485a5df4078e7cc3038864b4e7c380865968e75e1e51e62457a", size = 6027259, upload-time = "2026-04-23T11:55:42.122Z" }, + { url = "https://files.pythonhosted.org/packages/bf/58/12b19b17fb79ee064a8a77a865a031bb49f4ea813789ad63186458ea02c9/granian-2.7.4-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:abbab303b502a770355c13c93569e6c0c71ccc864ab41b59636720d5a643f6b3", size = 6760497, upload-time = "2026-04-23T11:55:44.116Z" }, + { url = "https://files.pythonhosted.org/packages/45/9f/572711f882423f599707aae577ccdbc1700cf0cc3ceb4e9500e00c6b8d14/granian-2.7.4-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:f0b0423fa33a1afb9730fbfb5700fef4dac16bf7a1b7a2a79d0349739c1b1f44", size = 6843897, upload-time = "2026-04-23T11:55:46.322Z" }, + { url = "https://files.pythonhosted.org/packages/31/f9/75d51721069a184cd00310c4b0b0d614a6370905c13a096ccee193432ba3/granian-2.7.4-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:efa0d4fc35ab42562747e4103124e1c4f21afab081c1591de6472174a3416802", size = 6876194, upload-time = "2026-04-23T11:55:48.231Z" }, + { url = "https://files.pythonhosted.org/packages/44/5e/fd81492529bc2b02dafc63c95d03c2c7faa26ac883ccd94aa93b21fc68c3/granian-2.7.4-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:03b5ce06df095b5db49bd4e976ac8d8419bb0e73dc160613fc3db5e5d5dcd1af", size = 7094066, upload-time = "2026-04-23T11:55:50.328Z" }, + { url = "https://files.pythonhosted.org/packages/c7/04/f2fa35dc2956edb9a5abaabc0840aed92b4121ce27adf684a1c75e3c70ac/granian-2.7.4-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:9247db25dd66f74766a6a9488f1279c9b40cf422c6d7a04010492fa1aa7c9019", size = 6892580, upload-time = "2026-04-23T11:55:52.084Z" }, + { url = "https://files.pythonhosted.org/packages/72/6f/f45fa86c36fcc34f6e613bb223b10fd36c6acd9f7aa43d4f65d0f1eff4cc/granian-2.7.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:efccd6818a1ac4cba7eededf5e2768f56d4a8c7c93bd5e3a8d7a901510976944", size = 3971242, upload-time = "2026-04-23T11:55:53.834Z" }, ] [[package]] @@ -3269,7 +3264,7 @@ wheels = [ [[package]] name = "litellm" -version = "1.87.0" +version = "1.88.0" source = { editable = "." } dependencies = [ { name = "aiohttp" }, @@ -3335,6 +3330,7 @@ proxy = [ { name = "rich" }, { name = "rq" }, { name = "soundfile" }, + { name = "starlette" }, { name = "uvicorn" }, { name = "uvloop", marker = "sys_platform != 'win32'" }, { name = "websockets" }, @@ -3455,82 +3451,83 @@ proxy-dev = [ [package.metadata] requires-dist = [ - { name = "a2a-sdk", marker = "extra == 'extra-proxy'", specifier = "==0.3.24" }, + { name = "a2a-sdk", marker = "extra == 'extra-proxy'", specifier = ">=0.3.24,<1.0" }, { name = "aiohttp", specifier = ">=3.10,<4.0" }, - { name = "anthropic", extras = ["vertex"], marker = "extra == 'proxy-runtime'", specifier = "==0.84.0" }, - { name = "apscheduler", marker = "extra == 'proxy'", specifier = "==3.11.2" }, + { name = "anthropic", extras = ["vertex"], marker = "extra == 'proxy-runtime'", specifier = ">=0.84.0,<1.0" }, + { name = "apscheduler", marker = "extra == 'proxy'", specifier = ">=3.11.2,<4.0" }, { name = "audioread", marker = "extra == 'stt-nvidia-riva'", specifier = ">=3.0.1" }, - { name = "aurelio-sdk", marker = "python_full_version < '3.14' and extra == 'semantic-router'", specifier = "==0.0.19" }, - { name = "azure-ai-contentsafety", marker = "extra == 'proxy-runtime'", specifier = "==1.0.0" }, - { name = "azure-identity", marker = "extra == 'extra-proxy'", specifier = "==1.25.2" }, - { name = "azure-identity", marker = "extra == 'proxy'", specifier = "==1.25.2" }, - { name = "azure-keyvault-secrets", marker = "extra == 'extra-proxy'", specifier = "==4.10.0" }, - { name = "azure-storage-blob", marker = "extra == 'proxy'", specifier = "==12.28.0" }, - { name = "azure-storage-file-datalake", marker = "extra == 'proxy-runtime'", specifier = "==12.20.0" }, - { name = "backoff", marker = "extra == 'proxy'", specifier = "==2.2.1" }, - { name = "boto3", marker = "extra == 'proxy'", specifier = "==1.43.1" }, + { name = "aurelio-sdk", marker = "python_full_version < '3.14' and extra == 'semantic-router'", specifier = ">=0.0.19,<1.0" }, + { name = "azure-ai-contentsafety", marker = "extra == 'proxy-runtime'", specifier = ">=1.0.0,<2.0" }, + { name = "azure-identity", marker = "extra == 'extra-proxy'", specifier = ">=1.25.2,<2.0" }, + { name = "azure-identity", marker = "extra == 'proxy'", specifier = ">=1.25.2,<2.0" }, + { name = "azure-keyvault-secrets", marker = "extra == 'extra-proxy'", specifier = ">=4.10.0,<5.0" }, + { name = "azure-storage-blob", marker = "extra == 'proxy'", specifier = ">=12.28.0,<13.0" }, + { name = "azure-storage-file-datalake", marker = "extra == 'proxy-runtime'", specifier = ">=12.20.0,<13.0" }, + { name = "backoff", marker = "extra == 'proxy'", specifier = ">=2.2.1,<3.0" }, + { name = "boto3", marker = "extra == 'proxy'", specifier = ">=1.43.1,<2.0" }, { name = "click", specifier = ">=8.0.0,<9.0" }, - { name = "cryptography", marker = "extra == 'proxy'", specifier = "==46.0.7" }, - { name = "ddtrace", marker = "extra == 'proxy-runtime'", specifier = "==2.19.0" }, - { name = "detect-secrets", marker = "extra == 'proxy-runtime'", specifier = "==1.5.0" }, - { name = "diskcache", marker = "extra == 'caching'", specifier = "==5.6.3" }, - { name = "fastapi", marker = "extra == 'proxy'", specifier = "==0.124.4" }, - { name = "fastapi-sso", marker = "extra == 'proxy'", specifier = "==0.19.0" }, + { name = "cryptography", marker = "extra == 'proxy'", specifier = ">=46.0.7,<47.0" }, + { name = "ddtrace", marker = "extra == 'proxy-runtime'", specifier = ">=2.19.0,<3.0" }, + { name = "detect-secrets", marker = "extra == 'proxy-runtime'", specifier = ">=1.5.0,<2.0" }, + { name = "diskcache", marker = "extra == 'caching'", specifier = ">=5.6.3,<6.0" }, + { name = "fastapi", marker = "extra == 'proxy'", specifier = ">=0.136.3,<1.0" }, + { name = "fastapi-sso", marker = "extra == 'proxy'", specifier = ">=0.19.0,<1.0" }, { name = "fastuuid", specifier = ">=0.14.0,<1.0" }, - { name = "google-cloud-aiplatform", marker = "extra == 'google'", specifier = "==1.133.0" }, - { name = "google-cloud-aiplatform", marker = "extra == 'proxy-runtime'", specifier = "==1.133.0" }, - { name = "google-cloud-iam", marker = "extra == 'extra-proxy'", specifier = "==2.19.1" }, - { name = "google-cloud-kms", marker = "extra == 'extra-proxy'", specifier = "==2.24.2" }, - { name = "google-genai", marker = "extra == 'proxy-runtime'", specifier = "==1.37.0" }, - { name = "granian", marker = "extra == 'proxy'", specifier = "==2.5.7" }, + { name = "google-cloud-aiplatform", marker = "extra == 'google'", specifier = ">=1.133.0,<2.0" }, + { name = "google-cloud-aiplatform", marker = "extra == 'proxy-runtime'", specifier = ">=1.133.0,<2.0" }, + { name = "google-cloud-iam", marker = "extra == 'extra-proxy'", specifier = ">=2.19.1,<3.0" }, + { name = "google-cloud-kms", marker = "extra == 'extra-proxy'", specifier = ">=2.24.2,<3.0" }, + { name = "google-genai", marker = "extra == 'proxy-runtime'", specifier = ">=1.37.0,<2.0" }, + { name = "granian", marker = "extra == 'proxy'", specifier = ">=2.7.4,<3.0" }, { name = "grpcio", marker = "extra == 'grpc'", specifier = "==1.78.0" }, { name = "grpcio", marker = "extra == 'proxy-runtime'", specifier = "==1.78.0" }, - { name = "gunicorn", marker = "extra == 'proxy'", specifier = "==23.0.0" }, + { name = "gunicorn", marker = "extra == 'proxy'", specifier = ">=23.0.0,<24.0" }, { name = "httpx", specifier = ">=0.28.0,<1.0" }, { name = "importlib-metadata", specifier = ">=8.0.0,<9.0" }, { name = "jinja2", specifier = ">=3.1.6,<4.0" }, { name = "jsonschema", specifier = ">=4.0.0,<5.0" }, - { name = "langfuse", marker = "extra == 'proxy-runtime'", specifier = "==2.59.7" }, + { name = "langfuse", marker = "extra == 'proxy-runtime'", specifier = ">=2.59.7,<3.0" }, { name = "litellm-enterprise", marker = "extra == 'proxy'", editable = "enterprise" }, { name = "litellm-proxy-extras", marker = "extra == 'proxy'", editable = "litellm-proxy-extras" }, - { name = "llm-sandbox", marker = "extra == 'proxy-runtime'", specifier = "==0.3.39" }, - { name = "mangum", marker = "extra == 'proxy-runtime'", specifier = "==0.17.0" }, - { name = "mcp", marker = "extra == 'proxy'", specifier = "==1.26.0" }, - { name = "mlflow", marker = "extra == 'mlflow'", specifier = "==3.11.1" }, + { name = "llm-sandbox", marker = "extra == 'proxy-runtime'", specifier = ">=0.3.39,<1.0" }, + { name = "mangum", marker = "extra == 'proxy-runtime'", specifier = ">=0.17.0,<1.0" }, + { name = "mcp", marker = "extra == 'proxy'", specifier = ">=1.26.0,<2.0" }, + { name = "mlflow", marker = "extra == 'mlflow'", specifier = ">=3.11.1,<4.0" }, { name = "numpy", marker = "extra == 'stt-nvidia-riva'", specifier = ">=1.26.0" }, - { name = "numpydoc", marker = "extra == 'utils'", specifier = "==1.8.0" }, + { name = "numpydoc", marker = "extra == 'utils'", specifier = ">=1.8.0,<2.0" }, { name = "nvidia-riva-client", marker = "extra == 'stt-nvidia-riva'", specifier = ">=2.15.0" }, { name = "openai", specifier = ">=2.20.0,<3.0.0" }, { name = "opentelemetry-api", marker = "extra == 'proxy-runtime'", specifier = "==1.28.0" }, { name = "opentelemetry-exporter-otlp", marker = "extra == 'proxy-runtime'", specifier = "==1.28.0" }, { name = "opentelemetry-sdk", marker = "extra == 'proxy-runtime'", specifier = "==1.28.0" }, - { name = "orjson", marker = "extra == 'proxy'", specifier = "==3.11.6" }, - { name = "polars", marker = "extra == 'proxy'", specifier = "==1.38.1" }, - { name = "prisma", marker = "extra == 'extra-proxy'", specifier = "==0.11.0" }, - { name = "prometheus-client", marker = "extra == 'proxy-runtime'", specifier = "==0.20.0" }, + { name = "orjson", marker = "extra == 'proxy'", specifier = ">=3.11.6,<4.0" }, + { name = "polars", marker = "extra == 'proxy'", specifier = ">=1.38.1,<2.0" }, + { name = "prisma", marker = "extra == 'extra-proxy'", specifier = ">=0.11.0,<1.0" }, + { name = "prometheus-client", marker = "extra == 'proxy-runtime'", specifier = ">=0.20.0,<1.0" }, { name = "pydantic", specifier = ">=2.10.0,<3.0.0" }, - { name = "pydantic-settings", marker = "extra == 'proxy'", specifier = ">=2.14.1" }, - { name = "pyjwt", marker = "extra == 'proxy'", specifier = "==2.12.0" }, - { name = "pynacl", marker = "extra == 'proxy'", specifier = "==1.6.2" }, - { name = "pypdf", marker = "python_full_version < '3.14' and extra == 'proxy-runtime'", specifier = "==6.10.2" }, - { name = "pyroscope-io", marker = "sys_platform != 'win32' and extra == 'proxy'", specifier = "==0.8.16" }, + { name = "pydantic-settings", marker = "extra == 'proxy'", specifier = ">=2.14.1,<3.0" }, + { name = "pyjwt", marker = "extra == 'proxy'", specifier = ">=2.12.0,<3.0" }, + { name = "pynacl", marker = "extra == 'proxy'", specifier = ">=1.6.2,<2.0" }, + { name = "pypdf", marker = "python_full_version < '3.14' and extra == 'proxy-runtime'", specifier = ">=6.10.2,<7.0" }, + { name = "pyroscope-io", marker = "sys_platform != 'win32' and extra == 'proxy'", specifier = ">=0.8.16,<1.0" }, { name = "python-dotenv", specifier = ">=1.0.0,<2.0" }, - { name = "python-multipart", marker = "extra == 'proxy'", specifier = "==0.0.27" }, - { name = "pyyaml", marker = "extra == 'proxy'", specifier = "==6.0.3" }, - { name = "redisvl", marker = "python_full_version < '3.14' and extra == 'extra-proxy'", specifier = "==0.4.1" }, - { name = "resend", marker = "extra == 'extra-proxy'", specifier = "==2.23.0" }, - { name = "restrictedpython", marker = "extra == 'proxy'", specifier = "==8.1" }, - { name = "rich", marker = "extra == 'proxy'", specifier = "==13.9.4" }, - { name = "rq", marker = "extra == 'proxy'", specifier = "==2.7.0" }, - { name = "semantic-router", marker = "python_full_version < '3.14' and extra == 'semantic-router'", specifier = "==0.1.12" }, - { name = "sentry-sdk", marker = "extra == 'proxy-runtime'", specifier = "==2.21.0" }, - { name = "soundfile", marker = "extra == 'proxy'", specifier = "==0.12.1" }, + { name = "python-multipart", marker = "extra == 'proxy'", specifier = ">=0.0.27,<1.0" }, + { name = "pyyaml", marker = "extra == 'proxy'", specifier = ">=6.0.3,<7.0" }, + { name = "redisvl", marker = "python_full_version < '3.14' and extra == 'extra-proxy'", specifier = ">=0.4.1,<1.0" }, + { name = "resend", marker = "extra == 'extra-proxy'", specifier = ">=2.23.0,<3.0" }, + { name = "restrictedpython", marker = "extra == 'proxy'", specifier = ">=8.1,<9.0" }, + { name = "rich", marker = "extra == 'proxy'", specifier = ">=13.9.4,<14.0" }, + { name = "rq", marker = "extra == 'proxy'", specifier = ">=2.7.0,<3.0" }, + { name = "semantic-router", marker = "python_full_version < '3.14' and extra == 'semantic-router'", specifier = ">=0.1.15,<1.0" }, + { name = "sentry-sdk", marker = "extra == 'proxy-runtime'", specifier = ">=2.21.0,<3.0" }, + { name = "soundfile", marker = "extra == 'proxy'", specifier = ">=0.12.1,<1.0" }, { name = "soundfile", marker = "extra == 'stt-nvidia-riva'", specifier = ">=0.12.1" }, + { name = "starlette", marker = "extra == 'proxy'", specifier = ">=1.0.1,<2.0" }, { name = "tiktoken", specifier = ">=0.8.0,<1.0" }, { name = "tokenizers", specifier = ">=0.21.0,<1.0" }, - { name = "uvicorn", marker = "extra == 'proxy'", specifier = "==0.33.0" }, - { name = "uvloop", marker = "sys_platform != 'win32' and extra == 'proxy'", specifier = "==0.21.0" }, - { name = "websockets", marker = "extra == 'proxy'", specifier = "==15.0.1" }, + { name = "uvicorn", marker = "extra == 'proxy'", specifier = ">=0.33.0,<1.0" }, + { name = "uvloop", marker = "sys_platform != 'win32' and extra == 'proxy'", specifier = ">=0.21.0,<1.0" }, + { name = "websockets", marker = "extra == 'proxy'", specifier = ">=15.0.1,<16.0" }, ] provides-extras = ["proxy", "extra-proxy", "utils", "caching", "semantic-router", "mlflow", "grpc", "stt-nvidia-riva", "google", "proxy-runtime"] @@ -3556,7 +3553,7 @@ ci = [ { name = "lunary", marker = "python_full_version >= '3.11'", specifier = "==1.4.37" }, { name = "pillow", specifier = "==12.2.0" }, { name = "psycopg2-binary", specifier = "==2.9.11" }, - { name = "pyarrow", specifier = "==22.0.0" }, + { name = "pyarrow", specifier = "==23.0.1" }, { name = "pygithub", specifier = "==2.8.1" }, { name = "pylint", specifier = "==4.0.5" }, { name = "pyright", specifier = "==1.1.408" }, @@ -5691,45 +5688,45 @@ wheels = [ [[package]] name = "pyarrow" -version = "22.0.0" +version = "23.0.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/30/53/04a7fdc63e6056116c9ddc8b43bc28c12cdd181b85cbeadb79278475f3ae/pyarrow-22.0.0.tar.gz", hash = "sha256:3d600dc583260d845c7d8a6db540339dd883081925da2bd1c5cb808f720b3cd9", size = 1151151, upload-time = "2025-10-24T12:30:00.762Z" } +sdist = { url = "https://files.pythonhosted.org/packages/88/22/134986a4cc224d593c1afde5494d18ff629393d74cc2eddb176669f234a4/pyarrow-23.0.1.tar.gz", hash = "sha256:b8c5873e33440b2bc2f4a79d2b47017a89c5a24116c055625e6f2ee50523f019", size = 1167336, upload-time = "2026-02-16T10:14:12.39Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d9/9b/cb3f7e0a345353def531ca879053e9ef6b9f38ed91aebcf68b09ba54dec0/pyarrow-22.0.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:77718810bd3066158db1e95a63c160ad7ce08c6b0710bc656055033e39cdad88", size = 34223968, upload-time = "2025-10-24T10:03:31.21Z" }, - { url = "https://files.pythonhosted.org/packages/6c/41/3184b8192a120306270c5307f105b70320fdaa592c99843c5ef78aaefdcf/pyarrow-22.0.0-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:44d2d26cda26d18f7af7db71453b7b783788322d756e81730acb98f24eb90ace", size = 35942085, upload-time = "2025-10-24T10:03:38.146Z" }, - { url = "https://files.pythonhosted.org/packages/d9/3d/a1eab2f6f08001f9fb714b8ed5cfb045e2fe3e3e3c0c221f2c9ed1e6d67d/pyarrow-22.0.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:b9d71701ce97c95480fecb0039ec5bb889e75f110da72005743451339262f4ce", size = 44964613, upload-time = "2025-10-24T10:03:46.516Z" }, - { url = "https://files.pythonhosted.org/packages/46/46/a1d9c24baf21cfd9ce994ac820a24608decf2710521b29223d4334985127/pyarrow-22.0.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:710624ab925dc2b05a6229d47f6f0dac1c1155e6ed559be7109f684eba048a48", size = 47627059, upload-time = "2025-10-24T10:03:55.353Z" }, - { url = "https://files.pythonhosted.org/packages/3a/4c/f711acb13075c1391fd54bc17e078587672c575f8de2a6e62509af026dcf/pyarrow-22.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f963ba8c3b0199f9d6b794c90ec77545e05eadc83973897a4523c9e8d84e9340", size = 47947043, upload-time = "2025-10-24T10:04:05.408Z" }, - { url = "https://files.pythonhosted.org/packages/4e/70/1f3180dd7c2eab35c2aca2b29ace6c519f827dcd4cfeb8e0dca41612cf7a/pyarrow-22.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:bd0d42297ace400d8febe55f13fdf46e86754842b860c978dfec16f081e5c653", size = 50206505, upload-time = "2025-10-24T10:04:15.786Z" }, - { url = "https://files.pythonhosted.org/packages/80/07/fea6578112c8c60ffde55883a571e4c4c6bc7049f119d6b09333b5cc6f73/pyarrow-22.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:00626d9dc0f5ef3a75fe63fd68b9c7c8302d2b5bbc7f74ecaedba83447a24f84", size = 28101641, upload-time = "2025-10-24T10:04:22.57Z" }, - { url = "https://files.pythonhosted.org/packages/2e/b7/18f611a8cdc43417f9394a3ccd3eace2f32183c08b9eddc3d17681819f37/pyarrow-22.0.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:3e294c5eadfb93d78b0763e859a0c16d4051fc1c5231ae8956d61cb0b5666f5a", size = 34272022, upload-time = "2025-10-24T10:04:28.973Z" }, - { url = "https://files.pythonhosted.org/packages/26/5c/f259e2526c67eb4b9e511741b19870a02363a47a35edbebc55c3178db22d/pyarrow-22.0.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:69763ab2445f632d90b504a815a2a033f74332997052b721002298ed6de40f2e", size = 35995834, upload-time = "2025-10-24T10:04:35.467Z" }, - { url = "https://files.pythonhosted.org/packages/50/8d/281f0f9b9376d4b7f146913b26fac0aa2829cd1ee7e997f53a27411bbb92/pyarrow-22.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:b41f37cabfe2463232684de44bad753d6be08a7a072f6a83447eeaf0e4d2a215", size = 45030348, upload-time = "2025-10-24T10:04:43.366Z" }, - { url = "https://files.pythonhosted.org/packages/f5/e5/53c0a1c428f0976bf22f513d79c73000926cb00b9c138d8e02daf2102e18/pyarrow-22.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:35ad0f0378c9359b3f297299c3309778bb03b8612f987399a0333a560b43862d", size = 47699480, upload-time = "2025-10-24T10:04:51.486Z" }, - { url = "https://files.pythonhosted.org/packages/95/e1/9dbe4c465c3365959d183e6345d0a8d1dc5b02ca3f8db4760b3bc834cf25/pyarrow-22.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8382ad21458075c2e66a82a29d650f963ce51c7708c7c0ff313a8c206c4fd5e8", size = 48011148, upload-time = "2025-10-24T10:04:59.585Z" }, - { url = "https://files.pythonhosted.org/packages/c5/b4/7caf5d21930061444c3cf4fa7535c82faf5263e22ce43af7c2759ceb5b8b/pyarrow-22.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:1a812a5b727bc09c3d7ea072c4eebf657c2f7066155506ba31ebf4792f88f016", size = 50276964, upload-time = "2025-10-24T10:05:08.175Z" }, - { url = "https://files.pythonhosted.org/packages/ae/f3/cec89bd99fa3abf826f14d4e53d3d11340ce6f6af4d14bdcd54cd83b6576/pyarrow-22.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:ec5d40dd494882704fb876c16fa7261a69791e784ae34e6b5992e977bd2e238c", size = 28106517, upload-time = "2025-10-24T10:05:14.314Z" }, - { url = "https://files.pythonhosted.org/packages/af/63/ba23862d69652f85b615ca14ad14f3bcfc5bf1b99ef3f0cd04ff93fdad5a/pyarrow-22.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:bea79263d55c24a32b0d79c00a1c58bb2ee5f0757ed95656b01c0fb310c5af3d", size = 34211578, upload-time = "2025-10-24T10:05:21.583Z" }, - { url = "https://files.pythonhosted.org/packages/b1/d0/f9ad86fe809efd2bcc8be32032fa72e8b0d112b01ae56a053006376c5930/pyarrow-22.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:12fe549c9b10ac98c91cf791d2945e878875d95508e1a5d14091a7aaa66d9cf8", size = 35989906, upload-time = "2025-10-24T10:05:29.485Z" }, - { url = "https://files.pythonhosted.org/packages/b4/a8/f910afcb14630e64d673f15904ec27dd31f1e009b77033c365c84e8c1e1d/pyarrow-22.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:334f900ff08ce0423407af97e6c26ad5d4e3b0763645559ece6fbf3747d6a8f5", size = 45021677, upload-time = "2025-10-24T10:05:38.274Z" }, - { url = "https://files.pythonhosted.org/packages/13/95/aec81f781c75cd10554dc17a25849c720d54feafb6f7847690478dcf5ef8/pyarrow-22.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:c6c791b09c57ed76a18b03f2631753a4960eefbbca80f846da8baefc6491fcfe", size = 47726315, upload-time = "2025-10-24T10:05:47.314Z" }, - { url = "https://files.pythonhosted.org/packages/bb/d4/74ac9f7a54cfde12ee42734ea25d5a3c9a45db78f9def949307a92720d37/pyarrow-22.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c3200cb41cdbc65156e5f8c908d739b0dfed57e890329413da2748d1a2cd1a4e", size = 47990906, upload-time = "2025-10-24T10:05:58.254Z" }, - { url = "https://files.pythonhosted.org/packages/2e/71/fedf2499bf7a95062eafc989ace56572f3343432570e1c54e6599d5b88da/pyarrow-22.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ac93252226cf288753d8b46280f4edf3433bf9508b6977f8dd8526b521a1bbb9", size = 50306783, upload-time = "2025-10-24T10:06:08.08Z" }, - { url = "https://files.pythonhosted.org/packages/68/ed/b202abd5a5b78f519722f3d29063dda03c114711093c1995a33b8e2e0f4b/pyarrow-22.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:44729980b6c50a5f2bfcc2668d36c569ce17f8b17bccaf470c4313dcbbf13c9d", size = 27972883, upload-time = "2025-10-24T10:06:14.204Z" }, - { url = "https://files.pythonhosted.org/packages/a6/d6/d0fac16a2963002fc22c8fa75180a838737203d558f0ed3b564c4a54eef5/pyarrow-22.0.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:e6e95176209257803a8b3d0394f21604e796dadb643d2f7ca21b66c9c0b30c9a", size = 34204629, upload-time = "2025-10-24T10:06:20.274Z" }, - { url = "https://files.pythonhosted.org/packages/c6/9c/1d6357347fbae062ad3f17082f9ebc29cc733321e892c0d2085f42a2212b/pyarrow-22.0.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:001ea83a58024818826a9e3f89bf9310a114f7e26dfe404a4c32686f97bd7901", size = 35985783, upload-time = "2025-10-24T10:06:27.301Z" }, - { url = "https://files.pythonhosted.org/packages/ff/c0/782344c2ce58afbea010150df07e3a2f5fdad299cd631697ae7bd3bac6e3/pyarrow-22.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:ce20fe000754f477c8a9125543f1936ea5b8867c5406757c224d745ed033e691", size = 45020999, upload-time = "2025-10-24T10:06:35.387Z" }, - { url = "https://files.pythonhosted.org/packages/1b/8b/5362443737a5307a7b67c1017c42cd104213189b4970bf607e05faf9c525/pyarrow-22.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:e0a15757fccb38c410947df156f9749ae4a3c89b2393741a50521f39a8cf202a", size = 47724601, upload-time = "2025-10-24T10:06:43.551Z" }, - { url = "https://files.pythonhosted.org/packages/69/4d/76e567a4fc2e190ee6072967cb4672b7d9249ac59ae65af2d7e3047afa3b/pyarrow-22.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cedb9dd9358e4ea1d9bce3665ce0797f6adf97ff142c8e25b46ba9cdd508e9b6", size = 48001050, upload-time = "2025-10-24T10:06:52.284Z" }, - { url = "https://files.pythonhosted.org/packages/01/5e/5653f0535d2a1aef8223cee9d92944cb6bccfee5cf1cd3f462d7cb022790/pyarrow-22.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:252be4a05f9d9185bb8c18e83764ebcfea7185076c07a7a662253af3a8c07941", size = 50307877, upload-time = "2025-10-24T10:07:02.405Z" }, - { url = "https://files.pythonhosted.org/packages/2d/f8/1d0bd75bf9328a3b826e24a16e5517cd7f9fbf8d34a3184a4566ef5a7f29/pyarrow-22.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:a4893d31e5ef780b6edcaf63122df0f8d321088bb0dee4c8c06eccb1ca28d145", size = 27977099, upload-time = "2025-10-24T10:08:07.259Z" }, - { url = "https://files.pythonhosted.org/packages/90/81/db56870c997805bf2b0f6eeeb2d68458bf4654652dccdcf1bf7a42d80903/pyarrow-22.0.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:f7fe3dbe871294ba70d789be16b6e7e52b418311e166e0e3cba9522f0f437fb1", size = 34336685, upload-time = "2025-10-24T10:07:11.47Z" }, - { url = "https://files.pythonhosted.org/packages/1c/98/0727947f199aba8a120f47dfc229eeb05df15bcd7a6f1b669e9f882afc58/pyarrow-22.0.0-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:ba95112d15fd4f1105fb2402c4eab9068f0554435e9b7085924bcfaac2cc306f", size = 36032158, upload-time = "2025-10-24T10:07:18.626Z" }, - { url = "https://files.pythonhosted.org/packages/96/b4/9babdef9c01720a0785945c7cf550e4acd0ebcd7bdd2e6f0aa7981fa85e2/pyarrow-22.0.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:c064e28361c05d72eed8e744c9605cbd6d2bb7481a511c74071fd9b24bc65d7d", size = 44892060, upload-time = "2025-10-24T10:07:26.002Z" }, - { url = "https://files.pythonhosted.org/packages/f8/ca/2f8804edd6279f78a37062d813de3f16f29183874447ef6d1aadbb4efa0f/pyarrow-22.0.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:6f9762274496c244d951c819348afbcf212714902742225f649cf02823a6a10f", size = 47504395, upload-time = "2025-10-24T10:07:34.09Z" }, - { url = "https://files.pythonhosted.org/packages/b9/f0/77aa5198fd3943682b2e4faaf179a674f0edea0d55d326d83cb2277d9363/pyarrow-22.0.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:a9d9ffdc2ab696f6b15b4d1f7cec6658e1d788124418cb30030afbae31c64746", size = 48066216, upload-time = "2025-10-24T10:07:43.528Z" }, - { url = "https://files.pythonhosted.org/packages/79/87/a1937b6e78b2aff18b706d738c9e46ade5bfcf11b294e39c87706a0089ac/pyarrow-22.0.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:ec1a15968a9d80da01e1d30349b2b0d7cc91e96588ee324ce1b5228175043e95", size = 50288552, upload-time = "2025-10-24T10:07:53.519Z" }, - { url = "https://files.pythonhosted.org/packages/60/ae/b5a5811e11f25788ccfdaa8f26b6791c9807119dffcf80514505527c384c/pyarrow-22.0.0-cp313-cp313t-win_amd64.whl", hash = "sha256:bba208d9c7decf9961998edf5c65e3ea4355d5818dd6cd0f6809bec1afb951cc", size = 28262504, upload-time = "2025-10-24T10:08:00.932Z" }, + { url = "https://files.pythonhosted.org/packages/bc/a8/24e5dc6855f50a62936ceb004e6e9645e4219a8065f304145d7fb8a79d5d/pyarrow-23.0.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:3fab8f82571844eb3c460f90a75583801d14ca0cc32b1acc8c361650e006fd56", size = 34307390, upload-time = "2026-02-16T10:08:08.654Z" }, + { url = "https://files.pythonhosted.org/packages/bc/8e/4be5617b4aaae0287f621ad31c6036e5f63118cfca0dc57d42121ff49b51/pyarrow-23.0.1-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:3f91c038b95f71ddfc865f11d5876c42f343b4495535bd262c7b321b0b94507c", size = 35853761, upload-time = "2026-02-16T10:08:17.811Z" }, + { url = "https://files.pythonhosted.org/packages/2e/08/3e56a18819462210432ae37d10f5c8eed3828be1d6c751b6e6a2e93c286a/pyarrow-23.0.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:d0744403adabef53c985a7f8a082b502a368510c40d184df349a0a8754533258", size = 44493116, upload-time = "2026-02-16T10:08:25.792Z" }, + { url = "https://files.pythonhosted.org/packages/f8/82/c40b68001dbec8a3faa4c08cd8c200798ac732d2854537c5449dc859f55a/pyarrow-23.0.1-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:c33b5bf406284fd0bba436ed6f6c3ebe8e311722b441d89397c54f871c6863a2", size = 47564532, upload-time = "2026-02-16T10:08:34.27Z" }, + { url = "https://files.pythonhosted.org/packages/20/bc/73f611989116b6f53347581b02177f9f620efdf3cd3f405d0e83cdf53a83/pyarrow-23.0.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:ddf743e82f69dcd6dbbcb63628895d7161e04e56794ef80550ac6f3315eeb1d5", size = 48183685, upload-time = "2026-02-16T10:08:42.889Z" }, + { url = "https://files.pythonhosted.org/packages/b0/cc/6c6b3ecdae2a8c3aced99956187e8302fc954cc2cca2a37cf2111dad16ce/pyarrow-23.0.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e052a211c5ac9848ae15d5ec875ed0943c0221e2fcfe69eee80b604b4e703222", size = 50605582, upload-time = "2026-02-16T10:08:51.641Z" }, + { url = "https://files.pythonhosted.org/packages/8d/94/d359e708672878d7638a04a0448edf7c707f9e5606cee11e15aaa5c7535a/pyarrow-23.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:5abde149bb3ce524782d838eb67ac095cd3fd6090eba051130589793f1a7f76d", size = 27521148, upload-time = "2026-02-16T10:08:58.077Z" }, + { url = "https://files.pythonhosted.org/packages/b0/41/8e6b6ef7e225d4ceead8459427a52afdc23379768f54dd3566014d7618c1/pyarrow-23.0.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:6f0147ee9e0386f519c952cc670eb4a8b05caa594eeffe01af0e25f699e4e9bb", size = 34302230, upload-time = "2026-02-16T10:09:03.859Z" }, + { url = "https://files.pythonhosted.org/packages/bf/4a/1472c00392f521fea03ae93408bf445cc7bfa1ab81683faf9bc188e36629/pyarrow-23.0.1-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:0ae6e17c828455b6265d590100c295193f93cc5675eb0af59e49dbd00d2de350", size = 35850050, upload-time = "2026-02-16T10:09:11.877Z" }, + { url = "https://files.pythonhosted.org/packages/0c/b2/bd1f2f05ded56af7f54d702c8364c9c43cd6abb91b0e9933f3d77b4f4132/pyarrow-23.0.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:fed7020203e9ef273360b9e45be52a2a47d3103caf156a30ace5247ffb51bdbd", size = 44491918, upload-time = "2026-02-16T10:09:18.144Z" }, + { url = "https://files.pythonhosted.org/packages/0b/62/96459ef5b67957eac38a90f541d1c28833d1b367f014a482cb63f3b7cd2d/pyarrow-23.0.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:26d50dee49d741ac0e82185033488d28d35be4d763ae6f321f97d1140eb7a0e9", size = 47562811, upload-time = "2026-02-16T10:09:25.792Z" }, + { url = "https://files.pythonhosted.org/packages/7d/94/1170e235add1f5f45a954e26cd0e906e7e74e23392dcb560de471f7366ec/pyarrow-23.0.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:3c30143b17161310f151f4a2bcfe41b5ff744238c1039338779424e38579d701", size = 48183766, upload-time = "2026-02-16T10:09:34.645Z" }, + { url = "https://files.pythonhosted.org/packages/0e/2d/39a42af4570377b99774cdb47f63ee6c7da7616bd55b3d5001aa18edfe4f/pyarrow-23.0.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:db2190fa79c80a23fdd29fef4b8992893f024ae7c17d2f5f4db7171fa30c2c78", size = 50607669, upload-time = "2026-02-16T10:09:44.153Z" }, + { url = "https://files.pythonhosted.org/packages/00/ca/db94101c187f3df742133ac837e93b1f269ebdac49427f8310ee40b6a58f/pyarrow-23.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:f00f993a8179e0e1c9713bcc0baf6d6c01326a406a9c23495ec1ba9c9ebf2919", size = 27527698, upload-time = "2026-02-16T10:09:50.263Z" }, + { url = "https://files.pythonhosted.org/packages/9a/4b/4166bb5abbfe6f750fc60ad337c43ecf61340fa52ab386da6e8dbf9e63c4/pyarrow-23.0.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:f4b0dbfa124c0bb161f8b5ebb40f1a680b70279aa0c9901d44a2b5a20806039f", size = 34214575, upload-time = "2026-02-16T10:09:56.225Z" }, + { url = "https://files.pythonhosted.org/packages/e1/da/3f941e3734ac8088ea588b53e860baeddac8323ea40ce22e3d0baa865cc9/pyarrow-23.0.1-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:7707d2b6673f7de054e2e83d59f9e805939038eebe1763fe811ee8fa5c0cd1a7", size = 35832540, upload-time = "2026-02-16T10:10:03.428Z" }, + { url = "https://files.pythonhosted.org/packages/88/7c/3d841c366620e906d54430817531b877ba646310296df42ef697308c2705/pyarrow-23.0.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:86ff03fb9f1a320266e0de855dee4b17da6794c595d207f89bba40d16b5c78b9", size = 44470940, upload-time = "2026-02-16T10:10:10.704Z" }, + { url = "https://files.pythonhosted.org/packages/2c/a5/da83046273d990f256cb79796a190bbf7ec999269705ddc609403f8c6b06/pyarrow-23.0.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:813d99f31275919c383aab17f0f455a04f5a429c261cc411b1e9a8f5e4aaaa05", size = 47586063, upload-time = "2026-02-16T10:10:17.95Z" }, + { url = "https://files.pythonhosted.org/packages/5b/3c/b7d2ebcff47a514f47f9da1e74b7949138c58cfeb108cdd4ee62f43f0cf3/pyarrow-23.0.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:bf5842f960cddd2ef757d486041d57c96483efc295a8c4a0e20e704cbbf39c67", size = 48173045, upload-time = "2026-02-16T10:10:25.363Z" }, + { url = "https://files.pythonhosted.org/packages/43/b2/b40961262213beaba6acfc88698eb773dfce32ecdf34d19291db94c2bd73/pyarrow-23.0.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:564baf97c858ecc03ec01a41062e8f4698abc3e6e2acd79c01c2e97880a19730", size = 50621741, upload-time = "2026-02-16T10:10:33.477Z" }, + { url = "https://files.pythonhosted.org/packages/f6/70/1fdda42d65b28b078e93d75d371b2185a61da89dda4def8ba6ba41ebdeb4/pyarrow-23.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:07deae7783782ac7250989a7b2ecde9b3c343a643f82e8a4df03d93b633006f0", size = 27620678, upload-time = "2026-02-16T10:10:39.31Z" }, + { url = "https://files.pythonhosted.org/packages/47/10/2cbe4c6f0fb83d2de37249567373d64327a5e4d8db72f486db42875b08f6/pyarrow-23.0.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:6b8fda694640b00e8af3c824f99f789e836720aa8c9379fb435d4c4953a756b8", size = 34210066, upload-time = "2026-02-16T10:10:45.487Z" }, + { url = "https://files.pythonhosted.org/packages/cb/4f/679fa7e84dadbaca7a65f7cdba8d6c83febbd93ca12fa4adf40ba3b6362b/pyarrow-23.0.1-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:8ff51b1addc469b9444b7c6f3548e19dc931b172ab234e995a60aea9f6e6025f", size = 35825526, upload-time = "2026-02-16T10:10:52.266Z" }, + { url = "https://files.pythonhosted.org/packages/f9/63/d2747d930882c9d661e9398eefc54f15696547b8983aaaf11d4a2e8b5426/pyarrow-23.0.1-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:71c5be5cbf1e1cb6169d2a0980850bccb558ddc9b747b6206435313c47c37677", size = 44473279, upload-time = "2026-02-16T10:11:01.557Z" }, + { url = "https://files.pythonhosted.org/packages/b3/93/10a48b5e238de6d562a411af6467e71e7aedbc9b87f8d3a35f1560ae30fb/pyarrow-23.0.1-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:9b6f4f17b43bc39d56fec96e53fe89d94bac3eb134137964371b45352d40d0c2", size = 47585798, upload-time = "2026-02-16T10:11:09.401Z" }, + { url = "https://files.pythonhosted.org/packages/5c/20/476943001c54ef078dbf9542280e22741219a184a0632862bca4feccd666/pyarrow-23.0.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:9fc13fc6c403d1337acab46a2c4346ca6c9dec5780c3c697cf8abfd5e19b6b37", size = 48179446, upload-time = "2026-02-16T10:11:17.781Z" }, + { url = "https://files.pythonhosted.org/packages/4b/b6/5dd0c47b335fcd8edba9bfab78ad961bd0fd55ebe53468cc393f45e0be60/pyarrow-23.0.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5c16ed4f53247fa3ffb12a14d236de4213a4415d127fe9cebed33d51671113e2", size = 50623972, upload-time = "2026-02-16T10:11:26.185Z" }, + { url = "https://files.pythonhosted.org/packages/d5/09/a532297c9591a727d67760e2e756b83905dd89adb365a7f6e9c72578bcc1/pyarrow-23.0.1-cp313-cp313-win_amd64.whl", hash = "sha256:cecfb12ef629cf6be0b1887f9f86463b0dd3dc3195ae6224e74006be4736035a", size = 27540749, upload-time = "2026-02-16T10:12:23.297Z" }, + { url = "https://files.pythonhosted.org/packages/a5/8e/38749c4b1303e6ae76b3c80618f84861ae0c55dd3c2273842ea6f8258233/pyarrow-23.0.1-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:29f7f7419a0e30264ea261fdc0e5fe63ce5a6095003db2945d7cd78df391a7e1", size = 34471544, upload-time = "2026-02-16T10:11:32.535Z" }, + { url = "https://files.pythonhosted.org/packages/a3/73/f237b2bc8c669212f842bcfd842b04fc8d936bfc9d471630569132dc920d/pyarrow-23.0.1-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:33d648dc25b51fd8055c19e4261e813dfc4d2427f068bcecc8b53d01b81b0500", size = 35949911, upload-time = "2026-02-16T10:11:39.813Z" }, + { url = "https://files.pythonhosted.org/packages/0c/86/b912195eee0903b5611bf596833def7d146ab2d301afeb4b722c57ffc966/pyarrow-23.0.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:cd395abf8f91c673dd3589cadc8cc1ee4e8674fa61b2e923c8dd215d9c7d1f41", size = 44520337, upload-time = "2026-02-16T10:11:47.764Z" }, + { url = "https://files.pythonhosted.org/packages/69/c2/f2a717fb824f62d0be952ea724b4f6f9372a17eed6f704b5c9526f12f2f1/pyarrow-23.0.1-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:00be9576d970c31defb5c32eb72ef585bf600ef6d0a82d5eccaae96639cf9d07", size = 47548944, upload-time = "2026-02-16T10:11:56.607Z" }, + { url = "https://files.pythonhosted.org/packages/84/a7/90007d476b9f0dc308e3bc57b832d004f848fd6c0da601375d20d92d1519/pyarrow-23.0.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:c2139549494445609f35a5cda4eb94e2c9e4d704ce60a095b342f82460c73a83", size = 48236269, upload-time = "2026-02-16T10:12:04.47Z" }, + { url = "https://files.pythonhosted.org/packages/b0/3f/b16fab3e77709856eb6ac328ce35f57a6d4a18462c7ca5186ef31b45e0e0/pyarrow-23.0.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:7044b442f184d84e2351e5084600f0d7343d6117aabcbc1ac78eb1ae11eb4125", size = 50604794, upload-time = "2026-02-16T10:12:11.797Z" }, + { url = "https://files.pythonhosted.org/packages/e9/a1/22df0620a9fac31d68397a75465c344e83c3dfe521f7612aea33e27ab6c0/pyarrow-23.0.1-cp313-cp313t-win_amd64.whl", hash = "sha256:a35581e856a2fafa12f3f54fce4331862b1cfb0bef5758347a858a4aa9d6bae8", size = 27660642, upload-time = "2026-02-16T10:12:17.746Z" }, ] [[package]] @@ -6954,7 +6951,7 @@ wheels = [ [[package]] name = "semantic-router" -version = "0.1.12" +version = "0.1.15" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp" }, @@ -6972,9 +6969,9 @@ dependencies = [ { name = "tornado" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/8d/d7/88a1330f53a26eaea25249b21a5b776cbabfa333a6107ed88ce8b881d14f/semantic_router-0.1.12.tar.gz", hash = "sha256:b63fbb8b9127dcb1763efea17dfa74ab409e626e87c8695b589131af12ef3a65", size = 93372, upload-time = "2025-11-18T13:22:44.848Z" } +sdist = { url = "https://files.pythonhosted.org/packages/dc/a9/1a689e916e8b280f1fd8fb335cc059be626a22fe4533baa045d32fcd6de5/semantic_router-0.1.15.tar.gz", hash = "sha256:328256ddc3c2b713101ec69561d6585aecbf1198ea3461e1486289d8c3a35288", size = 95605, upload-time = "2026-05-23T12:58:15.444Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/18/ad/4816aabd264b6b677002bde0cd4784f7f7f553f98e2ec01b96fda4ce5215/semantic_router-0.1.12-py3-none-any.whl", hash = "sha256:94658545f89cc63d2eb7dff6f74bc713b61bbcfe91146b0e4353a383f6790804", size = 126216, upload-time = "2025-11-18T13:22:43.655Z" }, + { url = "https://files.pythonhosted.org/packages/1e/c7/f4a20292aef9badd277efbb24d697c5b934693fb21e8e490d3ecb0fc83f0/semantic_router-0.1.15-py3-none-any.whl", hash = "sha256:c08978584c73c5ff8e75005202007ac8ee6593d77deaf8c7ec53f71e01e7f757", size = 128102, upload-time = "2026-05-23T12:58:14.295Z" }, ] [[package]] @@ -7352,15 +7349,15 @@ wheels = [ [[package]] name = "starlette" -version = "0.50.0" +version = "1.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ba/b8/73a0e6a6e079a9d9cfa64113d771e421640b6f679a52eeb9b32f72d871a1/starlette-0.50.0.tar.gz", hash = "sha256:a2a17b22203254bcbc2e1f926d2d55f3f9497f769416b3190768befe598fa3ca", size = 2646985, upload-time = "2025-11-01T15:25:27.516Z" } +sdist = { url = "https://files.pythonhosted.org/packages/95/66/4d20cdf39a8d6a51e663b7038e3b828ff211d3891a43a713fe7e4643f3a8/starlette-1.1.0.tar.gz", hash = "sha256:e83c7fe0ddecd8719c5b840080325aec0260acec86e9832899e377b91d65e90f", size = 2660060, upload-time = "2026-05-23T16:55:41.376Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d9/52/1064f510b141bd54025f9b55105e26d1fa970b9be67ad766380a3c9b74b0/starlette-0.50.0-py3-none-any.whl", hash = "sha256:9e5391843ec9b6e472eed1365a78c8098cfceb7a74bfd4d6b1c0c0095efb3bca", size = 74033, upload-time = "2025-11-01T15:25:25.461Z" }, + { url = "https://files.pythonhosted.org/packages/93/79/920b8e0a8b20f793e8d64855095cb8febabf6175b8550b6f7a547d813891/starlette-1.1.0-py3-none-any.whl", hash = "sha256:7f0dfd38e428aad5cb6f9f667f0ca1d2d8ca3f3385dccac8305f79ec98458382", size = 72899, upload-time = "2026-05-23T16:55:39.201Z" }, ] [[package]]