diff --git a/.circleci/config.yml b/.circleci/config.yml index cf69ff68da6..1462891fa7f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2400,6 +2400,11 @@ jobs: environment: DATABASE_URL: "postgresql://e2euser:e2epassword@localhost:5432/litellm_e2e" CI: "true" + # Boot the proxy with an external logout URL so proxyLogoutUrl.spec.ts can + # assert the redirect. Set at job level so both the proxy boot step and the + # Playwright step (whose skip guard reads this) see the same value. Safe for + # the rest of the suite: nothing else performs a logout. + PROXY_LOGOUT_URL: "https://www.example.com" steps: - checkout - setup_google_dns @@ -2476,7 +2481,8 @@ jobs: MOCK_LLM_URL: "http://127.0.0.1:8090/v1" DISABLE_SCHEMA_UPDATE: "true" SERVER_ROOT_PATH: "" - PROXY_LOGOUT_URL: "" + # PROXY_LOGOUT_URL is inherited from the job-level environment so the + # proxy and proxyLogoutUrl.spec.ts agree on the logout target. # LITELLM_LICENSE is forwarded from the project env so premium-gated # UI flows can be exercised. license.spec.ts asserts the resulting # JWT carries premium_user=true; if it ever stops being passed, that diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index f9ce9e5dcb8..99f79c0b272 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -10,9 +10,9 @@ **Please complete all items before asking a LiteLLM maintainer to review your PR** -- [ ] I have Added testing in the [`tests/test_litellm/`](https://github.com/BerriAI/litellm/tree/main/tests/test_litellm) directory, **Adding at least 1 test is a hard requirement** - [see details](https://docs.litellm.ai/docs/extras/contributing_code) +- [ ] I have added meaningful tests - [ ] My PR passes all unit tests on [`make test-unit`](https://docs.litellm.ai/docs/extras/contributing_code) -- [ ] My PR's scope is as isolated as possible, it only solves 1 specific problem +- [ ] My PR's scope is as isolated as possible; it only solves 1 specific problem - [ ] I have requested a Greptile review by commenting `@greptileai` and received a **Confidence Score of at least 4/5** before requesting a maintainer review ## Delays in PR merge? diff --git a/.github/workflows/create_daily_oss_agent_shin_branch.yml b/.github/workflows/create_daily_oss_agent_shin_branch.yml new file mode 100644 index 00000000000..d6118f3b53c --- /dev/null +++ b/.github/workflows/create_daily_oss_agent_shin_branch.yml @@ -0,0 +1,47 @@ +name: Create Daily oss-agent-shin Branch + +on: + schedule: + - cron: "0 0 * * *" # Runs every day at midnight UTC + workflow_dispatch: # Allow manual trigger + +jobs: + create-oss-agent-shin-branch: + if: github.repository == 'BerriAI/litellm' + runs-on: ubuntu-latest + permissions: + contents: write + + steps: + - name: Checkout repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Create daily oss-agent-shin branch + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + # Configure Git user + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + # Generate branch name with MM_DD_YYYY format + BRANCH_NAME="litellm_oss_agent_shin_$(date +'%m_%d_%Y')" + echo "Creating branch: $BRANCH_NAME" + + # Fetch all branches + git fetch --all + + # Check if the branch already exists + if git show-ref --verify --quiet refs/remotes/origin/$BRANCH_NAME; then + echo "Branch $BRANCH_NAME already exists. Skipping creation." + else + echo "Creating new branch: $BRANCH_NAME" + # Create the new branch from main + git checkout -b $BRANCH_NAME origin/main + # Push the new branch + git push origin $BRANCH_NAME + echo "Successfully created and pushed branch: $BRANCH_NAME" + fi diff --git a/.github/workflows/test-unit-proxy-endpoints.yml b/.github/workflows/test-unit-proxy-endpoints.yml index 1439b2c07f7..6b34a08a8e0 100644 --- a/.github/workflows/test-unit-proxy-endpoints.yml +++ b/.github/workflows/test-unit-proxy-endpoints.yml @@ -7,6 +7,7 @@ on: - litellm_internal_staging - litellm_oss_branch - "litellm_**" + workflow_dispatch: permissions: contents: read @@ -32,6 +33,7 @@ jobs: tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/vector_store_endpoints tests/test_litellm/proxy/agent_endpoints + tests/test_litellm/proxy/a2a tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/public_endpoints @@ -42,3 +44,16 @@ jobs: workers: 2 reruns: 2 artifact-name: proxy-endpoints + + # Behavior-pinning tests for litellm/proxy/proxy_server.py. Owns its + # own job (not a path on the proxy-endpoints job above) so its budget + # is independent and its coverage artifact is uploaded separately. + # See: https://www.notion.so/36c43b8acdab81ee845fd5365128a2fc + proxy-server: + uses: ./.github/workflows/_test-unit-base.yml + with: + test-path: tests/test_litellm/proxy/proxy_server + workers: 4 + reruns: 2 + timeout-minutes: 60 + artifact-name: proxy-server diff --git a/.github/workflows/test_server_root_path.yml b/.github/workflows/test_server_root_path.yml index 155445acdf6..57ff746c9c8 100644 --- a/.github/workflows/test_server_root_path.yml +++ b/.github/workflows/test_server_root_path.yml @@ -101,6 +101,31 @@ jobs: docker logs litellm-test exit 1 + - name: Setup Node for Playwright + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: "20" + + - name: Install UI deps and Chromium + working-directory: ui/litellm-dashboard + run: | + npm ci + npx playwright install --with-deps chromium + + - name: Run SERVER_ROOT_PATH redirect e2e + working-directory: ui/litellm-dashboard + env: + SERVER_ROOT_PATH: ${{ matrix.root_path }} + run: npx playwright test --config=e2e_tests/serverRootPath.config.ts + + - name: Upload Playwright artifacts on failure + if: failure() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: playwright-trace-${{ strategy.job-index }} + path: ui/litellm-dashboard/test-results/ + retention-days: 7 + - name: Cleanup if: always() run: | diff --git a/.gitignore b/.gitignore index a0b326bbafb..a303591c635 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,8 @@ litellm/tests/config_*.yaml litellm/tests/langfuse.log langfuse.log .langfuse.log +.pin_list.txt +.cov_new.xml litellm/tests/test_custom_logger.py litellm/tests/langfuse.log litellm/tests/dynamo*.log @@ -123,3 +125,4 @@ crash.*.log # .terraform.lock.hcl is intentionally NOT ignored — it pins provider versions # and should be committed. .vscode +.pin_list.txt diff --git a/AGENTS.md b/AGENTS.md index e99bf79d783..41921fdff4d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,293 +1 @@ -# INSTRUCTIONS FOR LITELLM - -This document provides comprehensive instructions for AI agents working in the LiteLLM repository. - -## OVERVIEW - -LiteLLM is a unified interface for 100+ LLMs that: -- Translates inputs to provider-specific completion, embedding, and image generation endpoints -- Provides consistent OpenAI-format output across all providers -- Includes retry/fallback logic across multiple deployments (Router) -- Offers a proxy server (LLM Gateway) with budgets, rate limits, and authentication -- Supports advanced features like function calling, streaming, caching, and observability - -## REPOSITORY STRUCTURE - -### Core Components -- `litellm/` - Main library code - - `llms/` - Provider-specific implementations (OpenAI, Anthropic, Azure, etc.) - - `proxy/` - Proxy server implementation (LLM Gateway) - - `router_utils/` - Load balancing and fallback logic - - `types/` - Type definitions and schemas - - `integrations/` - Third-party integrations (observability, caching, etc.) - -### Key Directories -- `tests/` - Comprehensive test suites -- `ui/litellm-dashboard/` - Admin dashboard UI -- `enterprise/` - Enterprise-specific features - -Documentation lives in the separate [BerriAI/litellm-docs](https://github.com/BerriAI/litellm-docs) repository and is served at [docs.litellm.ai](https://docs.litellm.ai). - -## DEVELOPMENT GUIDELINES - -### MAKING CODE CHANGES - -1. **Provider Implementations**: When adding/modifying LLM providers: - - Follow existing patterns in `litellm/llms/{provider}/` - - Implement proper transformation classes that inherit from `BaseConfig` - - Support both sync and async operations - - Handle streaming responses appropriately - - Include proper error handling with provider-specific exceptions - -2. **Type Safety**: - - Use proper type hints throughout - - Update type definitions in `litellm/types/` - - Ensure compatibility with both Pydantic v1 and v2 - -3. **Testing**: - - Add tests in appropriate `tests/` subdirectories - - Include both unit tests and integration tests - - Test provider-specific functionality thoroughly - - Consider adding load tests for performance-critical changes - -### MAKING CODE CHANGES FOR THE UI (IGNORE FOR BACKEND) - -1. **Always use `antd` for new UI components — Tremor is DEPRECATED** - - We are migrating off of `@tremor/react`. Do not introduce new `Badge`, `Text`, `Card`, `Grid`, `Title`, or other imports from `@tremor/react` in any new or modified file. - - Use `antd` equivalents: `Tag` for labels, plain ``/`
` with Tailwind classes (or `Typography.Text`) for text, `Card` from `antd`, etc. Note that `antd` has no `"yellow"` Tag color — use `"gold"` for amber/yellow. - - The only exception is the Tremor Table component and its required Tremor Table sub components. - -2. **Use Common Components as much as possible**: - - These are usually defined in the `common_components` directory - - Use these components as much as possible and avoid building new components unless needed - -3. **Testing**: - - The codebase uses **Vitest** and **React Testing Library** - - **Query Priority Order**: Use query methods in this order: `getByRole`, `getByLabelText`, `getByPlaceholderText`, `getByText`, `getByTestId` - - **Always use `screen`** instead of destructuring from `render()` (e.g., use `screen.getByText()` not `getByText`) - - **Wrap user interactions in `act()`**: Always wrap `fireEvent` calls with `act()` to ensure React state updates are properly handled - - **Use `query` methods for absence checks**: Use `queryBy*` methods (not `getBy*`) when expecting an element to NOT be present - - **Test names must start with "should"**: All test names should follow the pattern `it("should ...")` - - **Mock external dependencies**: Check `setupTests.ts` for global mocks and mock child components/networking calls as needed - - **Structure tests properly**: - - First test should verify the component renders successfully - - Subsequent tests should focus on functionality and user interactions - - Use `waitFor` for async operations that aren't already awaited - - **Avoid using `querySelector`**: Prefer React Testing Library queries over direct DOM manipulation - -### IMPORTANT PATTERNS - -1. **Function/Tool Calling**: - - LiteLLM standardizes tool calling across providers - - OpenAI format is the standard, with transformations for other providers - - See `litellm/llms/anthropic/chat/transformation.py` for complex tool handling - -2. **Streaming**: - - All providers should support streaming where possible - - Use consistent chunk formatting across providers - - Handle both sync and async streaming - -3. **Error Handling**: - - Use provider-specific exception classes - - Maintain consistent error formats across providers - - Include proper retry logic and fallback mechanisms - -4. **Configuration**: - - Support both environment variables and programmatic configuration - - Use `BaseConfig` classes for provider configurations - - Allow dynamic parameter passing - -## PROXY SERVER (LLM GATEWAY) - -The proxy server is a critical component that provides: -- Authentication and authorization -- Rate limiting and budget management -- Load balancing across multiple models/deployments -- Observability and logging -- Admin dashboard UI -- Enterprise features - -Key files: -- `litellm/proxy/proxy_server.py` - Main server implementation -- `litellm/proxy/auth/` - Authentication logic -- `litellm/proxy/management_endpoints/` - Admin API endpoints - -**Database (proxy)**: Use Prisma model methods (`prisma_client.db..upsert`, `.find_many`, `.find_unique`, etc.), not raw SQL (`execute_raw`/`query_raw`). See COMMON PITFALLS for details. - -## MCP (MODEL CONTEXT PROTOCOL) SUPPORT - -LiteLLM supports MCP for agent workflows: -- MCP server integration for tool calling -- Transformation between OpenAI and MCP tool formats -- Support for external MCP servers (Zapier, Jira, Linear, etc.) -- See `litellm/experimental_mcp_client/` and `litellm/proxy/_experimental/mcp_server/` - -## RUNNING SCRIPTS - -Use `uv run python script.py` to run Python scripts in the project environment (for non-test files). - -## GITHUB TEMPLATES - -When opening issues or pull requests, follow these templates: - -### Bug Reports (`.github/ISSUE_TEMPLATE/bug_report.yml`) -- Describe what happened vs. expected behavior -- Include relevant log output -- Specify LiteLLM version -- Indicate if you're part of an ML Ops team (helps with prioritization) - -### Feature Requests (`.github/ISSUE_TEMPLATE/feature_request.yml`) -- Clearly describe the feature -- Explain motivation and use case with concrete examples - -### Pull Requests (`.github/pull_request_template.md`) -- Add at least 1 test in `tests/litellm/` -- Ensure `make test-unit` passes - - -## TESTING CONSIDERATIONS - -1. **Provider Tests**: Test against real provider APIs when possible -2. **Proxy Tests**: Include authentication, rate limiting, and routing tests -3. **Performance Tests**: Load testing for high-throughput scenarios -4. **Integration Tests**: End-to-end workflows including tool calling - -## DOCUMENTATION - -- Keep documentation in sync with code changes -- Update provider documentation when adding new providers -- Include code examples for new features -- Update changelog and release notes - -## SECURITY CONSIDERATIONS - -- Handle API keys securely -- Validate all inputs, especially for proxy endpoints -- Consider rate limiting and abuse prevention -- Follow security best practices for authentication - -## ENTERPRISE FEATURES - -- Some features are enterprise-only -- Check `enterprise/` directory for enterprise-specific code -- Maintain compatibility between open-source and enterprise versions - -## COMMON PITFALLS TO AVOID - -1. **Breaking Changes**: LiteLLM has many users - avoid breaking existing APIs -2. **Provider Specifics**: Each provider has unique quirks - handle them properly -3. **Rate Limits**: Respect provider rate limits in tests -4. **Memory Usage**: Be mindful of memory usage in streaming scenarios -5. **Dependencies**: Keep dependencies minimal and well-justified -6. **UI/Backend Contract Mismatch**: When adding a new entity type to the UI, always check whether the backend endpoint accepts a single value or an array. Match the UI control accordingly (single-select vs. multi-select) to avoid silently dropping user selections -7. **Missing Tests for New Entity Types**: When adding a new entity type (e.g., in `EntityUsage`, `UsageViewSelect`), always add corresponding tests in the existing test files and update any icon/component mocks -8. **Raw SQL in proxy DB code**: Do not use `execute_raw` or `query_raw` for proxy database access. Use Prisma model methods (e.g. `prisma_client.db.litellm_tooltable.upsert()`, `.find_many()`, `.find_unique()`) so behavior stays consistent with the schema, the client stays mockable in tests, and you avoid the pitfalls of hand-written SQL (parameter ordering, type casting, schema drift) - -8. **Do not hardcode model-specific flags**: Put model-specific capability flags in `model_prices_and_context_window.json` and read them via `get_model_info` (or existing helpers like `supports_reasoning`). This prevents users from needing to upgrade LiteLLM each time a new model supports a feature. - - **Example of BAD** (hardcoded model checks): - - ```python - @staticmethod - def _is_effort_supported_model(model: str) -> bool: - """Check if the model supports the output_config.effort parameter...""" - model_lower = model.lower() - if AnthropicConfig._is_claude_4_6_model(model): - return True - return any( - v in model_lower for v in ("opus-4-5", "opus_4_5", "opus-4.5", "opus_4.5") - ) - ``` - - **Example of GOOD** (config-driven or helper that reads from config): - - ```python - if ( - "claude-3-7-sonnet" in model - or AnthropicConfig._is_claude_4_6_model(model) - or supports_reasoning( - model=model, - custom_llm_provider=self.custom_llm_provider, - ) - ): - ... - ``` - - Using helpers like `supports_reasoning` (which read from `model_prices_and_context_window.json` / `get_model_info`) allows future model updates to "just work" without code changes. - -9. **Never close HTTP/SDK clients on cache eviction**: Do not add `close()`, `aclose()`, or `create_task(close_fn())` inside `LLMClientCache._remove_key()` or any cache eviction path. Evicted clients may still be held by in-flight requests; closing them causes `RuntimeError: Cannot send a request, as the client has been closed.` in production after the cache TTL (1 hour) expires. Connection cleanup is handled at shutdown by `close_litellm_async_clients()`. See PR #22247 for the full incident history. - -## HELPFUL RESOURCES - -- Main documentation: https://docs.litellm.ai/ (source: [BerriAI/litellm-docs](https://github.com/BerriAI/litellm-docs)) -- Provider-specific docs: https://docs.litellm.ai/docs/providers/ -- Admin UI for testing proxy features - -## WHEN IN DOUBT - -- Follow existing patterns in the codebase -- Check similar provider implementations -- Ensure comprehensive test coverage -- Update documentation appropriately -- Consider backward compatibility impact - -## Cursor Cloud specific instructions - -### Environment - -- uv is installed in `~/.local/bin`; the update script ensures it is on `PATH`. -- Python 3.12, Node 22 are pre-installed. -- The project virtual environment lives under `.venv/`. - -### Running the proxy server - -Create a minimal config file and start the proxy: - -```yaml -# config.yaml -model_list: - - model_name: fake-openai-endpoint - litellm_params: - model: openai/fake-model - api_key: fake-key - api_base: https://fake-api.example.com - -general_settings: - master_key: sk-1234 - -litellm_settings: - drop_params: True - telemetry: False -``` - -```bash -uv run litellm --config config.yaml --port 4000 -``` - -The proxy takes ~15-20 seconds to fully start (it runs Prisma migrations on boot). Wait for `/health` to return before sending requests. Without a PostgreSQL `DATABASE_URL`, the proxy connects to a default Neon dev database embedded in the `litellm-proxy-extras` package. - -### Running tests - -See `CLAUDE.md` and the `Makefile` for standard commands. Key notes: - -- `uv sync --group proxy-dev --extra proxy` installs the Prisma and proxy-side test dependencies used by the standard local workflow. -- The `--timeout` pytest flag is NOT available; don't pass it. -- Unit tests: `uv run pytest tests/test_litellm/ -x -vv -n 4` -- **Before committing, always run `uv run black .` to format your code.** Black formatting is enforced in CI. -- If `uv sync` fails because the lockfile is outdated, run `uv lock` and retry. - -### Lint - -```bash -cd litellm && uv run ruff check . -``` - -Ruff is the primary fast linter. For the full lint suite (including mypy, black, circular imports), run `make lint` per `CLAUDE.md`. - -### UI Dashboard development - -- The UI is at `ui/litellm-dashboard/`. Run `npm run dev` from that directory for the Next.js dev server on port 3000. -- The proxy at port 4000 serves a **pre-built** static UI from `litellm/proxy/_experimental/out/`. After making UI code changes, you must run `npm run build` in the dashboard directory and copy the output: `cp -r ui/litellm-dashboard/out/* litellm/proxy/_experimental/out/` for the proxy to serve the updated UI. -- SVGs used as provider logos (loaded via `` tags) must NOT use `fill="currentColor"` — replace with an explicit color like `#000000` or use the `-color` variant from lobehub icons, since CSS color inheritance does not work inside `` elements. -- Provider logos live in `ui/litellm-dashboard/public/assets/logos/` (source) and `litellm/proxy/_experimental/out/assets/logos/` (pre-built). Both locations must have the file for it to work in dev and proxy-served modes. -- UI Vitest tests: `cd ui/litellm-dashboard && npx vitest run` +Read @CLAUDE.md for coding guidelines diff --git a/CLAUDE.md b/CLAUDE.md index baf23c90148..3477b71a621 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,181 +1,70 @@ -# CLAUDE.md +Do not write comments unless they are absolutely necessary to explain some very complex business logic. Please clean up if there are comments that are not absolutely necessary. Do not remove comments that are unrelated to the addition of the code of this PR -This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. +Explanation: code comments are, in a way, a violation of DRY code. You must update logic in two locations to change the code and "hard to change" is literally the definition of tech debt. We should instead aim to write code that is intuitive to the reader, while being both easy to maintain and high performance -## Documentation +Don't assume that the existing code is correct or the right way of doing things / good coding patterns. In fact, there are a lot of bad coding practices, overly complex code, code smells, etc. If something doesn't look right, speak up. Feel free to break existing patterns or question weird existing code to make new code high quality, as in: +- correct +- secure +- performant +- readable +- easy to maintain/change +- modern +In that order of importance -Documentation lives in a separate repository: [BerriAI/litellm-docs](https://github.com/BerriAI/litellm-docs). It is served at [docs.litellm.ai](https://docs.litellm.ai). Do not create or edit documentation files in this repository — open doc PRs against `BerriAI/litellm-docs` instead. +When adding new features, add meaningful tests. Don't add tests that don't check anything substantial and is there just to make the code coverage pass. Yes, code coverage is important, but I'd rather have no signal whether the code is working than tests that don't fail when code is broken. The goal is to have tests that would fail before the feature was added/if the code was mutated in a way that breaks the feature and succeed only when the feature is fully working. I should run mutation testing and see > 90% kill rate -## Development Commands +Same thing for bug fixes. The tests should make it so that this specific bug can never happen again without failing tests (i.e., regression) -### Installation -- `make install-dev` - Install core development dependencies -- `make install-proxy-dev` - Install proxy development dependencies with full feature set -- `make install-test-deps` - Install the full local test environment and generate the Prisma client +When creating PRs, don't set base to `main`. `litellm_internal_staging` serves that purpose -### Testing -- `make test` - Run all tests -- `make test-unit` - Run unit tests (tests/test_litellm) with 4 parallel workers -- `make test-integration` - Run integration tests (excludes unit tests) -- `pytest tests/` - Direct pytest execution +Always use @.github/pull_request_template.md as a guide for your PR body -### Code Quality -- `make lint` - Run all linting (Ruff, MyPy, Black, circular imports, import safety) -- `make format` - Apply Black code formatting -- `make lint-ruff` - Run Ruff linting only -- `make lint-mypy` - Run MyPy type checking only -- **Before committing, always run `uv run black .` to format your code.** Black formatting is enforced in CI. +Never use `pytest` commands or the like as "Screenshots / Proof of Fix". We prefer curl'ing a live proxy instance running on localhost:4000 (I like to run it with `python litellm/proxy/proxy_cli.py --config litellm/proxy/dev_config.yaml --detailed_debug --reload --use_v2_migration_resolver 2>&1 | tee litellm.log`) and showing both the command run and the output. Also, it should hit real LLM provider APIs, not mocks, and cost real $$$ because that is the most realistic test. The proof of fix should be exactly what the end user / customer would see / do. The run logs in PR #27703 is a prime example of how to do it (not a huge fan of using a python test script that future me and the team will have no visibility into; I prefer just curl commands or a short list of bash commands (e.g., using `for`)). If it's a UI thing, just tell me which URLs to go to (e.g., http://localhost:4000/ui/?page=logs), where to click, what fields to fill out, etc. along with the other commands to run in an ordered list, and I'll do it myself and post the screenshots after you make the PR -### Single Test Files -- `uv run pytest tests/path/to/test_file.py -v` - Run specific test file -- `uv run pytest tests/path/to/test_file.py::test_function -v` - Run specific test +If you ever make public-facing PR descriptions, comments, issues, commit messages, etc., always follow these guidelines to sound less AI-y: +- don't use emojis +- don't use "—". Instead, reach for ";", ".", etc. +- don't use the pattern "It's not X, it's Y", "You're not X, you're Y", etc. +- don't use bulleted or numbered lists unless it would be nonsensical not to. Instead, prefer prose -### Running Scripts -- `uv run python script.py` - Run Python scripts (use for non-test files) +Don't hesitate to use values in .env to get needed API keys and other secrets, as long as you never add them to conversation history, commit them, or include them in GitHub issues / PRs -### GitHub Issue & PR Templates -When contributing to the project, use the appropriate templates: +Run tests, format your code, and lint your code before each commit -**Bug Reports** (`.github/ISSUE_TEMPLATE/bug_report.yml`): -- Describe what happened vs. what you expected -- Include relevant log output -- Specify your LiteLLM version +Ask to commit and push your work when you're done (or if you're confident that your code is good and works, just do it) -**Feature Requests** (`.github/ISSUE_TEMPLATE/feature_request.yml`): -- Describe the feature clearly -- Explain the motivation and use case +When you must use real LLM models to, for example, write e2e tests, write a QA runbook, etc., make sure to use the latest models (doesn't have to be smartest, can also be a modern small, fast one. No strong preference for smart vs fast here, just use something modern) as of the year and month of the current date. Do a web search as necessary to figure that out -**Pull Requests** (`.github/pull_request_template.md`): -- Add at least 1 test in `tests/litellm/` -- Ensure `make test-unit` passes +If you're an internal contributor, when creating a new PR, the typical flow is to branch off litellm_internal_staging and create a branch prefixed with litellm_. Do not create a branch prefixed with claude/ and generally do not have / in your branch names -## Architecture Overview +Do not add `Co-Authored-By: Claude` or any Claude attribution to commit messages. Never use a `claude/` prefix or put a `/` in a branch name. Do not add "Generated with Claude Code" (or any similar attribution) to PR descriptions. Do not create a new PR/branch off the existing PR to fix/add something that is related and could've just been committed directly to the existing PR's branch -LiteLLM is a unified interface for 100+ LLM providers with two main components: +When working on a PR, keep the PR description in sync with new commits being made -### Core Library (`litellm/`) -- **Main entry point**: `litellm/main.py` - Contains core completion() function -- **Provider implementations**: `litellm/llms/` - Each provider has its own subdirectory -- **Router system**: `litellm/router.py` + `litellm/router_utils/` - Load balancing and fallback logic -- **Type definitions**: `litellm/types/` - Pydantic models and type hints -- **Integrations**: `litellm/integrations/` - Third-party observability, caching, logging -- **Caching**: `litellm/caching/` - Multiple cache backends (Redis, in-memory, S3, etc.) +Monkeypatching attributes of a class to do testing is an anti-pattern. Prefer dependency-injecting things into classes. That way, at unit test time, you can pass a mocked dependency in -### Proxy Server (`litellm/proxy/`) -- **Main server**: `proxy_server.py` - FastAPI application -- **Authentication**: `auth/` - API key management, JWT, OAuth2 -- **Database**: `db/` - Prisma ORM with PostgreSQL/SQLite support -- **Management endpoints**: `management_endpoints/` - Admin APIs for keys, teams, models -- **Pass-through endpoints**: `pass_through_endpoints/` - Provider-specific API forwarding -- **Guardrails**: `guardrails/` - Safety and content filtering hooks -- **UI Dashboard**: Served from `_experimental/out/` (Next.js build) +Do not put names of customers or customer company names in code, PRs, and issues. The codebase is public -## Key Patterns +CI supply-chain safety: Never pipe a remote script into a shell (`curl ... | bash`, `wget ... | sh`); download the artifact to a file, verify its SHA-256 checksum, then install. Pin every external tool to a specific version with a full URL (not `latest` or `stable`). Verify checksums for all downloaded binaries, using the provider's official `.sha256` / `.sha256sum` sidecar when available. These rules apply to every download in CI -### Provider Implementation -- Providers inherit from base classes in `litellm/llms/base.py` -- Each provider has transformation functions for input/output formatting -- Support both sync and async operations -- Handle streaming responses and function calling +## Think Before Coding -### Error Handling -- Provider-specific exceptions mapped to OpenAI-compatible errors -- Fallback logic handled by Router system -- Comprehensive logging through `litellm/_logging.py` +**Don't assume. Don't hide confusion. Surface tradeoffs.** -### Configuration -- YAML config files for proxy server (see `proxy/example_config_yaml/`) -- Environment variables for API keys and settings -- Database schema managed via Prisma (`proxy/schema.prisma`) +Before implementing: +- State your assumptions explicitly. If uncertain, ask. +- If multiple interpretations exist, present them. Don't pick silently. +- If a simpler approach exists, say so. Push back when warranted. +- If something is unclear, stop. Name what's confusing. Ask. -## Development Notes +## Simplicity First -### Code Style -- Uses Black formatter, Ruff linter, MyPy type checker -- Pydantic v2 for data validation -- Async/await patterns throughout -- Type hints required for all public APIs -- **Avoid imports within methods** — place all imports at the top of the file (module-level). Inline imports inside functions/methods make dependencies harder to trace and hurt readability. The only exception is avoiding circular imports where absolutely necessary. -- **Use dict spread for immutable copies** — prefer `{**original, "key": new_value}` over `dict(obj)` + mutation. The spread produces the final dict in one step and makes intent clear. -- **Guard at resolution time** — when resolving an optional value through a fallback chain (`a or b or ""`), raise immediately if the resolved result being empty is an error. Don't pass empty strings or sentinel values downstream for the callee to deal with. -- **Extract complex comprehensions to named helpers** — a set/dict comprehension that calls into the DB or manager (e.g. "which of these server IDs are OAuth2?") belongs in a named helper function, not inline in the caller. -- **FastAPI parameter declarations** — mark required query/form params with `= Query(...)` / `= Form(...)` explicitly when other params in the same handler are optional. Mixing `str` (required) with `Optional[str] = None` in the same signature causes silent 422s when the required param is missing. +**Minimum code that solves the problem. Nothing speculative.** -### Testing Strategy -- Unit tests in `tests/test_litellm/` -- Integration tests for each provider in `tests/llm_translation/` -- Proxy tests in `tests/proxy_unit_tests/` -- Load tests in `tests/load_tests/` -- **Always add tests when adding new entity types or features** — if the existing test file covers other entity types, add corresponding tests for the new one -- **Keep monkeypatch stubs in sync with real signatures** — when a function gains a new optional parameter, update every `fake_*` / `stub_*` in tests that patch it to also accept that kwarg (even as `**kwargs`). Stale stubs fail with `unexpected keyword argument` and mask real bugs. -- **Test all branches of name→ID resolution** — when adding server/resource lookup that resolves names to UUIDs, test: (1) name resolves and UUID is allowed, (2) name resolves but UUID is not allowed, (3) name does not resolve at all. The silent-fallback path is where access-control bugs hide. +- No features beyond what was asked. +- No abstractions for single-use code. +- No "flexibility" or "configurability" that wasn't requested. +- No error handling for impossible scenarios. +- If you write 200 lines and it could be 50, rewrite it. -### UI / Backend Consistency -- When wiring a new UI entity type to an existing backend endpoint, verify the backend API contract (single value vs. array, required vs. optional params) and ensure the UI controls match — e.g., use a single-select dropdown when the backend accepts a single value, not a multi-select - -### UI Component Library -- **Always use `antd` for new UI components** — we are migrating off of `@tremor/react`. Do not introduce new `Badge`, `Text`, `Card`, `Grid`, `Title`, or other imports from `@tremor/react` in any new or modified file. Use `antd` equivalents: `Tag` for labels, `Typography.Text` / `Typography.Title` / `Typography.Paragraph` for textual content (avoid plain text-only ``, `

`, `` when Typography fits), and `Card` from `antd`. Note that `antd` has no `"yellow"` Tag color — use `"gold"` for amber/yellow. - -### MCP OAuth / OpenAPI Transport Mapping -- **`available_on_public_internet: false` with `delegate_auth_to_upstream: true` (oauth2, interactive — not `client_credentials`)** — LiteLLM still allows the anonymous upstream PKCE path (no proxy API key for `/authorize` and matching MCP routes). The internal-only flag mainly affects other surfaces (e.g. IP-based discovery). Rely on the upstream IdP and network policy; the dashboard shows a warning when both are set, and the proxy logs a warning when the server is loaded from config or the database. -- `TRANSPORT.OPENAPI` is a UI-only concept. The backend only accepts `"http"`, `"sse"`, or `"stdio"`. Always map it to `"http"` before any API call (including pre-OAuth temp-session calls). -- FastAPI validation errors return `detail` as an array of `{loc, msg, type}` objects. Error extractors must handle: array (map `.msg`), string, nested `{error: string}`, and fallback. -- When an MCP server already has `authorization_url` stored, skip OAuth discovery (`_discovery_metadata`) — the server URL for OpenAPI MCPs is the spec file, not the API base, and fetching it causes timeouts. -- `client_id` should be optional in the `/authorize` endpoint — if the server has a stored `client_id` in credentials, use that. Never require callers to re-supply it. - -### MCP Credential Storage -- OAuth credentials and BYOK credentials share the `litellm_mcpusercredentials` table, distinguished by a `"type"` field in the JSON payload (`"oauth2"` vs plain string). -- When deleting OAuth credentials, check type before deleting to avoid accidentally deleting a BYOK credential for the same `(user_id, server_id)` pair. -- Always pass the raw `expires_at` timestamp to the client — never set it to `None` for expired credentials. Let the frontend compute the "Expired" display state from the timestamp. -- Use `RecordNotFoundError` (not bare `except Exception`) when catching "already deleted" in credential delete endpoints. - -### Browser Storage Safety (UI) -- Never write LiteLLM access tokens or API keys to `localStorage` — use `sessionStorage` only. `localStorage` survives browser close and is readable by any injected script (XSS). -- Shared utility functions (e.g. `extractErrorMessage`) belong in `src/utils/` — never define them inline in hooks or duplicate them across files. - -### Database Migrations -- Prisma handles schema migrations -- Migration files auto-generated with `prisma migrate dev` -- Always test migrations against both PostgreSQL and SQLite - -### Proxy database access -- **Do not write raw SQL** for proxy DB operations. Use Prisma model methods instead of `execute_raw` / `query_raw`. -- Use the generated client: `prisma_client.db.` (e.g. `litellm_tooltable`, `litellm_usertable`) with `.upsert()`, `.find_many()`, `.find_unique()`, `.update()`, `.update_many()` as appropriate. This avoids schema/client drift, keeps code testable with simple mocks, and matches patterns used in spend logs and other proxy code. -- **No N+1 queries.** Never query the DB inside a loop. Batch-fetch with `{"in": ids}` and distribute in-memory. -- **Batch writes.** Use `create_many`/`update_many`/`delete_many` instead of individual calls (these return counts only; `update_many`/`delete_many` no-op silently on missing rows). When multiple separate writes target the same table (e.g. in `batch_()`), order by primary key to avoid deadlocks. -- **Push work to the DB.** Filter, sort, group, and aggregate in SQL, not Python. Verify Prisma generates the expected SQL — e.g. prefer `group_by` over `find_many(distinct=...)` which does client-side processing. -- **Bound large result sets.** Prisma materializes full results in memory. For results over ~10 MB, paginate with `take`/`skip` or `cursor`/`take`, always with an explicit `order`. Prefer cursor-based pagination (`skip` is O(n)). Don't paginate naturally small result sets. -- **Limit fetched columns on wide tables.** Use `select` to fetch only needed fields — returns a partial object, so downstream code must not access unselected fields. -- **Check index coverage.** For new or modified queries, check `schema.prisma` for a supporting index. Prefer extending an existing index (e.g. `@@index([a])` → `@@index([a, b])`) over adding a new one, unless it's a `@@unique`. Only add indexes for large/frequent queries. -- **Keep schema files in sync.** Apply schema changes to all `schema.prisma` copies (`schema.prisma`, `litellm/proxy/`, `litellm-proxy-extras/`) with a migration under `litellm-proxy-extras/litellm_proxy_extras/migrations/`. - -### Setup Wizard (`litellm/setup_wizard.py`) -- The wizard is implemented as a single `SetupWizard` class with `@staticmethod` methods — keep it that way. No module-level functions except `run_setup_wizard()` (the public entrypoint) and pure helpers (color, ANSI). -- Use `litellm.utils.check_valid_key(model, api_key)` for credential validation — never roll a custom completion call. -- Do not hardcode provider env-key names or model lists that already exist in the codebase. Add a `test_model` field to each provider entry to drive `check_valid_key`; set it to `None` for providers that can't be validated with a single API key (Azure, Bedrock, Ollama). - -### Enterprise Features -- Enterprise-specific code in `enterprise/` directory -- Optional features enabled via environment variables -- Separate licensing and authentication for enterprise features - -### CI Supply-Chain Safety -- **Never pipe a remote script into a shell** (`curl ... | bash`, `wget ... | sh`). Download the artifact to a file, verify its SHA-256 checksum, then install. -- **Pin every external tool to a specific version** with a full URL (not `latest` or `stable`). Unversioned downloads silently change under you. -- **Verify checksums for all downloaded binaries.** Use the provider's official `.sha256` / `.sha256sum` sidecar file when available; otherwise compute and hardcode the digest. -- **Prefer reusable CircleCI commands** (`commands:` section) so a tool is installed and verified in exactly one place, then referenced everywhere with `- install_` or `- wait_for_service`. -- **Don't add tools just because they were there before.** Audit whether an external dependency is still needed. If it can be replaced with a shell one-liner or a tool already in the image, remove it. -- These rules apply to every download in CI: binaries, install scripts, language version managers, package repos. No exceptions. - -### HTTP Client Cache Safety -- **Never close HTTP/SDK clients on cache eviction.** `LLMClientCache._remove_key()` must not call `close()`/`aclose()` on evicted clients — they may still be used by in-flight requests. Doing so causes `RuntimeError: Cannot send a request, as the client has been closed.` after the 1-hour TTL expires. Cleanup happens at shutdown via `close_litellm_async_clients()`. - -### Troubleshooting: DB schema out of sync after proxy restart -`litellm-proxy-extras` runs `prisma migrate deploy` on startup using **its own** bundled migration files, which may lag behind schema changes in the current worktree. Symptoms: `Unknown column`, `Invalid prisma invocation`, or missing data on new fields. - -**Diagnose:** Run `\d "TableName"` in psql and compare against `schema.prisma` — missing columns confirm the issue. - -**Fix options:** -1. **Create a Prisma migration** (permanent) — run `prisma migrate dev --name ` in the worktree. The generated file will be picked up by `prisma migrate deploy` on next startup. -2. **Apply manually for local dev** — `psql -d litellm -c "ALTER TABLE ... ADD COLUMN IF NOT EXISTS ..."` after each proxy start. Fine for dev, not for production. -3. **Update litellm-proxy-extras** — if the package is installed from PyPI, its migration directory must include the new file. Either update the package or run the migration manually until the next release ships it. +Ask yourself: "Would a senior engineer say this is overcomplicated?" If yes, simplify. diff --git a/GEMINI.md b/GEMINI.md index 9e950d89b33..41921fdff4d 100644 --- a/GEMINI.md +++ b/GEMINI.md @@ -1,108 +1 @@ -# GEMINI.md - -This file provides guidance to Gemini when working with code in this repository. - -## Development Commands - -### Installation -- `make install-dev` - Install core development dependencies -- `make install-proxy-dev` - Install proxy development dependencies with full feature set -- `make install-test-deps` - Install all test dependencies - -### Testing -- `make test` - Run all tests -- `make test-unit` - Run unit tests (tests/test_litellm) with 4 parallel workers -- `make test-integration` - Run integration tests (excludes unit tests) -- `pytest tests/` - Direct pytest execution - -### Code Quality -- `make lint` - Run all linting (Ruff, MyPy, Black, circular imports, import safety) -- `make format` - Apply Black code formatting -- `make lint-ruff` - Run Ruff linting only -- `make lint-mypy` - Run MyPy type checking only - -### Single Test Files -- `uv run pytest tests/path/to/test_file.py -v` - Run specific test file -- `uv run pytest tests/path/to/test_file.py::test_function -v` - Run specific test - -### Running Scripts -- `uv run python script.py` - Run Python scripts (use for non-test files) - -### GitHub Issue & PR Templates -When contributing to the project, use the appropriate templates: - -**Bug Reports** (`.github/ISSUE_TEMPLATE/bug_report.yml`): -- Describe what happened vs. what you expected -- Include relevant log output -- Specify your LiteLLM version - -**Feature Requests** (`.github/ISSUE_TEMPLATE/feature_request.yml`): -- Describe the feature clearly -- Explain the motivation and use case - -**Pull Requests** (`.github/pull_request_template.md`): -- Add at least 1 test in `tests/litellm/` -- Ensure `make test-unit` passes - -## Architecture Overview - -LiteLLM is a unified interface for 100+ LLM providers with two main components: - -### Core Library (`litellm/`) -- **Main entry point**: `litellm/main.py` - Contains core completion() function -- **Provider implementations**: `litellm/llms/` - Each provider has its own subdirectory -- **Router system**: `litellm/router.py` + `litellm/router_utils/` - Load balancing and fallback logic -- **Type definitions**: `litellm/types/` - Pydantic models and type hints -- **Integrations**: `litellm/integrations/` - Third-party observability, caching, logging -- **Caching**: `litellm/caching/` - Multiple cache backends (Redis, in-memory, S3, etc.) - -### Proxy Server (`litellm/proxy/`) -- **Main server**: `proxy_server.py` - FastAPI application -- **Authentication**: `auth/` - API key management, JWT, OAuth2 -- **Database**: `db/` - Prisma ORM with PostgreSQL/SQLite support -- **Management endpoints**: `management_endpoints/` - Admin APIs for keys, teams, models -- **Pass-through endpoints**: `pass_through_endpoints/` - Provider-specific API forwarding -- **Guardrails**: `guardrails/` - Safety and content filtering hooks -- **UI Dashboard**: Served from `_experimental/out/` (Next.js build) - -## Key Patterns - -### Provider Implementation -- Providers inherit from base classes in `litellm/llms/base.py` -- Each provider has transformation functions for input/output formatting -- Support both sync and async operations -- Handle streaming responses and function calling - -### Error Handling -- Provider-specific exceptions mapped to OpenAI-compatible errors -- Fallback logic handled by Router system -- Comprehensive logging through `litellm/_logging.py` - -### Configuration -- YAML config files for proxy server (see `proxy/example_config_yaml/`) -- Environment variables for API keys and settings -- Database schema managed via Prisma (`proxy/schema.prisma`) - -## Development Notes - -### Code Style -- Uses Black formatter, Ruff linter, MyPy type checker -- Pydantic v2 for data validation -- Async/await patterns throughout -- Type hints required for all public APIs - -### Testing Strategy -- Unit tests in `tests/test_litellm/` -- Integration tests for each provider in `tests/llm_translation/` -- Proxy tests in `tests/proxy_unit_tests/` -- Load tests in `tests/load_tests/` - -### Database Migrations -- Prisma handles schema migrations -- Migration files auto-generated with `prisma migrate dev` -- Always test migrations against both PostgreSQL and SQLite - -### Enterprise Features -- Enterprise-specific code in `enterprise/` directory -- Optional features enabled via environment variables -- Separate licensing and authentication for enterprise features +Read @CLAUDE.md for coding guidelines diff --git a/backend/Dockerfile b/backend/Dockerfile index c08014fc0ef..2cfdde8a517 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -12,17 +12,27 @@ USER root COPY --from=uvbin /uv /uvx /usr/local/bin/ -RUN apk add --no-cache bash gcc python3 python3-dev openssl openssl-dev libsndfile +# nodejs/npm so `prisma generate` uses Wolfi's Node via PRISMA_USE_GLOBAL_NODE +# instead of nodeenv downloading one whose dynamic deps may not be in Wolfi +# (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes. +RUN for i in 1 2 3; do \ + apk add --no-cache bash gcc python3 python3-dev openssl openssl-dev libsndfile nodejs npm && break; \ + [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \ + sleep 5; \ + done # UV_COMPILE_BYTECODE=1 precompiles .pyc at install time → faster cold start. # UV_LINK_MODE=copy avoids hardlink warnings when uv installs from a # BuildKit cache mount (different filesystem). # UV_PYTHON_DOWNLOADS=0 force uv to use the apk-installed CPython instead of # silently pulling a managed interpreter. +# PRISMA_USE_GLOBAL_NODE explicit (matches default) so an env override can't +# silently re-enable nodeenv's Node download. ENV UV_PROJECT_ENVIRONMENT=/app/.venv \ UV_LINK_MODE=copy \ UV_COMPILE_BYTECODE=1 \ UV_PYTHON_DOWNLOADS=0 \ + PRISMA_USE_GLOBAL_NODE=true \ PATH="/app/.venv/bin:${PATH}" # Stage 1 — install dependencies only. @@ -58,7 +68,11 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime USER root -RUN apk add --no-cache bash openssl tzdata python3 libsndfile libatomic +RUN for i in 1 2 3; do \ + apk add --no-cache bash openssl tzdata python3 libsndfile libatomic && break; \ + [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \ + sleep 5; \ + done # wolfi-base ships an unprivileged `nonroot` account (UID/GID 65532) with # /home/nonroot. We run the backend as that user diff --git a/cookbook/gollem_go_agent_framework/go.mod b/cookbook/gollem_go_agent_framework/go.mod index 89d9033aa22..a8dc9365d7f 100644 --- a/cookbook/gollem_go_agent_framework/go.mod +++ b/cookbook/gollem_go_agent_framework/go.mod @@ -1,5 +1,5 @@ module github.com/BerriAI/litellm/cookbook/gollem_go_agent_framework -go 1.25.1 +go 1.26.3 require github.com/fugue-labs/gollem v0.1.0 diff --git a/deploy/charts/litellm-helm/templates/deployment.yaml b/deploy/charts/litellm-helm/templates/deployment.yaml index aefe2a564bb..b9cd1be06ec 100644 --- a/deploy/charts/litellm-helm/templates/deployment.yaml +++ b/deploy/charts/litellm-helm/templates/deployment.yaml @@ -30,7 +30,7 @@ spec: checksum/config: {{ include (print $.Template.BasePath "/configmap-litellm.yaml") . | sha256sum }} {{- end }} {{- with .Values.podAnnotations }} - {{- toYaml . | nindent 8 }} + {{- tpl (toYaml .) $ | nindent 8 }} {{- end }} labels: {{- include "litellm.labels" . | nindent 8 }} diff --git a/deploy/charts/litellm-helm/tests/deployment_tests.yaml b/deploy/charts/litellm-helm/tests/deployment_tests.yaml index df6d1345644..f3d62651d8f 100644 --- a/deploy/charts/litellm-helm/tests/deployment_tests.yaml +++ b/deploy/charts/litellm-helm/tests/deployment_tests.yaml @@ -377,3 +377,28 @@ tests: content: name: sidecar-tpl image: "ghcr.io/berriai/litellm-database:test" + - it: should support tpl in podAnnotations + template: deployment.yaml + set: + image: + repository: ghcr.io/berriai/litellm-database + tag: test + # Mirrors the real-world scenario this feature unblocks: + # user disables the built-in ConfigMap (and its built-in checksum/config + # annotation) and re-implements checksum/config themselves via tpl. + proxyConfigMap: + create: false + podAnnotations: + checksum/config: "{{ .Values.image.tag }}" + example.com/some-key: "{{ .Values.image.repository }}" + example.com/literal: "plain-string-value" + asserts: + - equal: + path: spec.template.metadata.annotations["checksum/config"] + value: "test" + - equal: + path: spec.template.metadata.annotations["example.com/some-key"] + value: "ghcr.io/berriai/litellm-database" + - equal: + path: spec.template.metadata.annotations["example.com/literal"] + value: "plain-string-value" diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root index 2729babb6d6..8717e5b3fcd 100644 --- a/docker/Dockerfile.non_root +++ b/docker/Dockerfile.non_root @@ -55,22 +55,10 @@ COPY . . # Set non-root flag for build time consistency ENV LITELLM_NON_ROOT=true -# Stage the pre-built Admin UI from the checked-in Next.js static export. -# _experimental/out/ is regenerated as part of the release runbook. -# Restructure extensionless routes (foo.html -> foo/index.html) to match the layout -# proxy_server.py expects, and drop a readiness marker. RUN mkdir -p /var/lib/litellm/ui /var/lib/litellm/assets && \ cp -r /app/litellm/proxy/_experimental/out/. /var/lib/litellm/ui/ && \ cp /app/litellm/proxy/logo.jpg /var/lib/litellm/assets/logo.jpg && \ - ( cd /var/lib/litellm/ui && \ - for html_file in *.html; do \ - if [ "$html_file" != "index.html" ] && [ -f "$html_file" ]; then \ - folder_name="${html_file%.html}" && \ - mkdir -p "$folder_name" && \ - mv "$html_file" "$folder_name/index.html"; \ - fi; \ - done && \ - touch .litellm_ui_ready ) + touch /var/lib/litellm/ui/.litellm_ui_ready RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \ if [ "$PROXY_EXTRAS_SOURCE" = "published" ]; then \ diff --git a/enterprise/litellm_enterprise/enterprise_callbacks/send_emails/resend_email.py b/enterprise/litellm_enterprise/enterprise_callbacks/send_emails/resend_email.py index 7593e66aa47..3fad5601f52 100644 --- a/enterprise/litellm_enterprise/enterprise_callbacks/send_emails/resend_email.py +++ b/enterprise/litellm_enterprise/enterprise_callbacks/send_emails/resend_email.py @@ -19,12 +19,26 @@ RESEND_API_ENDPOINT = "https://api.resend.com/emails" class ResendEmailLogger(BaseEmailLogger): + """ + Send emails using Resend's API. + + Required env vars: + - RESEND_API_KEY + + Optional env vars: + - RESEND_FROM_EMAIL: Override the default sender address. Must be on a + domain verified in your Resend account. When unset, falls back to the + `from_email` argument passed by the caller (which defaults to + `notifications@alerts.litellm.ai` and only works on LiteLLM Cloud). + """ + def __init__(self, internal_usage_cache=None, **kwargs): super().__init__(internal_usage_cache=internal_usage_cache, **kwargs) self.async_httpx_client = get_async_httpx_client( llm_provider=httpxSpecialProvider.LoggingCallback ) self.resend_api_key = os.getenv("RESEND_API_KEY") + self.resend_from_email = os.getenv("RESEND_FROM_EMAIL") async def send_email( self, @@ -33,13 +47,14 @@ class ResendEmailLogger(BaseEmailLogger): subject: str, html_body: str, ): + sender_email = self.resend_from_email or from_email verbose_logger.debug( - f"Sending email from {from_email} to {to_email} with subject {subject}" + f"Sending email from {sender_email} to {to_email} with subject {subject}" ) response = await self.async_httpx_client.post( url=RESEND_API_ENDPOINT, json={ - "from": from_email, + "from": sender_email, "to": to_email, "subject": subject, "html": html_body, diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py index 5ed49070347..ae5905f9cdf 100644 --- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py @@ -658,7 +658,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): if isinstance(content, str): continue for c in content: - if c["type"] == "file": + if c.get("type") == "file": file_object = cast(ChatCompletionFileObject, c) file_object_file_field = file_object["file"] file_id = file_object_file_field.get("file_id") diff --git a/enterprise/pyproject.toml b/enterprise/pyproject.toml index 9f37b52d94c..d0432448433 100644 --- a/enterprise/pyproject.toml +++ b/enterprise/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm-enterprise" -version = "0.1.41" +version = "0.1.42" description = "Package for LiteLLM Enterprise features" readme = "README.md" requires-python = ">=3.9" @@ -26,7 +26,7 @@ required-version = ">=0.10.9" module-root = "" [tool.commitizen] -version = "0.1.41" +version = "0.1.42" version_files = [ "pyproject.toml:^version", "../pyproject.toml:litellm-enterprise==", diff --git a/gateway/Dockerfile b/gateway/Dockerfile index a2ca3d3f83f..19c8a10fdfe 100644 --- a/gateway/Dockerfile +++ b/gateway/Dockerfile @@ -12,17 +12,27 @@ USER root COPY --from=uvbin /uv /uvx /usr/local/bin/ -RUN apk add --no-cache bash gcc python3 python3-dev openssl openssl-dev libsndfile +# nodejs/npm so `prisma generate` uses Wolfi's Node via PRISMA_USE_GLOBAL_NODE +# instead of nodeenv downloading one whose dynamic deps may not be in Wolfi +# (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes. +RUN for i in 1 2 3; do \ + apk add --no-cache bash gcc python3 python3-dev openssl openssl-dev libsndfile nodejs npm && break; \ + [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \ + sleep 5; \ + done # UV_COMPILE_BYTECODE=1 precompiles .pyc at install time → faster cold start. # UV_LINK_MODE=copy avoids hardlink warnings when uv installs from a # BuildKit cache mount (different filesystem). # UV_PYTHON_DOWNLOADS=0 force uv to use the apk-installed CPython instead of # silently pulling a managed interpreter. +# PRISMA_USE_GLOBAL_NODE explicit (matches default) so an env override can't +# silently re-enable nodeenv's Node download. ENV UV_PROJECT_ENVIRONMENT=/app/.venv \ UV_LINK_MODE=copy \ UV_COMPILE_BYTECODE=1 \ UV_PYTHON_DOWNLOADS=0 \ + PRISMA_USE_GLOBAL_NODE=true \ PATH="/app/.venv/bin:${PATH}" # Stage 1 — install dependencies only. @@ -58,7 +68,11 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime USER root -RUN apk add --no-cache bash openssl tzdata python3 libsndfile libatomic +RUN for i in 1 2 3; do \ + apk add --no-cache bash openssl tzdata python3 libsndfile libatomic && break; \ + [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \ + sleep 5; \ + done # wolfi-base ships an unprivileged `nonroot` account (UID/GID 65532) with # /home/nonroot. We run the proxy as that user. diff --git a/helm/litellm/templates/_helpers.tpl b/helm/litellm/templates/_helpers.tpl index e2faf42b766..4319907883e 100644 --- a/helm/litellm/templates/_helpers.tpl +++ b/helm/litellm/templates/_helpers.tpl @@ -56,16 +56,34 @@ app.kubernetes.io/component: ui {{- end -}} {{/* -Shared ServiceAccount name used by all three component Deployments. When -`serviceAccount.create` is true and `serviceAccount.name` is empty, default -to the chart fullname. When `create` is false, fall back to the provided -name or the namespace's `default` SA. +Per-component ServiceAccount name helpers. + +Each component (gateway, backend, ui) has its own SA config under +.Values.serviceAccounts.. When `create` is true and `name` is +empty the chart defaults to "-litellm-". When `create` +is false the chart uses the provided name, or the namespace `default` SA. */}} -{{- define "litellm.serviceAccountName" -}} -{{- if .Values.serviceAccount.create -}} -{{ default (include "litellm.fullname" .) .Values.serviceAccount.name }} +{{- define "litellm.gateway.serviceAccountName" -}} +{{- if .Values.serviceAccounts.gateway.create -}} +{{ default (include "litellm.gateway.fullname" .) .Values.serviceAccounts.gateway.name }} {{- else -}} -{{ default "default" .Values.serviceAccount.name }} +{{ default "default" .Values.serviceAccounts.gateway.name }} +{{- end -}} +{{- end -}} + +{{- define "litellm.backend.serviceAccountName" -}} +{{- if .Values.serviceAccounts.backend.create -}} +{{ default (include "litellm.backend.fullname" .) .Values.serviceAccounts.backend.name }} +{{- else -}} +{{ default "default" .Values.serviceAccounts.backend.name }} +{{- end -}} +{{- end -}} + +{{- define "litellm.ui.serviceAccountName" -}} +{{- if .Values.serviceAccounts.ui.create -}} +{{ default (include "litellm.ui.fullname" .) .Values.serviceAccounts.ui.name }} +{{- else -}} +{{ default "default" .Values.serviceAccounts.ui.name }} {{- end -}} {{- end -}} diff --git a/helm/litellm/templates/backend/deployment.yaml b/helm/litellm/templates/backend/deployment.yaml index e761409f8c4..3b59c58c8bf 100644 --- a/helm/litellm/templates/backend/deployment.yaml +++ b/helm/litellm/templates/backend/deployment.yaml @@ -19,7 +19,8 @@ spec: labels: {{- include "litellm.backend.selectorLabels" . | nindent 8 }} spec: - serviceAccountName: {{ include "litellm.serviceAccountName" . }} + serviceAccountName: {{ include "litellm.backend.serviceAccountName" . }} + automountServiceAccountToken: {{ .Values.serviceAccounts.backend.automount }} {{- with .Values.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} diff --git a/helm/litellm/templates/gateway/deployment.yaml b/helm/litellm/templates/gateway/deployment.yaml index 935d432342e..05ea4052159 100644 --- a/helm/litellm/templates/gateway/deployment.yaml +++ b/helm/litellm/templates/gateway/deployment.yaml @@ -22,7 +22,8 @@ spec: labels: {{- include "litellm.gateway.selectorLabels" . | nindent 8 }} spec: - serviceAccountName: {{ include "litellm.serviceAccountName" . }} + serviceAccountName: {{ include "litellm.gateway.serviceAccountName" . }} + automountServiceAccountToken: {{ .Values.serviceAccounts.gateway.automount }} {{- with .Values.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} diff --git a/helm/litellm/templates/migrations-job.yaml b/helm/litellm/templates/migrations-job.yaml index f3dc2ae0236..92671388546 100644 --- a/helm/litellm/templates/migrations-job.yaml +++ b/helm/litellm/templates/migrations-job.yaml @@ -28,7 +28,7 @@ spec: app.kubernetes.io/component: migrations spec: restartPolicy: Never - serviceAccountName: {{ include "litellm.serviceAccountName" . }} + serviceAccountName: {{ include "litellm.backend.serviceAccountName" . }} {{- with .Values.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} diff --git a/helm/litellm/templates/serviceaccount.yaml b/helm/litellm/templates/serviceaccount.yaml index 3c998448ae5..a2fc52f47c0 100644 --- a/helm/litellm/templates/serviceaccount.yaml +++ b/helm/litellm/templates/serviceaccount.yaml @@ -1,13 +1,51 @@ -{{- if .Values.serviceAccount.create -}} +{{- $prev := false -}} +{{- if .Values.serviceAccounts.gateway.create -}} +{{- $prev = true }} apiVersion: v1 kind: ServiceAccount metadata: - name: {{ include "litellm.serviceAccountName" . }} + name: {{ include "litellm.gateway.serviceAccountName" . }} labels: {{- include "litellm.commonLabels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} + app.kubernetes.io/component: gateway + {{- with .Values.serviceAccounts.gateway.annotations }} annotations: {{- toYaml . | nindent 4 }} {{- end }} -automountServiceAccountToken: {{ .Values.serviceAccount.automount }} +automountServiceAccountToken: {{ .Values.serviceAccounts.gateway.automount }} +{{- end }} +{{- if .Values.serviceAccounts.backend.create }} +{{- if $prev }} +--- +{{- end }} +{{- $prev = true }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "litellm.backend.serviceAccountName" . }} + labels: + {{- include "litellm.commonLabels" . | nindent 4 }} + app.kubernetes.io/component: backend + {{- with .Values.serviceAccounts.backend.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: {{ .Values.serviceAccounts.backend.automount }} +{{- end }} +{{- if .Values.serviceAccounts.ui.create }} +{{- if $prev }} +--- +{{- end }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "litellm.ui.serviceAccountName" . }} + labels: + {{- include "litellm.commonLabels" . | nindent 4 }} + app.kubernetes.io/component: ui + {{- with .Values.serviceAccounts.ui.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: {{ .Values.serviceAccounts.ui.automount }} {{- end }} diff --git a/helm/litellm/templates/ui/deployment.yaml b/helm/litellm/templates/ui/deployment.yaml index 549bf61a0dd..b40b44cca53 100644 --- a/helm/litellm/templates/ui/deployment.yaml +++ b/helm/litellm/templates/ui/deployment.yaml @@ -19,7 +19,8 @@ spec: labels: {{- include "litellm.ui.selectorLabels" . | nindent 8 }} spec: - serviceAccountName: {{ include "litellm.serviceAccountName" . }} + serviceAccountName: {{ include "litellm.ui.serviceAccountName" . }} + automountServiceAccountToken: {{ .Values.serviceAccounts.ui.automount }} {{- with .Values.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} diff --git a/helm/litellm/values.yaml b/helm/litellm/values.yaml index 92477616a9a..934661643bd 100644 --- a/helm/litellm/values.yaml +++ b/helm/litellm/values.yaml @@ -14,16 +14,33 @@ ingress: host: "" # optional; if set, becomes the rule's host tls: [] -# Shared ServiceAccount used by all three component Deployments. Set -# `create: true` to have the chart provision it (e.g. when wiring an EKS -# Pod Identity association by SA name). Set `name` to use an existing SA -# (chart-created or out-of-band). When both are empty / false, pods run -# with the namespace's `default` SA. -serviceAccount: - create: false - automount: true - annotations: {} - name: "" +# Per-component ServiceAccounts for gateway, backend, and ui. +# +# Each section mirrors the old shared serviceAccount shape. Set `create: +# true` to have the chart provision the SA (useful for EKS Pod Identity / +# GKE Workload Identity annotations). Set `name` to bind an existing SA. +# When both are unset the component pod runs with the namespace `default` SA. +# +# The UI SA deliberately defaults to `automount: false` — the static nginx +# container does not need the K8s API and should not carry a projected +# ServiceAccount token that a compromised container could use to call the +# cloud-provider metadata service or the K8s API. +serviceAccounts: + gateway: + create: false + automount: true + annotations: {} + name: "" + backend: + create: false + automount: true + annotations: {} + name: "" + ui: + create: false + automount: false + annotations: {} + name: "" # Pre-install / pre-upgrade Helm hook that runs `prisma migrate deploy` # against the writer database, creating the LiteLLM schema (tables that diff --git a/litellm/__init__.py b/litellm/__init__.py index 7c92623358d..56d516536e8 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -225,6 +225,11 @@ use_chat_completions_url_for_anthropic_messages: bool = bool( route_all_chat_openai_to_responses: bool = ( os.getenv("LITELLM_ROUTE_ALL_CHAT_OPENAI_TO_RESPONSES", "false").lower() == "true" ) # When True, routes all OpenAI /chat/completions requests through the Responses API bridge +# When True, Gemini/Vertex Live setup is deferred until client `session.update`. +# Default False preserves historical behavior (auto-send setup on connect). +gemini_live_defer_setup: bool = ( + os.getenv("LITELLM_GEMINI_LIVE_DEFER_SETUP", "false").lower() == "true" +) use_legacy_interactions_schema: bool = ( os.getenv("LITELLM_USE_LEGACY_INTERACTIONS_SCHEMA", "false").lower() == "true" ) # When True, sends Api-Revision: 2026-05-07 to Google so responses use the legacy `outputs` diff --git a/litellm/_service_logger.py b/litellm/_service_logger.py index 1a3be203fec..5531c418799 100644 --- a/litellm/_service_logger.py +++ b/litellm/_service_logger.py @@ -24,6 +24,22 @@ else: UserAPIKeyAuth = Any +def _get_otel_v2_class() -> Optional[type]: + """Return the ``OpenTelemetryV2`` class, or ``None`` if the OTel SDK is absent. + + Imported lazily: ``litellm.integrations.otel.logger`` imports the OpenTelemetry + SDK at module scope, so importing it eagerly would break installs without the + SDK. The V2 logger only exists when ``LITELLM_OTEL_V2`` is enabled (which + requires the SDK), so a failed import simply means "no V2 logger in play". + """ + try: + from litellm.integrations.otel.logger import OpenTelemetryV2 + + return OpenTelemetryV2 + except Exception: + return None + + class ServiceLogging(CustomLogger): """ Separate class used for monitoring health of litellm-adjacent services (redis/postgres). @@ -38,6 +54,37 @@ class ServiceLogging(CustomLogger): if "prometheus_system" in litellm.service_callback: self.prometheusServicesLogger = PrometheusServicesLogger() + def _resolve_otel_service_logger(self, callback: Any) -> Optional[Any]: + """Resolve the OTel logger (legacy or V2) to emit a service span on. + + Returns the logger instance whose ``async_service_*_hook`` should fire for + this ``callback``, or ``None`` when ``callback`` is not an OTel callback. + + The V2 ``OpenTelemetryV2`` logger is a plain ``CustomLogger`` and is NOT a + subclass of the legacy ``OpenTelemetry``, so the legacy ``isinstance`` + check alone misses it — which is why redis/postgres service spans never + showed up under ``LITELLM_OTEL_V2``. Match both the legacy and V2 types, + whether the callback is the logger instance itself or the ``"otel"`` string + (which routes to the proxy's registered ``open_telemetry_logger``). + """ + otel_v2_cls = _get_otel_v2_class() + + def _is_otel_logger(obj: Any) -> bool: + if isinstance(obj, OpenTelemetry): + return True + return otel_v2_cls is not None and isinstance(obj, otel_v2_cls) + + if _is_otel_logger(callback): + return callback + if callback == "otel": + from litellm.proxy.proxy_server import open_telemetry_logger + + if open_telemetry_logger is not None and _is_otel_logger( + open_telemetry_logger + ): + return open_telemetry_logger + return None + def service_success_hook( self, service: ServiceTypes, @@ -129,6 +176,13 @@ class ServiceLogging(CustomLogger): event_metadata=event_metadata, ) + # OTel loggers already fired this event. ``service_callback`` can hold more + # than one reference that resolves to the *same* logger — the ``"otel"`` + # string AND the registered instance both map to ``open_telemetry_logger`` + # (the V2 logger self-registers its instance even when the string is + # present, unlike V1). Without this guard each such reference emits its own + # span, so a single DB call shows up as duplicate ``postgres ...`` spans. + emitted_otel_logger_ids: set = set() for callback in litellm.service_callback: if callback == "prometheus_system": await self.init_prometheus_services_logger_if_none() @@ -144,19 +198,18 @@ class ServiceLogging(CustomLogger): end_time=end_time, event_metadata=event_metadata, ) - elif callback == "otel" or isinstance(callback, OpenTelemetry): - _otel_logger_to_use: Optional[OpenTelemetry] = None - if isinstance(callback, OpenTelemetry): - _otel_logger_to_use = callback - else: - from litellm.proxy.proxy_server import open_telemetry_logger - - if open_telemetry_logger is not None and isinstance( - open_telemetry_logger, OpenTelemetry - ): - _otel_logger_to_use = open_telemetry_logger - - if _otel_logger_to_use is not None and parent_otel_span is not None: + else: + _otel_logger_to_use = self._resolve_otel_service_logger(callback) + # No ``parent_otel_span is not None`` gate: a background service + # call (no request on the stack) has no parent, and dropping it + # here is what hid those calls from traces entirely. The OTel + # logger decides what to do with a missing parent — legacy V1 + # no-ops, V2 emits a root span (and skips metrics-only pings). + if ( + _otel_logger_to_use is not None + and id(_otel_logger_to_use) not in emitted_otel_logger_ids + ): + emitted_otel_logger_ids.add(id(_otel_logger_to_use)) await _otel_logger_to_use.async_service_success_hook( payload=payload, parent_otel_span=parent_otel_span, @@ -238,6 +291,9 @@ class ServiceLogging(CustomLogger): event_metadata=event_metadata, ) + # Dedupe OTel loggers per event — see ``async_service_success_hook`` for why + # the same logger can be referenced twice in ``service_callback``. + emitted_otel_logger_ids: set = set() for callback in litellm.service_callback: if callback == "prometheus_system": await self.init_prometheus_services_logger_if_none() @@ -255,22 +311,19 @@ class ServiceLogging(CustomLogger): end_time=end_time, event_metadata=event_metadata, ) - elif callback == "otel" or isinstance(callback, OpenTelemetry): - _otel_logger_to_use: Optional[OpenTelemetry] = None - if isinstance(callback, OpenTelemetry): - _otel_logger_to_use = callback - else: - from litellm.proxy.proxy_server import open_telemetry_logger - - if open_telemetry_logger is not None and isinstance( - open_telemetry_logger, OpenTelemetry - ): - _otel_logger_to_use = open_telemetry_logger + else: + _otel_logger_to_use = self._resolve_otel_service_logger(callback) if not isinstance(error, str): error = str(error) - if _otel_logger_to_use is not None and parent_otel_span is not None: + # See the success hook: no parent gate, so background failures + # are traced too. V1 no-ops without a parent; V2 emits a root. + if ( + _otel_logger_to_use is not None + and id(_otel_logger_to_use) not in emitted_otel_logger_ids + ): + emitted_otel_logger_ids.add(id(_otel_logger_to_use)) await _otel_logger_to_use.async_service_failure_hook( payload=payload, error=error, diff --git a/litellm/a2a_protocol/litellm_completion_bridge/handler.py b/litellm/a2a_protocol/litellm_completion_bridge/handler.py index 4e66fe4ba67..67ffcf4f8f7 100644 --- a/litellm/a2a_protocol/litellm_completion_bridge/handler.py +++ b/litellm/a2a_protocol/litellm_completion_bridge/handler.py @@ -107,6 +107,14 @@ class A2ACompletionBridgeHandler: if k not in ("model", "custom_llm_provider") and k not in _AGENT_ONLY_PARAMS } completion_params.update(litellm_params_to_add) + # Apply forward metadata AFTER the litellm_params merge so the helper + # sees any agent-owner-configured ``extra_body.metadata`` and can keep + # those keys authoritative over the client-supplied A2A metadata. + A2ACompletionBridgeTransformation.apply_forward_metadata_to_completion_params( + completion_params=completion_params, + a2a_message=message, + params=params, + ) # Call litellm.acompletion response = await litellm.acompletion(**completion_params) @@ -214,6 +222,14 @@ class A2ACompletionBridgeHandler: if k not in ("model", "custom_llm_provider") and k not in _AGENT_ONLY_PARAMS } completion_params.update(litellm_params_to_add) + # Apply forward metadata AFTER the litellm_params merge so the helper + # sees any agent-owner-configured ``extra_body.metadata`` and can keep + # those keys authoritative over the client-supplied A2A metadata. + A2ACompletionBridgeTransformation.apply_forward_metadata_to_completion_params( + completion_params=completion_params, + a2a_message=message, + params=params, + ) # 1. Emit initial task event (kind: "task", status: "submitted") task_event = A2ACompletionBridgeTransformation.create_task_event(ctx) diff --git a/litellm/a2a_protocol/litellm_completion_bridge/transformation.py b/litellm/a2a_protocol/litellm_completion_bridge/transformation.py index 8a03569f689..06c0a8fc82f 100644 --- a/litellm/a2a_protocol/litellm_completion_bridge/transformation.py +++ b/litellm/a2a_protocol/litellm_completion_bridge/transformation.py @@ -45,10 +45,80 @@ class A2ACompletionBridgeTransformation: Static methods for transforming between A2A and OpenAI message formats. """ + @staticmethod + def _extract_text_from_a2a_parts(parts: List[Dict[str, Any]]) -> str: + """Extract text from A2A parts (with or without explicit ``kind``).""" + content_parts: List[str] = [] + for part in parts: + if not isinstance(part, dict): + continue + kind = part.get("kind") + text = part.get("text") + if text is None: + continue + if kind in (None, "", "text"): + content_parts.append(str(text)) + return "\n".join(content_parts) + + @staticmethod + def get_forward_metadata( + a2a_message: Dict[str, Any], + params: Optional[Dict[str, Any]] = None, + ) -> Optional[Dict[str, Any]]: + """ + Merge A2A metadata from MessageSendParams and the message for downstream providers. + + Forwarded once on the LangGraph run payload (``metadata``), not duplicated on + each input message — see ``apply_forward_metadata_to_completion_params``. + """ + merged: Dict[str, Any] = {} + if params and isinstance(params.get("metadata"), dict): + merged.update(params["metadata"]) + message_metadata = a2a_message.get("metadata") + if isinstance(message_metadata, dict): + merged.update(message_metadata) + return merged or None + + @staticmethod + def apply_forward_metadata_to_completion_params( + completion_params: Dict[str, Any], + a2a_message: Dict[str, Any], + params: Optional[Dict[str, Any]] = None, + ) -> None: + """ + Attach A2A metadata to completion kwargs for provider bridges (e.g. LangGraph). + + Uses ``extra_body`` so we do not collide with LiteLLM's spend-log ``metadata`` kwarg. + """ + forward_metadata = A2ACompletionBridgeTransformation.get_forward_metadata( + a2a_message=a2a_message, + params=params, + ) + if not forward_metadata: + return + + extra_body = completion_params.get("extra_body") + if not isinstance(extra_body, dict): + extra_body = {} + # Layer client-supplied A2A metadata under any agent-owner-configured + # ``extra_body.metadata`` so the configured keys remain authoritative + # and an A2A caller cannot overwrite server-set run metadata. + existing_metadata = extra_body.get("metadata") + existing_dict: Dict[str, Any] = ( + existing_metadata if isinstance(existing_metadata, dict) else {} + ) + merged_metadata: Dict[str, Any] = {**forward_metadata, **existing_dict} + extra_body = {**extra_body, "metadata": merged_metadata} + completion_params["extra_body"] = extra_body + + verbose_logger.debug( + f"A2A -> completion forward metadata keys={list(forward_metadata.keys())}" + ) + @staticmethod def a2a_message_to_openai_messages( a2a_message: Dict[str, Any], - ) -> List[Dict[str, str]]: + ) -> List[Dict[str, Any]]: """ Transform an A2A message to OpenAI message format. @@ -70,21 +140,20 @@ class A2ACompletionBridgeTransformation: elif role == "system": openai_role = "system" - # Extract text content from parts - content_parts = [] - for part in parts: - kind = part.get("kind", "") - if kind == "text": - text = part.get("text", "") - content_parts.append(text) + if not isinstance(parts, list): + parts = [] - content = "\n".join(content_parts) if content_parts else "" + content = A2ACompletionBridgeTransformation._extract_text_from_a2a_parts(parts) + + # Do not attach A2A message.metadata here — the completion bridge forwards it + # once at run level via extra_body.metadata (LangGraph POST /runs/wait shape). + openai_message: Dict[str, Any] = {"role": openai_role, "content": content} verbose_logger.debug( f"A2A -> OpenAI transform: role={role} -> {openai_role}, content_length={len(content)}" ) - return [{"role": openai_role, "content": content}] + return [openai_message] @staticmethod def openai_response_to_a2a_response( @@ -110,6 +179,7 @@ class A2ACompletionBridgeTransformation: # Build A2A message a2a_message = { + "kind": "message", "role": "agent", "parts": [{"kind": "text", "text": content}], "messageId": uuid4().hex, @@ -119,9 +189,7 @@ class A2ACompletionBridgeTransformation: a2a_response = { "jsonrpc": "2.0", "id": request_id, - "result": { - "message": a2a_message, - }, + "result": a2a_message, } verbose_logger.debug(f"OpenAI -> A2A transform: content_length={len(content)}") @@ -235,50 +303,3 @@ class A2ACompletionBridgeTransformation: "taskId": ctx.task_id, }, } - - @staticmethod - def openai_chunk_to_a2a_chunk( - chunk: Any, - request_id: Optional[str] = None, - is_final: bool = False, - ) -> Optional[Dict[str, Any]]: - """ - Transform a LiteLLM streaming chunk to A2A streaming format. - - NOTE: This method is deprecated for streaming. Use the event-based - methods (create_task_event, create_status_update_event, - create_artifact_update_event) instead for proper A2A streaming. - - Args: - chunk: LiteLLM ModelResponse chunk - request_id: Original A2A request ID - is_final: Whether this is the final chunk - - Returns: - A2A streaming chunk dict or None if no content - """ - # Extract delta content - content = "" - if chunk is not None and hasattr(chunk, "choices") and chunk.choices: - choice = chunk.choices[0] - if hasattr(choice, "delta") and choice.delta: - content = choice.delta.content or "" - - if not content and not is_final: - return None - - # Build A2A streaming chunk (legacy format) - a2a_chunk = { - "jsonrpc": "2.0", - "id": request_id, - "result": { - "message": { - "role": "agent", - "parts": [{"kind": "text", "text": content}], - "messageId": uuid4().hex, - }, - "final": is_final, - }, - } - - return a2a_chunk diff --git a/litellm/a2a_protocol/providers/litellm_completion/README.md b/litellm/a2a_protocol/providers/litellm_completion/README.md deleted file mode 100644 index a809e9bf55e..00000000000 --- a/litellm/a2a_protocol/providers/litellm_completion/README.md +++ /dev/null @@ -1,74 +0,0 @@ -# A2A to LiteLLM Completion Bridge - -Routes A2A protocol requests through `litellm.acompletion`, enabling any LiteLLM-supported provider to be invoked via A2A. - -## Flow - -``` -A2A Request → Transform → litellm.acompletion → Transform → A2A Response -``` - -## SDK Usage - -Use the existing `asend_message` and `asend_message_streaming` functions with `litellm_params`: - -```python -from litellm.a2a_protocol import asend_message, asend_message_streaming -from a2a.types import SendMessageRequest, SendStreamingMessageRequest, MessageSendParams -from uuid import uuid4 - -# Non-streaming -request = SendMessageRequest( - id=str(uuid4()), - params=MessageSendParams( - message={"role": "user", "parts": [{"kind": "text", "text": "Hello!"}], "messageId": uuid4().hex} - ) -) -response = await asend_message( - request=request, - api_base="http://localhost:2024", - litellm_params={"custom_llm_provider": "langgraph", "model": "agent"}, -) - -# Streaming -stream_request = SendStreamingMessageRequest( - id=str(uuid4()), - params=MessageSendParams( - message={"role": "user", "parts": [{"kind": "text", "text": "Hello!"}], "messageId": uuid4().hex} - ) -) -async for chunk in asend_message_streaming( - request=stream_request, - api_base="http://localhost:2024", - litellm_params={"custom_llm_provider": "langgraph", "model": "agent"}, -): - print(chunk) -``` - -## Proxy Usage - -Configure an agent with `custom_llm_provider` in `litellm_params`: - -```yaml -agents: - - agent_name: my-langgraph-agent - agent_card_params: - name: "LangGraph Agent" - url: "http://localhost:2024" # Used as api_base - litellm_params: - custom_llm_provider: langgraph - model: agent -``` - -When an A2A request hits `/a2a/{agent_id}/message/send`, the bridge: - -1. Detects `custom_llm_provider` in agent's `litellm_params` -2. Transforms A2A message → OpenAI messages -3. Calls `litellm.acompletion(model="langgraph/agent", api_base="http://localhost:2024")` -4. Transforms response → A2A format - -## Classes - -- `A2ACompletionBridgeTransformation` - Static methods for message format conversion -- `A2ACompletionBridgeHandler` - Static methods for handling requests (streaming/non-streaming) - diff --git a/litellm/a2a_protocol/providers/litellm_completion/__init__.py b/litellm/a2a_protocol/providers/litellm_completion/__init__.py deleted file mode 100644 index fc2fc17f54f..00000000000 --- a/litellm/a2a_protocol/providers/litellm_completion/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -""" -LiteLLM Completion bridge provider for A2A protocol. - -Routes A2A requests through litellm.acompletion based on custom_llm_provider. -""" diff --git a/litellm/a2a_protocol/providers/litellm_completion/handler.py b/litellm/a2a_protocol/providers/litellm_completion/handler.py deleted file mode 100644 index 730f8f6b36f..00000000000 --- a/litellm/a2a_protocol/providers/litellm_completion/handler.py +++ /dev/null @@ -1,301 +0,0 @@ -""" -Handler for A2A to LiteLLM completion bridge. - -Routes A2A requests through litellm.acompletion based on custom_llm_provider. - -A2A Streaming Events (in order): -1. Task event (kind: "task") - Initial task creation with status "submitted" -2. Status update (kind: "status-update") - Status change to "working" -3. Artifact update (kind: "artifact-update") - Content/artifact delivery -4. Status update (kind: "status-update") - Final status "completed" with final=true -""" - -from typing import Any, AsyncIterator, Dict, Optional - -import litellm -from litellm._logging import verbose_logger -from litellm.a2a_protocol.litellm_completion_bridge.pydantic_ai_transformation import ( - PydanticAITransformation, -) -from litellm.a2a_protocol.litellm_completion_bridge.transformation import ( - A2ACompletionBridgeTransformation, - A2AStreamingContext, -) - - -class A2ACompletionBridgeHandler: - """ - Static methods for handling A2A requests via LiteLLM completion. - """ - - @staticmethod - async def handle_non_streaming( - request_id: str, - params: Dict[str, Any], - litellm_params: Dict[str, Any], - api_base: Optional[str] = None, - ) -> Dict[str, Any]: - """ - Handle non-streaming A2A request via litellm.acompletion. - - Args: - request_id: A2A JSON-RPC request ID - params: A2A MessageSendParams containing the message - litellm_params: Agent's litellm_params (custom_llm_provider, model, etc.) - api_base: API base URL from agent_card_params - - Returns: - A2A SendMessageResponse dict - """ - # Check if this is a Pydantic AI agent request - custom_llm_provider = litellm_params.get("custom_llm_provider") - if custom_llm_provider == "pydantic_ai_agents": - if api_base is None: - raise ValueError("api_base is required for Pydantic AI agents") - - verbose_logger.info( - f"Pydantic AI: Routing to Pydantic AI agent at {api_base}" - ) - - # Send request directly to Pydantic AI agent - response_data = await PydanticAITransformation.send_non_streaming_request( - api_base=api_base, - request_id=request_id, - params=params, - ) - - return response_data - - # Extract message from params - message = params.get("message", {}) - - # Transform A2A message to OpenAI format - openai_messages = ( - A2ACompletionBridgeTransformation.a2a_message_to_openai_messages(message) - ) - - # Get completion params - custom_llm_provider = litellm_params.get("custom_llm_provider") - model = litellm_params.get("model", "agent") - - # Build full model string if provider specified - # Skip prepending if model already starts with the provider prefix - if custom_llm_provider and not model.startswith(f"{custom_llm_provider}/"): - full_model = f"{custom_llm_provider}/{model}" - else: - full_model = model - - verbose_logger.info( - f"A2A completion bridge: model={full_model}, api_base={api_base}" - ) - - # Build completion params dict - completion_params = { - "model": full_model, - "messages": openai_messages, - "api_base": api_base, - "stream": False, - } - # Add litellm_params (contains api_key, client_id, client_secret, tenant_id, etc.) - litellm_params_to_add = { - k: v - for k, v in litellm_params.items() - if k not in ("model", "custom_llm_provider") - } - completion_params.update(litellm_params_to_add) - - # Call litellm.acompletion - response = await litellm.acompletion(**completion_params) - - # Transform response to A2A format - a2a_response = ( - A2ACompletionBridgeTransformation.openai_response_to_a2a_response( - response=response, - request_id=request_id, - ) - ) - - verbose_logger.info(f"A2A completion bridge completed: request_id={request_id}") - - return a2a_response - - @staticmethod - async def handle_streaming( - request_id: str, - params: Dict[str, Any], - litellm_params: Dict[str, Any], - api_base: Optional[str] = None, - ) -> AsyncIterator[Dict[str, Any]]: - """ - Handle streaming A2A request via litellm.acompletion with stream=True. - - Emits proper A2A streaming events: - 1. Task event (kind: "task") - Initial task with status "submitted" - 2. Status update (kind: "status-update") - Status "working" - 3. Artifact update (kind: "artifact-update") - Content delivery - 4. Status update (kind: "status-update") - Final "completed" status - - Args: - request_id: A2A JSON-RPC request ID - params: A2A MessageSendParams containing the message - litellm_params: Agent's litellm_params (custom_llm_provider, model, etc.) - api_base: API base URL from agent_card_params - - Yields: - A2A streaming response events - """ - # Check if this is a Pydantic AI agent request - custom_llm_provider = litellm_params.get("custom_llm_provider") - if custom_llm_provider == "pydantic_ai_agents": - if api_base is None: - raise ValueError("api_base is required for Pydantic AI agents") - - verbose_logger.info( - f"Pydantic AI: Faking streaming for Pydantic AI agent at {api_base}" - ) - - # Get non-streaming response first - response_data = await PydanticAITransformation.send_non_streaming_request( - api_base=api_base, - request_id=request_id, - params=params, - ) - - # Convert to fake streaming - async for chunk in PydanticAITransformation.fake_streaming_from_response( - response_data=response_data, - request_id=request_id, - ): - yield chunk - - return - - # Extract message from params - message = params.get("message", {}) - - # Create streaming context - ctx = A2AStreamingContext( - request_id=request_id, - input_message=message, - ) - - # Transform A2A message to OpenAI format - openai_messages = ( - A2ACompletionBridgeTransformation.a2a_message_to_openai_messages(message) - ) - - # Get completion params - custom_llm_provider = litellm_params.get("custom_llm_provider") - model = litellm_params.get("model", "agent") - - # Build full model string if provider specified - # Skip prepending if model already starts with the provider prefix - if custom_llm_provider and not model.startswith(f"{custom_llm_provider}/"): - full_model = f"{custom_llm_provider}/{model}" - else: - full_model = model - - verbose_logger.info( - f"A2A completion bridge streaming: model={full_model}, api_base={api_base}" - ) - - # Build completion params dict - completion_params = { - "model": full_model, - "messages": openai_messages, - "api_base": api_base, - "stream": True, - } - # Add litellm_params (contains api_key, client_id, client_secret, tenant_id, etc.) - litellm_params_to_add = { - k: v - for k, v in litellm_params.items() - if k not in ("model", "custom_llm_provider") - } - completion_params.update(litellm_params_to_add) - - # 1. Emit initial task event (kind: "task", status: "submitted") - task_event = A2ACompletionBridgeTransformation.create_task_event(ctx) - yield task_event - - # 2. Emit status update (kind: "status-update", status: "working") - working_event = A2ACompletionBridgeTransformation.create_status_update_event( - ctx=ctx, - state="working", - final=False, - message_text="Processing request...", - ) - yield working_event - - # Call litellm.acompletion with streaming - response = await litellm.acompletion(**completion_params) - - # 3. Accumulate content and emit artifact update - accumulated_text = "" - chunk_count = 0 - async for chunk in response: # type: ignore[union-attr] - chunk_count += 1 - - # Extract delta content - content = "" - if chunk is not None and hasattr(chunk, "choices") and chunk.choices: - choice = chunk.choices[0] - if hasattr(choice, "delta") and choice.delta: - content = choice.delta.content or "" - - if content: - accumulated_text += content - - # Emit artifact update with accumulated content - if accumulated_text: - artifact_event = ( - A2ACompletionBridgeTransformation.create_artifact_update_event( - ctx=ctx, - text=accumulated_text, - ) - ) - yield artifact_event - - # 4. Emit final status update (kind: "status-update", status: "completed", final: true) - completed_event = A2ACompletionBridgeTransformation.create_status_update_event( - ctx=ctx, - state="completed", - final=True, - ) - yield completed_event - - verbose_logger.info( - f"A2A completion bridge streaming completed: request_id={request_id}, chunks={chunk_count}" - ) - - -# Convenience functions that delegate to the class methods -async def handle_a2a_completion( - request_id: str, - params: Dict[str, Any], - litellm_params: Dict[str, Any], - api_base: Optional[str] = None, -) -> Dict[str, Any]: - """Convenience function for non-streaming A2A completion.""" - return await A2ACompletionBridgeHandler.handle_non_streaming( - request_id=request_id, - params=params, - litellm_params=litellm_params, - api_base=api_base, - ) - - -async def handle_a2a_completion_streaming( - request_id: str, - params: Dict[str, Any], - litellm_params: Dict[str, Any], - api_base: Optional[str] = None, -) -> AsyncIterator[Dict[str, Any]]: - """Convenience function for streaming A2A completion.""" - async for chunk in A2ACompletionBridgeHandler.handle_streaming( - request_id=request_id, - params=params, - litellm_params=litellm_params, - api_base=api_base, - ): - yield chunk diff --git a/litellm/a2a_protocol/providers/litellm_completion/transformation.py b/litellm/a2a_protocol/providers/litellm_completion/transformation.py deleted file mode 100644 index 8a03569f689..00000000000 --- a/litellm/a2a_protocol/providers/litellm_completion/transformation.py +++ /dev/null @@ -1,284 +0,0 @@ -""" -Transformation utilities for A2A <-> OpenAI message format conversion. - -A2A Message Format: -{ - "role": "user", - "parts": [{"kind": "text", "text": "Hello!"}], - "messageId": "abc123" -} - -OpenAI Message Format: -{"role": "user", "content": "Hello!"} - -A2A Streaming Events: -- Task event (kind: "task") - Initial task creation with status "submitted" -- Status update (kind: "status-update") - Status changes (working, completed) -- Artifact update (kind: "artifact-update") - Content/artifact delivery -""" - -from datetime import datetime, timezone -from typing import Any, Dict, List, Optional -from uuid import uuid4 - -from litellm._logging import verbose_logger - - -class A2AStreamingContext: - """ - Context holder for A2A streaming state. - Tracks task_id, context_id, and message accumulation. - """ - - def __init__(self, request_id: str, input_message: Dict[str, Any]): - self.request_id = request_id - self.task_id = str(uuid4()) - self.context_id = str(uuid4()) - self.input_message = input_message - self.accumulated_text = "" - self.has_emitted_task = False - self.has_emitted_working = False - - -class A2ACompletionBridgeTransformation: - """ - Static methods for transforming between A2A and OpenAI message formats. - """ - - @staticmethod - def a2a_message_to_openai_messages( - a2a_message: Dict[str, Any], - ) -> List[Dict[str, str]]: - """ - Transform an A2A message to OpenAI message format. - - Args: - a2a_message: A2A message with role, parts, and messageId - - Returns: - List of OpenAI-format messages - """ - role = a2a_message.get("role", "user") - parts = a2a_message.get("parts", []) - - # Map A2A roles to OpenAI roles - openai_role = role - if role == "user": - openai_role = "user" - elif role == "assistant": - openai_role = "assistant" - elif role == "system": - openai_role = "system" - - # Extract text content from parts - content_parts = [] - for part in parts: - kind = part.get("kind", "") - if kind == "text": - text = part.get("text", "") - content_parts.append(text) - - content = "\n".join(content_parts) if content_parts else "" - - verbose_logger.debug( - f"A2A -> OpenAI transform: role={role} -> {openai_role}, content_length={len(content)}" - ) - - return [{"role": openai_role, "content": content}] - - @staticmethod - def openai_response_to_a2a_response( - response: Any, - request_id: Optional[str] = None, - ) -> Dict[str, Any]: - """ - Transform a LiteLLM ModelResponse to A2A SendMessageResponse format. - - Args: - response: LiteLLM ModelResponse object - request_id: Original A2A request ID - - Returns: - A2A SendMessageResponse dict - """ - # Extract content from response - content = "" - if hasattr(response, "choices") and response.choices: - choice = response.choices[0] - if hasattr(choice, "message") and choice.message: - content = choice.message.content or "" - - # Build A2A message - a2a_message = { - "role": "agent", - "parts": [{"kind": "text", "text": content}], - "messageId": uuid4().hex, - } - - # Build A2A response - a2a_response = { - "jsonrpc": "2.0", - "id": request_id, - "result": { - "message": a2a_message, - }, - } - - verbose_logger.debug(f"OpenAI -> A2A transform: content_length={len(content)}") - - return a2a_response - - @staticmethod - def _get_timestamp() -> str: - """Get current timestamp in ISO format with timezone.""" - return datetime.now(timezone.utc).isoformat() - - @staticmethod - def create_task_event( - ctx: A2AStreamingContext, - ) -> Dict[str, Any]: - """ - Create the initial task event with status 'submitted'. - - This is the first event emitted in an A2A streaming response. - """ - return { - "id": ctx.request_id, - "jsonrpc": "2.0", - "result": { - "contextId": ctx.context_id, - "history": [ - { - "contextId": ctx.context_id, - "kind": "message", - "messageId": ctx.input_message.get("messageId", uuid4().hex), - "parts": ctx.input_message.get("parts", []), - "role": ctx.input_message.get("role", "user"), - "taskId": ctx.task_id, - } - ], - "id": ctx.task_id, - "kind": "task", - "status": { - "state": "submitted", - }, - }, - } - - @staticmethod - def create_status_update_event( - ctx: A2AStreamingContext, - state: str, - final: bool = False, - message_text: Optional[str] = None, - ) -> Dict[str, Any]: - """ - Create a status update event. - - Args: - ctx: Streaming context - state: Status state ('working', 'completed') - final: Whether this is the final event - message_text: Optional message text for 'working' status - """ - status: Dict[str, Any] = { - "state": state, - "timestamp": A2ACompletionBridgeTransformation._get_timestamp(), - } - - # Add message for 'working' status - if state == "working" and message_text: - status["message"] = { - "contextId": ctx.context_id, - "kind": "message", - "messageId": str(uuid4()), - "parts": [{"kind": "text", "text": message_text}], - "role": "agent", - "taskId": ctx.task_id, - } - - return { - "id": ctx.request_id, - "jsonrpc": "2.0", - "result": { - "contextId": ctx.context_id, - "final": final, - "kind": "status-update", - "status": status, - "taskId": ctx.task_id, - }, - } - - @staticmethod - def create_artifact_update_event( - ctx: A2AStreamingContext, - text: str, - ) -> Dict[str, Any]: - """ - Create an artifact update event with content. - - Args: - ctx: Streaming context - text: The text content for the artifact - """ - return { - "id": ctx.request_id, - "jsonrpc": "2.0", - "result": { - "artifact": { - "artifactId": str(uuid4()), - "name": "response", - "parts": [{"kind": "text", "text": text}], - }, - "contextId": ctx.context_id, - "kind": "artifact-update", - "taskId": ctx.task_id, - }, - } - - @staticmethod - def openai_chunk_to_a2a_chunk( - chunk: Any, - request_id: Optional[str] = None, - is_final: bool = False, - ) -> Optional[Dict[str, Any]]: - """ - Transform a LiteLLM streaming chunk to A2A streaming format. - - NOTE: This method is deprecated for streaming. Use the event-based - methods (create_task_event, create_status_update_event, - create_artifact_update_event) instead for proper A2A streaming. - - Args: - chunk: LiteLLM ModelResponse chunk - request_id: Original A2A request ID - is_final: Whether this is the final chunk - - Returns: - A2A streaming chunk dict or None if no content - """ - # Extract delta content - content = "" - if chunk is not None and hasattr(chunk, "choices") and chunk.choices: - choice = chunk.choices[0] - if hasattr(choice, "delta") and choice.delta: - content = choice.delta.content or "" - - if not content and not is_final: - return None - - # Build A2A streaming chunk (legacy format) - a2a_chunk = { - "jsonrpc": "2.0", - "id": request_id, - "result": { - "message": { - "role": "agent", - "parts": [{"kind": "text", "text": content}], - "messageId": uuid4().hex, - }, - "final": is_final, - }, - } - - return a2a_chunk diff --git a/litellm/a2a_protocol/providers/pydantic_ai_agents/transformation.py b/litellm/a2a_protocol/providers/pydantic_ai_agents/transformation.py index e73b17ac3c0..bf68a01d98c 100644 --- a/litellm/a2a_protocol/providers/pydantic_ai_agents/transformation.py +++ b/litellm/a2a_protocol/providers/pydantic_ai_agents/transformation.py @@ -289,16 +289,16 @@ class PydanticAITransformation: Transform Pydantic AI task response to standard A2A non-streaming format. Pydantic AI returns a task with history/artifacts, but the standard A2A - non-streaming format expects: + non-streaming format expects ``result`` to be the Message directly + (``kind="message"``), per the A2A spec / ``SendMessageResponse``: { "jsonrpc": "2.0", "id": "...", "result": { - "message": { - "role": "agent", - "parts": [{"kind": "text", "text": "..."}], - "messageId": "..." - } + "kind": "message", + "role": "agent", + "parts": [{"kind": "text", "text": "..."}], + "messageId": "..." } } @@ -316,6 +316,7 @@ class PydanticAITransformation: # Build standard A2A message a2a_message = { + "kind": "message", "role": "agent", "parts": parts if parts else [{"kind": "text", "text": full_text}], "messageId": message_id, @@ -325,9 +326,7 @@ class PydanticAITransformation: return { "jsonrpc": "2.0", "id": request_id, - "result": { - "message": a2a_message, - }, + "result": a2a_message, } @staticmethod diff --git a/litellm/a2a_protocol/utils.py b/litellm/a2a_protocol/utils.py index 1cdbde97755..0dbd1eefc63 100644 --- a/litellm/a2a_protocol/utils.py +++ b/litellm/a2a_protocol/utils.py @@ -60,6 +60,12 @@ class A2ARequestUtils: if not isinstance(result, dict): return "" + # Direct message format (A2A spec): detect by explicit kind tag only. + # The "parts" heuristic is too broad and would match any future result + # type that happens to include a "parts" field. + if result.get("kind") == "message": + return A2ARequestUtils.extract_text_from_message(result) + message = result.get("message", {}) return A2ARequestUtils.extract_text_from_message(message) diff --git a/litellm/anthropic_interface/messages/__init__.py b/litellm/anthropic_interface/messages/__init__.py index 0996d62c866..f71279b226d 100644 --- a/litellm/anthropic_interface/messages/__init__.py +++ b/litellm/anthropic_interface/messages/__init__.py @@ -10,7 +10,7 @@ This is an __init__.py file to allow the following interface """ -from typing import Any, AsyncIterator, Coroutine, Dict, List, Optional, Union +from typing import Any, AsyncIterator, Coroutine, Dict, Iterator, List, Optional, Union from litellm.llms.anthropic.experimental_pass_through.messages.handler import ( anthropic_messages as _async_anthropic_messages, @@ -100,8 +100,11 @@ def create( **kwargs, ) -> Union[ AnthropicMessagesResponse, + Iterator[bytes], AsyncIterator[Any], - Coroutine[Any, Any, Union[AnthropicMessagesResponse, AsyncIterator[Any]]], + Coroutine[ + Any, Any, Union[AnthropicMessagesResponse, AsyncIterator[Any], Iterator[bytes]] + ], ]: """ Async wrapper for Anthropic's messages API diff --git a/litellm/constants.py b/litellm/constants.py index fb765c0226c..ae98b37d6e6 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1147,6 +1147,7 @@ BEDROCK_CONVERSE_MODELS = [ "openai.gpt-oss-120b-1:0", "anthropic.claude-haiku-4-5-20251001-v1:0", "anthropic.claude-sonnet-4-5-20250929-v1:0", + "anthropic.claude-opus-4-8", "anthropic.claude-opus-4-7", "anthropic.claude-opus-4-6-v1:0", "anthropic.claude-opus-4-6-v1", @@ -1408,6 +1409,13 @@ LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME = "litellm_internal_jobs" # Prometheus metrics, audit trails, or any other downstream consumer. LITELLM_PROXY_MASTER_KEY_ALIAS = "litellm_proxy_master_key" +# Marker placed in ``model_call_details`` on a synthetic ``Logging`` object that +# records a proxy-gate error (auth/rate-limit rejection) for a request that never +# reached an upstream provider. Tracing callbacks key off it to avoid fabricating +# an LLM-call span for a call that did not happen. See +# ``ProxyLogging._handle_logging_proxy_only_error``. +LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL = "litellm_no_upstream_llm_call" + # Key Rotation Constants LITELLM_KEY_ROTATION_ENABLED = os.getenv("LITELLM_KEY_ROTATION_ENABLED", "false") LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS = int( diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 98e00cf5788..9a4b158b622 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -24,6 +24,7 @@ from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import from litellm.litellm_core_utils.llm_cost_calc.utils import ( CostCalculatorUtils, _generic_cost_per_character, + _get_regional_uplift_multiplier, _get_service_tier_cost_key, _parse_prompt_tokens_details, calculate_cost_component, @@ -132,6 +133,8 @@ _VIDEO_CALL_TYPES = frozenset( { CallTypes.create_video.value, CallTypes.acreate_video.value, + CallTypes.video_edit.value, + CallTypes.avideo_edit.value, CallTypes.video_remix.value, CallTypes.avideo_remix.value, } @@ -312,6 +315,10 @@ def cost_per_token( # noqa: PLR0915 audio_transcription_file_duration: float = 0.0, # for audio transcription calls - the file time in seconds ### SERVICE TIER ### service_tier: Optional[str] = None, # for OpenAI service tier pricing + ### DATA RESIDENCY ### + data_residency: Optional[ + str + ] = None, # for OpenAI regional-processing uplift (e.g. "eu", "us") response: Optional[Any] = None, ### REQUEST MODEL ### request_model: Optional[str] = None, # original request model for router detection @@ -412,9 +419,36 @@ def cost_per_token( # noqa: PLR0915 prompt_tokens_cost_usd_dollar: float = 0 completion_tokens_cost_usd_dollar: float = 0 model_cost_ref = litellm.model_cost + # Only callers that explicitly pass `custom_llm_provider` get the + # dedup/prefix-join treatment. When provider is omitted, preserve legacy + # behavior: `model_with_provider` stays equal to the raw `model` string + # (provider is detected below for downstream use only). + caller_supplied_provider = custom_llm_provider is not None + + # `model` is normally a string, but callers that mock the transport can pass + # non-string objects. Only run the string-based dedup/prefix-join when it is + # actually a string — e.g. a MagicMock's `.startswith()` is always truthy and + # its slices return new mocks, which would spin the dedup loop forever. + model_is_str = isinstance(model, str) + + # Router/proxy deployments may repeat the provider segment (e.g. model_name + # "openai/openai/gpt-5.5"). Strip duplicated `{provider}/` chains before joining. + if caller_supplied_provider and model_is_str: + _dup_prefix = f"{custom_llm_provider}/" + while model.startswith(_dup_prefix): + _remainder = model[len(_dup_prefix) :] + if _remainder.startswith(_dup_prefix): + model = _remainder + else: + break + model_with_provider = model - if custom_llm_provider is not None: - model_with_provider = custom_llm_provider + "/" + model + if caller_supplied_provider: + _prov_prefix = f"{custom_llm_provider}/" + if model_is_str and model.startswith(_prov_prefix): + model_with_provider = model + else: + model_with_provider = f"{custom_llm_provider}/{model}" if region_name is not None: model_with_provider_and_region = ( f"{custom_llm_provider}/{region_name}/{model}" @@ -425,6 +459,9 @@ def cost_per_token( # noqa: PLR0915 model_with_provider = model_with_provider_and_region else: _, custom_llm_provider, _, _ = litellm.get_llm_provider(model=model) + + assert custom_llm_provider is not None # caller-supplied or get_llm_provider + model_without_prefix = model model_parts = model.split("/", 1) if len(model_parts) > 1: @@ -493,6 +530,7 @@ def cost_per_token( # noqa: PLR0915 usage=usage_block, custom_llm_provider=custom_llm_provider, service_tier=service_tier, + data_residency=data_residency, ) return prompt_cost, completion_cost @@ -521,7 +559,10 @@ def cost_per_token( # noqa: PLR0915 or call_type == CallTypes.retrieve_batch ): return batch_cost_calculator( - usage=usage_block, model=model, custom_llm_provider=custom_llm_provider + usage=usage_block, + model=model, + custom_llm_provider=custom_llm_provider, + data_residency=data_residency, ) elif call_type == "atranscription" or call_type == "transcription": if _transcription_usage_has_token_details(usage_block): @@ -529,6 +570,7 @@ def cost_per_token( # noqa: PLR0915 model=model_without_prefix, usage=usage_block, service_tier=service_tier, + data_residency=data_residency, ) return openai_cost_per_second( @@ -579,7 +621,10 @@ def cost_per_token( # noqa: PLR0915 ) elif custom_llm_provider == "openai": return openai_cost_per_token( - model=model, usage=usage_block, service_tier=service_tier + model=model, + usage=usage_block, + service_tier=service_tier, + data_residency=data_residency, ) elif custom_llm_provider == "databricks": return databricks_cost_per_token(model=model, usage=usage_block) @@ -631,6 +676,7 @@ def cost_per_token( # noqa: PLR0915 usage=usage_block, custom_llm_provider=custom_llm_provider, service_tier=service_tier, + data_residency=data_residency, ) if ( @@ -1117,6 +1163,10 @@ def completion_cost( # noqa: PLR0915 litellm_logging_obj: Optional[LitellmLoggingObject] = None, ### SERVICE TIER ### service_tier: Optional[str] = None, # for OpenAI service tier pricing + ### DATA RESIDENCY ### + data_residency: Optional[ + str + ] = None, # for OpenAI regional-processing uplift (e.g. "eu", "us") ) -> float: """ Calculate the cost of a given completion call fot GPT-3.5-turbo, llama2, any litellm supported llm. @@ -1516,6 +1566,7 @@ def completion_cost( # noqa: PLR0915 combined_usage_object=cost_per_token_usage_object, custom_llm_provider=custom_llm_provider, litellm_model_name=model, + data_residency=data_residency, ) elif call_type == _MCP_CALL_TYPE: from litellm.proxy._experimental.mcp_server.cost_calculator import ( @@ -1600,6 +1651,7 @@ def completion_cost( # noqa: PLR0915 audio_transcription_file_duration=audio_transcription_file_duration, rerank_billed_units=rerank_billed_units, service_tier=service_tier, + data_residency=data_residency, response=completion_response, request_model=request_model_for_cost, ) @@ -1811,6 +1863,10 @@ def response_cost_calculator( litellm_logging_obj: Optional[LitellmLoggingObject] = None, ### SERVICE TIER ### service_tier: Optional[str] = None, # for OpenAI service tier pricing + ### DATA RESIDENCY ### + data_residency: Optional[ + str + ] = None, # for OpenAI regional-processing uplift (e.g. "eu", "us") ) -> float: """ Returns @@ -1844,6 +1900,7 @@ def response_cost_calculator( router_model_id=router_model_id, litellm_logging_obj=litellm_logging_obj, service_tier=service_tier, + data_residency=data_residency, ) return response_cost except Exception as e: @@ -2202,6 +2259,7 @@ def batch_cost_calculator( model: str, custom_llm_provider: Optional[str] = None, model_info: Optional[ModelInfo] = None, + data_residency: Optional[str] = None, ) -> Tuple[float, float]: """ Calculate the cost of a batch job. @@ -2286,6 +2344,11 @@ def batch_cost_calculator( usage.completion_tokens * (output_cost_per_token) / 2 ) # batch cost is usually half of the regular token cost + uplift = _get_regional_uplift_multiplier(model_info, data_residency) + if uplift != 1.0: + total_prompt_cost *= uplift + total_completion_cost *= uplift + return total_prompt_cost, total_completion_cost @@ -2431,6 +2494,7 @@ def handle_realtime_stream_cost_calculation( combined_usage_object: Usage, custom_llm_provider: str, litellm_model_name: str, + data_residency: Optional[str] = None, ) -> float: """ Handles the cost calculation for realtime stream responses. @@ -2461,6 +2525,7 @@ def handle_realtime_stream_cost_calculation( model=model_name, usage=combined_usage_object, custom_llm_provider=custom_llm_provider, + data_residency=data_residency, ) except Exception: continue diff --git a/litellm/integrations/arize/arize_phoenix.py b/litellm/integrations/arize/arize_phoenix.py index b8cd04836c3..d48dba8e7bb 100644 --- a/litellm/integrations/arize/arize_phoenix.py +++ b/litellm/integrations/arize/arize_phoenix.py @@ -1,5 +1,7 @@ import os -from typing import TYPE_CHECKING, Any, Optional, Union +import threading +from collections import OrderedDict +from typing import TYPE_CHECKING, Any, Optional, Tuple, Union from litellm._logging import verbose_logger from litellm.integrations.arize import _utils @@ -8,8 +10,10 @@ from litellm.types.integrations.arize_phoenix import ArizePhoenixConfig if TYPE_CHECKING: from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import SpanProcessor from opentelemetry.trace import Span as _Span from opentelemetry.trace import SpanKind + from opentelemetry.trace import Tracer from litellm.integrations.opentelemetry import OpenTelemetry as _OpenTelemetry from litellm.integrations.opentelemetry import ( @@ -21,20 +25,27 @@ if TYPE_CHECKING: OpenTelemetryConfig = _OpenTelemetryConfig Span = Union[_Span, Any] OpenTelemetry = _OpenTelemetry + LITELLM_TRACER_NAME: str else: Protocol = Any OpenTelemetryConfig = Any Span = Any + Tracer = Any TracerProvider = Any SpanKind = Any - # Import OpenTelemetry at runtime + SpanProcessor = Any try: - from litellm.integrations.opentelemetry import OpenTelemetry + from litellm.integrations.opentelemetry import ( + LITELLM_TRACER_NAME, + OpenTelemetry, + ) except ImportError: + LITELLM_TRACER_NAME = "litellm" OpenTelemetry = None # type: ignore ARIZE_HOSTED_PHOENIX_ENDPOINT = "https://otlp.arize.com/v1/traces" +_MAX_PROJECT_PROVIDERS = 64 class ArizePhoenixLogger(OpenTelemetry): # type: ignore @@ -48,37 +59,142 @@ class ArizePhoenixLogger(OpenTelemetry): # type: ignore def _init_tracing(self, tracer_provider): """ - Override to always create a *private* TracerProvider for Arize Phoenix. + Override to create per-project TracerProviders (LRU-cached) for Arize Phoenix. The base ``OpenTelemetry._init_tracing`` falls back to the global TracerProvider when one already exists. That causes whichever integration initialises second to silently reuse the first one's exporter, so spans only reach one destination. - - By creating our own provider we guarantee Arize Phoenix always gets - its own exporter pipeline, regardless of initialisation order. """ - from opentelemetry.sdk.trace import TracerProvider from opentelemetry.trace import SpanKind if tracer_provider is not None: - # Explicitly supplied (e.g. in tests) — honour it. - self.tracer = tracer_provider.get_tracer("litellm") + self._use_injected_tracer_provider = True + self._shared_span_processor = None + self.tracer = tracer_provider.get_tracer(LITELLM_TRACER_NAME) self.span_kind = SpanKind return - # Always create a dedicated provider — never touch the global one. - provider = TracerProvider(resource=self._get_litellm_resource(self.config)) - provider.add_span_processor(self._get_span_processor()) - self.tracer = provider.get_tracer("litellm") + self._use_injected_tracer_provider = False + self._project_providers: OrderedDict[str, TracerProvider] = OrderedDict() + self._project_providers_lock = threading.Lock() + self._shared_span_processor = self._get_span_processor() self.span_kind = SpanKind + + default_project = self._resolve_project_name({}) + self.tracer = self._get_tracer_for(default_project) verbose_logger.debug( - "ArizePhoenixLogger: Created dedicated TracerProvider " - "(endpoint=%s, exporter=%s)", + "ArizePhoenixLogger: Initialized per-project TracerProvider cache " + "(default_project=%s, endpoint=%s, exporter=%s)", + default_project, self.config.endpoint, self.config.exporter, ) + def flush_tracer_providers(self) -> None: + """ + Flush all cached per-project providers and the shared span processor. + + Call on graceful proxy shutdown. Do not call on LRU eviction — in-flight + spans may still reference evicted providers. + """ + if getattr(self, "_use_injected_tracer_provider", False): + return + + shared_processor = getattr(self, "_shared_span_processor", None) + if shared_processor is not None: + try: + shared_processor.force_flush() + except Exception as e: + verbose_logger.debug( + "ArizePhoenixLogger: shared span processor force_flush failed: %s", + e, + ) + + with getattr(self, "_project_providers_lock", threading.Lock()): + providers = list(getattr(self, "_project_providers", {}).values()) + + for provider in providers: + try: + provider.force_flush() + except Exception as e: + verbose_logger.debug( + "ArizePhoenixLogger: TracerProvider force_flush failed: %s", e + ) + + def _get_litellm_resource_for_project(self, project_name: str): + """ + Build an OTEL Resource with project routing attrs that win over env detector. + + Phoenix uses ``openinference.project.name``; Arize AX uses ``model_id`` and + ``service.name``. Project attrs are merged last so OTEL_RESOURCE_ATTRIBUTES + from init does not pin every provider to one project. + """ + from opentelemetry.sdk.resources import OTELResourceDetector, Resource + + project_attributes: dict[str, str] = { + "openinference.project.name": project_name, + "model_id": project_name, + "service.name": project_name, + } + deployment_environment = getattr(self.config, "deployment_environment", None) + if deployment_environment is not None: + project_attributes["deployment.environment"] = deployment_environment + + env_resource = OTELResourceDetector().detect() + project_resource = Resource.create(project_attributes) # type: ignore[arg-type] + return env_resource.merge(project_resource) + + def _build_tracer_provider_for_project(self, project_name: str) -> TracerProvider: + """Create a TracerProvider for *project_name* (caller holds no cache lock).""" + from opentelemetry.sdk.trace import TracerProvider + + provider = TracerProvider( + resource=self._get_litellm_resource_for_project(project_name) + ) + provider.add_span_processor(self._shared_span_processor) + return provider + + def _get_tracer_for(self, project_name: str) -> Tracer: + """Return a tracer for *project_name*, creating/caching a provider on miss.""" + if getattr(self, "_use_injected_tracer_provider", False): + return self.tracer + + with self._project_providers_lock: + if project_name in self._project_providers: + self._project_providers.move_to_end(project_name) + return self._project_providers[project_name].get_tracer( + LITELLM_TRACER_NAME + ) + + # OTELResourceDetector().detect() is synchronous; build outside the lock so + # concurrent requests for other projects are not blocked on cache misses. + new_provider = self._build_tracer_provider_for_project(project_name) + + with self._project_providers_lock: + if project_name in self._project_providers: + self._project_providers.move_to_end(project_name) + return self._project_providers[project_name].get_tracer( + LITELLM_TRACER_NAME + ) + + if len(self._project_providers) >= _MAX_PROJECT_PROVIDERS: + self._project_providers.popitem(last=False) + + self._project_providers[project_name] = new_provider + return new_provider.get_tracer(LITELLM_TRACER_NAME) + + def _resolve_tracer_for_kwargs(self, kwargs: dict) -> Tuple[str, Tracer]: + """Resolve project name once and return the matching tracer.""" + project_name = self._resolve_project_name(kwargs) + return project_name, self._get_tracer_for(project_name) + + def get_tracer_to_use_for_request(self, kwargs: dict) -> Tracer: + """Route guardrail/raw-request spans to the same per-project tracer as the request.""" + if getattr(self, "_use_injected_tracer_provider", False): + return self.tracer + return self._resolve_tracer_for_kwargs(kwargs)[1] + def _init_otel_logger_on_litellm_proxy(self): """ Override: Arize Phoenix should NOT overwrite the proxy's @@ -93,56 +209,109 @@ class ArizePhoenixLogger(OpenTelemetry): # type: ignore @staticmethod def set_arize_phoenix_attributes(span: Span, kwargs, response_obj): - from litellm.integrations.opentelemetry_utils.base_otel_llm_obs_attributes import ( - safe_set_attribute, - ) - _utils.set_attributes(span, kwargs, response_obj, ArizeOTELAttributes) - - # Dynamic project name: check metadata first, then fall back to env var config - dynamic_project_name = ArizePhoenixLogger._get_dynamic_project_name(kwargs) - if dynamic_project_name: - safe_set_attribute(span, "openinference.project.name", dynamic_project_name) - else: - # Fall back to static config from env var - config = ArizePhoenixLogger.get_arize_phoenix_config() - if config.project_name: - safe_set_attribute( - span, "openinference.project.name", config.project_name - ) - return @staticmethod - def _get_dynamic_project_name(kwargs) -> Optional[str]: - """ - Retrieve dynamic Phoenix project name from request metadata. + def _normalize_project_name(name: Optional[str]) -> Optional[str]: + if name is None: + return None + normalized = str(name).strip() + return normalized if normalized else None - Users can set `metadata.phoenix_project_name` in their request to route - traces to different Phoenix projects dynamically. - """ - standard_logging_payload = kwargs.get("standard_logging_object") - if isinstance(standard_logging_payload, dict): - metadata = standard_logging_payload.get("metadata") + @staticmethod + def _iter_metadata_dicts_from_kwargs(kwargs: dict): + """Yield request metadata dicts; standard_logging_object before litellm_params.""" + for key in ("standard_logging_object", "litellm_params"): + found_key = kwargs.get(key) + if not isinstance(found_key, dict): + continue + metadata = found_key.get("metadata") if isinstance(metadata, dict): - project_name = metadata.get("phoenix_project_name") - if project_name: - return str(project_name) + yield metadata - # Also check litellm_params.metadata for SDK usage + @staticmethod + def _is_proxy_request(kwargs: dict) -> bool: + """True when the call is routed through the LiteLLM proxy. + + Proxy mode is determined solely by the server-set ``proxy_server_request`` + field in ``litellm_params``. Checking request metadata for + ``user_api_key_auth_metadata`` is intentionally avoided: that field is + user-supplied and would let an authenticated caller fake proxy-mode + detection to route their telemetry into arbitrary Arize/Phoenix projects. + """ litellm_params = kwargs.get("litellm_params") - if isinstance(litellm_params, dict): - metadata = litellm_params.get("metadata") or {} - else: - metadata = {} - if isinstance(metadata, dict): - project_name = metadata.get("phoenix_project_name") - if project_name: - return str(project_name) + return isinstance(litellm_params, dict) and bool( + litellm_params.get("proxy_server_request") + ) + @staticmethod + def _project_from_metadata_dict( + metadata: dict, metadata_key: str, *, proxy_mode: bool + ) -> Optional[str]: + """ + Read a Phoenix project field from proxy/SDK metadata. + + On the proxy, only ``user_api_key_auth_metadata`` (team/key config) may + select the project. SDK callers may still set project fields directly on + ``metadata``. + """ + auth_metadata = metadata.get("user_api_key_auth_metadata") + if isinstance(auth_metadata, dict): + project = ArizePhoenixLogger._normalize_project_name( + auth_metadata.get(metadata_key) + ) + if project: + return project + + if not proxy_mode: + return ArizePhoenixLogger._normalize_project_name( + metadata.get(metadata_key) + ) return None - def _get_phoenix_context(self, kwargs): + @staticmethod + def _metadata_project_from_kwargs(kwargs: dict, metadata_key: str) -> Optional[str]: + proxy_mode = ArizePhoenixLogger._is_proxy_request(kwargs) + for metadata in ArizePhoenixLogger._iter_metadata_dicts_from_kwargs(kwargs): + project = ArizePhoenixLogger._project_from_metadata_dict( + metadata, metadata_key, proxy_mode=proxy_mode + ) + if project: + return project + return None + + @staticmethod + def _resolve_project_name(kwargs: dict) -> str: + """ + Resolve the target Phoenix/Arize project for this request. + + Proxy priority: ``user_api_key_auth_metadata.phoenix_project_name_override``, + ``user_api_key_auth_metadata.phoenix_project_name``, env, then ``default``. + SDK priority: request metadata fields, then env, then ``default``. + """ + override = ArizePhoenixLogger._metadata_project_from_kwargs( + kwargs, "phoenix_project_name_override" + ) + if override: + return override + + phoenix_name = ArizePhoenixLogger._metadata_project_from_kwargs( + kwargs, "phoenix_project_name" + ) + if phoenix_name: + return phoenix_name + + env_name = ArizePhoenixLogger._normalize_project_name( + os.environ.get("PHOENIX_PROJECT_NAME") + or os.environ.get("ARIZE_PROJECT_NAME") + ) + if env_name: + return env_name + + return "default" + + def _get_phoenix_context(self, kwargs, tracer: Optional[Tracer] = None): """ Build a trace context for Phoenix's dedicated TracerProvider. @@ -159,11 +328,13 @@ class ArizePhoenixLogger(OpenTelemetry): # type: ignore """ from opentelemetry import trace + if tracer is None: + tracer = self._resolve_tracer_for_kwargs(kwargs)[1] + litellm_params = kwargs.get("litellm_params", {}) or {} proxy_server_request = litellm_params.get("proxy_server_request", {}) or {} headers = proxy_server_request.get("headers", {}) or {} - # Propagate distributed trace context if the caller sent a traceparent traceparent_ctx = ( self.get_traceparent_from_header(headers=headers) if headers.get("traceparent") @@ -173,10 +344,8 @@ class ArizePhoenixLogger(OpenTelemetry): # type: ignore is_proxy_mode = bool(proxy_server_request) if is_proxy_mode: - # Create a parent span on Phoenix's own tracer so both parent - # and child are exported to Phoenix. start_time_val = kwargs.get("start_time", kwargs.get("api_call_start_time")) - parent_span = self.tracer.start_span( + parent_span = tracer.start_span( name="litellm_proxy_request", start_time=( self._to_ns(start_time_val) if start_time_val is not None else None @@ -187,100 +356,77 @@ class ArizePhoenixLogger(OpenTelemetry): # type: ignore ctx = trace.set_span_in_context(parent_span) return ctx, parent_span - # SDK mode — no parent span needed return traceparent_ctx, None def _handle_success(self, kwargs, response_obj, start_time, end_time): - """ - Override to always create spans on ArizePhoenixLogger's dedicated TracerProvider. - - The base class's ``_get_span_context`` would find the parent span created by - the ``otel`` callback on the *global* TracerProvider. That span is invisible - in Phoenix (different exporter pipeline), so we ignore it and build our own - hierarchy via ``_get_phoenix_context``. - """ - from opentelemetry.trace import Status, StatusCode - - verbose_logger.debug( - "ArizePhoenixLogger: Logging kwargs: %s, OTEL config settings=%s", - kwargs, - self.config, + self._handle_phoenix_trace( + kwargs, response_obj, start_time, end_time, success=True ) - ctx, parent_span = self._get_phoenix_context(kwargs) - - # Create litellm_request span (child of our parent when in proxy mode) - span = self.tracer.start_span( - name=self._get_span_name(kwargs), - start_time=self._to_ns(start_time), - context=ctx, - ) - span.set_status(Status(StatusCode.OK)) - self.set_attributes(span, kwargs, response_obj) - - # Raw-request sub-span (if enabled) — must be created before - # ending the parent span so the hierarchy is valid. - self._maybe_log_raw_request(kwargs, response_obj, start_time, end_time, span) - span.end(end_time=self._to_ns(end_time)) - - # Guardrail span - self._create_guardrail_span(kwargs=kwargs, context=ctx) - - # Annotate and close our proxy parent span - if parent_span is not None: - parent_span.set_status(Status(StatusCode.OK)) - self.set_attributes(parent_span, kwargs, response_obj) - parent_span.end(end_time=self._to_ns(end_time)) - - # Metrics & cost recording - self._record_metrics(kwargs, response_obj, start_time, end_time) - - # Semantic logs - if self.config.enable_events: - self._emit_semantic_logs(kwargs, response_obj, span) - def _handle_failure(self, kwargs, response_obj, start_time, end_time): - """ - Override to always create failure spans on ArizePhoenixLogger's dedicated - TracerProvider. Mirrors ``_handle_success`` but sets ERROR status. - """ + self._handle_phoenix_trace( + kwargs, response_obj, start_time, end_time, success=False + ) + + def _handle_phoenix_trace( + self, + kwargs, + response_obj, + start_time, + end_time, + *, + success: bool, + ): from opentelemetry.trace import Status, StatusCode verbose_logger.debug( - "ArizePhoenixLogger: Failure - Logging kwargs: %s, OTEL config settings=%s", + "ArizePhoenixLogger: %s - kwargs: %s, OTEL config settings=%s", + "success" if success else "failure", kwargs, self.config, ) - ctx, parent_span = self._get_phoenix_context(kwargs) + _project_name, tracer = self._resolve_tracer_for_kwargs(kwargs) + ctx, parent_span = self._get_phoenix_context(kwargs, tracer=tracer) - # Create litellm_request span (child of our parent when in proxy mode) - span = self.tracer.start_span( + status = Status(StatusCode.OK if success else StatusCode.ERROR) + + span = tracer.start_span( name=self._get_span_name(kwargs), start_time=self._to_ns(start_time), context=ctx, ) - span.set_status(Status(StatusCode.ERROR)) + span.set_status(status) self.set_attributes(span, kwargs, response_obj) - self._record_exception_on_span(span=span, kwargs=kwargs) + if not success: + self._record_exception_on_span(span=span, kwargs=kwargs) + + if success: + self._maybe_log_raw_request( + kwargs, response_obj, start_time, end_time, span + ) span.end(end_time=self._to_ns(end_time)) - # Guardrail span self._create_guardrail_span(kwargs=kwargs, context=ctx) - # Annotate and close our proxy parent span if parent_span is not None: - parent_span.set_status(Status(StatusCode.ERROR)) + parent_span.set_status(status) self.set_attributes(parent_span, kwargs, response_obj) - self._record_exception_on_span(span=parent_span, kwargs=kwargs) + if not success: + self._record_exception_on_span(span=parent_span, kwargs=kwargs) parent_span.end(end_time=self._to_ns(end_time)) + if success: + self._record_metrics(kwargs, response_obj, start_time, end_time) + + if self.config.enable_events: + self._emit_semantic_logs(kwargs, response_obj, span) + @staticmethod def get_arize_phoenix_config() -> ArizePhoenixConfig: """ Retrieves the Arize Phoenix configuration based on environment variables. Returns: - ArizePhoenixConfig: A Pydantic model containing Arize Phoenix configuration. """ api_key = os.environ.get("PHOENIX_API_KEY", None) @@ -295,18 +441,15 @@ class ArizePhoenixLogger(OpenTelemetry): # type: ignore protocol: Protocol = "otlp_http" if collector_endpoint: - # Parse the endpoint to determine protocol if collector_endpoint.startswith("grpc://") or ( ":4317" in collector_endpoint and "/v1/traces" not in collector_endpoint ): endpoint = collector_endpoint protocol = "otlp_grpc" else: - # Phoenix Cloud endpoints (app.phoenix.arize.com) include the space in the URL if "app.phoenix.arize.com" in collector_endpoint: endpoint = collector_endpoint protocol = "otlp_http" - # For other HTTP endpoints, ensure they have the correct path elif "/v1/traces" not in collector_endpoint: if collector_endpoint.endswith("/v1"): endpoint = collector_endpoint + "/traces" @@ -318,7 +461,6 @@ class ArizePhoenixLogger(OpenTelemetry): # type: ignore endpoint = collector_endpoint protocol = "otlp_http" else: - # If no endpoint specified, self hosted phoenix endpoint = "http://localhost:6006/v1/traces" protocol = "otlp_http" verbose_logger.debug( @@ -329,12 +471,11 @@ class ArizePhoenixLogger(OpenTelemetry): # type: ignore if api_key is not None: otlp_auth_headers = f"Authorization=Bearer {api_key}" elif "app.phoenix.arize.com" in endpoint: - # Phoenix Cloud requires an API key raise ValueError( "PHOENIX_API_KEY must be set when using Phoenix Cloud (app.phoenix.arize.com)." ) - project_name = os.environ.get("PHOENIX_PROJECT_NAME", "default") + project_name = os.environ.get("PHOENIX_PROJECT_NAME") or "default" return ArizePhoenixConfig( otlp_auth_headers=otlp_auth_headers, @@ -343,8 +484,6 @@ class ArizePhoenixLogger(OpenTelemetry): # type: ignore project_name=project_name, ) - ## cannot suppress additional proxy server spans, removed previous methods. - async def async_health_check(self): config = self.get_arize_phoenix_config() diff --git a/litellm/integrations/datadog/datadog_cost_management.py b/litellm/integrations/datadog/datadog_cost_management.py index a961d4f9244..0f954eb1ce0 100644 --- a/litellm/integrations/datadog/datadog_cost_management.py +++ b/litellm/integrations/datadog/datadog_cost_management.py @@ -2,10 +2,17 @@ import asyncio import os import time from datetime import datetime -from typing import Dict, List, Optional, Tuple +from typing import Any, Dict, List, Optional, Tuple, cast from litellm._logging import verbose_logger from litellm.integrations.custom_batch_logger import CustomBatchLogger +from litellm.integrations.datadog.datadog_handler import ( + get_datadog_env, + get_datadog_hostname, + get_datadog_pod_name, + get_datadog_service, +) +from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, @@ -15,9 +22,30 @@ from litellm.types.integrations.datadog_cost_management import ( ) from litellm.types.utils import StandardLoggingPayload +# Reserved tag keys whose values come from trusted sources (infra env, LiteLLM +# core payload fields, or proxy-controlled auth metadata). User-supplied +# request_tags / metadata cannot overwrite these, even when the key is +# allowlisted via cost_tag_keys, because that would let an authenticated caller +# spoof cost attribution (e.g. request_tags=["team:victim-team"]). +_RESERVED_TAG_KEYS: frozenset = frozenset( + { + "env", + "service", + "host", + "pod_name", + "provider", + "model", + "model_id", + "team", + "user", + "model_group", + } +) + class DatadogCostManagementLogger(CustomBatchLogger): - def __init__(self, **kwargs): + def __init__(self, cost_tag_keys: Optional[List[str]] = None, **kwargs): + self.cost_tag_keys: List[str] = list(cost_tag_keys) if cost_tag_keys else [] self.dd_api_key = os.getenv("DD_API_KEY") self.dd_app_key = os.getenv("DD_APP_KEY") self.dd_site = os.getenv("DD_SITE", "datadoghq.com") @@ -68,20 +96,21 @@ class DatadogCostManagementLogger(CustomBatchLogger): if not self.log_queue: return + batch_to_send = self.log_queue[:] + self.log_queue = [] + try: - # Aggregate costs from the batch - aggregated_entries = self._aggregate_costs(self.log_queue) - + aggregated_entries = self._aggregate_costs(batch_to_send) if not aggregated_entries: + verbose_logger.debug( + "Datadog Cost Management: batch produced no aggregable entries; " + "dropping %d log(s) from queue.", + len(batch_to_send), + ) return - - # Send to Datadog await self._upload_to_datadog(aggregated_entries) - - # Clear queue only on success (or if we decide to drop on failure) - # CustomBatchLogger clears queue in flush_queue, so we just process here - except Exception as e: + self.log_queue = batch_to_send + self.log_queue verbose_logger.exception( f"Datadog Cost Management: Error in async_send_batch: {str(e)}" ) @@ -151,45 +180,81 @@ class DatadogCostManagementLogger(CustomBatchLogger): return list(aggregator.values()) def _extract_tags(self, log: StandardLoggingPayload) -> Dict[str, str]: - from litellm.integrations.datadog.datadog_handler import ( - get_datadog_env, - get_datadog_hostname, - get_datadog_pod_name, - get_datadog_service, - ) - - tags = { + tags: Dict[str, str] = { "env": get_datadog_env(), "service": get_datadog_service(), "host": get_datadog_hostname(), "pod_name": get_datadog_pod_name(), } - # Add metadata as tags - metadata = log.get("metadata", {}) - if metadata: - # Add user info - # Add user info - if metadata.get("user_api_key_alias"): - tags["user"] = str(metadata["user_api_key_alias"]) + # Always-on canonical FOCUS dimensions from top-level payload fields. + # Non-sensitive and required for Datadog Custom Costs per-model attribution. + self._add_tag(tags, "provider", log.get("custom_llm_provider")) + self._add_tag(tags, "model", log.get("model")) + self._add_tag(tags, "model_id", log.get("model_id")) - # Add Team Tag - team_tag = ( - metadata.get("user_api_key_team_alias") - or metadata.get("team_alias") # type: ignore - or metadata.get("user_api_key_team_id") - or metadata.get("team_id") # type: ignore - ) + # cast because StandardLoggingMetadata is a TypedDict; we iterate it + # as a generic mapping below. + metadata: Dict[str, Any] = cast(Dict[str, Any], log.get("metadata") or {}) - if team_tag: - tags["team"] = str(team_tag) - # model_group is not in StandardLoggingMetadata TypedDict, so we need to access it via dict.get() - model_group = metadata.get("model_group") # type: ignore[misc] - if model_group: - tags["model_group"] = str(model_group) + # Backwards-compat: team/user/model_group preserved regardless of allowlist. + if metadata.get("user_api_key_alias"): + tags["user"] = str(metadata["user_api_key_alias"]) + team_tag = ( + metadata.get("user_api_key_team_alias") + or metadata.get("team_alias") + or metadata.get("user_api_key_team_id") + or metadata.get("team_id") + ) + if team_tag: + tags["team"] = str(team_tag) + if metadata.get("model_group"): + tags["model_group"] = str(metadata["model_group"]) + + # Allowlist-gated: request_tags (split on `:`) and arbitrary metadata.*. + # Reserved keys are hard-blocked here regardless of allowlist membership — + # see _RESERVED_TAG_KEYS for the rationale. + if self.cost_tag_keys: + allow = set(self.cost_tag_keys) + for rt in log.get("request_tags") or []: + if not isinstance(rt, str) or ":" not in rt: + continue + k, _, v = rt.partition(":") + if k in allow and v: + self._set_custom_tag(tags, k, v) + for k, v in metadata.items(): + if k in allow and v is not None and not isinstance(v, (dict, list)): + self._set_custom_tag(tags, k, str(v)) + for nested_key in ("spend_logs_metadata", "requester_metadata"): + nested = metadata.get(nested_key) + if isinstance(nested, dict): + for k, v in nested.items(): + if ( + k in allow + and v is not None + and not isinstance(v, (dict, list)) + ): + self._set_custom_tag(tags, k, str(v)) return tags + @staticmethod + def _set_custom_tag(tags: Dict[str, str], key: str, value: str) -> None: + if key in _RESERVED_TAG_KEYS: + verbose_logger.debug( + "Datadog Cost Management: dropping user-supplied tag %r=%r — " + "key is reserved for trusted cost attribution.", + key, + value, + ) + return + tags[key] = value + + @staticmethod + def _add_tag(tags: Dict[str, str], key: str, value: Any) -> None: + if value: + tags[key] = str(value) + async def _upload_to_datadog(self, payload: List[Dict]): if not self.dd_api_key or not self.dd_app_key: return @@ -201,8 +266,6 @@ class DatadogCostManagementLogger(CustomBatchLogger): } # The API endpoint expects a list of objects directly in the body (file content behavior) - from litellm.litellm_core_utils.safe_json_dumps import safe_dumps - data_json = safe_dumps(payload) response = await self.async_client.put( diff --git a/litellm/integrations/datadog/datadog_metrics.py b/litellm/integrations/datadog/datadog_metrics.py index fcf40701e28..d7847027d7e 100644 --- a/litellm/integrations/datadog/datadog_metrics.py +++ b/litellm/integrations/datadog/datadog_metrics.py @@ -144,7 +144,26 @@ class DatadogMetricsLogger(CustomBatchLogger): } self.log_queue.append(series_llm_latency) - # 3. Request Count / Status Code + # 3. LiteLLM Overhead Latency Metric (total - llm_api time) + hidden_params = log.get("hidden_params", {}) or {} + litellm_overhead_time_ms = hidden_params.get("litellm_overhead_time_ms") + if litellm_overhead_time_ms is not None: + overhead_tags = self._extract_tags(log) # no status_code on latency metric + series_overhead: DatadogMetricSeries = { + "metric": "litellm.overhead.latency", + "type": 3, # gauge + "points": [ + { + "timestamp": timestamp, + "value": litellm_overhead_time_ms + / 1000, # convert ms → seconds + } + ], + "tags": overhead_tags, + } + self.log_queue.append(series_overhead) + + # 4. Request Count / Status Code series_count: DatadogMetricSeries = { "metric": "litellm.llm_api.request_count", "type": 1, # count diff --git a/litellm/integrations/galileo.py b/litellm/integrations/galileo.py index e99d5f23a4c..a598124f612 100644 --- a/litellm/integrations/galileo.py +++ b/litellm/integrations/galileo.py @@ -1,18 +1,29 @@ +import json import os -from typing import Any, Dict, List, Optional +import re +from typing import Any, Dict, List, Optional, Tuple, cast from pydantic import BaseModel, Field import litellm from litellm._logging import verbose_logger from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + convert_content_list_to_str, + get_content_from_model_response, +) from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, ) +from litellm.types.llms.openai import AllMessageValues + +GALILEO_CLOUD_API_BASE_URL = "https://api.galileo.ai" +# Cap the in-memory buffer so persistent flush failures (e.g. Galileo +# unavailable, invalid credentials) cannot leak memory unboundedly. +GALILEO_MAX_IN_MEMORY_RECORDS = 1000 -# from here: https://docs.rungalileo.io/galileo/gen-ai-studio-products/galileo-observe/how-to/logging-data-via-restful-apis#structuring-your-records class LLMResponse(BaseModel): latency_ms: int status_code: int @@ -37,65 +48,190 @@ class GalileoObserve(CustomLogger): def __init__(self) -> None: self.in_memory_records: List[dict] = [] self.batch_size = 1 - self.base_url = os.getenv("GALILEO_BASE_URL", None) - self.project_id = os.getenv("GALILEO_PROJECT_ID", None) + self.api_key = os.getenv("GALILEO_API_KEY") + self.project_id = os.getenv("GALILEO_PROJECT_ID") + self.log_stream_id = os.getenv("GALILEO_LOG_STREAM_ID") + self.username = os.getenv("GALILEO_USERNAME") + self.password = os.getenv("GALILEO_PASSWORD") + self.base_url = self._normalize_base_url(os.getenv("GALILEO_BASE_URL")) + if self.api_key and not self.base_url: + self.base_url = GALILEO_CLOUD_API_BASE_URL + self.use_v2_api = bool(self.api_key) self.headers: Optional[Dict[str, str]] = None self.async_httpx_handler = get_async_httpx_client( llm_provider=httpxSpecialProvider.LoggingCallback ) - pass - def set_galileo_headers(self): - # following https://docs.rungalileo.io/galileo/gen-ai-studio-products/galileo-observe/how-to/logging-data-via-restful-apis#logging-your-records + @staticmethod + def _normalize_base_url(base_url: Optional[str]) -> Optional[str]: + if base_url: + return base_url.rstrip("/") + return None - headers = { - "accept": "application/json", - "Content-Type": "application/x-www-form-urlencoded", - } - galileo_login_response = litellm.module_level_client.post( + def _is_configured(self) -> bool: + if not self.project_id or not self.base_url: + return False + if self.use_v2_api: + return bool(self.api_key) + return bool(self.username and self.password) + + async def async_set_galileo_headers(self) -> None: + galileo_login_response = await self.async_httpx_handler.post( url=f"{self.base_url}/login", - headers=headers, + headers={ + "accept": "application/json", + "Content-Type": "application/x-www-form-urlencoded", + }, data={ - "username": os.getenv("GALILEO_USERNAME"), - "password": os.getenv("GALILEO_PASSWORD"), + "username": self.username, + "password": self.password, }, ) - + galileo_login_response.raise_for_status() access_token = galileo_login_response.json()["access_token"] - self.headers = { "accept": "application/json", "Content-Type": "application/json", "Authorization": f"Bearer {access_token}", } - def get_output_str_from_response(self, response_obj, kwargs): - output = None - if response_obj is not None and ( - kwargs.get("call_type", None) == "embedding" - or isinstance(response_obj, litellm.EmbeddingResponse) - ): - output = None - elif response_obj is not None and isinstance( - response_obj, litellm.ModelResponse - ): - output = response_obj["choices"][0]["message"].json() - elif response_obj is not None and isinstance( - response_obj, litellm.TextCompletionResponse - ): - output = response_obj.choices[0].text - elif response_obj is not None and isinstance( - response_obj, litellm.ImageResponse - ): - output = response_obj["data"] + async def _ensure_headers(self) -> bool: + if self.headers is not None: + return True - return output + if self.use_v2_api: + if not self.api_key: + return False + self.headers = { + "accept": "application/json", + "Content-Type": "application/json", + "Galileo-API-Key": self.api_key, + } + return True + + if not (self.username and self.password and self.base_url): + return False + + try: + await self.async_set_galileo_headers() + return True + except Exception as e: + verbose_logger.debug("Galileo Logger: failed to authenticate: %s", e) + return False + + @staticmethod + def _galileo_input_messages( + messages: Optional[List[Any]], input_text: str + ) -> List[Dict[str, str]]: + if not messages: + return [{"role": "user", "content": input_text}] + + galileo_messages: List[Dict[str, str]] = [] + for message in messages: + if not isinstance(message, dict): + continue + role = message.get("role") + if not role: + continue + galileo_messages.append( + { + "role": str(role), + "content": convert_content_list_to_str( + message=cast(AllMessageValues, message) + ), + } + ) + + if galileo_messages: + return galileo_messages + return [{"role": "user", "content": input_text}] + + @staticmethod + def _record_to_v2_span(record: Dict[str, Any]) -> Dict[str, Any]: + created_at = record.get("created_at", "") + if created_at and not re.search(r"(Z|[+-]\d{2}:?\d{2})$", created_at): + created_at = f"{created_at}Z" + + span: Dict[str, Any] = { + "type": "llm", + "name": record.get("node_type", "litellm"), + "created_at": created_at, + "input": GalileoObserve._galileo_input_messages( + record.get("messages"), record.get("input_text", "") + ), + "output": { + "role": "assistant", + "content": record.get("output_text", ""), + }, + "status_code": record.get("status_code", 200), + "model": record.get("model"), + "metrics": { + "duration_ns": int(record.get("latency_ms", 0)) * 1_000_000, + "num_input_tokens": record.get("num_input_tokens"), + "num_output_tokens": record.get("num_output_tokens"), + }, + } + if record.get("tags"): + span["tags"] = record["tags"] + return span + + def _get_ingest_request(self) -> Optional[Tuple[str, Dict[str, Any]]]: + if not self.base_url or not self.project_id: + return None + + # Snapshot the records to be sent into a new list so concurrent appends + # during the network round-trip (across the await points in + # flush_in_memory_records) aren't silently dropped when we later clear + # the in-memory buffer. + records = list(self.in_memory_records) + + if self.use_v2_api: + payload: Dict[str, Any] = { + "spans": [self._record_to_v2_span(record) for record in records], + "reliable": False, + } + if self.log_stream_id: + payload["log_stream_id"] = self.log_stream_id + return ( + f"{self.base_url}/v2/projects/{self.project_id}/spans", + payload, + ) + + return ( + f"{self.base_url}/projects/{self.project_id}/observe/ingest", + {"records": records}, + ) + + def get_output_str_from_response( + self, response_obj: Any, kwargs: Dict[str, Any] + ) -> Optional[str]: + if response_obj is None: + return None + if kwargs.get("call_type", None) == "embedding" or isinstance( + response_obj, litellm.EmbeddingResponse + ): + return None + if isinstance(response_obj, litellm.TextCompletionResponse): + return response_obj.choices[0].text + if isinstance(response_obj, litellm.ImageResponse): + return json.dumps(response_obj["data"], default=str) + if isinstance(response_obj, (litellm.ModelResponse, dict)): + return get_content_from_model_response(response_obj) + return None async def async_log_success_event( self, kwargs: Any, response_obj: Any, start_time: Any, end_time: Any ): verbose_logger.debug("On Async Success") + if not self._is_configured(): + verbose_logger.debug( + "Galileo Logger: skipping flush — set GALILEO_PROJECT_ID and " + "either GALILEO_API_KEY (hosted) or GALILEO_USERNAME/GALILEO_PASSWORD " + "(enterprise Observe)." + ) + return + _latency_ms = int((end_time - start_time).total_seconds() * 1000) _call_type = kwargs.get("call_type", "litellm") input_text = litellm.utils.get_formatted_prompt( @@ -125,26 +261,69 @@ class GalileoObserve(CustomLogger): ), # timestamp str constructed in "%Y-%m-%dT%H:%M:%S" format ) - # dump to dict request_dict = request_record.model_dump() + messages = kwargs.get("messages") + if messages: + request_dict["messages"] = messages self.in_memory_records.append(request_dict) + # Bound the buffer so persistent flush failures cannot grow it + # without limit. Drop the oldest records once we exceed the cap. + if len(self.in_memory_records) > GALILEO_MAX_IN_MEMORY_RECORDS: + dropped = len(self.in_memory_records) - GALILEO_MAX_IN_MEMORY_RECORDS + self.in_memory_records = self.in_memory_records[ + -GALILEO_MAX_IN_MEMORY_RECORDS: + ] + verbose_logger.warning( + "Galileo Logger: in-memory buffer exceeded %s records; " + "dropped %s oldest record(s). Check Galileo connectivity/credentials.", + GALILEO_MAX_IN_MEMORY_RECORDS, + dropped, + ) + if len(self.in_memory_records) >= self.batch_size: await self.flush_in_memory_records() async def flush_in_memory_records(self): - verbose_logger.debug("flushing in memory records") - response = await self.async_httpx_handler.post( - url=f"{self.base_url}/projects/{self.project_id}/observe/ingest", - headers=self.headers, - json={"records": self.in_memory_records}, - ) + if not self.in_memory_records: + return - if response.status_code == 200: + # Capture the number of records that will be sent BEFORE any await so + # that concurrent appends made by other asyncio tasks during the + # network round-trip aren't silently dropped on the success-clear. + records_in_payload = len(self.in_memory_records) + + ingest_request = self._get_ingest_request() + if ingest_request is None: verbose_logger.debug( - "Galileo Logger:successfully flushed in memory records" + "Galileo Logger: missing GALILEO_BASE_URL or GALILEO_PROJECT_ID" ) - self.in_memory_records = [] + return + + if not await self._ensure_headers(): + verbose_logger.debug("Galileo Logger: could not set request headers") + return + + url, payload = ingest_request + verbose_logger.debug("flushing in memory records to %s", url) + + try: + response = await self.async_httpx_handler.post( + url=url, + headers=self.headers, + json=payload, + ) + except Exception as e: + verbose_logger.debug( + "Galileo Logger: failed to flush in memory records: %s", e + ) + return + + if response.is_success: + verbose_logger.debug( + "Galileo Logger: successfully flushed in memory records" + ) + del self.in_memory_records[:records_in_payload] else: verbose_logger.debug("Galileo Logger: failed to flush in memory records") verbose_logger.debug( @@ -152,6 +331,13 @@ class GalileoObserve(CustomLogger): response.text, response.status_code, ) + # Legacy enterprise auth caches a bearer token obtained from + # /login. If the request was rejected for auth reasons, drop the + # cached headers so the next flush re-authenticates instead of + # silently failing forever on a stale token. The v2 API key path + # uses a long-lived static key, so leave its headers in place. + if not self.use_v2_api and response.status_code in (401, 403): + self.headers = None async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): verbose_logger.debug("On Async Failure") diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 81fdc5a1e21..814da344f03 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -64,6 +64,9 @@ HTTP_RESPONSE_STATUS_CODE_ATTRIBUTE = "http.response.status_code" HTTP_ROUTE_ATTRIBUTE = "http.route" URL_PATH_ATTRIBUTE = "url.path" PREPROCESSING_DURATION_MS_ATTRIBUTE = "litellm.preprocessing.duration_ms" +TEAM_METADATA_ATTRIBUTE = "litellm.team.metadata" +MODEL_GROUP_ATTRIBUTE = "litellm.model_group" +PROVIDER_MODEL_ATTRIBUTE = "litellm.provider.model" # Remove the hardcoded LITELLM_RESOURCE dictionary - we'll create it properly later RAW_REQUEST_SPAN_NAME = "raw_gen_ai_request" LITELLM_REQUEST_SPAN_NAME = "litellm_request" @@ -1213,6 +1216,68 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): ): self._set_team_attributes_from_kwargs(proxy_span, kwargs) + def _set_inference_identity_attributes( + self, + span: Span, + standard_logging_payload: StandardLoggingPayload, + litellm_params: dict, + ) -> None: + """Stamp request-identity attributes onto an inference span so every + LLM-call span is filterable by the route it came in on, the team's + metadata, and both the user-facing (model_group alias) and the + dispatched (provider) model names. Empty/absent values are skipped. + """ + metadata = standard_logging_payload.get("metadata") or {} + + http_route = metadata.get("user_api_key_request_route") + if http_route: + self.safe_set_attribute( + span=span, key=HTTP_ROUTE_ATTRIBUTE, value=http_route + ) + + # ``user_api_key_team_metadata`` is dropped from the standard logging + # payload metadata, so read it from the raw request metadata in kwargs. + # ``metadata`` and ``litellm_metadata`` are alternate names for the same + # full metadata dict (the name varies by endpoint), so first-truthy wins. + raw_metadata = ( + litellm_params.get("metadata") + or litellm_params.get("litellm_metadata") + or {} + ) + team_metadata = self._team_metadata_json( + raw_metadata.get("user_api_key_team_metadata") + ) + if team_metadata: + self.safe_set_attribute( + span=span, key=TEAM_METADATA_ATTRIBUTE, value=team_metadata + ) + + model_group = standard_logging_payload.get("model_group") + if model_group: + self.safe_set_attribute( + span=span, key=MODEL_GROUP_ATTRIBUTE, value=model_group + ) + + hidden_params = standard_logging_payload.get("hidden_params") or {} + provider_model = hidden_params.get( + "litellm_model_name" + ) or standard_logging_payload.get("model") + if provider_model: + self.safe_set_attribute( + span=span, key=PROVIDER_MODEL_ATTRIBUTE, value=provider_model + ) + + @staticmethod + def _team_metadata_json(value: Any) -> Optional[str]: + """JSON-serialize a team's metadata dict for a single span attribute. + + Returns ``None`` for a missing, non-dict, or empty mapping so the + empty case is dropped rather than stamping a useless ``"{}"``. + """ + if not isinstance(value, dict) or not value: + return None + return safe_dumps(value) + def _record_metrics(self, kwargs, response_obj, start_time, end_time): duration_s = (end_time - start_time).total_seconds() params = kwargs.get("litellm_params") or {} @@ -2023,6 +2088,12 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): key="hidden_params", value=safe_dumps(hidden_params), ) + + self._set_inference_identity_attributes( + span=span, + standard_logging_payload=standard_logging_payload, + litellm_params=litellm_params, + ) # Cost breakdown tracking cost_breakdown: Optional[CostBreakdown] = standard_logging_payload.get( "cost_breakdown" diff --git a/litellm/integrations/otel/README.md b/litellm/integrations/otel/README.md new file mode 100644 index 00000000000..99b3ecea162 --- /dev/null +++ b/litellm/integrations/otel/README.md @@ -0,0 +1,258 @@ +# OpenTelemetry instrumentation + +This package produces OpenTelemetry traces for LiteLLM. It is enabled by the +`LITELLM_OTEL_V2` environment variable (`is_otel_v2_enabled()` in +[`config.py`](./model/config.py)); when unset, nothing in this package runs. + +## What gets traced + +A traced proxy request produces one trace with two kinds of spans: + +``` +SERVER span "POST /v1/chat/completions" ← FastAPI instrumentation +├── INTERNAL span "auth /v1/chat/completions" ← auth phase ┐ +│ ├── CLIENT span "postgres get_key_object" ← datastore call │ +│ └── CLIENT span "postgres get_team_membership" │ +├── INTERNAL span "execute_guardrail …" ← guardrail │ this package +├── CLIENT span "chat gpt-4o" ← LLM call │ +└── CLIENT span "batch_write_to_db …" ← spend write ┘ +``` + +The gen-ai spans are siblings under the server span. In particular the guardrail +span is a sibling of the LLM call, not a child of it: pre/during/post-call +guardrail hooks are part of the request lifecycle (a pre-call guardrail runs +before the LLM call even starts), so they belong directly under the server span, +alongside the LLM call. + +Request-level spans (LLM call, guardrail) parent to the server span via an +**explicit anchor** — `context.set_request_root_span` captures the server span +once at request entry, and `resolve_request_span_context` reads it — rather than +to whatever span is momentarily active. Ambient-only parenting was wrong at two +boundaries: inside the live `auth` phase span the active span is `auth` (so the +span would nest under auth), and a pass-through request closes its span from a +detached `asyncio.create_task` where the server span is no longer active (so the +span orphaned into its own trace). The anchor — a contextvar inherited by those +child tasks — gives a stable parent in both cases. DB/service spans keep ambient +parenting so an auth DB lookup still nests under `auth`. + +**Which service calls become spans (`spans.span_role_for_service`).** LiteLLM's +service-logging layer instruments many internal functions, but only some are +traceable units of work: + +- **`DB_CALL` (CLIENT)** — outbound datastore calls (redis, postgres, + `batch_write_to_db`), carrying `db.system.name` / `db.operation.name` semconv. +- **`SERVICE` (INTERNAL)** — genuine internal work worth a span (background + budget/reset jobs, pod-lock manager). +- **metrics-only (no span)** — `self` (the `track_llm_api_timing` wrapper, which + duplicates the LLM-call span), `router` (duplicates the request), and + `proxy_pre_call` (a guardrail's real span is `execute_guardrail …`). These + still feed Prometheus/Datadog through their own hooks; they just never enter + the trace. `auth` is also excluded here because it gets a **live phase span** + instead (see below). + +Spans are named `"{service} {call_type}"` (e.g. `"redis set"`) so repeated calls +to one service stay distinguishable. Like every other span they parent to the +**ambient** context, falling back to the threaded `litellm_parent_otel_span` only +when ambient has no live span; a background job with neither starts its own root +trace. Caller-supplied `event_metadata` is **sanitized** before it reaches a span +(primitives only, no live objects, no secrets/headers, bounded) — see +`payloads.sanitize_event_metadata`. + +**Live phase spans.** `auth` is wrapped in a real, active span +(`logger.phase_span`) for the duration of authentication, so the DB lookups it +triggers nest **under** it instead of flattening onto the server span. Identity +Baggage (team/key/user) is seeded once the key resolves, so every post-auth span +inherits it; auth-internal DB lookups that run before the key is known stay +unlabeled, which is correct. + +**Status.** On success a span's status is left `UNSET` (the semconv default, +matching the FastAPI server span); only a genuine error sets `ERROR`. + +- **Server spans** (one per HTTP route) are created by the + `opentelemetry-instrumentation-fastapi` package. It stamps `http.*` attributes + and extracts inbound `traceparent` headers. This package does **not** create + or modify server spans — request routes never touch spans. +- **Gen-AI spans** (LLM calls, guardrails, internal service calls) are created + by this package from LiteLLM's logging callbacks. Request-level spans parent to + the server span via the captured anchor; DB/service spans parent to the active + span (ambient) so they nest under the request phase that triggered them. + +Both kinds share a single `TracerProvider`, so they belong to the same trace +and export through the same configured exporters. FastAPI middleware can only be +added before the app starts serving, so the app is instrumented at +import time **without** a provider — it binds to the OTel global +`ProxyTracerProvider`. Once config (and the callbacks) is loaded, the proxy +publishes the chosen logger's `TracerProvider` as the global via +`trace.set_tracer_provider(...)`, and the server spans delegate to it. When a +preset callback (`arize`, `langfuse_otel`, …) is configured, its provider +becomes the global, so server spans export to that backend too. + +## How a request flows + +1. **App creation** (`proxy_server` import): when the gate is on, + `mount.instrument_fastapi_app(app)` calls `FastAPIInstrumentor.instrument_app` + with no provider (the middleware stack is frozen once the app serves, so this + can't wait for startup). It binds to the OTel global `ProxyTracerProvider`. Noisy + non-LLM routes are excluded by default (`mount._DEFAULT_EXCLUDED_ROUTES`): health + checks (`/health*`), the Prometheus scrape (`/metrics`), and static UI/docs assets + (`/litellm-asset-prefix`, `/_next`, `/ui`, `/swagger`, `/docs`, `/redoc`, + `/openapi.json`, favicons, `/.well-known`) — so load-balancer polling, metric + scrapes, and asset fetches don't flood traces. Entries are substring-matched, so + `/metrics` also drops the `/model/metrics` admin-analytics spans. Set + `OTEL_PYTHON_FASTAPI_EXCLUDED_URLS` to override the whole set (e.g. `""` to trace + everything, or your own comma-separated path list). +2. **Startup** (`proxy_server.proxy_startup_event`): after the config (and + callbacks) is loaded, the already-registered preset `OpenTelemetryV2` logger + is reused — or a generic one reading `OTEL_*` envs is built when no preset is + configured — and its `TracerProvider` is published as the OTel global with + `trace.set_tracer_provider(...)`. The proxy tracer then delegates to it, so + server spans and gen-ai spans share one provider and the same trace. +3. **Request**: the FastAPI instrumentation starts the server span and makes it + the active context for the request task. The proxy's first call into the V2 + logger (`create_litellm_proxy_request_started_span`, at the auth boundary) + **captures it as the request anchor** (`set_request_root_span`), so every later + request-level span has a stable explicit parent regardless of what is active + when it emits. +4. **LLM call span (born at the boundary)**: `OpenTelemetryV2.log_pre_api_call` + runs synchronously in the request task, just before the upstream call, and + **opens** the LLM-call span there, parented to the anchored server span + (`resolve_request_span_context`). The open span is held in a bounded cache keyed + by `litellm_call_id` (a primitive the callback kwargs carry at both `pre_call` + and close), so no live `Span` ever travels through a `litellm_params` metadata + dict. For the boundary hook to fire at all, the logger is registered into + `litellm.input_callback` — the list `Logging.pre_call` iterates. The async + success/failure callback later + **closes** it: it builds an `LLMCallSpanData` from the typed + `standard_logging_object` (token usage and cost are computed only by then), + stamps the attributes, sets status, and ends the span. The sync callback is a + no-op (closing is async-only). When `pre_call` runs off the request task — a + sync-only provider driven through a thread pool, where contextvars (and so the + anchor) don't follow — no parent is visible there, so creation is **deferred** + to the async callback, whose worker context was copied from the request task at + enqueue and so still carries the anchor. **Pass-through** endpoints call + `logging_obj.pre_call` in the request task too, then close from a detached + `asyncio.create_task`; the anchor (not the by-then-inactive server span) keeps + their LLM-call span in the request's trace. `pre_call` is litellm's generic + "log the attempt" hook, so it also fires for synthetic proxy-gate error logs + (auth/rate-limit rejections); those carry `LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL` + and are skipped, so a request rejected before reaching a provider never produces + a phantom CLIENT span. +5. **Guardrails / services**: the post-call and service hooks emit guardrail and + service spans the same way — typed data → engine → span. Service spans + (Redis/Postgres) are dispatched by `litellm/_service_logger.py`, which + recognizes the V2 `OpenTelemetryV2` logger (a plain `CustomLogger`, not a + subclass of the legacy `OpenTelemetry`). It hands every service call to the + logger — including calls with no parent span — and the V2 adapter decides the + role (`DB_CALL` vs `SERVICE`), the parent (ambient → threaded → root), and + whether the call is a traceable operation or a metrics-only ping. Guardrail + span data is built from the typed, provider-agnostic + `StandardLoggingGuardrailInformation` — no single provider's field shape is + assumed. +6. **Export**: each span ends and is handed to the provider's span processors, + which export to the configured backends (OTLP, console, in-memory, …). + +## Components + +### Sources of truth (`model/`, no OpenTelemetry import) + +These define the shape of a span without depending on the OTel SDK, so they can +be imported anywhere. They live in [`model/`](./model) and form a closed set — +nothing here imports outside it: + +- [`semconv.py`](./model/semconv.py) — attribute-key constants (`gen_ai.*`, `http.*`, + `litellm.*`), the GenAI operation/provider enums, and the functions that map + LiteLLM provider/call-type strings onto convention values. +- [`spans.py`](./model/spans.py) — the span registry: every span role, its OTel span + kind, its place in the hierarchy, and its name builder. +- [`payloads.py`](./model/payloads.py) — frozen dataclasses (`LLMCallSpanData`, + `GuardrailSpanData`, `ServiceSpanData`, …) built from heterogeneous logging + payloads via `from_*` classmethods. +- [`config.py`](./model/config.py) — `OpenTelemetryV2Config`, a pydantic-settings + model that reads `OTEL_*` / `LITELLM_OTEL_*` env vars, plus the feature gate. + `capture_span_content` gates whether prompt/response bodies may be written as + span attributes; it defaults **off** (`no_content`). The Baggage allowlists are + configurable, not hard-coded: set `LITELLM_OTEL_BAGGAGE_PROMOTED_KEYS` / + `LITELLM_OTEL_BAGGAGE_METADATA_KEYS` (comma-separated) as env vars, or + `baggage_promoted_keys` / `baggage_metadata_keys` (YAML lists) under + `callback_settings.otel` in `config.yaml` — the latter reach the config through + the logger's constructor kwargs. +- [`baggage.py`](./model/baggage.py) — the single definition of which request-identity + values are promoted into Baggage (so child spans inherit them) and under which + attribute keys. +- [`utils.py`](./model/utils.py) — value coercion, JSON serialization, and + extractor-table application, shared across the package. + +### Engine + +- [`emitter.py`](./emitter.py) — `SpanEmitter.emit(role, data)`: dedupe → start + the span → run the mapper chain to stamp attributes → set status → end. It + owns no attribute keys. The dedupe set (which coalesces the sync+async firing + of one request) is a bounded LRU so it can't grow without limit. +- [`mappers/`](./mappers) — each mapper turns typed span data into a flat + `{attribute key: value}` dict. They compose: listing several mapper names in + the config layers multiple attribute vocabularies onto the same span. + - `genai` — the canonical OpenTelemetry GenAI vocabulary, always present. + - `legacy` — an additional vocabulary using the older semconv-ai / Traceloop + attribute key names, for backends that read those. + - `openinference`, `langfuse`, `weave`, `langtrace` — vendor vocabularies. + - `resolve_mappers(names)` turns config names into mapper instances. + +### Plumbing (`plumbing/`) + +The OTel-SDK wiring. Everything here imports only `model/` and each other; it +lives in [`plumbing/`](./plumbing): + +- [`providers.py`](./plumbing/providers.py) — builds the `TracerProvider`, its exporters + (from `ExporterSpec`s), and the span processor that copies allowlisted Baggage + entries onto every span. `register_exporter_factory(kind, factory)` lets a + preset contribute a custom exporter `kind` (e.g. one that fetches an auth + token lazily) without coupling this module to any vendor. +- [`context.py`](./plumbing/context.py) — trace-context and Baggage read/write helpers. +- [`routing.py`](./plumbing/routing.py) — `TenantTracerCache`: when a request carries + team/key-scoped vendor credentials, route its spans through a credential-keyed + `TracerProvider` so one logger serves many tenants. The cache is a bounded LRU + that flushes + shuts down evicted providers, since the key derives from + request-supplied credentials and must not grow (or leak threads) without limit. +- [`metrics.py`](./plumbing/metrics.py) — GenAI client metric instruments. + +### Adapter + +- [`logger.py`](./logger.py) — `OpenTelemetryV2`, a `CustomLogger` that + translates LiteLLM's logging callbacks into typed span data and hands them to + the engine. The LLM-call span is opened at the `log_pre_api_call` boundary + (parented to the live server span via ambient context) and closed at the async + success/failure callback; the open span is held in a bounded cache keyed by + `litellm_call_id`, never threaded through a metadata dict. The logger registers + itself into `litellm.input_callback` so `Logging.pre_call` fires the boundary + hook. +- [`mount.py`](./mount.py) — `instrument_fastapi_app(app)`, the single call site + that attaches `opentelemetry-instrumentation-fastapi` for SERVER spans. It owns + the health-check exclusion default (`OTEL_PYTHON_FASTAPI_EXCLUDED_URLS`) and the + passthrough span-naming hook (`PASSTHROUGH_PREFIXES`) so `proxy_server` carries + no OTel detail. A safe no-op when the gate is off or the instrumentation package + is absent; must be called at app-creation time (the middleware stack freezes + once the app serves). + +### Presets + +- [`presets/`](./presets) — each preset reads one integration's env vars and + returns an `OpenTelemetryV2Config` (exporter destination + mapper vocabularies + + resource attributes). `PRESET_BY_CALLBACK` maps a callback name (`"arize"`, + `"langfuse_otel"`, …) to its preset. Integrations that support team/key-scoped + credentials also provide a per-request OTLP header builder + (`DYNAMIC_HEADERS_BY_CALLBACK`). Presets do **no** network I/O at build time: + AgentOps, for example, mints its JWT lazily inside a custom exporter on the + first export (in the `BatchSpanProcessor` worker thread), never on the event + loop. + +## Extending + +- **A new attribute vocabulary for a backend**: add a mapper in `mappers/` + (a class with a `map(data) -> AttributeMap` method, typically built from + `key -> extractor` tables) and register it in `mappers/__init__._MAPPER_BY_NAME`. +- **A new integration**: add a preset in `presets/` that returns an + `OpenTelemetryV2Config`, and register it in `presets/__init__.PRESET_BY_CALLBACK`. + If it supports dynamic credentials, add a header builder to + `DYNAMIC_HEADERS_BY_CALLBACK`. +- **A new span kind**: add a role to `spans.py` (registry entry + name builder), + a payload dataclass in `payloads.py`, and a branch in the relevant mapper(s). diff --git a/litellm/integrations/otel/__init__.py b/litellm/integrations/otel/__init__.py new file mode 100644 index 00000000000..42a84a85fbd --- /dev/null +++ b/litellm/integrations/otel/__init__.py @@ -0,0 +1,102 @@ +"""Typed, semconv-aligned OpenTelemetry instrumentation for LiteLLM. + +The three sources of truth — attribute keys (:mod:`semconv`), the span and +hierarchy registry (:mod:`spans`), and the typed span-data inputs +(:mod:`payloads`) — plus :mod:`config` are exported here and are free of any +``opentelemetry`` import. The engine layer (``emitter``, ``providers``, +``context``, ``metrics``) and the ``CustomLogger`` adapter (``logger``) are +reached via their submodule paths so that importing this package never +requires the OTel SDK. + +The ``LITELLM_OTEL_V2`` env var gates whether the factory in +``litellm_core_utils.litellm_logging`` constructs the ``OpenTelemetryV2`` +class (from :mod:`logger`). +""" + +from litellm.integrations.otel.model.config import ( + OTEL_V2_ENV, + OpenTelemetryV2Config, + is_otel_v2_enabled, +) +from litellm.integrations.otel.model.baggage import ( + BAGGAGE_PROMOTED_KEYS, + DEFAULT_BAGGAGE_METADATA_KEYS, + promoted_baggage, +) +from litellm.integrations.otel.model.metadata import ( + RequestContext, + RequestIdentity, +) +from litellm.integrations.otel.model.payloads import ( + GuardrailSpanData, + LLMCallSpanData, + LLMRequestParams, + LLMUsage, + ProxyRequestSpanData, + ServerInfo, + ServiceSpanData, + SpanError, +) +from litellm.integrations.otel.model.semconv import ( + DB, + Error, + GenAI, + GenAIOperation, + GenAIProvider, + HTTP, + LiteLLM, + Metric, + Server, + resolve_operation, + resolve_provider, +) +from litellm.integrations.otel.model.spans import ( + SPAN_REGISTRY, + LiteLLMSpanKind, + SpanRole, + SpanSpec, + db_system, + span_role_for_service, + validate_registry, +) + +__all__ = [ + # config + "OTEL_V2_ENV", + "OpenTelemetryV2Config", + "is_otel_v2_enabled", + # semconv + "BAGGAGE_PROMOTED_KEYS", + "DB", + "DEFAULT_BAGGAGE_METADATA_KEYS", + "Error", + "GenAI", + "GenAIOperation", + "GenAIProvider", + "HTTP", + "LiteLLM", + "Metric", + "Server", + "resolve_operation", + "resolve_provider", + # spans + "SPAN_REGISTRY", + "LiteLLMSpanKind", + "SpanRole", + "SpanSpec", + "db_system", + "span_role_for_service", + "validate_registry", + # payloads + "GuardrailSpanData", + "LLMCallSpanData", + "LLMRequestParams", + "LLMUsage", + "ProxyRequestSpanData", + "RequestContext", + "RequestIdentity", + "ServerInfo", + "ServiceSpanData", + "SpanError", + "promoted_baggage", +] diff --git a/litellm/integrations/otel/emitter.py b/litellm/integrations/otel/emitter.py new file mode 100644 index 00000000000..cae6514efdf --- /dev/null +++ b/litellm/integrations/otel/emitter.py @@ -0,0 +1,175 @@ +"""The span engine: dedup, start, run the mapper chain, set status, end.""" + +from collections import OrderedDict +from typing import Callable, Sequence + +from opentelemetry.context import Context +from opentelemetry.trace import Span, Tracer +from opentelemetry.trace.status import Status, StatusCode + +from litellm.integrations.otel.model.config import OpenTelemetryV2Config +from litellm.integrations.otel.mappers import resolve_mappers +from litellm.integrations.otel.mappers.base import AttributeMapper, SpanData +from litellm.integrations.otel.model.payloads import ( + GuardrailSpanData, + LLMCallSpanData, + ServiceSpanData, +) +from litellm.integrations.otel.plumbing.providers import to_otel_span_kind +from litellm.integrations.otel.model.semconv import Error +from litellm.integrations.otel.model.spans import ( + SPAN_REGISTRY, + SpanRole, + guardrail_span_name, + llm_call_span_name, + service_span_name, +) + +# Roles emit() knows how to name and emit. PROXY_REQUEST and the management +# routes are SERVER spans owned by the mounted FastAPI instrumentor, so they +# have no builder here. +_NAME_BUILDERS: dict[SpanRole, Callable[..., str]] = { + SpanRole.LLM_CALL: llm_call_span_name, + SpanRole.GUARDRAIL: guardrail_span_name, + # DB_CALL and SERVICE are both built from ServiceSpanData; they differ only in + # span kind (CLIENT vs INTERNAL) and attribute vocabulary, not in naming. + SpanRole.DB_CALL: service_span_name, + SpanRole.SERVICE: service_span_name, +} + +# Cap on the dedup cache. It only needs to coalesce the sync+async firing window +# of a single in-flight request, so a bounded LRU keeps memory flat on a +# long-running proxy while still covering every concurrently-open call. +_DEDUP_CACHE_MAX = 10_000 + + +class SpanEmitter: + def __init__( + self, + tracer: Tracer, + config: OpenTelemetryV2Config, + mappers: Sequence[AttributeMapper] | None = None, + ) -> None: + self._tracer = tracer + self._config = config + # The mapper chain is the sole source of span attributes. When not + # passed in, resolve it from the config so there's one source of truth. + self._mappers: list[AttributeMapper] = ( + list(mappers) + if mappers is not None + else resolve_mappers(config.mapper_names) + ) + # Bounded LRU (ordered by insertion / most-recent touch). Storing keys + # only — the value is unused — so it behaves like a capped set. + self._emitted: "OrderedDict[tuple[str, SpanRole], None]" = OrderedDict() + + # -- low-level helpers --------------------------------------------------- # + + def start_span( + self, + role: SpanRole, + name: str, + parent_context: Context | None = None, + start_time_ns: int | None = None, + *, + tracer: Tracer | None = None, + ) -> Span: + """Start a span for ``role`` without dedup or attribute mapping. + + For callers that own and manage their own span lifecycle. ``tracer`` + overrides the bound tracer for this span only, used for per-request + multi-tenant credential routing. + """ + return (tracer or self._tracer).start_span( + name, + context=parent_context, + kind=to_otel_span_kind(SPAN_REGISTRY[role].kind), + start_time=start_time_ns, + ) + + def _seen(self, dedup_key: str | None, role: SpanRole) -> bool: + """Return True once a ``(dedup_key, role)`` pair has been emitted. + + Guards against emitting the same span twice when a streaming call + fires both a sync and an async logging callback. + """ + if not dedup_key: + return False + marker = (dedup_key, role) + if marker in self._emitted: + self._emitted.move_to_end(marker) + return True + self._emitted[marker] = None + if len(self._emitted) > _DEDUP_CACHE_MAX: + self._emitted.popitem(last=False) # evict least-recently-used + return False + + # -- the engine ---------------------------------------------------------- # + + def emit( + self, + role: SpanRole, + data: SpanData, + parent_context: Context | None = None, + *, + start_time_ns: int | None = None, + end_time_ns: int | None = None, + tracer: Tracer | None = None, + ) -> Span | None: + """Emit one complete span: dedup, start, map attributes, status, end. + + Return the span, or ``None`` if it was deduplicated away. ``tracer`` + overrides the bound tracer for this span, used for per-request routing. + """ + # Only LLM-call spans carry a dedup key; LLM-call and service spans + # carry an ``error`` field. ``isinstance`` narrows the type for mypy and + # keeps the engine free of duck-typed attribute reads. + dedup_key = data.identity.call_id if isinstance(data, LLMCallSpanData) else None + if self._seen(dedup_key, role): + return None + span = self.start_span( + role, + _NAME_BUILDERS[role](data), + parent_context=parent_context, + start_time_ns=start_time_ns, + tracer=tracer, + ) + self.finish_span(role, span, data, end_time_ns=end_time_ns) + return span + + def finish_span( + self, + role: SpanRole, + span: Span, + data: SpanData, + *, + end_time_ns: int | None = None, + ) -> None: + """Stamp attributes + status on an already-started ``span`` and end it. + + The counterpart to :meth:`start_span` for callers that own a span's + lifecycle — the LLM-call span is opened at the request's ``pre_call`` + boundary (so it parents to the live server span via real ambient context, + never a span threaded through a metadata dict) and closed here once the + typed payload is available. The span name is (re)built from the now-known + data, since the boundary opener only has a provisional name. + """ + span.update_name(_NAME_BUILDERS[role](data)) + for mapper in self._mappers: + for key, value in mapper.map(data).items(): + span.set_attribute(key, value) + error = ( + data.error + if isinstance(data, (LLMCallSpanData, ServiceSpanData, GuardrailSpanData)) + else None + ) + if error and (error.error_type or error.message): + span.set_attribute(Error.TYPE, error.error_type or "error") + span.set_status( + Status(StatusCode.ERROR, error.message or error.error_type or "error") + ) + # On success leave the status UNSET (the semconv default) rather than + # forcing OK — that matches the FastAPI server span and avoids implying a + # span-level health signal litellm doesn't actually evaluate. Only a + # genuine error sets a status. + span.end(end_time=end_time_ns) diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py new file mode 100644 index 00000000000..007b41df0a3 --- /dev/null +++ b/litellm/integrations/otel/logger.py @@ -0,0 +1,495 @@ +"""``CustomLogger`` adapter on the OpenTelemetry span engine.""" + +from collections import OrderedDict +from contextlib import contextmanager +from datetime import datetime +from typing import TYPE_CHECKING, Any, Iterator, Mapping, cast + +from opentelemetry.context import attach, get_current +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.trace import Span, Tracer, get_current_span, use_span + +import litellm +from litellm.integrations.custom_logger import CustomLogger +from litellm.integrations.otel.model.baggage import promoted_baggage +from litellm.integrations.otel.model.config import OpenTelemetryV2Config +from litellm.integrations.otel.plumbing.context import ( + is_recordable_span, + resolve_parent_context, + resolve_request_span_context, + set_request_baggage, + set_request_root_span, +) +from litellm.integrations.otel.emitter import SpanEmitter +from litellm.integrations.otel.mappers import resolve_mappers +from litellm.integrations.otel.model.metadata import ( + LLMCallEvent, + RequestIdentity, + guardrail_entries_from_request_data, + model_from_request_data, +) +from litellm.integrations.otel.model.payloads import ( + GuardrailSpanData, + LLMCallSpanData, + ServiceSpanData, + SpanError, +) +from litellm.integrations.otel.plumbing.providers import ( + build_tracer_provider, + get_tracer, +) +from litellm.integrations.otel.plumbing.routing import TenantTracerCache +from litellm.integrations.otel.model.spans import SpanRole, span_role_for_service +from litellm.integrations.otel.model.utils import to_ns + +if TYPE_CHECKING: + from litellm.types.utils import StandardLoggingGuardrailInformation + +LITELLM_TRACER_NAME = "litellm" + +# Any callback whose class belongs to one of these modules is "the OTel +# callback" for proxy-global-registration purposes. +_OTEL_MODULES = ( + "litellm.integrations.otel", + "litellm.integrations.opentelemetry", +) + + +# Cap on the open-call carrier map. A span opened at ``pre_call`` that never +# reaches a success/failure callback (e.g. a stream that only fires stream +# events) would otherwise linger; bounding the map evicts the oldest so memory +# stays flat on a long-running proxy while covering every concurrent in-flight +# call. +_OPEN_CALLS_MAX = 10_000 + + +class _LLMCallSpan: + """The state carried from the ``pre_call`` boundary to span close. + + ``span`` is the live span when it could be opened at the boundary (the server + span was ambient), or ``None`` when creation was deferred because no ambient + parent was visible — in which case the async callback creates it against its + own (worker-copied) ambient context using ``start_time_ns``. The presence of + a carrier for a call at all is the proof that ``pre_call`` ran, i.e. that an + upstream call was actually attempted. + """ + + __slots__ = ("span", "start_time_ns") + + def __init__(self, span: "Span | None", start_time_ns: int | None) -> None: + self.span = span + self.start_time_ns = start_time_ns + + +class OpenTelemetryV2(CustomLogger): + """The ``CustomLogger`` for OpenTelemetry.""" + + def __init__( + self, + config: OpenTelemetryV2Config | None = None, + callback_name: str | None = None, + tracer_provider: TracerProvider | None = None, + logger_provider: Any | None = None, # reserved for OTel logs + meter_provider: Any | None = None, # reserved for metrics + **kwargs: Any, + ) -> None: + super().__init__(**kwargs) + self.config: OpenTelemetryV2Config = config or OpenTelemetryV2Config(**kwargs) + self.callback_name = callback_name + self._tracer_provider: TracerProvider = ( + tracer_provider + if tracer_provider is not None + else build_tracer_provider(self.config) + ) + self.tracer: Tracer = get_tracer(self._tracer_provider, LITELLM_TRACER_NAME) + self._emitter = SpanEmitter( + self.tracer, self.config, mappers=resolve_mappers(self.config.mapper_names) + ) + self._tenant_tracers = TenantTracerCache( + self.config, callback_name, LITELLM_TRACER_NAME + ) + self._open_llm_calls: "OrderedDict[str, _LLMCallSpan]" = OrderedDict() + self._init_otel_logger_on_litellm_proxy() + + # ====================================================================== # + # Proxy global registration + # ====================================================================== # + + def _register_in_callback_list(self, callbacks: list) -> None: + already_otel = any( + cb.__class__.__module__.startswith(_OTEL_MODULES) + for cb in callbacks + if hasattr(cb, "__class__") + ) + if not already_otel: + callbacks.append(self) + + def _init_otel_logger_on_litellm_proxy(self) -> None: + try: + from litellm.proxy import proxy_server + except Exception: + return + try: + self._register_in_callback_list(litellm.service_callback) + self._register_in_callback_list(litellm.input_callback) + self._register_in_callback_list(litellm._async_success_callback) + self._register_in_callback_list(litellm._async_failure_callback) + except Exception: + pass + if getattr(proxy_server, "open_telemetry_logger", None) is None: + setattr(proxy_server, "open_telemetry_logger", self) + + # ====================================================================== # + # LLM-call callbacks — the span is opened at the ``pre_call`` boundary and + # closed here. See ``log_pre_api_call``. + # ====================================================================== # + + def log_pre_api_call(self, model, messages, kwargs): + """Open the LLM-call span at the call boundary. + + Runs synchronously inside the request task, before the upstream call — + the one place where the live server span is genuinely the ambient OTel + context — so the span parents to it natively, with no span threaded + through a metadata dict. The open span is stashed on the per-request + ``LiteLLMLoggingObj`` (a typed object) and closed in the async callback. + + When no recordable parent is visible (``pre_call`` was driven from a thread + pool for a sync-only provider, where contextvars — and so the anchor — + don't follow), creation is deferred: only the start time is recorded, and + the async callback — whose worker context was copied from the request task + and so still carries the anchor — creates the span then. + + Synthetic proxy-gate error logs (auth/rate-limit rejections) also fire this + hook but never made an upstream call; they are tagged and skipped so no + phantom LLM-call span is produced. + """ + call = LLMCallEvent.from_dict(kwargs) + if call.is_no_upstream_call: + return + call_id = call.call_id + if call_id is None: + return + # Idempotent: a retried call may re-enter ``pre_call`` with the same + # call id; keep the first span so its start time is the true one. + if call_id in self._open_llm_calls: + return + start_time_ns = to_ns(datetime.now()) + span: Span | None = None + # Parent to the request's anchored root span (stable across the request), + # falling back to ambient on the SDK path. Open the span live only when + # that resolves to a recordable parent; otherwise defer to the close + # callback (the thread-pool case, where the anchor isn't visible here). + parent_context = resolve_request_span_context() + if is_recordable_span(get_current_span(parent_context)): + span = self._emitter.start_span( + SpanRole.LLM_CALL, + call.provisional_span_name, + parent_context=parent_context, + start_time_ns=start_time_ns, + tracer=self._tenant_tracers.tracer_for( + self.tracer, call.dynamic_params + ), + ) + self._open_llm_calls[call_id] = _LLMCallSpan( + span=span, start_time_ns=start_time_ns + ) + # Evict the oldest open call if the map is over budget. A call that opens + # but never closes (a stream that only fires stream events) would linger + # otherwise; the evicted span is simply dropped (never exported). + if len(self._open_llm_calls) > _OPEN_CALLS_MAX: + self._open_llm_calls.popitem(last=False) + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + self._close_llm_call(kwargs, start_time, end_time) + + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + self._close_llm_call(kwargs, start_time, end_time) + + def _close_llm_call( + self, + kwargs: Mapping[str, Any], + start_time: datetime | float | None, + end_time: datetime | float | None, + ) -> Span | None: + """Finish the LLM-call span opened at ``pre_call`` (or create it deferred). + + No carrier for this call id means ``pre_call`` never ran — the request was + rejected at the gate or blocked by a pre-call guardrail before any upstream + call — so there is nothing to record and no phantom span. + """ + call = LLMCallEvent.from_dict(kwargs) + call_id = call.call_id + # ``pop`` is the dedup: this method runs from both the success and failure + # paths, and whichever fires first removes the carrier and closes the span. + carrier = self._open_llm_calls.pop(call_id, None) if call_id else None + if carrier is None: + return None + payload = call.payload + if payload is None: + if carrier.span is not None: + # Opened at the boundary but the payload never materialized — end + # it (named provisionally) so it isn't leaked as an open span. + carrier.span.end(end_time=to_ns(end_time)) + return None + data = LLMCallSpanData.from_standard_logging_payload( + payload, capture_content=self.config.capture_span_content + ) + end_time_ns = to_ns(end_time) + if carrier.span is not None: + # Born at the boundary: stamp attributes from the typed payload, set + # status, and end it. Its parent (the server span) was captured at + # creation from real ambient context. + self._emitter.finish_span( + SpanRole.LLM_CALL, carrier.span, data, end_time_ns=end_time_ns + ) + return carrier.span + # Deferred: ``pre_call`` saw no recordable parent, so create the span now. + # The worker copied the request task's context, which carries the anchored + # root span — parent to it (ambient fallback on the SDK path). Seed identity + # Baggage so the span — and the SDK path, which has none — is labeled + # consistently. + parent_ctx = resolve_request_span_context() + bag = promoted_baggage( + data.identity, + data.request_model, + promoted_keys=tuple(self.config.baggage_promoted_keys), + metadata_keys=tuple(self.config.baggage_metadata_keys), + ) + if bag: + parent_ctx = set_request_baggage(bag, context=parent_ctx) + return self._emitter.emit( + SpanRole.LLM_CALL, + data, + parent_context=parent_ctx, + start_time_ns=carrier.start_time_ns, + end_time_ns=end_time_ns, + tracer=self._tenant_tracers.tracer_for(self.tracer, call.dynamic_params), + ) + + # ====================================================================== # + # Service hooks + # ====================================================================== # + + async def async_service_success_hook( + self, + payload: Any, + parent_otel_span: Span | None = None, + start_time: datetime | float | None = None, + end_time: datetime | float | None = None, + event_metadata: dict | None = None, + ) -> None: + self._emit_service( + payload, + parent_otel_span=parent_otel_span, + start_time=start_time, + end_time=end_time, + event_metadata=event_metadata, + error_override=None, + ) + + async def async_service_failure_hook( + self, + payload: Any, + error: str | None = "", + parent_otel_span: Span | None = None, + start_time: datetime | float | None = None, + end_time: datetime | float | None = None, + event_metadata: dict | None = None, + ) -> None: + self._emit_service( + payload, + parent_otel_span=parent_otel_span, + start_time=start_time, + end_time=end_time, + event_metadata=event_metadata, + error_override=error or "error", + ) + + def _emit_service( + self, + payload: Any, + *, + parent_otel_span: Span | None, + start_time: datetime | float | None, + end_time: datetime | float | None, + event_metadata: dict | None, + error_override: str | None, + ) -> Span | None: + data = ServiceSpanData.from_payload(payload, event_metadata=event_metadata) + # Decide whether this service call is a span at all, and of what kind. + # ``None`` means metrics-only (framework instrumentation that duplicates a + # gen-AI span — ``self``/``router``/``proxy_pre_call`` — or ``auth``, which + # gets a live phase span instead). Those still feed Prometheus/Datadog via + # their own hooks; they just never enter the trace. + role = span_role_for_service(data.service_name) + if role is None: + return None + # A metrics-only ping with neither timing nor a parent (in-memory queue + # gauges) is not a traceable operation; a span for it would be a + # zero-duration root with no context, so skip it. Real background work + # (budget/reset jobs, spend flush) passes start/end times and still emits + # as a root; anything with a parent emits regardless. + if ( + error_override is None + and start_time is None + and end_time is None + and parent_otel_span is None + ): + return None + if error_override is not None and data.error is None: + data = ServiceSpanData( + service_name=data.service_name, + call_type=data.call_type, + error=SpanError(message=error_override), + event_metadata=data.event_metadata, + ) + # Parent like every other span: ambient context first (so identity Baggage + # rides along and the call nests under whatever request phase is active — + # e.g. a DB lookup under the live ``auth`` span), falling back to the + # server span the proxy threaded as ``parent_otel_span``. A background + # service call has neither, so it starts its own root trace. + parent_context = resolve_parent_context(threaded=parent_otel_span) + return self._emitter.emit( + role, + data, + parent_context=parent_context, + start_time_ns=to_ns(start_time), + end_time_ns=to_ns(end_time), + ) + + # ====================================================================== # + # async_post_call_* hooks — emit guardrail spans. The server span's status + # / errors are the FastAPI instrumentor's job, so we don't touch it here. + # ====================================================================== # + + def seed_request_identity(self, user_api_key_dict: Any, model: Any = None) -> None: + """Attach request-identity Baggage to the current context + server span. + + Seeding identity into Baggage makes **every** span emitted afterwards for + this request — LLM call, guardrail, DB call — inherit it via + ``LiteLLMBaggageSpanProcessor``. Called once at the auth boundary (as soon + as the key resolves) so post-auth spans are labeled consistently; the + Baggage rides the request task's contextvar from there on. Auth-internal + DB lookups that run before the key is known stay unlabeled — identity + isn't determined yet, which is correct. + """ + try: + identity = RequestIdentity.from_user_api_key_auth(user_api_key_dict) + bag = promoted_baggage( + identity, + model, + promoted_keys=tuple(self.config.baggage_promoted_keys), + metadata_keys=tuple(self.config.baggage_metadata_keys), + ) + if bag: + # Attach (no detach): the contextvar is scoped to this request's + # asyncio task and is reclaimed when the task ends. + attach(set_request_baggage(bag, context=get_current())) + # The server span was started by the instrumentor before this ran, + # so the Baggage processor (which only fires at span start) won't + # backfill it — stamp identity on it directly. + server_span = get_current_span() + if is_recordable_span(server_span): + # Re-capture the anchor here too: this runs post-auth with the + # server span active and covers entrypoints that bypass + # ``create_litellm_proxy_request_started_span`` (e.g. the SDK + # path's ``async_pre_call_hook``). Idempotent. + set_request_root_span(server_span) + for key, value in bag.items(): + server_span.set_attribute(key, value) + except Exception: + pass + + @contextmanager + def start_phase_span(self, name: str) -> "Iterator[Span]": + span = self._emitter.start_span(SpanRole.SERVICE, name) + with use_span(span, end_on_exit=True): + yield span + + async def async_pre_call_hook( + self, + user_api_key_dict: Any, + cache: Any, + data: dict, + call_type: Any, + ) -> dict: + self.seed_request_identity( + user_api_key_dict, + model=model_from_request_data(data), + ) + return data + + async def async_post_call_success_hook( + self, + data: Mapping[str, Any], + user_api_key_dict: Any, + response: Any, + ) -> Any: + self._emit_guardrail_spans(data) + return response + + async def async_post_call_failure_hook( + self, + request_data: Mapping[str, Any], + original_exception: BaseException | None, + user_api_key_dict: Any, + traceback_str: str | None = None, + ) -> None: + self._emit_guardrail_spans(request_data) + + def _emit_guardrail_spans(self, request_data: Mapping[str, Any]) -> None: + # A guardrail is a sibling of the LLM call under the request's root span, + # so parent it to the explicit anchor — not the active span, which on the + # failure path can be the live ``auth`` phase span (post-call failure hooks + # run from inside it on an auth rejection). Emit with the guardrail's actual + # execution window so a pre_call guardrail is placed before the LLM call + # rather than at post-call emission time. + guardrails = guardrail_entries_from_request_data(request_data) + if not guardrails: + return + parent_ctx = resolve_request_span_context() + for entry in guardrails: + data = GuardrailSpanData.from_logging_entry( + cast("StandardLoggingGuardrailInformation", entry) + ) + self._emitter.emit( + SpanRole.GUARDRAIL, + data, + parent_context=parent_ctx, + start_time_ns=to_ns(data.start_time), + end_time_ns=to_ns(data.end_time), + ) + + def create_litellm_proxy_request_started_span( + self, start_time: datetime, headers: Mapping[str, str] | None + ) -> Span | None: + span = get_current_span() + if not is_recordable_span(span): + return None + set_request_root_span(span) + return span + + +def _registered_v2_logger() -> "OpenTelemetryV2 | None": + try: + from litellm.proxy import proxy_server + except Exception: + return None + logger = getattr(proxy_server, "open_telemetry_logger", None) + return logger if isinstance(logger, OpenTelemetryV2) else None + + +def seed_request_identity(user_api_key_dict: Any, model: Any = None) -> None: + logger = _registered_v2_logger() + if logger is not None: + logger.seed_request_identity(user_api_key_dict, model=model) + + +@contextmanager +def phase_span(name: str) -> "Iterator[Span | None]": + logger = _registered_v2_logger() + if logger is None: + yield None + return + with logger.start_phase_span(name) as span: + yield span diff --git a/litellm/integrations/otel/mappers/__init__.py b/litellm/integrations/otel/mappers/__init__.py new file mode 100644 index 00000000000..012e63f1bee --- /dev/null +++ b/litellm/integrations/otel/mappers/__init__.py @@ -0,0 +1,58 @@ +"""Attribute mappers: pure ``LLMCallSpanData -> {attribute key: value}`` functions. + +Composition over inheritance: vocabularies layer onto the same span. Listing +``["genai", "openinference"]`` in ``config.mapper_names`` makes every span +carry both the canonical ``gen_ai.*`` keys and the OpenInference (Arize + +Phoenix) keys. Add ``"langfuse"`` and it works for all three backends at once. +""" + +from typing import Callable, Iterable + +from litellm.integrations.otel.mappers.base import ( + AttributeMap, + AttributeMapper, + AttrValue, +) +from litellm.integrations.otel.mappers.genai import GenAIMapper +from litellm.integrations.otel.mappers.langfuse import LangfuseMapper +from litellm.integrations.otel.mappers.langtrace import LangtraceMapper +from litellm.integrations.otel.mappers.legacy import LegacyMapper +from litellm.integrations.otel.mappers.openinference import OpenInferenceMapper +from litellm.integrations.otel.mappers.weave import WeaveMapper + +# Registry keyed by ``config.mapper_names`` entries. +_MAPPER_BY_NAME: dict[str, Callable[[], AttributeMapper]] = { + "genai": GenAIMapper, + "legacy": LegacyMapper, + "openinference": OpenInferenceMapper, + "langfuse": LangfuseMapper, + "weave": WeaveMapper, + "langtrace": LangtraceMapper, +} + + +def resolve_mappers(names: Iterable[str]) -> list[AttributeMapper]: + """Resolve mapper names to instances. Unknown names raise ``ValueError``.""" + out: list[AttributeMapper] = [] + for name in names: + factory = _MAPPER_BY_NAME.get(name) + if factory is None: + raise ValueError( + f"unknown mapper name {name!r}; known: " f"{sorted(_MAPPER_BY_NAME)}" + ) + out.append(factory()) + return out + + +__all__ = [ + "AttributeMap", + "AttributeMapper", + "AttrValue", + "GenAIMapper", + "LangfuseMapper", + "LangtraceMapper", + "LegacyMapper", + "OpenInferenceMapper", + "WeaveMapper", + "resolve_mappers", +] diff --git a/litellm/integrations/otel/mappers/base.py b/litellm/integrations/otel/mappers/base.py new file mode 100644 index 00000000000..e8fb5af9797 --- /dev/null +++ b/litellm/integrations/otel/mappers/base.py @@ -0,0 +1,36 @@ +"""Mapper protocol and attribute value types.""" + +from typing import Sequence + +from typing_extensions import Protocol, runtime_checkable + +from litellm.integrations.otel.model.payloads import ( + GuardrailSpanData, + LLMCallSpanData, + ServiceSpanData, +) + +AttrScalar = str | bool | int | float +# Mirrors ``opentelemetry.util.types.AttributeValue`` (homogeneous sequences) +# without importing the SDK, so mappers stay OTel-free. +AttrValue = ( + AttrScalar | Sequence[str] | Sequence[bool] | Sequence[int] | Sequence[float] +) +AttributeMap = dict[str, AttrValue] + +# The closed set of span-data types the engine routes through the mapper chain. +# Server spans (PROXY_REQUEST + management routes) belong to the mounted FastAPI +# instrumentor, not the mapper chain. +SpanData = LLMCallSpanData | GuardrailSpanData | ServiceSpanData + + +@runtime_checkable +class AttributeMapper(Protocol): + """Maps a typed span input to a flat dict of OTel span attributes. + + One method per mapper, dispatched internally on the ``data`` type. The + engine calls this uniformly for every span kind — mappers that don't speak + a given type return ``{}``. This is why the engine contains no attribute keys. + """ + + def map(self, data: SpanData) -> AttributeMap: ... diff --git a/litellm/integrations/otel/mappers/genai.py b/litellm/integrations/otel/mappers/genai.py new file mode 100644 index 00000000000..57fa51ea1fb --- /dev/null +++ b/litellm/integrations/otel/mappers/genai.py @@ -0,0 +1,137 @@ +"""Canonical OpenTelemetry GenAI semantic-convention mapper (always active). + +Owns the attribute schema for every span kind the engine emits — LLM call, +guardrail, and service — so the engine itself never references attribute keys. + +Each span kind declares its schema as a flat ``attribute key -> extractor`` +table: one lambda per mapping operation, applied against the typed span data. +""" + +from typing import Callable + +from litellm.integrations.otel.mappers.base import AttributeMap, AttrValue, SpanData +from litellm.integrations.otel.mappers.utils import collect, drop_none +from litellm.integrations.otel.model.payloads import ( + GuardrailSpanData, + LLMCallSpanData, + ServiceSpanData, + ToolDefinition, +) +from litellm.integrations.otel.model.semconv import DB, Error, GenAI, LiteLLM, Server +from litellm.integrations.otel.model.spans import db_system + + +class GenAIMapper: + + _LLM_CALL_ATTRS: dict[str, Callable[[LLMCallSpanData], AttrValue | None]] = { + GenAI.OPERATION_NAME: lambda d: d.operation.value, + GenAI.PROVIDER_NAME: lambda d: d.provider or None, + GenAI.REQUEST_MODEL: lambda d: d.request_model or None, + GenAI.REQUEST_TEMPERATURE: lambda d: d.request_params.temperature, + GenAI.REQUEST_TOP_P: lambda d: d.request_params.top_p, + GenAI.REQUEST_TOP_K: lambda d: d.request_params.top_k, + GenAI.REQUEST_MAX_TOKENS: lambda d: d.request_params.max_tokens, + GenAI.REQUEST_FREQUENCY_PENALTY: lambda d: d.request_params.frequency_penalty, + GenAI.REQUEST_PRESENCE_PENALTY: lambda d: d.request_params.presence_penalty, + GenAI.REQUEST_STOP_SEQUENCES: lambda d: ( + list(d.request_params.stop_sequences) + if d.request_params.stop_sequences + else None + ), + GenAI.REQUEST_SEED: lambda d: d.request_params.seed, + GenAI.RESPONSE_MODEL: lambda d: d.response_model, + GenAI.RESPONSE_ID: lambda d: d.response_id, + GenAI.RESPONSE_FINISH_REASONS: lambda d: ( + list(d.finish_reasons) if d.finish_reasons else None + ), + GenAI.USAGE_INPUT_TOKENS: lambda d: d.usage.input_tokens, + GenAI.USAGE_OUTPUT_TOKENS: lambda d: d.usage.output_tokens, + Error.TYPE: lambda d: d.error.error_type if d.error else None, + Server.ADDRESS: lambda d: d.server.address if d.server else None, + Server.PORT: lambda d: d.server.port if d.server else None, + LiteLLM.CALL_ID: lambda d: d.identity.call_id or None, + # The provider/underlying model is only known once routing has picked a + # deployment, so it can't ride identity Baggage (seeded at auth, before + # routing) onto the boundary-born LLM span — stamp it directly here. + LiteLLM.PROVIDER_MODEL: lambda d: d.identity.provider_model or None, + f"{LiteLLM.COST_PREFIX}total": lambda d: d.response_cost, + LiteLLM.REQUEST_STREAMING: lambda d: d.is_streaming, + } + + _TOOL_ATTRS: dict[str, Callable[[ToolDefinition], AttrValue | None]] = { + "name": lambda t: t.name, + "description": lambda t: t.description or None, + "parameters": lambda t: t.parameters_json or None, + } + + _GUARDRAIL_ATTRS: dict[str, Callable[[GuardrailSpanData], AttrValue | None]] = { + LiteLLM.GUARDRAIL_NAME: lambda d: d.guardrail_name, + LiteLLM.GUARDRAIL_MODE: lambda d: d.mode, + LiteLLM.GUARDRAIL_STATUS: lambda d: d.status, + LiteLLM.GUARDRAIL_PROVIDER: lambda d: d.provider, + LiteLLM.GUARDRAIL_ACTION: lambda d: d.action, + LiteLLM.GUARDRAIL_RESPONSE: lambda d: d.response_json, + LiteLLM.GUARDRAIL_VIOLATION_CATEGORIES: lambda d: ( + list(d.violation_categories) if d.violation_categories else None + ), + LiteLLM.GUARDRAIL_CONFIDENCE_SCORE: lambda d: d.confidence_score, + LiteLLM.GUARDRAIL_RISK_SCORE: lambda d: d.risk_score, + LiteLLM.GUARDRAIL_MASKED_ENTITY_COUNT: lambda d: d.masked_entity_count, + LiteLLM.GUARDRAIL_DURATION: lambda d: d.duration, + LiteLLM.GUARDRAIL_ID: lambda d: d.guardrail_id, + LiteLLM.GUARDRAIL_POLICY_TEMPLATE: lambda d: d.policy_template, + LiteLLM.GUARDRAIL_DETECTION_METHOD: lambda d: d.detection_method, + } + + _SERVICE_ATTRS: dict[str, Callable[[ServiceSpanData], AttrValue | None]] = { + LiteLLM.SERVICE_NAME: lambda d: d.service_name, + LiteLLM.SERVICE_CALL_TYPE: lambda d: d.call_type, + } + + def map(self, data: SpanData) -> AttributeMap: + match data: + case LLMCallSpanData(): + return self._llm_call(data) + case GuardrailSpanData(): + return self._guardrail(data) + case ServiceSpanData(): + return self._service(data) + case _: + return {} + + @classmethod + def _llm_call(cls, data: LLMCallSpanData) -> AttributeMap: + attrs = collect(cls._LLM_CALL_ATTRS, data) + attrs.update( + drop_none( + { + f"gen_ai.tool.{idx}.{suffix}": extract(tool) + for idx, tool in enumerate(data.tools) + for suffix, extract in cls._TOOL_ATTRS.items() + } + ) + ) + return attrs + + @classmethod + def _guardrail(cls, data: GuardrailSpanData) -> AttributeMap: + return collect(cls._GUARDRAIL_ATTRS, data) + + @classmethod + def _service(cls, data: ServiceSpanData) -> AttributeMap: + attrs = collect(cls._SERVICE_ATTRS, data) + # An outbound datastore call (DB_CALL / CLIENT span) also carries db.* + # semconv. Internal services (router, budget jobs, …) have no db.system, + # so they get only the litellm.service.* keys above. + system = db_system(data.service_name) + if system is not None: + attrs[DB.SYSTEM_NAME] = system + if data.call_type: + attrs[DB.OPERATION_NAME] = data.call_type + attrs.update( + { + f"{LiteLLM.METADATA_PREFIX}{key}": value + for key, value in data.event_metadata.items() + } + ) + return attrs diff --git a/litellm/integrations/otel/mappers/langfuse.py b/litellm/integrations/otel/mappers/langfuse.py new file mode 100644 index 00000000000..14c9fd01d05 --- /dev/null +++ b/litellm/integrations/otel/mappers/langfuse.py @@ -0,0 +1,84 @@ +"""Langfuse OTLP attribute mapper. + +Langfuse ingests OTLP spans and reads from its own vendor namespace +(``langfuse.observation.*``, ``langfuse.trace.*``). Compose this mapper after +``GenAIMapper`` to send canonical + Langfuse-flavored spans simultaneously. + +Every attribute is declared as a ``key -> extractor`` table entry (one callable +per mapping operation): ``_LLM_CALL_ATTRS`` for scalars and ``_BLOB_ATTRS`` for +the JSON-serialized payloads. ``_llm_call`` just applies both tables. +""" + +import json +from typing import Callable + +from litellm.integrations.otel.mappers.base import AttributeMap, AttrValue, SpanData +from litellm.integrations.otel.mappers.utils import ( + collect, + json_if, + output_messages, + serialize_messages, +) +from litellm.integrations.otel.model.payloads import ( + LLMCallSpanData, + LLMRequestParams, + LLMUsage, +) + + +class LangfuseMapper: + + _LLM_CALL_ATTRS: dict[str, Callable[[LLMCallSpanData], AttrValue | None]] = { + "langfuse.observation.type": lambda d: "generation", + "langfuse.observation.model.name": lambda d: d.request_model or None, + "langfuse.observation.metadata.provider": lambda d: d.provider or None, + "langfuse.observation.id": lambda d: d.identity.call_id or None, + "langfuse.trace.metadata.team_id": lambda d: d.identity.team_id or None, + "langfuse.trace.metadata.team_alias": lambda d: d.identity.team_alias or None, + } + + # Sub-tables folded into their respective JSON blobs. + _MODEL_PARAMS: dict[str, Callable[[LLMRequestParams], AttrValue | None]] = { + "temperature": lambda rp: rp.temperature, + "top_p": lambda rp: rp.top_p, + "max_tokens": lambda rp: rp.max_tokens, + "frequency_penalty": lambda rp: rp.frequency_penalty, + "presence_penalty": lambda rp: rp.presence_penalty, + "seed": lambda rp: rp.seed, + } + _USAGE_FIELDS: dict[str, Callable[[LLMUsage], AttrValue | None]] = { + "input": lambda u: u.input_tokens, + "output": lambda u: u.output_tokens, + "total": lambda u: u.total_tokens, + } + + # JSON-payload attributes: each builder returns the serialized blob or None. + _BLOB_ATTRS: dict[str, Callable[[LLMCallSpanData], AttrValue | None]] = { + "langfuse.observation.model.parameters": lambda d: json_if( + collect(LangfuseMapper._MODEL_PARAMS, d.request_params) + ), + "langfuse.observation.input": lambda d: serialize_messages(d.messages_in), + "langfuse.observation.output": lambda d: serialize_messages(output_messages(d)), + "langfuse.observation.usage_details": lambda d: json_if( + collect(LangfuseMapper._USAGE_FIELDS, d.usage) + ), + "langfuse.observation.cost_details": lambda d: ( + json.dumps({"total": d.response_cost}) + if d.response_cost is not None + else None + ), + } + + def map(self, data: SpanData) -> AttributeMap: + match data: + case LLMCallSpanData(): + return self._llm_call(data) + case _: + return {} + + @classmethod + def _llm_call(cls, data: LLMCallSpanData) -> AttributeMap: + return { + **collect(cls._LLM_CALL_ATTRS, data), + **collect(cls._BLOB_ATTRS, data), + } diff --git a/litellm/integrations/otel/mappers/langtrace.py b/litellm/integrations/otel/mappers/langtrace.py new file mode 100644 index 00000000000..7c0f30e57dd --- /dev/null +++ b/litellm/integrations/otel/mappers/langtrace.py @@ -0,0 +1,64 @@ +"""Langtrace attribute mapper. + +Produces Langtrace's attribute vocabulary so a span can be ingested by a +Langtrace backend. Compose it alongside other mappers like any other +vocabulary. + +Scalar attributes are declared as a flat ``key -> extractor`` table (one lambda +per mapping operation); the prompt/completion blobs are serialized as a tail. +""" + +from typing import Callable + +from litellm.integrations.otel.mappers.base import AttributeMap, AttrValue, SpanData +from litellm.integrations.otel.mappers.utils import ( + collect, + json_or_none, + output_messages, +) +from litellm.integrations.otel.model.payloads import LLMCallSpanData + + +class LangtraceMapper: + + _LLM_CALL_ATTRS: dict[str, Callable[[LLMCallSpanData], AttrValue | None]] = { + "gen_ai.operation.name": lambda d: "chat", + "langtrace.service.name": lambda d: d.provider or None, + "llm.model": lambda d: d.request_model or None, + "gen_ai.response.model": lambda d: d.response_model or None, + "gen_ai.response_id": lambda d: d.response_id or None, + "gen_ai.system_fingerprint": lambda d: d.system_fingerprint or None, + "llm.temperature": lambda d: d.request_params.temperature, + "llm.top_p": lambda d: d.request_params.top_p, + "llm.top_k": lambda d: d.request_params.top_k, + "llm.max_tokens": lambda d: d.request_params.max_tokens, + "llm.frequency_penalty": lambda d: d.request_params.frequency_penalty, + "llm.presence_penalty": lambda d: d.request_params.presence_penalty, + "llm.stream": lambda d: d.is_streaming, + "llm.token.counts.prompt": lambda d: d.usage.input_tokens, + "llm.token.counts.completion": lambda d: d.usage.output_tokens, + "llm.token.counts.total": lambda d: d.usage.total_tokens, + } + + _BLOB_ATTRS: dict[str, Callable[[LLMCallSpanData], AttrValue | None]] = { + "llm.prompts": lambda d: ( + json_or_none(list(d.messages_in)) if d.messages_in else None + ), + "llm.completions": lambda d: ( + json_or_none(output_messages(d)) if d.choices_out else None + ), + } + + def map(self, data: SpanData) -> AttributeMap: + match data: + case LLMCallSpanData(): + return self._llm_call(data) + case _: + return {} + + @classmethod + def _llm_call(cls, data: LLMCallSpanData) -> AttributeMap: + return { + **collect(cls._LLM_CALL_ATTRS, data), + **collect(cls._BLOB_ATTRS, data), + } diff --git a/litellm/integrations/otel/mappers/legacy.py b/litellm/integrations/otel/mappers/legacy.py new file mode 100644 index 00000000000..20ffe8b0dd8 --- /dev/null +++ b/litellm/integrations/otel/mappers/legacy.py @@ -0,0 +1,97 @@ +"""Mapper for the older semantic-convention attribute vocabulary. + +Emits attributes under the semconv-ai / Traceloop key names (e.g. +``gen_ai.system``, ``gen_ai.usage.prompt_tokens``, ``llm.is_streaming``) plus a +few bare, unprefixed service keys (``service``, ``call_type``, ``error``), for +backends that consume those names. + +Like ``GenAIMapper``, each span kind declares its schema as a flat +``attribute key -> extractor`` table: one lambda per mapping operation. +""" + +from typing import Callable, Final + +from litellm.integrations.otel.mappers.base import AttributeMap, AttrValue, SpanData +from litellm.integrations.otel.mappers.utils import collect, drop_none +from litellm.integrations.otel.model.payloads import ( + LLMCallSpanData, + ServiceSpanData, + ToolDefinition, +) + +# Attribute keys in the semconv-ai / Traceloop vocabulary. +_LEGACY_SYSTEM: Final = "gen_ai.system" +_LEGACY_PROMPT_TOKENS: Final = "gen_ai.usage.prompt_tokens" +_LEGACY_COMPLETION_TOKENS: Final = "gen_ai.usage.completion_tokens" +_LEGACY_TOTAL_TOKENS: Final = "gen_ai.usage.total_tokens" +_LEGACY_IS_STREAMING: Final = "llm.is_streaming" +_LEGACY_TOP_K: Final = "llm.top_k" +_LEGACY_FREQUENCY_PENALTY: Final = "llm.frequency_penalty" +_LEGACY_PRESENCE_PENALTY: Final = "llm.presence_penalty" +_LEGACY_STOP_SEQUENCES: Final = "llm.chat.stop_sequences" +_LEGACY_SERVICE: Final = "service" +_LEGACY_CALL_TYPE: Final = "call_type" +_LEGACY_ERROR: Final = "error" + + +class LegacyMapper: + """Emits LLM-call and service attributes under the older key names.""" + + _LLM_CALL_ATTRS: dict[str, Callable[[LLMCallSpanData], AttrValue | None]] = { + _LEGACY_SYSTEM: lambda d: d.provider or None, + _LEGACY_PROMPT_TOKENS: lambda d: d.usage.input_tokens, + _LEGACY_COMPLETION_TOKENS: lambda d: d.usage.output_tokens, + _LEGACY_TOTAL_TOKENS: lambda d: d.usage.total_tokens, + _LEGACY_IS_STREAMING: lambda d: d.is_streaming, + _LEGACY_TOP_K: lambda d: d.request_params.top_k, + _LEGACY_FREQUENCY_PENALTY: lambda d: d.request_params.frequency_penalty, + _LEGACY_PRESENCE_PENALTY: lambda d: d.request_params.presence_penalty, + _LEGACY_STOP_SEQUENCES: lambda d: ( + list(d.request_params.stop_sequences) + if d.request_params.stop_sequences + else None + ), + } + + _TOOL_ATTRS: dict[str, Callable[[ToolDefinition], AttrValue | None]] = { + "name": lambda t: t.name, + "description": lambda t: t.description or None, + "parameters": lambda t: t.parameters_json or None, + } + + _SERVICE_ATTRS: dict[str, Callable[[ServiceSpanData], AttrValue | None]] = { + _LEGACY_SERVICE: lambda d: d.service_name, + _LEGACY_CALL_TYPE: lambda d: d.call_type, + _LEGACY_ERROR: lambda d: ( + d.error.message if d.error is not None and d.error.message else None + ), + } + + def map(self, data: SpanData) -> AttributeMap: + match data: + case LLMCallSpanData(): + return self._llm_call(data) + case ServiceSpanData(): + return self._service(data) + case _: + return {} + + @classmethod + def _llm_call(cls, data: LLMCallSpanData) -> AttributeMap: + attrs = collect(cls._LLM_CALL_ATTRS, data) + attrs.update( + drop_none( + { + f"llm.request.functions.{idx}.{suffix}": extract(tool) + for idx, tool in enumerate(data.tools) + for suffix, extract in cls._TOOL_ATTRS.items() + } + ) + ) + return attrs + + @classmethod + def _service(cls, data: ServiceSpanData) -> AttributeMap: + attrs = collect(cls._SERVICE_ATTRS, data) + attrs.update(dict(data.event_metadata)) + return attrs diff --git a/litellm/integrations/otel/mappers/openinference.py b/litellm/integrations/otel/mappers/openinference.py new file mode 100644 index 00000000000..d8195cbe03d --- /dev/null +++ b/litellm/integrations/otel/mappers/openinference.py @@ -0,0 +1,128 @@ +"""OpenInference attribute mapper (Arize + Arize-Phoenix shared vocabulary). + +Spec: https://github.com/Arize-ai/openinference/tree/main/spec — the standard +both Arize and Phoenix consume. Composing this mapper after ``GenAIMapper`` +gives the same span both vocabularies, so a single trace lights up Arize + +Phoenix + any other OpenInference-aware backend simultaneously. +""" + +import json +from typing import Callable, Sequence + +from litellm.integrations.otel.mappers.base import AttributeMap, AttrValue, SpanData +from litellm.integrations.otel.mappers.utils import ( + collect, + drop_none, + json_if, + message_content, + output_messages, +) +from litellm.integrations.otel.model.payloads import ( + LLMCallSpanData, + LLMRequestParams, + ToolDefinition, +) + + +class OpenInferenceMapper: + """Emits OpenInference attributes for LLM_CALL spans. + + Key families (per the OpenInference spec): + - ``openinference.span.kind`` — discriminator (``"LLM"`` here) + - ``llm.model_name`` / ``llm.provider`` / ``llm.invocation_parameters`` + - ``llm.input_messages.{i}.message.role`` / ``...content`` + - ``llm.output_messages.{i}.message.role`` / ``...content`` + - ``llm.token_count.prompt`` / ``...completion`` / ``...total`` + - ``input.value`` / ``output.value`` — JSON-serialized request / response + """ + + _LLM_CALL_ATTRS: dict[str, Callable[[LLMCallSpanData], AttrValue | None]] = { + "openinference.span.kind": lambda d: "LLM", + "llm.model_name": lambda d: d.request_model or None, + "llm.provider": lambda d: d.provider or None, + "llm.token_count.prompt": lambda d: d.usage.input_tokens, + "llm.token_count.completion": lambda d: d.usage.output_tokens, + "llm.token_count.total": lambda d: d.usage.total_tokens, + } + + # Folded into the ``llm.invocation_parameters`` JSON blob. + _INVOCATION_PARAMS: dict[str, Callable[[LLMRequestParams], AttrValue | None]] = { + "temperature": lambda rp: rp.temperature, + "top_p": lambda rp: rp.top_p, + "top_k": lambda rp: rp.top_k, + "max_tokens": lambda rp: rp.max_tokens, + "frequency_penalty": lambda rp: rp.frequency_penalty, + "presence_penalty": lambda rp: rp.presence_penalty, + "seed": lambda rp: rp.seed, + } + + # Per-tool extractors, keyed by the ``llm.tools.{idx}.*`` suffix. + _TOOL_ATTRS: dict[str, Callable[[ToolDefinition], AttrValue | None]] = { + "tool.name": lambda t: t.name, + "tool.description": lambda t: t.description or None, + "tool.json_schema": lambda t: t.parameters_json or None, + } + + # JSON-payload attributes: each builder returns the serialized blob or None. + _BLOB_ATTRS: dict[str, Callable[[LLMCallSpanData], AttrValue | None]] = { + "llm.invocation_parameters": lambda d: json_if( + collect(OpenInferenceMapper._INVOCATION_PARAMS, d.request_params) + ), + } + + def map(self, data: SpanData) -> AttributeMap: + match data: + case LLMCallSpanData(): + return self._llm_call(data) + case _: + return {} + + @classmethod + def _llm_call(cls, data: LLMCallSpanData) -> AttributeMap: + return { + **collect(cls._LLM_CALL_ATTRS, data), + **collect(cls._BLOB_ATTRS, data), + **cls._messages("llm.input_messages", "input.value", data.messages_in), + **cls._messages( + "llm.output_messages", "output.value", output_messages(data) + ), + **cls._tools(data), + } + + @staticmethod + def _messages( + prefix: str, value_key: str, messages: Sequence[object] + ) -> AttributeMap: + """Per-message ``{prefix}.{idx}.message.*`` keys + the ``value_key`` blob.""" + parsed = [ + (m.get("role") if isinstance(m, dict) else None, message_content(m)) + for m in messages + ] + attrs = drop_none( + { + key: value + for idx, (role, content) in enumerate(parsed) + for key, value in ( + ( + f"{prefix}.{idx}.message.role", + role if isinstance(role, str) else None, + ), + (f"{prefix}.{idx}.message.content", content), + ) + } + ) + if parsed: + attrs[value_key] = json.dumps( + [{"role": role, "content": content} for role, content in parsed] + ) + return attrs + + @classmethod + def _tools(cls, data: LLMCallSpanData) -> AttributeMap: + return drop_none( + { + f"llm.tools.{idx}.{suffix}": extract(tool) + for idx, tool in enumerate(data.tools) + for suffix, extract in cls._TOOL_ATTRS.items() + } + ) diff --git a/litellm/integrations/otel/mappers/utils.py b/litellm/integrations/otel/mappers/utils.py new file mode 100644 index 00000000000..6228fc8bbe7 --- /dev/null +++ b/litellm/integrations/otel/mappers/utils.py @@ -0,0 +1,76 @@ +"""Shared helpers for the attribute mappers. + +Small, mapper-agnostic utilities — JSON serialization, message extraction, and +extractor-table application — pulled out of the individual mapper modules so +they live in one place. +""" + +import json +from typing import Callable, Mapping, Sequence + +from litellm.integrations.otel.mappers.base import AttributeMap, AttrValue +from litellm.integrations.otel.model.payloads import LLMCallSpanData + + +def drop_none(values: Mapping[str, AttrValue | None]) -> AttributeMap: + """Return ``values`` with ``None``-valued entries removed.""" + return {k: v for k, v in values.items() if v is not None} + + +def collect(table: Mapping[str, Callable], source: object) -> AttributeMap: + """Apply an extractor table to ``source``, dropping ``None`` results.""" + return drop_none({key: extract(source) for key, extract in table.items()}) + + +def json_if(payload: Mapping[str, object]) -> str | None: + """JSON-serialize ``payload`` only when it's non-empty; else ``None``.""" + return json.dumps(payload) if payload else None + + +def json_or_none(value: object) -> str | None: + """JSON-serialize ``value`` (falling back to ``str``); ``None`` on failure.""" + try: + return json.dumps(value, default=str) + except Exception: + return None + + +def stringify_message(message: object) -> str | None: + """JSON-serialize a chat message dict; ``None`` if not a dict or on failure.""" + if not isinstance(message, dict): + return None + try: + return json.dumps(message, default=str) + except Exception: + return None + + +def serialize_messages(messages: Sequence[object]) -> str | None: + """Round-trip a sequence of message dicts through ``stringify_message``.""" + serialized = [ + json.loads(s) for s in (stringify_message(m) for m in messages) if s is not None + ] + return json.dumps(serialized) if serialized else None + + +def message_content(message: object) -> str | None: + """Extract the textual ``content`` from a chat message dict.""" + if not isinstance(message, dict): + return None + content = message.get("content") + if isinstance(content, str): + return content + if isinstance(content, list): + # multimodal: concatenate text parts only + parts = [ + part.get("text", "") + for part in content + if isinstance(part, dict) and part.get("type") == "text" + ] + return "".join(p for p in parts if isinstance(p, str)) or None + return None + + +def output_messages(data: LLMCallSpanData) -> list: + """The ``message`` payload of each response choice.""" + return [c.get("message") for c in data.choices_out if isinstance(c, dict)] diff --git a/litellm/integrations/otel/mappers/weave.py b/litellm/integrations/otel/mappers/weave.py new file mode 100644 index 00000000000..54b07299271 --- /dev/null +++ b/litellm/integrations/otel/mappers/weave.py @@ -0,0 +1,48 @@ +"""Weave (W&B) attribute mapper. + +Weave consumes OpenInference + a small set of Weave-specific keys (display +name, thread id, output value). This mapper layers the latter on top of +OpenInference's vocabulary — compose ``["genai", "openinference", "weave"]`` +to feed a Weave backend. +""" + +from typing import Callable + +from litellm.integrations.otel.mappers.base import AttributeMap, AttrValue, SpanData +from litellm.integrations.otel.mappers.utils import collect, json_or_none +from litellm.integrations.otel.model.payloads import LLMCallSpanData + + +class WeaveMapper: + """Maps ``LLMCallSpanData`` to Weave's vendor attributes.""" + + _LLM_CALL_ATTRS: dict[str, Callable[[LLMCallSpanData], AttrValue | None]] = { + # ``display_name`` has the form ``"{operation} {model}"``. The span + # name already covers that, but Weave reads this attribute too. + "weave.display_name": lambda d: ( + f"{d.operation.value} {d.request_model}" if d.request_model else None + ), + "weave.call_id": lambda d: d.identity.call_id or None, + } + + # JSON-payload attributes: each builder returns the serialized blob or None. + _BLOB_ATTRS: dict[str, Callable[[LLMCallSpanData], AttrValue | None]] = { + # Weave treats the response choices as the "output" payload. + "weave.output": lambda d: ( + json_or_none(list(d.choices_out)) if d.choices_out else None + ), + } + + def map(self, data: SpanData) -> AttributeMap: + match data: + case LLMCallSpanData(): + return self._llm_call(data) + case _: + return {} + + @classmethod + def _llm_call(cls, data: LLMCallSpanData) -> AttributeMap: + return { + **collect(cls._LLM_CALL_ATTRS, data), + **collect(cls._BLOB_ATTRS, data), + } diff --git a/litellm/integrations/otel/model/__init__.py b/litellm/integrations/otel/model/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/integrations/otel/model/baggage.py b/litellm/integrations/otel/model/baggage.py new file mode 100644 index 00000000000..67dd64e3914 --- /dev/null +++ b/litellm/integrations/otel/model/baggage.py @@ -0,0 +1,76 @@ +"""Baggage promotion: request-identity values carried across child spans. + +A bounded set of identity values is written into OpenTelemetry Baggage on the +LLM-call span so that child spans (guardrail, service) inherit them. +``providers.LiteLLMBaggageSpanProcessor`` reads Baggage at span start and stamps +the allowlisted keys onto every span. + +This module is the single place baggage is defined: ``_PROMOTABLE`` maps each +promotable attribute key to how its value is read, and the two ``*_KEYS`` +defaults select what is promoted unless the config overrides them. +""" + +from collections.abc import Callable +from typing import Final + +from litellm.integrations.otel.model.metadata import RequestIdentity +from litellm.integrations.otel.model.semconv import GenAI, LiteLLM + +# Attribute key -> value extractor over (identity, request_model). The single +# definition of what may be promoted and under which key. +_PROMOTABLE: Final[dict[str, Callable[[RequestIdentity, str | None], str | None]]] = { + LiteLLM.TEAM_ID: lambda identity, model: identity.team_id, + LiteLLM.TEAM_ALIAS: lambda identity, model: identity.team_alias, + LiteLLM.TEAM_METADATA: lambda identity, model: identity.team_metadata, + LiteLLM.KEY_HASH: lambda identity, model: identity.key_hash, + LiteLLM.END_USER: lambda identity, model: identity.end_user, + GenAI.REQUEST_MODEL: lambda identity, model: model, + LiteLLM.PROVIDER_MODEL: lambda identity, model: identity.provider_model, +} + +# Keys promoted by default (a subset of ``_PROMOTABLE``). ``END_USER`` is +# promotable but off by default — it identifies an individual user, so stamping +# it onto every span is opt-in via ``config.baggage_promoted_keys``. +BAGGAGE_PROMOTED_KEYS: Final[tuple[str, ...]] = ( + LiteLLM.TEAM_ID, + LiteLLM.TEAM_ALIAS, + LiteLLM.TEAM_METADATA, + LiteLLM.KEY_HASH, + GenAI.REQUEST_MODEL, + LiteLLM.PROVIDER_MODEL, +) + +# Metadata sub-keys eligible for promotion under the ``litellm.metadata.*`` +# namespace. The full metadata blob is never promoted; only this allowlist is. +DEFAULT_BAGGAGE_METADATA_KEYS: Final[tuple[str, ...]] = ( + "user_api_key_org_id", + "user_api_key_user_id", + "user_api_key_alias", + "user_api_key_end_user_id", + "requester_ip_address", +) + + +def promoted_baggage( + identity: RequestIdentity, + request_model: str | None, + promoted_keys: tuple[str, ...], + metadata_keys: tuple[str, ...] = DEFAULT_BAGGAGE_METADATA_KEYS, +) -> dict[str, str]: + """Identity values to write into Baggage, filtered to ``promoted_keys``. + + ``promoted_keys`` selects from ``_PROMOTABLE``; ``metadata_keys`` selects + sub-keys of ``identity.metadata`` to promote under ``litellm.metadata.*``. + Empty values are dropped. + """ + out: dict[str, str] = {} + for key, extract in _PROMOTABLE.items(): + if key in promoted_keys: + value = extract(identity, request_model) + if value: + out[key] = value + for meta_key in metadata_keys: + value = identity.metadata.get(meta_key) + if value: + out[f"{LiteLLM.METADATA_PREFIX}{meta_key}"] = value + return out diff --git a/litellm/integrations/otel/model/config.py b/litellm/integrations/otel/model/config.py new file mode 100644 index 00000000000..f78e1515ea1 --- /dev/null +++ b/litellm/integrations/otel/model/config.py @@ -0,0 +1,236 @@ +"""Typed configuration for the OpenTelemetry instrumentation.""" + +from typing import Any, List + +from pydantic import AliasChoices, BaseModel, Field, field_validator, model_validator +from pydantic_settings import BaseSettings, NoDecode, SettingsConfigDict +from typing_extensions import Annotated + +from litellm.integrations.otel.model.baggage import ( + BAGGAGE_PROMOTED_KEYS, + DEFAULT_BAGGAGE_METADATA_KEYS, +) + +#: Master feature-flag env var. The logger is inert until this is truthy. +OTEL_V2_ENV = "LITELLM_OTEL_V2" + + +class CaptureMessageContent(str): + NO_CONTENT = "no_content" + SPAN_ONLY = "span_only" + EVENT_ONLY = "event_only" + SPAN_AND_EVENT = "span_and_event" + + +class _OTelV2Flag(BaseSettings): + model_config = SettingsConfigDict(extra="ignore") + + enabled: bool = Field(default=False, validation_alias=AliasChoices(OTEL_V2_ENV)) + + +def is_otel_v2_enabled() -> bool: + return _OTelV2Flag().enabled + + +class ExporterSpec(BaseModel): + """One span-export destination. + + The shared ``TracerProvider`` attaches one ``SpanProcessor`` per spec, so + listing several specs sends every span to all of them at once (e.g. Arize + + Phoenix + your own Honeycomb). + """ + + model_config = {"extra": "forbid"} + + kind: str = Field( + default="console", + description="console | in_memory | otlp_http | otlp_grpc | ", + ) + endpoint: str | None = None + headers: str | None = None + options: dict[str, str] | None = Field( + default=None, + description=( + "Factory-specific configuration for a custom exporter ``kind`` " + "registered via ``providers.register_exporter_factory`` (e.g. an " + "API key a lazy-auth exporter fetches a token with). Ignored by the " + "built-in console/in_memory/otlp exporters." + ), + ) + use_simple_processor: bool | None = Field( + default=None, + description=( + "Force SimpleSpanProcessor regardless of exporter kind. Default: " + "auto (Simple for console/in_memory, Batch otherwise)." + ), + ) + + +class OpenTelemetryV2Config(BaseSettings): + model_config = SettingsConfigDict(populate_by_name=True, extra="ignore") + + # ----- single-destination shorthand, read from standard OTEL_* envs ----- # + exporter: str = Field( + default="console", + validation_alias=AliasChoices("OTEL_EXPORTER", "OTEL_EXPORTER_OTLP_PROTOCOL"), + description=( + "Exporter kind for the single-destination shorthand. The model " + "validator folds this (with ``endpoint`` / ``headers``) into a " + "one-entry ``exporters`` list when ``exporters`` is empty; set " + "``exporters`` directly for multiple destinations." + ), + ) + endpoint: str | None = Field( + default=None, + validation_alias=AliasChoices("OTEL_ENDPOINT", "OTEL_EXPORTER_OTLP_ENDPOINT"), + ) + headers: str | None = Field( + default=None, + validation_alias=AliasChoices("OTEL_HEADERS", "OTEL_EXPORTER_OTLP_HEADERS"), + ) + service_name: str = Field( + default="litellm", validation_alias=AliasChoices("OTEL_SERVICE_NAME") + ) + deployment_environment: str | None = Field( + default=None, validation_alias=AliasChoices("OTEL_ENVIRONMENT_NAME") + ) + + enable_metrics: bool = Field( + default=False, + validation_alias=AliasChoices("LITELLM_OTEL_INTEGRATION_ENABLE_METRICS"), + ) + enable_events: bool = Field( + default=False, + validation_alias=AliasChoices("LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS"), + ) + capture_message_content: str = Field( + default=CaptureMessageContent.NO_CONTENT, + validation_alias=AliasChoices( + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT" + ), + ) + legacy_compat: bool = Field( + default=True, validation_alias=AliasChoices("LITELLM_OTEL_LEGACY_COMPAT") + ) + + # ----- explicit multi-destination / vocabulary configuration ------------ # + + exporters: list[ExporterSpec] = Field( + default_factory=list, + description=( + "One destination per spec. The shared TracerProvider attaches a " + "SpanProcessor per entry. When empty, the model validator folds " + "the ``exporter`` / ``endpoint`` / ``headers`` shorthand into a " + "single spec so there is always at least one destination." + ), + ) + + mapper_names: Annotated[List[str], NoDecode] = Field( + default_factory=lambda: ["genai"], + description=( + "Ordered attribute vocabularies to emit. ``genai`` is the " + "canonical OTel GenAI vocabulary and is always placed first. " + "Vendor names: ``openinference`` (Arize + Phoenix), ``langfuse``, " + "``weave``, ``langtrace``." + ), + ) + + resource_attributes: dict[str, str] = Field( + default_factory=dict, + description=( + "Extra Resource attributes beyond ``service.name`` and " + "``deployment.environment`` (e.g. integration-specific markers)." + ), + ) + + baggage_promoted_keys: Annotated[List[str], NoDecode] = Field( + default_factory=lambda: list(BAGGAGE_PROMOTED_KEYS), + validation_alias=AliasChoices( + "baggage_promoted_keys", "LITELLM_OTEL_BAGGAGE_PROMOTED_KEYS" + ), + description=( + "Identity attribute keys written into Baggage and stamped on every " + "child span (e.g. ``litellm.team.id``). Configure via the " + "``LITELLM_OTEL_BAGGAGE_PROMOTED_KEYS`` env var (comma-separated) or " + "``callback_settings.otel.baggage_promoted_keys`` in config.yaml (a " + "YAML list)." + ), + ) + baggage_metadata_keys: Annotated[List[str], NoDecode] = Field( + default_factory=lambda: list(DEFAULT_BAGGAGE_METADATA_KEYS), + validation_alias=AliasChoices( + "baggage_metadata_keys", "LITELLM_OTEL_BAGGAGE_METADATA_KEYS" + ), + description=( + "Metadata sub-keys promoted under the ``litellm.metadata.*`` " + "namespace. Configure via the ``LITELLM_OTEL_BAGGAGE_METADATA_KEYS`` " + "env var (comma-separated) or " + "``callback_settings.otel.baggage_metadata_keys`` in config.yaml." + ), + ) + + @field_validator( + "baggage_promoted_keys", + "baggage_metadata_keys", + "mapper_names", + mode="before", + ) + @classmethod + def _split_csv(cls, value: Any) -> Any: + """Accept a comma-separated string for list fields. + + Env vars are strings, but these fields are lists. Pydantic-settings would + otherwise require JSON for a list env var; splitting on commas here lets + an operator write ``LITELLM_OTEL_BAGGAGE_PROMOTED_KEYS=litellm.team.id,litellm.api_key.hash``. + YAML lists (from ``callback_settings.otel.*``) and real lists pass through + unchanged. + """ + if isinstance(value, str): + return [item.strip() for item in value.split(",") if item.strip()] + return value + + @model_validator(mode="after") + def _normalize(self) -> "OpenTelemetryV2Config": + # An endpoint with the default exporter kind implies OTLP/HTTP. + if self.endpoint and self.exporter == "console": + self.exporter = "otlp_http" + # When no explicit destinations are given, fold the single-destination + # shorthand into one spec so the provider always has a destination. + if not self.exporters: + self.exporters = [ + ExporterSpec( + kind=self.exporter, + endpoint=self.endpoint, + headers=self.headers, + ) + ] + # Ensure ``genai`` is always present and first. + names = list(self.mapper_names) + if "genai" in names: + names = ["genai"] + [n for n in names if n != "genai"] + else: + names = ["genai"] + names + # When enabled, also emit attribute keys under their semconv-ai / + # Traceloop names via the ``legacy`` mapper. Append it at the tail so + # the canonical ``genai`` keys win on any conflict. + if self.legacy_compat and "legacy" not in names: + names.append("legacy") + self.mapper_names = names + return self + + @property + def capture_span_content(self) -> bool: + """Whether prompt/response content may be stamped as span attributes. + + Defaults off (``no_content``): an operator must opt in before message + bodies leave the process, so a user request can never force its prompt + or completion into the configured backend while capture is disabled. + """ + return self.capture_message_content in ( + CaptureMessageContent.SPAN_ONLY, + CaptureMessageContent.SPAN_AND_EVENT, + ) + + @classmethod + def from_env(cls) -> "OpenTelemetryV2Config": + return cls() diff --git a/litellm/integrations/otel/model/metadata.py b/litellm/integrations/otel/model/metadata.py new file mode 100644 index 00000000000..f0ea0a608c6 --- /dev/null +++ b/litellm/integrations/otel/model/metadata.py @@ -0,0 +1,315 @@ +"""The single translation layer between a request's metadata and the spans. + +Every relevant field litellm exposes about a request — the user-facing model, +the model actually dispatched to the provider, the deployment, and the caller's +identity (team, key, end-user) — is parsed **once**, here, out of the +``StandardLoggingPayload`` (or a ``UserAPIKeyAuth`` at the auth boundary). Span +data, baggage promotion, and the mappers then read these typed fields instead of +each digging into the raw ``metadata`` / ``hidden_params`` dicts. + +Two models live here because a request's identity is known *before* its model +resolution is: + +* :class:`RequestIdentity` — team / key / end-user, seeded into Baggage at the + auth boundary (``from_user_api_key_auth``), before routing has picked a + deployment. ``provider_model`` is therefore absent from that early seed and is + only filled in from the payload once the call closes. +* :class:`RequestContext` — the full picture available at close: the resolved + request vs. provider model split, plus the response model, model group, model + id, and api base, wrapping the :class:`RequestIdentity`. + +The request-vs-provider model split is the subtle part. On the proxy a caller +asks for a *model group* (e.g. ``gpt-4o``) that routes to a concrete deployment +(e.g. ``azure/my-deployment``); the two are distinct and both worth recording. +``StandardLoggingPayload`` exposes them as: + +* ``model_group`` — the user-facing name the caller requested. +* ``model`` — already reconstructed (see ``reconstruct_model_name``) to the name + litellm dispatched to the provider (the deployment, provider-prefixed). +* ``hidden_params.litellm_model_name`` — a secondary source for the dispatched + model (populated only on some call paths, e.g. files). + +So ``gen_ai.request.model`` is the *group* (falling back to the call model on the +SDK path, which has no group), and ``litellm.provider.model`` is the *dispatched* +model. They coincide on the SDK path, which is correct. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any, Mapping, cast + +from litellm.constants import LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL +from litellm.integrations.otel.model.semconv import resolve_operation +from litellm.integrations.otel.model.utils import as_str + +if TYPE_CHECKING: + from litellm.types.utils import StandardLoggingPayload + + +@dataclass(frozen=True) +class RequestIdentity: + call_id: str | None = None + team_id: str | None = None + team_alias: str | None = None + # The team's free-form metadata dict, JSON-serialized (empty/missing -> None). + team_metadata: str | None = None + key_hash: str | None = None + end_user: str | None = None + # The model litellm dispatched to the provider. Only known once the call + # completes (routing has picked a deployment), so it's absent from the + # auth-time seed and filled only from the payload. + provider_model: str | None = None + metadata: Mapping[str, str] = field(default_factory=dict) + + @classmethod + def from_payload(cls, payload: "StandardLoggingPayload") -> "RequestIdentity": + """Parse caller identity out of a closed request's payload metadata. + + ``provider_model`` is resolved here too (see :func:`resolve_provider_model`) + so the identity carried into Baggage labels every span with the dispatched + model, not just the user-facing one. + """ + raw_meta = cast(Mapping[str, object], payload.get("metadata") or {}) + metadata = { + key: str(value) + for key, value in raw_meta.items() + if isinstance(value, (str, bool, int, float)) + } + return cls( + call_id=as_str(payload.get("litellm_call_id")) or as_str(payload.get("id")), + # StandardLoggingMetadata's canonical key is ``user_api_key_team_id``; + # the bare ``team_id`` is a legacy alias and is often empty, so prefer + # the canonical key and fall back to the alias. + team_id=as_str(raw_meta.get("user_api_key_team_id")) + or as_str(raw_meta.get("team_id")), + team_alias=as_str(raw_meta.get("user_api_key_team_alias")) + or as_str(raw_meta.get("team_alias")), + team_metadata=_team_metadata_json( + raw_meta.get("user_api_key_team_metadata") + ), + key_hash=as_str(raw_meta.get("user_api_key_hash")), + end_user=as_str(payload.get("end_user")) + or as_str(raw_meta.get("user_api_key_end_user_id")), + provider_model=resolve_provider_model(payload), + metadata=metadata, + ) + + @classmethod + def from_user_api_key_auth(cls, auth: object) -> "RequestIdentity": + """Identity from a ``UserAPIKeyAuth`` (duck-typed to keep this module + free of a proxy import). + + Used in the pre-call hook to seed Baggage early — before any LLM, + guardrail, or service span is created — so the whole request's spans + inherit identity, not just the LLM-call span. Metadata sub-keys use the + ``user_api_key_*`` names that ``baggage.DEFAULT_BAGGAGE_METADATA_KEYS`` + promotes. + """ + get = lambda name: getattr(auth, name, None) # noqa: E731 + metadata = { + meta_key: str(value) + for meta_key, attr in ( + ("user_api_key_user_id", "user_id"), + ("user_api_key_org_id", "org_id"), + ("user_api_key_alias", "key_alias"), + ("user_api_key_end_user_id", "end_user_id"), + ) + if (value := get(attr)) + } + return cls( + team_id=as_str(get("team_id")), + team_alias=as_str(get("team_alias")), + team_metadata=_team_metadata_json(get("team_metadata")), + key_hash=as_str(get("api_key")), + end_user=as_str(get("end_user_id")), + # ``provider_model`` is unknown at the auth boundary — routing hasn't + # picked a deployment yet — so it's only populated from the payload. + metadata=metadata, + ) + + +@dataclass(frozen=True) +class RequestContext: + """The fully-resolved view of a closed request, parsed once from the payload. + + ``request_model`` is the user-facing requested model and ``provider_model`` + (on :attr:`identity`) is the model litellm dispatched to the provider; the two + differ on the proxy (group vs. deployment) and coincide on the SDK path. + """ + + request_model: str + response_model: str | None + model_group: str | None + model_id: str | None + api_base: str | None + identity: RequestIdentity + + @property + def provider_model(self) -> str | None: + """The dispatched-model name, carried on the identity for Baggage.""" + return self.identity.provider_model + + @classmethod + def from_standard_logging_payload( + cls, payload: "StandardLoggingPayload" + ) -> "RequestContext": + raw_meta = cast(Mapping[str, object], payload.get("metadata") or {}) + hidden = cast(Mapping[str, object], payload.get("hidden_params") or {}) + raw_response = payload.get("response") + response = cast( + Mapping[str, object], raw_response if isinstance(raw_response, dict) else {} + ) + model_group = as_str(payload.get("model_group")) or as_str( + raw_meta.get("model_group") + ) + return cls( + # The user asked for the group; fall back to the call model on the SDK + # path, which has no group. Empty string (never None) so the span name + # builder and the mapper see a plain string. + request_model=model_group or as_str(payload.get("model")) or "", + response_model=as_str(response.get("model")), + model_group=model_group, + model_id=as_str(payload.get("model_id")) + or _model_info_id(raw_meta.get("model_info")), + api_base=as_str(payload.get("api_base")) or as_str(hidden.get("api_base")), + identity=RequestIdentity.from_payload(payload), + ) + + +# --- live-callback kwargs parsing ------------------------------------------- # +# +# The model and helpers below parse the *live* callback ``kwargs`` god object (and +# the raw pre/post-call ``data`` dicts) — the untyped request state that reaches a +# ``CustomLogger`` before, or instead of, a ``StandardLoggingPayload``. They live +# here, with the payload/auth parsers, so every read out of a request's raw dicts +# is in one place rather than scattered across the ``CustomLogger``. + + +@dataclass(frozen=True) +class LLMCallEvent: + """The typed view of the live callback ``kwargs`` (``model_call_details``). + + litellm hands every callback an untyped ``kwargs`` god object. The fields the + OTel logger needs out of it are parsed **once**, here, so the ``CustomLogger`` + reads typed attributes instead of digging into the dict at each boundary. + """ + + # The ``litellm_call_id`` correlating ``pre_call`` with the close callback. + # Present in ``model_call_details`` at ``pre_call`` and in both the kwargs and + # the ``standard_logging_object`` at success/failure, so it's a stable key for + # the open-call carrier — no back-reference to the logging object required (the + # object isn't reachable from the callback kwargs at ``pre_call`` time). + call_id: str | None + # The ``StandardLoggingPayload`` carried on a success/failure callback; ``None`` + # at ``pre_call``, or when the call closed before any payload materialized (so + # there is nothing to stamp on the span). + payload: "StandardLoggingPayload | None" + # The ``standard_callback_dynamic_params`` routing the call to a per-tenant + # tracer (its own exporter/endpoint), or ``None`` when the call isn't scoped. + dynamic_params: Any + # True for synthetic proxy-gate logs (auth / rate-limit rejections): they fire + # the ``pre_call`` hook but never made an upstream call, so they get no span. + is_no_upstream_call: bool + # A best-effort ``"{operation} {model}"`` name known at ``pre_call`` time. The + # span is renamed from the typed payload at close (``finish_span``); this only + # needs to be reasonable for a span that never gets closed (a leak). + provisional_span_name: str + + @classmethod + def from_dict(cls, kwargs: Mapping[str, Any]) -> "LLMCallEvent": + raw_payload = kwargs.get("standard_logging_object") + payload = cast("StandardLoggingPayload", raw_payload) if raw_payload else None + operation = resolve_operation(as_str(kwargs.get("call_type"))) + model = as_str(kwargs.get("model")) or "" + return cls( + call_id=_call_id(payload, kwargs), + payload=payload, + dynamic_params=kwargs.get("standard_callback_dynamic_params"), + is_no_upstream_call=bool(kwargs.get(LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL)), + provisional_span_name=f"{operation.value} {model}".strip(), + ) + + +def _call_id( + payload: "StandardLoggingPayload | None", kwargs: Mapping[str, Any] +) -> str | None: + """The call id from the payload (when closed) or the bare kwargs (at pre_call).""" + if payload is not None: + call_id = as_str(payload.get("litellm_call_id")) or as_str(payload.get("id")) + if call_id: + return call_id + return as_str(kwargs.get("litellm_call_id")) + + +def model_from_request_data(data: object) -> str | None: + """The user-facing ``model`` from a pre-call ``data`` dict (``None`` if absent). + + Read at the auth boundary to label early Baggage before routing has resolved + a deployment; ``data`` is duck-typed since it arrives untyped from the proxy. + """ + if isinstance(data, Mapping): + return as_str(data.get("model")) + return None + + +def guardrail_entries_from_request_data( + request_data: Mapping[str, Any], +) -> list[dict]: + """The guardrail-information dicts buried in ``metadata`` of a post-call dict. + + ``standard_logging_guardrail_information`` is stored as either a single dict + or a list of them; normalize to a list of dicts (dropping non-dict noise) so + the caller just iterates. Empty list when none are present. + """ + metadata = request_data.get("metadata") + if not isinstance(metadata, Mapping): + return [] + info = metadata.get("standard_logging_guardrail_information") + if isinstance(info, Mapping): + return [cast(dict, info)] + if isinstance(info, list): + return [entry for entry in info if isinstance(entry, dict)] + return [] + + +def resolve_provider_model(payload: "StandardLoggingPayload") -> str | None: + """The model litellm dispatched to the provider, from the payload. + + Prefers the explicit ``hidden_params.litellm_model_name`` (set on call paths + that know it, e.g. files), then the top-level ``model`` — which + ``reconstruct_model_name`` has already resolved to the deployment's + provider-prefixed name. Returns ``None`` only when neither is present. + """ + raw_meta = cast(Mapping[str, object], payload.get("metadata") or {}) + hidden = cast(Mapping[str, object], payload.get("hidden_params") or {}) + return ( + # ``deployment`` survives only on paths that don't strip it from metadata; + # harmless (and most precise) to prefer it when present. + as_str(raw_meta.get("deployment")) + or as_str(hidden.get("litellm_model_name")) + or as_str(payload.get("model")) + ) + + +def _model_info_id(model_info: object) -> str | None: + """The deployment id from a ``metadata.model_info`` sub-dict, if present.""" + if isinstance(model_info, Mapping): + return as_str(model_info.get("id")) + return None + + +def _team_metadata_json(value: object) -> str | None: + """JSON-serialize a team's metadata dict for a single Baggage value. + + Returns ``None`` for a missing, non-dict, or empty mapping so the empty case + is dropped rather than promoting a useless ``"{}"``. Keys are sorted for a + stable, diff-friendly serialization. + """ + if not isinstance(value, Mapping) or not value: + return None + try: + return json.dumps(value, default=str, sort_keys=True) + except Exception: + return None diff --git a/litellm/integrations/otel/model/payloads.py b/litellm/integrations/otel/model/payloads.py new file mode 100644 index 00000000000..65b50d0fc12 --- /dev/null +++ b/litellm/integrations/otel/model/payloads.py @@ -0,0 +1,468 @@ +"""Typed span-data inputs: frozen dataclasses the engine and mappers consume.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from enum import Enum +from typing import TYPE_CHECKING, ClassVar, Mapping, cast +from urllib.parse import urlsplit + +from litellm.integrations.otel.model.metadata import ( + RequestContext, + RequestIdentity, +) +from litellm.integrations.otel.model.semconv import ( + GenAIOperation, + resolve_operation, + resolve_provider, +) +from litellm.integrations.otel.model.utils import ( + as_bool, + as_float, + as_int, + as_str, + as_str_tuple, +) + +# ``RequestIdentity`` and the request-metadata translation now live in +# :mod:`metadata`; re-exported here so existing ``model.payloads`` imports keep +# resolving it. +__all__ = [ + "RequestContext", + "RequestIdentity", + "GuardrailSpanData", + "LLMCallSpanData", + "LLMRequestParams", + "LLMUsage", + "ProxyRequestSpanData", + "ServerInfo", + "ServiceSpanData", + "SpanError", + "ToolDefinition", +] + +if TYPE_CHECKING: + from litellm.types.services import ServiceLoggerPayload + from litellm.types.utils import ( + StandardLoggingGuardrailInformation, + StandardLoggingPayload, + ) + + +# --- typed sub-structures ---------------------------------------------------- # + + +@dataclass(frozen=True) +class LLMRequestParams: + temperature: float | None = None + top_p: float | None = None + top_k: int | None = None + max_tokens: int | None = None + frequency_penalty: float | None = None + presence_penalty: float | None = None + stop_sequences: tuple[str, ...] | None = None + seed: int | None = None + + @classmethod + def from_model_parameters(cls, params: Mapping[str, object]) -> "LLMRequestParams": + max_tokens = as_int(params.get("max_tokens")) + if max_tokens is None: + max_tokens = as_int(params.get("max_completion_tokens")) + return cls( + temperature=as_float(params.get("temperature")), + top_p=as_float(params.get("top_p")), + top_k=as_int(params.get("top_k")), + max_tokens=max_tokens, + frequency_penalty=as_float(params.get("frequency_penalty")), + presence_penalty=as_float(params.get("presence_penalty")), + stop_sequences=as_str_tuple(params.get("stop")), + seed=as_int(params.get("seed")), + ) + + +@dataclass(frozen=True) +class LLMUsage: + input_tokens: int | None = None + output_tokens: int | None = None + total_tokens: int | None = None + + +@dataclass(frozen=True) +class SpanError: + error_type: str | None = None + message: str | None = None + + +@dataclass(frozen=True) +class ServerInfo: + address: str | None = None + port: int | None = None + + @classmethod + def from_api_base(cls, api_base: str | None) -> ServerInfo | None: + if not api_base: + return None + parsed = urlsplit(api_base if "://" in api_base else f"//{api_base}") + if not parsed.hostname: + return None + return cls(address=parsed.hostname, port=parsed.port) + + +@dataclass(frozen=True) +class GuardrailSpanData: + guardrail_name: str + mode: str | None = None + status: str | None = None + masked_entity_count: int | None = None + provider: str | None = None + action: str | None = None + # The guardrail verdict / provider response (e.g. the moderation result), + # JSON-serialized. This is the detail that belongs on the guardrail span. + response_json: str | None = None + violation_categories: tuple[str, ...] = () + confidence_score: float | None = None + risk_score: float | None = None + duration: float | None = None + # Actual execution window (epoch seconds) from the logging entry, so the span + # is placed when the guardrail really ran — a pre_call guardrail before the + # LLM call — rather than at post-call emission time. + start_time: float | None = None + end_time: float | None = None + # Provider-agnostic configuration/detection metadata (see + # ``StandardLoggingGuardrailInformation``). Present for any guardrail that + # populates them, not just one provider's shape. + guardrail_id: str | None = None + policy_template: str | None = None + detection_method: str | None = None + # Set when the guardrail intervened/blocked or failed, so the emitter marks + # the span ERROR — a blocking guardrail is an error outcome for that span. + error: SpanError | None = None + + # Guardrail statuses that mean the guardrail did not pass the request through. + _ERROR_STATUSES: ClassVar[frozenset[str]] = frozenset( + {"guardrail_intervened", "guardrail_failed_to_respond"} + ) + + @classmethod + def from_logging_entry( + cls, entry: "StandardLoggingGuardrailInformation" + ) -> "GuardrailSpanData": + """Build from one ``standard_logging_guardrail_information`` entry. + + Reads the canonical, provider-agnostic ``StandardLoggingGuardrailInformation`` + keys only — no guessing at a single provider's field names. Values that are + typed as enums or lists (e.g. ``guardrail_mode``) are normalized to a + stable string rather than assumed to already be plain strings. + """ + get = cast(Mapping[str, object], entry).get + status = as_str(get("guardrail_status")) + response = get("guardrail_response") + error = ( + SpanError(error_type=status, message=as_str(get("guardrail_action"))) + if status in cls._ERROR_STATUSES + else None + ) + return cls( + guardrail_name=as_str(get("guardrail_name")) or "guardrail", + mode=_guardrail_mode_str(get("guardrail_mode")), + status=status, + masked_entity_count=_total_masked_entities(get("masked_entity_count")), + provider=as_str(get("guardrail_provider")), + action=as_str(get("guardrail_action")), + response_json=_json_or_none(response) if response is not None else None, + violation_categories=as_str_tuple(get("violation_categories")) or (), + confidence_score=as_float(get("confidence_score")), + risk_score=as_float(get("risk_score")), + duration=as_float(get("duration")), + start_time=as_float(get("start_time")), + end_time=as_float(get("end_time")), + guardrail_id=as_str(get("guardrail_id")), + policy_template=as_str(get("policy_template")), + detection_method=as_str(get("detection_method")), + error=error, + ) + + +@dataclass(frozen=True) +class ServiceSpanData: + service_name: str + call_type: str | None = None + error: SpanError | None = None + # Caller-supplied attributes to stamp on the service span, passed through + # from ``async_service_*_hook(event_metadata=...)``. The mapper owns how + # these are namespaced: the canonical vocabulary uses ``litellm.metadata.*`` + # keys, the semconv-ai / Traceloop vocabulary uses the bare key names. + event_metadata: Mapping[str, str] = field(default_factory=dict) + + @classmethod + def from_payload( + cls, + payload: "ServiceLoggerPayload", + event_metadata: Mapping[str, object] | None = None, + ) -> "ServiceSpanData": + # ``payload.service`` is a ``ServiceTypes(str, Enum)`` and ``error`` is + # ``Optional[str]`` on the Pydantic model — no defensive reads needed. + # ``event_metadata`` is sanitized: the legacy service decorators pass raw + # call-site data (live objects, full request metadata, response headers), + # none of which belongs on a span. + return cls( + service_name=payload.service.value, + call_type=payload.call_type, + error=SpanError(message=payload.error) if payload.error else None, + event_metadata=sanitize_event_metadata(event_metadata), + ) + + +@dataclass(frozen=True) +class ProxyRequestSpanData: + http_method: str + route: str + url_path: str | None = None + status_code: int | None = None + identity: RequestIdentity | None = None + + +# --- the primary LLM-call model ---------------------------------------------- # + + +@dataclass(frozen=True) +class ToolDefinition: + """A single function/tool declared on a chat-completion request.""" + + name: str + description: str | None = None + parameters_json: str | None = ( + None # JSON-serialized schema (str so it's an AttrValue) + ) + + +@dataclass(frozen=True) +class LLMCallSpanData: + operation: GenAIOperation + provider: str + request_model: str + response_model: str | None + response_id: str | None + request_params: LLMRequestParams + usage: LLMUsage + finish_reasons: tuple[str, ...] + error: SpanError | None + response_cost: float | None + server: ServerInfo | None + identity: RequestIdentity + is_streaming: bool | None = None + tools: tuple[ToolDefinition, ...] = () + # Raw messages and response, needed by vendor mappers (OpenInference, + # Langfuse, Weave) that stamp message-level attributes. ``messages_in`` is + # the request payload; ``choices_out`` mirrors ``response.choices`` from + # the StandardLoggingPayload. Both are tuples of immutable mappings so the + # dataclass stays hashable and frozen. + messages_in: tuple[Mapping[str, object], ...] = () + choices_out: tuple[Mapping[str, object], ...] = () + system_fingerprint: str | None = None + + @classmethod + def from_standard_logging_payload( + cls, payload: "StandardLoggingPayload", capture_content: bool = False + ) -> "LLMCallSpanData": + params = cast(Mapping[str, object], payload.get("model_parameters") or {}) + # The single parse of the request's metadata — the request-vs-provider + # model split, the response model, api base, and identity all come from + # here rather than being re-derived from the raw payload dicts. + context = RequestContext.from_standard_logging_payload(payload) + # Normalize ``response`` to a dict once so the content/id reads below are a + # plain ``.get`` — no repeated ``isinstance`` guards. + raw_response = payload.get("response") + response = cast( + Mapping[str, object], raw_response if isinstance(raw_response, dict) else {} + ) + choices_out = _dicts(response.get("choices")) + # ``finish_reasons`` is metadata, not content, so derive it from + # ``choices_out`` before gating. The raw message/choice bodies are only + # retained when content capture is enabled (see ``capture_span_content``); + # otherwise the content-bearing mappers receive empty sequences and emit + # no prompt/response text. + finish_reasons = _finish_reasons(choices_out) + return cls( + operation=resolve_operation(as_str(payload.get("call_type"))), + provider=resolve_provider(as_str(payload.get("custom_llm_provider"))), + request_model=context.request_model, + response_model=context.response_model, + response_id=as_str(response.get("id")), + request_params=LLMRequestParams.from_model_parameters(params), + usage=LLMUsage( + input_tokens=as_int(payload.get("prompt_tokens")), + output_tokens=as_int(payload.get("completion_tokens")), + total_tokens=as_int(payload.get("total_tokens")), + ), + finish_reasons=finish_reasons, + error=_parse_error(payload), + response_cost=as_float(payload.get("response_cost")), + server=ServerInfo.from_api_base(context.api_base), + identity=context.identity, + is_streaming=as_bool(payload.get("stream")), + tools=_extract_tools(params), + messages_in=_dicts(payload.get("messages")) if capture_content else (), + choices_out=choices_out if capture_content else (), + system_fingerprint=as_str(response.get("system_fingerprint")), + ) + + +# --- service event_metadata sanitization ------------------------------------ # + +# Substrings (case-insensitive) of keys that must never reach a span: secrets, +# tokens, and raw request/response dumps the legacy service decorators pass. +_SENSITIVE_METADATA_SUBSTRINGS: tuple[str, ...] = ( + "api_key", + "token", + "secret", + "password", + "cookie", + "authorization", + "header", + "hidden_params", +) +# Keys that carry raw call-site internals — live objects, full kwargs/args. The +# operation name is already the span's ``call_type``, so ``function_name`` is +# redundant. +_DROP_METADATA_KEYS: frozenset = frozenset( + {"function_kwargs", "function_args", "function_name"} +) +_MAX_METADATA_VALUE_LEN = 1024 +_MAX_METADATA_ITEMS = 32 + + +def sanitize_event_metadata( + event_metadata: Mapping[str, object] | None, +) -> dict[str, str]: + """Reduce caller-supplied ``event_metadata`` to span-safe string attributes. + + Keeps only primitive values (str/int/float/bool) under non-sensitive keys — + never ``repr()``-ing objects, dicts, or lists, never stamping secrets/headers, + and bounding the count and per-value length. This is the single chokepoint: + both the GenAI and legacy mappers read the cleaned result. + """ + if not event_metadata: + return {} + clean: dict[str, str] = {} + for key, value in event_metadata.items(): + if len(clean) >= _MAX_METADATA_ITEMS: + break + if not isinstance(key, str) or key in _DROP_METADATA_KEYS: + continue + lowered = key.lower() + if any(token in lowered for token in _SENSITIVE_METADATA_SUBSTRINGS): + continue + # ``bool`` is a subclass of ``int``, so it's covered. Non-primitive values + # (objects, dicts, lists) are dropped rather than stringified. + if isinstance(value, (str, int, float)): + clean[key] = str(value)[:_MAX_METADATA_VALUE_LEN] + return clean + + +def _json_or_none(value: object) -> str | None: + """JSON-serialize ``value`` (already-string values pass through). ``None`` on failure.""" + if isinstance(value, str): + return value + try: + return json.dumps(value, default=str) + except Exception: + return None + + +def _guardrail_mode_str(value: object) -> str | None: + """Normalize ``guardrail_mode`` to a stable string. + + ``guardrail_mode`` is typed as a ``GuardrailEventHooks`` enum, a list of them, + or a ``GuardrailMode`` — not a plain string. Emit the enum *value* (e.g. + ``"pre_call"``) rather than ``str(enum)`` (``"GuardrailEventHooks.pre_call"``), + and join a list of modes so a guardrail that runs at multiple hooks is + represented faithfully. + """ + if value is None: + return None + if isinstance(value, (list, tuple)): + parts: list[str] = [] + for item in value: + if item is None: + continue + part = as_str(item.value) if isinstance(item, Enum) else as_str(item) + if part: + parts.append(part) + return ",".join(parts) or None + if isinstance(value, Enum): + return as_str(value.value) + return as_str(value) + + +def _total_masked_entities(value: object) -> int | None: + """``masked_entity_count`` is a ``{entity_type: count}`` map — sum to a total.""" + if isinstance(value, Mapping): + total = sum(v for v in value.values() if isinstance(v, int)) + return total or None + return as_int(value) + + +def _dicts(value: object) -> tuple[Mapping[str, object], ...]: + """The dict items of ``value`` (when it's a list), as a tuple. Else empty.""" + if not isinstance(value, list): + return () + return tuple(item for item in value if isinstance(item, dict)) + + +def _finish_reasons(choices: tuple[Mapping[str, object], ...]) -> tuple[str, ...]: + """Non-empty ``finish_reason`` of each response choice.""" + return tuple(r for c in choices if (r := as_str(c.get("finish_reason")))) + + +def _parse_error(payload: "StandardLoggingPayload") -> SpanError | None: + """A ``SpanError`` for a failed request, or ``None`` on success.""" + if payload.get("status") != "failure": + return None + info = cast(Mapping[str, object], payload.get("error_information") or {}) + return SpanError( + error_type=as_str(info.get("error_class")) or as_str(info.get("error_code")), + message=as_str(info.get("error_message")) or as_str(payload.get("error_str")), + ) + + +def _tool_from_entry(entry: object) -> ToolDefinition | None: + """One ``tools``/``functions`` entry → ``ToolDefinition``, or ``None`` if unusable.""" + if not isinstance(entry, dict): + return None + fn = entry.get("function") if "function" in entry else entry + if not isinstance(fn, dict): + return None + name = as_str(fn.get("name")) + if not name: + return None + params = fn.get("parameters") + parameters_json: str | None = None + if params is not None: + try: + parameters_json = json.dumps(params, default=str) + except Exception: + parameters_json = None + return ToolDefinition( + name=name, + description=as_str(fn.get("description")), + parameters_json=parameters_json, + ) + + +def _extract_tools( + model_parameters: Mapping[str, object], +) -> tuple[ToolDefinition, ...]: + """Pull declared tools from request params (OpenAI / Anthropic shape). + + Accepts the chat-completion ``tools=[{"type":"function", "function": + {...}}, ...]`` shape, and falls back to the ``functions=[...]`` shape. + Returns an empty tuple when neither is present. + """ + raw_tools = model_parameters.get("tools") + if not isinstance(raw_tools, list): + raw_tools = model_parameters.get("functions") # ``functions`` shape + if not isinstance(raw_tools, list): + return () + return tuple(t for entry in raw_tools if (t := _tool_from_entry(entry)) is not None) diff --git a/litellm/integrations/otel/model/semconv.py b/litellm/integrations/otel/model/semconv.py new file mode 100644 index 00000000000..1c6c30eda0d --- /dev/null +++ b/litellm/integrations/otel/model/semconv.py @@ -0,0 +1,201 @@ +""" +Keys follow the OpenTelemetry GenAI semantic conventions (experimental). Anything +without a semconv equivalent lives under the ``litellm.*`` vendor namespace. +""" + +from enum import Enum +from typing import Final + + +class GenAIOperation(str, Enum): + """Values for ``gen_ai.operation.name``.""" + + CHAT = "chat" + TEXT_COMPLETION = "text_completion" + EMBEDDINGS = "embeddings" + GENERATE_CONTENT = "generate_content" + CREATE_AGENT = "create_agent" # reserved for future agent spans + INVOKE_AGENT = "invoke_agent" # reserved for future agent spans + EXECUTE_TOOL = "execute_tool" # reserved for future tool spans + + +class GenAIProvider(str, Enum): + """Common values for the ``gen_ai.provider.name`` attribute.""" + + OPENAI = "openai" + ANTHROPIC = "anthropic" + AWS_BEDROCK = "aws.bedrock" + AZURE_AI_OPENAI = "azure.ai.openai" + AZURE_AI_INFERENCE = "azure.ai.inference" + GCP_GEMINI = "gcp.gemini" + GCP_VERTEX_AI = "gcp.vertex_ai" + COHERE = "cohere" + MISTRAL_AI = "mistral_ai" + DEEPSEEK = "deepseek" + GROQ = "groq" + PERPLEXITY = "perplexity" + X_AI = "x_ai" + IBM_WATSONX_AI = "ibm.watsonx.ai" + + +class GenAI: + """Canonical OTel GenAI span-attribute keys.""" + + # request + OPERATION_NAME: Final = "gen_ai.operation.name" + PROVIDER_NAME: Final = "gen_ai.provider.name" + REQUEST_MODEL: Final = "gen_ai.request.model" + REQUEST_TEMPERATURE: Final = "gen_ai.request.temperature" + REQUEST_TOP_P: Final = "gen_ai.request.top_p" + REQUEST_TOP_K: Final = "gen_ai.request.top_k" + REQUEST_MAX_TOKENS: Final = "gen_ai.request.max_tokens" + REQUEST_FREQUENCY_PENALTY: Final = "gen_ai.request.frequency_penalty" + REQUEST_PRESENCE_PENALTY: Final = "gen_ai.request.presence_penalty" + REQUEST_STOP_SEQUENCES: Final = "gen_ai.request.stop_sequences" + REQUEST_SEED: Final = "gen_ai.request.seed" + REQUEST_CHOICE_COUNT: Final = "gen_ai.request.choice.count" + REQUEST_ENCODING_FORMATS: Final = "gen_ai.request.encoding_formats" + # response + RESPONSE_ID: Final = "gen_ai.response.id" + RESPONSE_MODEL: Final = "gen_ai.response.model" + RESPONSE_FINISH_REASONS: Final = "gen_ai.response.finish_reasons" + # usage + USAGE_INPUT_TOKENS: Final = "gen_ai.usage.input_tokens" + USAGE_OUTPUT_TOKENS: Final = "gen_ai.usage.output_tokens" + # content (opt-in, gated by capture mode) + INPUT_MESSAGES: Final = "gen_ai.input.messages" + OUTPUT_MESSAGES: Final = "gen_ai.output.messages" + SYSTEM_INSTRUCTIONS: Final = "gen_ai.system_instructions" + OUTPUT_TYPE: Final = "gen_ai.output.type" + CONVERSATION_ID: Final = "gen_ai.conversation.id" + # agent / tool (reserved) + AGENT_ID: Final = "gen_ai.agent.id" + AGENT_NAME: Final = "gen_ai.agent.name" + TOOL_NAME: Final = "gen_ai.tool.name" + TOOL_CALL_ID: Final = "gen_ai.tool.call.id" + + +class Error: + TYPE: Final = "error.type" + + +class Server: + ADDRESS: Final = "server.address" + PORT: Final = "server.port" + + +class DB: + """Database / cache client-span keys (OTel ``db.*`` semconv). + + Stamped on ``DB_CALL`` spans (redis / postgres), which are CLIENT spans for + outbound datastore calls — not on the INTERNAL ``SERVICE`` spans. + """ + + SYSTEM_NAME: Final = "db.system.name" + OPERATION_NAME: Final = "db.operation.name" + + +class HTTP: + """HTTP server-span keys. Belong on the SERVER span only (never promoted).""" + + REQUEST_METHOD: Final = "http.request.method" + ROUTE: Final = "http.route" + RESPONSE_STATUS_CODE: Final = "http.response.status_code" + URL_PATH: Final = "url.path" + + +class LiteLLM: + """Vendor-extension keys (no semconv equivalent). Always ``litellm.*``.""" + + CALL_ID: Final = "litellm.call_id" + COST_PREFIX: Final = "litellm.cost." + METADATA_PREFIX: Final = "litellm.metadata." + TEAM_ID: Final = "litellm.team.id" + TEAM_ALIAS: Final = "litellm.team.alias" + # The team's free-form metadata dict, JSON-serialized into a single value. + TEAM_METADATA: Final = "litellm.team.metadata" + KEY_HASH: Final = "litellm.api_key.hash" + END_USER: Final = "litellm.end_user.id" + # The model string litellm actually sent to the provider (the deployment's + # ``litellm_params.model``), distinct from the user-facing ``gen_ai.request.model``. + PROVIDER_MODEL: Final = "litellm.provider.model" + REQUEST_STREAMING: Final = "litellm.request.streaming" + GUARDRAIL_NAME: Final = "litellm.guardrail.name" + GUARDRAIL_MODE: Final = "litellm.guardrail.mode" + GUARDRAIL_STATUS: Final = "litellm.guardrail.status" + GUARDRAIL_PROVIDER: Final = "litellm.guardrail.provider" + GUARDRAIL_ACTION: Final = "litellm.guardrail.action" + GUARDRAIL_RESPONSE: Final = "litellm.guardrail.response" + GUARDRAIL_VIOLATION_CATEGORIES: Final = "litellm.guardrail.violation_categories" + GUARDRAIL_CONFIDENCE_SCORE: Final = "litellm.guardrail.confidence_score" + GUARDRAIL_RISK_SCORE: Final = "litellm.guardrail.risk_score" + GUARDRAIL_MASKED_ENTITY_COUNT: Final = "litellm.guardrail.masked_entity_count" + GUARDRAIL_DURATION: Final = "litellm.guardrail.duration" + GUARDRAIL_ID: Final = "litellm.guardrail.id" + GUARDRAIL_POLICY_TEMPLATE: Final = "litellm.guardrail.policy_template" + GUARDRAIL_DETECTION_METHOD: Final = "litellm.guardrail.detection_method" + SERVICE_NAME: Final = "litellm.service.name" + SERVICE_CALL_TYPE: Final = "litellm.service.call_type" + PREPROCESSING_MS: Final = "litellm.preprocessing.duration_ms" + + +class Metric: + """GenAI metric instrument names.""" + + TOKEN_USAGE: Final = "gen_ai.client.token.usage" + OPERATION_DURATION: Final = "gen_ai.client.operation.duration" + + +# litellm ``custom_llm_provider`` -> ``gen_ai.provider.name`` value. +_PROVIDER_BY_LITELLM: dict[str, GenAIProvider] = { + "openai": GenAIProvider.OPENAI, + "text-completion-openai": GenAIProvider.OPENAI, + "azure": GenAIProvider.AZURE_AI_OPENAI, + "azure_ai": GenAIProvider.AZURE_AI_INFERENCE, + "anthropic": GenAIProvider.ANTHROPIC, + "bedrock": GenAIProvider.AWS_BEDROCK, + "bedrock_converse": GenAIProvider.AWS_BEDROCK, + "vertex_ai": GenAIProvider.GCP_VERTEX_AI, + "vertex_ai_beta": GenAIProvider.GCP_VERTEX_AI, + "gemini": GenAIProvider.GCP_GEMINI, + "cohere": GenAIProvider.COHERE, + "cohere_chat": GenAIProvider.COHERE, + "mistral": GenAIProvider.MISTRAL_AI, + "deepseek": GenAIProvider.DEEPSEEK, + "groq": GenAIProvider.GROQ, + "perplexity": GenAIProvider.PERPLEXITY, + "xai": GenAIProvider.X_AI, + "watsonx": GenAIProvider.IBM_WATSONX_AI, +} + +# litellm ``call_type`` -> ``gen_ai.operation.name``. +_OPERATION_BY_CALL_TYPE: dict[str, GenAIOperation] = { + "completion": GenAIOperation.CHAT, + "acompletion": GenAIOperation.CHAT, + "completion_with_retries": GenAIOperation.CHAT, + "text_completion": GenAIOperation.TEXT_COMPLETION, + "atext_completion": GenAIOperation.TEXT_COMPLETION, + "embedding": GenAIOperation.EMBEDDINGS, + "aembedding": GenAIOperation.EMBEDDINGS, + "responses": GenAIOperation.CHAT, + "aresponses": GenAIOperation.CHAT, +} + + +def resolve_provider(custom_llm_provider: str | None) -> str: + """Map a litellm provider string to a ``gen_ai.provider.name`` value. + + Unknown providers pass through verbatim — the convention explicitly allows + provider-specific values, so an unmapped name is still valid. + """ + if not custom_llm_provider: + return "" + mapped = _PROVIDER_BY_LITELLM.get(custom_llm_provider.lower()) + return mapped.value if mapped is not None else custom_llm_provider + + +def resolve_operation(call_type: str | None) -> GenAIOperation: + """Map a litellm ``call_type`` to a ``gen_ai.operation.name`` value.""" + if not call_type: + return GenAIOperation.CHAT + return _OPERATION_BY_CALL_TYPE.get(call_type.lower(), GenAIOperation.CHAT) diff --git a/litellm/integrations/otel/model/spans.py b/litellm/integrations/otel/model/spans.py new file mode 100644 index 00000000000..e4876f4ee58 --- /dev/null +++ b/litellm/integrations/otel/model/spans.py @@ -0,0 +1,203 @@ +""" +This module declares every span the instrumentation can emit and the hierarchy. + +Span-name patterns live here as typed builder functions. + +Canonical hierarchy:: + + PROXY_REQUEST (SERVER, root) # owned by the FastAPI instrumentor + ├── SERVICE (INTERNAL) # auth phase span (live; see logger.phase_span) + │ └── DB_CALL (CLIENT) # its key/user/team lookups nest here + ├── GUARDRAIL (INTERNAL) # request-lifecycle hook, sibling of LLM_CALL + ├── LLM_CALL (CLIENT) + └── DB_CALL (CLIENT) # e.g. the spend-log write + +Guardrails parent to PROXY_REQUEST, not LLM_CALL: pre/during/post-call guardrail +hooks are orchestrated by the request lifecycle (a pre-call guardrail runs +before the LLM call even starts), so a guardrail is a sibling of the LLM call, +not a child of it. The emitter parents every span to the ambient OTel context +(the active server span), which matches this. + +Not every service call becomes a span — :func:`span_role_for_service` decides: + +- ``DB_CALL`` (CLIENT) — outbound datastores (redis, postgres, + ``batch_write_to_db``), carrying ``db.*`` semconv. +- ``SERVICE`` (INTERNAL) — genuine internal work worth a span (background + budget/reset jobs, pod-lock manager). +- ``None`` (metrics-only) — framework instrumentation that duplicates a gen-AI + span (``self`` = the ``track_llm_api_timing`` wrapper, ``router``, + ``proxy_pre_call``) or ``auth`` (which gets a live phase span instead). These + still feed Prometheus/Datadog; they just never enter the trace. + +``DB_CALL`` and ``SERVICE`` are built from the same ``ServiceSpanData``; only the +role (hence span kind and attribute vocabulary) differs. A service call can fire +outside any request (a background job), in which case it parents to no server +span and starts its own root trace rather than being dropped. + +Management/admin endpoints are ordinary FastAPI routes — their SERVER spans are +owned by the instrumentor too, so they don't appear as a role here. +""" + +from dataclasses import dataclass +from enum import Enum +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from litellm.integrations.otel.model.payloads import ( + GuardrailSpanData, + LLMCallSpanData, + ProxyRequestSpanData, + ServiceSpanData, + ) + + +class SpanRole(str, Enum): + PROXY_REQUEST = "proxy_request" + LLM_CALL = "llm_call" + GUARDRAIL = "guardrail" + DB_CALL = "db_call" + SERVICE = "service" + + +class LiteLLMSpanKind(str, Enum): + SERVER = "server" + CLIENT = "client" + INTERNAL = "internal" + PRODUCER = "producer" + CONSUMER = "consumer" + + +@dataclass(frozen=True) +class SpanSpec: + role: SpanRole + kind: LiteLLMSpanKind + parent: SpanRole | None + + +SPAN_REGISTRY: dict[SpanRole, SpanSpec] = { + SpanRole.PROXY_REQUEST: SpanSpec( + SpanRole.PROXY_REQUEST, LiteLLMSpanKind.SERVER, parent=None + ), + SpanRole.LLM_CALL: SpanSpec( + SpanRole.LLM_CALL, LiteLLMSpanKind.CLIENT, parent=SpanRole.PROXY_REQUEST + ), + SpanRole.GUARDRAIL: SpanSpec( + SpanRole.GUARDRAIL, LiteLLMSpanKind.INTERNAL, parent=SpanRole.PROXY_REQUEST + ), + SpanRole.DB_CALL: SpanSpec( + SpanRole.DB_CALL, LiteLLMSpanKind.CLIENT, parent=SpanRole.PROXY_REQUEST + ), + SpanRole.SERVICE: SpanSpec( + SpanRole.SERVICE, LiteLLMSpanKind.INTERNAL, parent=SpanRole.PROXY_REQUEST + ), +} + + +# ``ServiceTypes`` value -> ``db.system.name``. These are outbound datastore +# calls and become CLIENT ``DB_CALL`` spans; ``redis_``-prefixed names cover the +# redis-backed spend queues. Any service not mapped here is litellm-internal work +# and stays an INTERNAL ``SERVICE`` span. This table is the single source of +# datastore knowledge — both the role classifier and the mapper read it. +_DB_SYSTEM_BY_SERVICE: dict[str, str] = { + "redis": "redis", + "postgres": "postgresql", + "batch_write_to_db": "postgresql", +} + + +def db_system(service_name: str) -> str | None: + """The ``db.system.name`` for a datastore service, else ``None``. + + ``None`` means the service is not an outbound datastore call. Redis-backed + spend queues (``redis_*``) map to ``redis``. + """ + if service_name in _DB_SYSTEM_BY_SERVICE: + return _DB_SYSTEM_BY_SERVICE[service_name] + if service_name.startswith("redis_"): + return "redis" + return None + + +# ``ServiceTypes`` values that are NOT emitted as spans — they are framework +# instrumentation that either duplicates a gen-AI span or has a better home as a +# Prometheus/Datadog metric. They still flow to those metric backends via their +# own hooks; the v2 logger just does not put them in the trace: +# +# - ``self`` — ``track_llm_api_timing`` wraps the LLM call; the +# ``chat {model}`` CLIENT span already represents it. +# - ``router`` — wraps the whole request; duplicates the server span. +# - ``proxy_pre_call`` — per-callback pre-call timing; a guardrail's real span +# is ``execute_guardrail {name}``. +# - ``auth`` — emitted instead as a live phase span (see +# ``logger.phase_span``) so its DB lookups nest under it, +# not as a flat post-hoc service span. +_METRICS_ONLY_SERVICES: frozenset[str] = frozenset( + {"self", "router", "proxy_pre_call", "auth"} +) + + +def span_role_for_service(service_name: str) -> SpanRole | None: + """The span role for a service call, or ``None`` when it must not be a span. + + ``DB_CALL`` for outbound datastores, ``SERVICE`` for genuine internal work + worth a span (background jobs), and ``None`` for framework instrumentation + that duplicates a gen-AI span or belongs in metrics only + (see ``_METRICS_ONLY_SERVICES``). + """ + if service_name in _METRICS_ONLY_SERVICES: + return None + return SpanRole.DB_CALL if db_system(service_name) is not None else SpanRole.SERVICE + + +# --- span name builders (the naming convention, per role) ------------------- # + + +# The name the FastAPI instrumentor gives the root server span. V2 never creates +# this span (the instrumentor owns it), but it anchors request-level spans to it +# and tests assert against it by name, so the literal lives here with the rest of +# the span vocabulary rather than being duplicated at each call site. +LITELLM_PROXY_REQUEST_SPAN_NAME = "Received Proxy Server Request" + + +def llm_call_span_name(data: "LLMCallSpanData") -> str: + """``"{operation} {model}"`` e.g. ``"chat gpt-4o"`` (GenAI semconv).""" + model = data.request_model or "" + return f"{data.operation.value} {model}".strip() + + +def proxy_request_span_name(data: "ProxyRequestSpanData") -> str: + """``"{method} {route}"`` (HTTP semconv).""" + return f"{data.http_method} {data.route}".strip() + + +def guardrail_span_name(data: "GuardrailSpanData") -> str: + return f"execute_guardrail {data.guardrail_name}".strip() + + +def service_span_name(data: "ServiceSpanData") -> str: + """``"{service} {call_type}"`` e.g. ``"redis set"`` — service name alone when + no call type is known, so identically-named calls stay distinguishable.""" + return f"{data.service_name} {data.call_type or ''}".strip() + + +def root_roles() -> list[SpanRole]: + """Roles that start a new trace (no in-process parent).""" + return [role for role, spec in SPAN_REGISTRY.items() if spec.parent is None] + + +def child_roles(parent: SpanRole) -> list[SpanRole]: + return [role for role, spec in SPAN_REGISTRY.items() if spec.parent == parent] + + +def validate_registry( + registry: dict[SpanRole, SpanSpec] | None = None, +) -> None: + reg = registry if registry is not None else SPAN_REGISTRY + for role, spec in reg.items(): + if spec.role is not role: + raise ValueError(f"SPAN_REGISTRY[{role}] has mismatched role {spec.role}") + if spec.parent is not None and spec.parent not in reg: + raise ValueError(f"span role {role} declares unknown parent {spec.parent}") + missing = [role for role in SpanRole if role not in reg] + if missing: + raise ValueError(f"SPAN_REGISTRY is missing roles: {missing}") diff --git a/litellm/integrations/otel/model/utils.py b/litellm/integrations/otel/model/utils.py new file mode 100644 index 00000000000..f37afc97879 --- /dev/null +++ b/litellm/integrations/otel/model/utils.py @@ -0,0 +1,103 @@ +"""Shared, OpenTelemetry-free helpers for the otel integration. + +Generic value coercion (for reading heterogeneous logging-payload dicts), time +conversion, and header parsing — pulled out of the individual modules so they +live in one place. Deliberately free of any ``opentelemetry`` import so the +OTel-free sources of truth (payloads, semconv, spans, config) can use it too. +""" + +from datetime import datetime + + +def as_str(value: object) -> str | None: + if value is None: + return None + if isinstance(value, str): + return value + return str(value) + + +def as_int(value: object) -> int | None: + if isinstance(value, bool): + return int(value) + if isinstance(value, int): + return value + if isinstance(value, float): + return int(value) + if isinstance(value, str): + try: + return int(value) + except ValueError: + return None + return None + + +def as_float(value: object) -> float | None: + if isinstance(value, bool): + return float(value) + if isinstance(value, (int, float)): + return float(value) + if isinstance(value, str): + try: + return float(value) + except ValueError: + return None + return None + + +def as_bool(value: object) -> bool | None: + if value is None: + return None + if isinstance(value, bool): + return value + return bool(value) + + +def as_str_tuple(value: object) -> tuple[str, ...] | None: + if value is None: + return None + if isinstance(value, str): + return (value,) + if isinstance(value, (list, tuple)): + return tuple(str(v) for v in value) + return None + + +def to_ns(value: datetime | float | int | None) -> int | None: + """Coerce a datetime / epoch value to integer nanoseconds.""" + if value is None: + return None + if isinstance(value, datetime): + return int(value.timestamp() * 1e9) + if isinstance(value, (int, float)) and not isinstance(value, bool): + return int(float(value) * 1e9) + return None + + +def to_seconds(value: datetime | float | int | str | None) -> float | None: + """Coerce a datetime / epoch / formatted-string value to epoch seconds.""" + if value is None: + return None + if isinstance(value, datetime): + return value.timestamp() + if isinstance(value, (int, float)) and not isinstance(value, bool): + return float(value) + if isinstance(value, str): + for fmt in ("%Y-%m-%d %H:%M:%S.%f", "%Y-%m-%d %H:%M:%S"): + try: + return datetime.strptime(value, fmt).timestamp() + except ValueError: + continue + return None + + +def parse_headers(raw: str | None) -> dict[str, str]: + """Parse an OTLP ``"k=v,k=v"`` header string into a dict.""" + headers: dict[str, str] = {} + if not raw: + return headers + for pair in raw.split(","): + if "=" in pair: + key, _, value = pair.partition("=") + headers[key.strip()] = value.strip() + return headers diff --git a/litellm/integrations/otel/mount.py b/litellm/integrations/otel/mount.py new file mode 100644 index 00000000000..ebcf4aa35af --- /dev/null +++ b/litellm/integrations/otel/mount.py @@ -0,0 +1,130 @@ +"""FastAPI server-span instrumentation — the proxy mounts this at app creation. + +``opentelemetry-instrumentation-fastapi`` creates the SERVER span for each HTTP +route and extracts inbound ``traceparent`` headers. This module owns the one call +site that attaches it to the proxy app, plus the passthrough span-naming hook, so +``proxy_server`` stays free of OTel details. + +The ``FastAPIInstrumentor`` import is kept lazy (inside :func:`instrument_fastapi_app`, +after the gate check) so importing this module never requires the optional +``opentelemetry-instrumentation-fastapi`` package and pulls in nothing OTel-related +when the feature gate is off. +""" + +import os +from typing import Any + +from litellm._logging import verbose_logger +from litellm.integrations.otel.model.config import is_otel_v2_enabled + +# Routes excluded from server-span tracing by default: high-frequency pollers and +# static UI/docs assets, none of which are LLM traffic. Entries are substring-matched +# against the request path (unanchored, so they survive a ``server_root_path`` prefix +# and each entry also covers everything beneath it — e.g. ``/health`` covers +# ``/health/readiness``). Operators override the whole set via the standard +# ``OTEL_PYTHON_FASTAPI_EXCLUDED_URLS`` env var (set "" to trace everything). +_DEFAULT_EXCLUDED_ROUTES = ( + "/health", # load-balancer liveness/readiness polling + "/metrics", # Prometheus scrape (also drops the /model/metrics admin analytics) + "/litellm-asset-prefix", # hashed UI asset bundles + "/_next", # Next.js static JS/CSS chunks (root-level mount) + "/ui", # admin UI single-page app + "/swagger", # static Swagger UI assets + "/docs", # FastAPI Swagger docs page + "/redoc", # FastAPI ReDoc docs page + "/openapi.json", # OpenAPI schema + "favicon", # /favicon.ico + /get_favicon + "/.well-known", # UI config discovery +) +_DEFAULT_EXCLUDED_URLS = ",".join(_DEFAULT_EXCLUDED_ROUTES) + +# Passthrough routes are catch-alls (e.g. "/openai/{endpoint:path}"), so the +# default OTel server-span name "{method} {route}" collapses every upstream +# endpoint into "POST /openai/{endpoint:path}". The hook below renames those spans +# to the real request path so each endpoint is distinguishable. Non-catch-all +# routes keep their low-cardinality template name. +PASSTHROUGH_PREFIXES = frozenset( + { + "openai", + "openai_passthrough", + "anthropic", + "azure", + "azure_ai", + "bedrock", + "cohere", + "cursor", + "gemini", + "mistral", + "vllm", + "vertex_ai", + "vertex-ai", + "assemblyai", + "eu.assemblyai", + "milvus", + } +) + + +def _passthrough_span_name_hook(span: Any, scope: dict) -> None: + """FastAPI ``server_request_hook``: give passthrough server spans a useful name. + + The instrumentation matches the route at span creation, so both the span name + and ``http.route`` are set to the catch-all template (``/openai/{endpoint:path}``) + before this hook runs. Rewrite both to the real request path so each upstream + endpoint is distinguishable. (The ASGI ``http receive``/``http send`` sub-spans + can't be renamed from here — their name is captured at creation — so they are + dropped via ``exclude_spans`` at instrumentation time.) + """ + try: + if span is None or not span.is_recording(): + return + path = scope.get("path") or "" + method = scope.get("method") or "" + first_segment = path.lstrip("/").split("/", 1)[0] + if first_segment in PASSTHROUGH_PREFIXES: + span.update_name(f"{method} {path}".strip()) + span.set_attribute("http.route", path) + except Exception: + pass + + +def instrument_fastapi_app(app: Any) -> None: + """Attach OTel server-span instrumentation to the proxy FastAPI app. + + Safe no-op when the V2 gate is off or ``opentelemetry-instrumentation-fastapi`` + is unavailable. This MUST be called at app-creation time — once the lifespan + runs, the middleware stack is frozen and ``instrument_app`` raises "Cannot add + middleware after an application has started". + + No ``TracerProvider`` is passed, so the instrumentation binds to the OTel global + ``ProxyTracerProvider``; the proxy publishes the real provider as the global + after config load (see ``proxy_startup_event``), and the proxy delegates to it. + That way server spans and gen-ai spans share one provider and the same trace. + """ + try: + if not is_otel_v2_enabled(): + return + + # Lazy: only the V2-enabled path needs the optional + # ``opentelemetry-instrumentation-fastapi`` package, which is not part of the + # base ``litellm[proxy]`` install. Importing it at module top would make + # ``proxy_server``'s unconditional ``import`` of this module crash when the + # package is absent, even with the gate off. + from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor + + excluded_urls = ( + os.environ.get("OTEL_PYTHON_FASTAPI_EXCLUDED_URLS") + if "OTEL_PYTHON_FASTAPI_EXCLUDED_URLS" in os.environ + else _DEFAULT_EXCLUDED_URLS + ) + FastAPIInstrumentor.instrument_app( + app, + excluded_urls=excluded_urls, + server_request_hook=_passthrough_span_name_hook, + # Drop the ASGI "http receive"/"http send" lifecycle sub-spans: they + # are low-value noise and (for passthrough) carry the catch-all route + # template in their name, which can't be rewritten from a hook. + exclude_spans=["receive", "send"], + ) + except Exception as e: + verbose_logger.debug("Skipping OTel V2 FastAPI instrumentation: %s", e) diff --git a/litellm/integrations/otel/plumbing/__init__.py b/litellm/integrations/otel/plumbing/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/integrations/otel/plumbing/context.py b/litellm/integrations/otel/plumbing/context.py new file mode 100644 index 00000000000..64790da814b --- /dev/null +++ b/litellm/integrations/otel/plumbing/context.py @@ -0,0 +1,127 @@ +"""Trace-context + Baggage helpers.""" + +from contextvars import ContextVar +from typing import Mapping + +from opentelemetry import baggage +from opentelemetry.context import Context, get_current +from opentelemetry.trace import Span, get_current_span, set_span_in_context +from opentelemetry.trace.propagation.tracecontext import ( + TraceContextTextMapPropagator, +) + +_PROPAGATOR = TraceContextTextMapPropagator() + +# The request's root span — the FastAPI-owned SERVER span — captured ONCE when the +# proxy first resolves it, so request-level spans (the LLM call, guardrails) can +# parent to it EXPLICITLY instead of to whatever span happens to be active at the +# instant they are emitted. Ambient-only parenting (``get_current_span()``) is +# wrong at two boundaries: +# * inside the ``auth`` phase span the active span is the auth span, so an LLM / +# guardrail span emitted there would nest under auth instead of being its +# sibling; and +# * in a detached success task (pass-through logs success from a fire-and-forget +# ``asyncio.create_task``) the server span may not be active at all, orphaning +# the span into a brand-new trace. +# A ``ContextVar`` (not a request attribute) so it rides the request task's context +# and is inherited by ``asyncio.create_task`` children — i.e. the async logging +# callbacks that close the span. It is never reset: the contextvar dies with the +# request task, so there is nothing to leak. +_request_root_span: "ContextVar[Span | None]" = ContextVar( + "litellm_otel_request_root_span", default=None +) + + +def set_request_root_span(span: Span) -> None: + """Anchor the request's root (server) span for explicit child parenting. + + No-ops for a non-recordable span so a bad capture can never replace a good one + with a phantom parent. Idempotent — the proxy captures the same server span at + more than one entry point. + """ + if is_recordable_span(span): + _request_root_span.set(span) + + +def request_root_span() -> "Span | None": + """The anchored request root span, or ``None`` outside a proxy request.""" + span = _request_root_span.get() + return span if is_recordable_span(span) else None + + +def set_request_baggage( + values: Mapping[str, str], context: Context | None = None +) -> Context: + """Return a context with ``values`` written into Baggage.""" + ctx = context + for key, value in values.items(): + ctx = baggage.set_baggage(key, value, context=ctx) + return ctx if ctx is not None else (context or get_current()) + + +def get_baggage_attributes(context: Context | None = None) -> dict[str, str]: + """All Baggage entries on ``context`` as strings.""" + return {key: str(value) for key, value in baggage.get_all(context).items()} + + +def context_from_span(span: Span, context: Context | None = None) -> Context: + """A context with ``span`` as the active span (for explicit parenting).""" + return set_span_in_context(span, context=context) + + +def resolve_parent_context(threaded: Span | None = None) -> Context: + """The context a child span should parent under. + + Ambient-first: parent to the active OTel context (the server span, restored + by the logging worker or active in the request task), falling back to a span + passed explicitly (``threaded``) only when the ambient context has no + recordable span — e.g. a background service call with no request on the + stack. When neither is recordable the ambient context is returned unchanged, + so the span starts a new root trace. + + Only service/DB spans pass ``threaded`` (the ``parent_otel_span`` handed to + the service hook). Request-level spans — the LLM call and guardrails — are + created where the server span is genuinely ambient, so they never need it. + """ + ctx = get_current() + if is_recordable_span(threaded) and not is_recordable_span(get_current_span(ctx)): + ctx = context_from_span(threaded, context=ctx) # type: ignore[arg-type] + return ctx + + +def resolve_request_span_context() -> Context: + """The parent context for a request-level span (the LLM call, a guardrail). + + These are direct children of the request's root server span — siblings of the + ``auth`` phase span and of each other, never nested under whatever span is + momentarily active. So prefer the explicitly anchored root span; fall back to + ambient context only when there is no anchor (the SDK / no-proxy path), where + the span legitimately starts its own root trace. + + Unlike :func:`resolve_parent_context` (used by DB/service spans, which DO want + to nest under the active phase span, e.g. an auth DB lookup under ``auth``), + this never returns the active span when an anchor exists. + """ + root = request_root_span() + if root is not None: + return context_from_span(root) + return get_current() + + +def is_recordable_span(obj: object) -> bool: + """True if ``obj`` is a live span with a valid context (safe to parent under).""" + if not isinstance(obj, Span): + return False + try: + ctx = obj.get_span_context() + except Exception: + return False + return ctx is not None and ctx.is_valid + + +def extract_traceparent(headers: Mapping[str, str]) -> Context | None: + """Extract a remote parent context from incoming HTTP headers, if present.""" + if not any(key.lower() == "traceparent" for key in headers): + return None + carrier = {str(key).lower(): value for key, value in headers.items()} + return _PROPAGATOR.extract(carrier) diff --git a/litellm/integrations/otel/plumbing/metrics.py b/litellm/integrations/otel/plumbing/metrics.py new file mode 100644 index 00000000000..edd120f91e6 --- /dev/null +++ b/litellm/integrations/otel/plumbing/metrics.py @@ -0,0 +1,28 @@ +"""GenAI client metrics (token usage + operation duration histograms).""" + +from dataclasses import dataclass + +from opentelemetry.metrics import Histogram, Meter + +from litellm.integrations.otel.model.semconv import Metric + + +@dataclass(frozen=True) +class GenAIMetrics: + token_usage: Histogram + operation_duration: Histogram + + +def create_genai_metrics(meter: Meter) -> GenAIMetrics: + return GenAIMetrics( + token_usage=meter.create_histogram( + name=Metric.TOKEN_USAGE, + unit="{token}", + description="Number of tokens used per GenAI request.", + ), + operation_duration=meter.create_histogram( + name=Metric.OPERATION_DURATION, + unit="s", + description="GenAI operation duration.", + ), + ) diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py new file mode 100644 index 00000000000..40a0e41b905 --- /dev/null +++ b/litellm/integrations/otel/plumbing/providers.py @@ -0,0 +1,220 @@ +"""Provider / exporter factory + the Baggage span processor.""" + +from typing import Callable, Iterable + +from opentelemetry import baggage +from opentelemetry.context import Context +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import ReadableSpan, SpanProcessor, TracerProvider +from opentelemetry.sdk.trace.export import ( + BatchSpanProcessor, + ConsoleSpanExporter, + SimpleSpanProcessor, + SpanExporter, +) +from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, +) +from opentelemetry.trace import Span, SpanKind, Tracer + +from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config +from litellm.integrations.otel.model.semconv import LiteLLM +from litellm.integrations.otel.model.spans import LiteLLMSpanKind + +# Re-exported so ``providers.parse_headers`` remains a stable entry point. +from litellm.integrations.otel.model.utils import parse_headers as parse_headers + +_SPAN_KIND_BY_ROLE_KIND: dict[LiteLLMSpanKind, SpanKind] = { + LiteLLMSpanKind.SERVER: SpanKind.SERVER, + LiteLLMSpanKind.CLIENT: SpanKind.CLIENT, + LiteLLMSpanKind.INTERNAL: SpanKind.INTERNAL, + LiteLLMSpanKind.PRODUCER: SpanKind.PRODUCER, + LiteLLMSpanKind.CONSUMER: SpanKind.CONSUMER, +} + + +def to_otel_span_kind(kind: LiteLLMSpanKind) -> SpanKind: + return _SPAN_KIND_BY_ROLE_KIND[kind] + + +# Custom exporter factories keyed by ``ExporterSpec.kind``. A preset registers +# one here when its destination needs construction logic the built-in kinds +# can't express — e.g. an exporter that fetches an auth token lazily on its +# first export (off the event loop) instead of blocking at config-build time. +# Keeping the registry here lets this module stay vendor-agnostic: the factory +# lives with the integration that needs it. +_EXPORTER_FACTORIES: dict[str, Callable[[ExporterSpec], SpanExporter]] = {} + + +def register_exporter_factory( + kind: str, factory: Callable[[ExporterSpec], SpanExporter] +) -> None: + """Register a custom exporter ``factory`` for the exporter ``kind``.""" + _EXPORTER_FACTORIES[kind.lower()] = factory + + +class LiteLLMBaggageSpanProcessor(SpanProcessor): + """Stamps an allowlisted set of Baggage entries onto every span at start.""" + + def __init__( + self, + allowed_keys: Iterable[str], + allowed_prefixes: tuple[str, ...] = (LiteLLM.METADATA_PREFIX,), + ) -> None: + self._allowed_keys = frozenset(allowed_keys) + self._allowed_prefixes = tuple(allowed_prefixes) + + def _is_allowed(self, key: str) -> bool: + return key in self._allowed_keys or any( + key.startswith(prefix) for prefix in self._allowed_prefixes + ) + + def on_start(self, span: Span, parent_context: Context | None = None) -> None: + for key, value in baggage.get_all(parent_context).items(): + if self._is_allowed(key) and isinstance(value, (str, bool, int, float)): + span.set_attribute(key, value) + + def on_end(self, span: ReadableSpan) -> None: # noqa: D401 - no-op + return None + + def shutdown(self) -> None: + return None + + def force_flush(self, timeout_millis: int = 30000) -> bool: + return True + + +def _otlp_traces_endpoint(endpoint: str | None) -> str | None: + """Point an OTLP/HTTP base endpoint at the ``/v1/traces`` signal path. + + ``OTEL_EXPORTER_OTLP_ENDPOINT`` is a base URL (e.g. ``http://host:4318``). + The OTLP/HTTP exporter only appends the ``/v1/traces`` path when it reads + that env var itself; when an endpoint is passed explicitly it is used + verbatim, so a base URL would POST to the root and the collector returns + 404. Append the signal path here (leaving an already-correct path intact). + """ + if not endpoint: + return endpoint + endpoint = endpoint.rstrip("/") + # Splunk Observability uses ``/v2/trace/otlp``; never rewrite it. + if endpoint.endswith("/v1/traces") or "/v2/trace/otlp" in endpoint: + return endpoint + for other_signal in ("/v1/logs", "/v1/metrics"): + if endpoint.endswith(other_signal): + return endpoint[: -len(other_signal)] + "/v1/traces" + return endpoint + "/v1/traces" + + +def _exporter_from_spec(spec: ExporterSpec) -> SpanExporter: + kind = (spec.kind or "console").lower() + factory = _EXPORTER_FACTORIES.get(kind) + if factory is not None: + return factory(spec) + if kind in ("in_memory", "inmemory", "memory"): + return InMemorySpanExporter() + if kind in ("otlp_http", "http", "http/protobuf", "http/json"): + from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( + OTLPSpanExporter as HTTPExporter, + ) + + return HTTPExporter( + endpoint=_otlp_traces_endpoint(spec.endpoint), + headers=parse_headers(spec.headers), + ) + if kind in ("otlp_grpc", "grpc"): + from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( + OTLPSpanExporter as GRPCExporter, + ) + + return GRPCExporter(endpoint=spec.endpoint, headers=parse_headers(spec.headers)) + return ConsoleSpanExporter() + + +def _processor_for(exporter: SpanExporter, use_simple: bool | None) -> SpanProcessor: + """Pick a Simple or Batch span processor for ``exporter``. + + When ``use_simple`` is unset, default to Simple for console and in-memory + exporters (spans export synchronously, which tests rely on) and Batch for + everything else (the right export semantics for production). + """ + if use_simple is None: + use_simple = isinstance(exporter, (ConsoleSpanExporter, InMemorySpanExporter)) + return SimpleSpanProcessor(exporter) if use_simple else BatchSpanProcessor(exporter) + + +def build_span_exporter(config: OpenTelemetryV2Config) -> SpanExporter: + """Build a single exporter from the top-level config fields. + + Convenience for the common single-exporter case (and for tests): reads the + ``exporter`` / ``endpoint`` / ``headers`` fields. To configure multiple + exporters, populate ``config.exporters`` directly. + """ + return _exporter_from_spec( + ExporterSpec( + kind=config.exporter, endpoint=config.endpoint, headers=config.headers + ) + ) + + +def build_resource(config: OpenTelemetryV2Config) -> Resource: + attributes: dict[str, str] = {"service.name": config.service_name} + if config.deployment_environment: + attributes["deployment.environment"] = config.deployment_environment + attributes.update(config.resource_attributes) + return Resource.create(attributes) + + +def build_tracer_provider( + config: OpenTelemetryV2Config, + exporter: SpanExporter | None = None, + baggage_processor: SpanProcessor | None = None, + use_simple_processor: bool | None = None, +) -> TracerProvider: + """Build the shared :class:`TracerProvider`. + + Attach the Baggage processor first (so identity attributes land on each + span before any export decision), then add one ``SpanProcessor`` per + ``config.exporters`` entry — this is what fans spans out to multiple + backends. ``exporter`` and ``use_simple_processor`` are explicit overrides: + pass a single exporter to attach exactly that one (used by tests). + """ + provider = TracerProvider(resource=build_resource(config)) + if baggage_processor is None: + baggage_processor = LiteLLMBaggageSpanProcessor( + allowed_keys=config.baggage_promoted_keys + ) + provider.add_span_processor(baggage_processor) + + if exporter is not None: + provider.add_span_processor(_processor_for(exporter, use_simple_processor)) + return provider + + # ``config._normalize`` guarantees at least one spec (it folds the top-level + # ``exporter``/``endpoint``/``headers`` fields in when ``exporters`` is empty). + for spec in config.exporters: + exp = _exporter_from_spec(spec) + provider.add_span_processor( + _processor_for( + exp, + ( + spec.use_simple_processor + if spec.use_simple_processor is not None + else use_simple_processor + ), + ) + ) + return provider + + +def get_tracer(provider: TracerProvider, name: str = "litellm") -> Tracer: + return provider.get_tracer(name) + + +def in_memory_provider( + config: OpenTelemetryV2Config | None = None, +) -> tuple[TracerProvider, InMemorySpanExporter]: + """Convenience for tests: a provider exporting to an in-memory buffer.""" + cfg = config or OpenTelemetryV2Config(exporter="in_memory") + exporter = InMemorySpanExporter() + provider = build_tracer_provider(cfg, exporter=exporter) + return provider, exporter diff --git a/litellm/integrations/otel/plumbing/routing.py b/litellm/integrations/otel/plumbing/routing.py new file mode 100644 index 00000000000..4d0943a263a --- /dev/null +++ b/litellm/integrations/otel/plumbing/routing.py @@ -0,0 +1,101 @@ +"""Per-request multi-tenant tracer routing. + +When a request carries team/key vendor credentials in +``standard_callback_dynamic_params``, its spans must export through a +``TracerProvider`` whose OTLP headers carry those credentials. +``TenantTracerCache`` builds and caches one provider per distinct credential +set, and otherwise hands back the logger's default tracer. This lets a single +logger fan requests out to many tenants without needing a logger per tenant. +""" + +from collections import OrderedDict +from typing import Any, Mapping + +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.trace import Tracer + +from litellm._logging import verbose_logger +from litellm.integrations.otel.model.config import OpenTelemetryV2Config +from litellm.integrations.otel.presets import dynamic_otlp_headers +from litellm.integrations.otel.plumbing.providers import ( + build_tracer_provider, + get_tracer, +) + +# Exporter kinds that ignore headers — never rewritten with dynamic credentials. +_NON_OTLP_KINDS = ("console", "in_memory", "inmemory", "memory") + +# Cap on distinct credential-scoped providers held at once. ``dynamic_params`` +# can be populated from request metadata, so an unbounded cache lets a caller +# spawn one ``TracerProvider`` (plus its ``BatchSpanProcessor`` background +# thread) per unique credential set and exhaust the proxy. The LRU bound keeps +# the working set of active tenants resident while flushing and shutting down +# evicted providers so their threads are reclaimed. +_MAX_CACHED_PROVIDERS = 256 + + +def _shutdown_provider(provider: TracerProvider) -> None: + """Flush + stop an evicted provider's processors (reclaims their threads). + + ``TracerProvider.shutdown`` force-flushes each ``SpanProcessor`` before + stopping it, so any spans already handed to a ``BatchSpanProcessor`` are + exported rather than dropped. Best-effort: a shutdown failure must not break + the request that triggered the eviction. + """ + try: + provider.shutdown() + except Exception as e: # pragma: no cover - defensive + verbose_logger.debug("OTel V2: error shutting down evicted provider: %s", e) + + +class TenantTracerCache: + """Credential-scoped ``TracerProvider`` cache keyed by the dynamic headers.""" + + def __init__( + self, + config: OpenTelemetryV2Config, + callback_name: str | None, + tracer_name: str, + ) -> None: + self._config = config + self._callback_name = callback_name + self._tracer_name = tracer_name + self._providers: "OrderedDict[tuple[tuple[str, str], ...], TracerProvider]" = ( + OrderedDict() + ) + + def tracer_for(self, default: Tracer, dynamic_params: Any) -> Tracer: + """Return the tracer for this request. + + Use ``default`` unless the request's dynamic credentials require a + credential-scoped tracer, in which case build (or reuse) one. The cache + is a bounded LRU: the least-recently-used provider is flushed and shut + down on overflow so its exporter threads don't accumulate. + """ + headers = dynamic_otlp_headers(self._callback_name, dynamic_params) + if not headers: + return default + cache_key = tuple(sorted(headers.items())) + provider = self._providers.get(cache_key) + if provider is not None: + self._providers.move_to_end(cache_key) + else: + provider = build_tracer_provider(self._config_with_headers(headers)) + self._providers[cache_key] = provider + if len(self._providers) > _MAX_CACHED_PROVIDERS: + _, evicted = self._providers.popitem(last=False) + _shutdown_provider(evicted) + return get_tracer(provider, self._tracer_name) + + def _config_with_headers(self, headers: Mapping[str, str]) -> OpenTelemetryV2Config: + """Clone the config, replacing OTLP exporter headers with ``headers``.""" + header_str = ",".join(f"{key}={value}" for key, value in headers.items()) + exporters = [ + ( + spec + if spec.kind.lower() in _NON_OTLP_KINDS + else spec.model_copy(update={"headers": header_str}) + ) + for spec in self._config.exporters + ] + return self._config.model_copy(update={"exporters": exporters}) diff --git a/litellm/integrations/otel/presets/__init__.py b/litellm/integrations/otel/presets/__init__.py new file mode 100644 index 00000000000..c69d257ab52 --- /dev/null +++ b/litellm/integrations/otel/presets/__init__.py @@ -0,0 +1,78 @@ +"""Integration presets — each one returns an :class:`OpenTelemetryV2Config`. + +A preset is a callable that reads an integration's env vars and returns an +``OpenTelemetryV2Config`` describing the exporter destination, the mapper +vocabularies to apply, and any resource attributes. ``PRESET_BY_CALLBACK`` +maps a callback name (``"arize"``, ``"langfuse_otel"``, ...) to its preset so +the factory in ``litellm_logging`` can resolve a name and build a single +``OpenTelemetryV2`` instance from the result. +""" + +from typing import Callable + +from litellm.integrations.otel.presets.agentops import agentops_preset +from litellm.integrations.otel.presets.arize import arize_dynamic_headers, arize_preset +from litellm.integrations.otel.presets.base import Preset +from litellm.integrations.otel.presets.langfuse import ( + langfuse_dynamic_headers, + langfuse_preset, +) +from litellm.integrations.otel.presets.langtrace import langtrace_preset +from litellm.integrations.otel.presets.levo import levo_preset +from litellm.integrations.otel.presets.phoenix import phoenix_preset +from litellm.integrations.otel.presets.weave import weave_dynamic_headers, weave_preset +from litellm.types.utils import StandardCallbackDynamicParams + +#: Callback name → preset. The ``Preset`` annotation makes mypy verify every +#: registered value matches the preset interface. +PRESET_BY_CALLBACK: dict[str, Preset] = { + "agentops": agentops_preset, + "arize": arize_preset, + "arize_phoenix": phoenix_preset, + "langfuse_otel": langfuse_preset, + "langtrace": langtrace_preset, + "levo": levo_preset, + "weave_otel": weave_preset, +} + +#: Callback name → per-request OTLP header builder (team/key multi-tenant +#: routing). Only integrations that support dynamic credentials appear here — +#: Arize-Phoenix/Langtrace/Levo/AgentOps don't, so they use the logger's +#: default tracer. +DYNAMIC_HEADERS_BY_CALLBACK: dict[ + str, Callable[[StandardCallbackDynamicParams], dict[str, str]] +] = { + "arize": arize_dynamic_headers, + "langfuse_otel": langfuse_dynamic_headers, + "weave_otel": weave_dynamic_headers, +} + + +def dynamic_otlp_headers( + callback_name: str | None, + dynamic_params: StandardCallbackDynamicParams | None, +) -> dict[str, str] | None: + """Per-request OTLP headers for ``callback_name``, or ``None`` if N/A. + + ``None`` means "no per-request routing" — the caller uses its default tracer. + """ + builder = DYNAMIC_HEADERS_BY_CALLBACK.get(callback_name or "") + if builder is None or not dynamic_params: + return None + headers = builder(dynamic_params) + return headers or None + + +__all__ = [ + "PRESET_BY_CALLBACK", + "DYNAMIC_HEADERS_BY_CALLBACK", + "Preset", + "dynamic_otlp_headers", + "agentops_preset", + "arize_preset", + "langfuse_preset", + "langtrace_preset", + "levo_preset", + "phoenix_preset", + "weave_preset", +] diff --git a/litellm/integrations/otel/presets/agentops.py b/litellm/integrations/otel/presets/agentops.py new file mode 100644 index 00000000000..5a12818fd99 --- /dev/null +++ b/litellm/integrations/otel/presets/agentops.py @@ -0,0 +1,139 @@ +"""AgentOps preset — OTLP/HTTP to AgentOps' endpoint with a lazily-fetched JWT. + +AgentOps authenticates with a short-lived JWT minted from the API key. Fetching +it is blocking network I/O, so it must never run on the event loop: callback +construction (where presets are built) can run inside the proxy's async startup +or, in the SDK, on the first request. Instead of fetching at config-build time, +this preset registers a custom exporter (``kind="agentops"``) that mints the JWT +**on its first export** — which the ``BatchSpanProcessor`` runs in its own +worker thread, off any event loop — and caches it for the process lifetime. +""" + +from typing import Any + +import httpx +from pydantic import Field +from pydantic_settings import BaseSettings, SettingsConfigDict + +from litellm._logging import verbose_logger +from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config +from litellm.integrations.otel.plumbing.providers import register_exporter_factory + +_AGENTOPS_ENDPOINT = "https://otlp.agentops.cloud/v1/traces" +_AGENTOPS_AUTH_ENDPOINT = "https://api.agentops.ai/v3/auth/token" +_AGENTOPS_EXPORTER_KIND = "agentops" + + +class _AgentOpsSettings(BaseSettings): + model_config = SettingsConfigDict(case_sensitive=False, extra="ignore") + + api_key: str | None = Field(default=None, validation_alias="AGENTOPS_API_KEY") + service_name: str = Field( + default="agentops", validation_alias="AGENTOPS_SERVICE_NAME" + ) + environment: str | None = Field( + default=None, validation_alias="AGENTOPS_ENVIRONMENT" + ) + + +def agentops_preset( + *, + config_overrides: OpenTelemetryV2Config | None = None, +) -> OpenTelemetryV2Config: + """Build the AgentOps config without any network I/O. + + The ``agentops`` exporter mints (and caches) the JWT lazily on its first + export, so this stays non-blocking. ``project.id`` is therefore not a + resource attribute — it is encoded in the JWT, which AgentOps uses to route + the trace to the right project. + """ + settings = _AgentOpsSettings() + base = config_overrides or OpenTelemetryV2Config() + return base.model_copy( + update={ + "exporters": [ + *base.exporters, + ExporterSpec( + kind=_AGENTOPS_EXPORTER_KIND, + endpoint=_AGENTOPS_ENDPOINT, + options=( + {"api_key": settings.api_key} if settings.api_key else None + ), + ), + ], + "resource_attributes": { + **base.resource_attributes, + "service.name": settings.service_name, + "telemetry.sdk.name": "agentops", + **( + {"deployment.environment": settings.environment} + if settings.environment + else {} + ), + }, + } + ) + + +def _build_agentops_exporter(spec: ExporterSpec) -> Any: + """Factory for the ``agentops`` exporter kind: a lazy-auth OTLP/HTTP exporter.""" + from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( + OTLPSpanExporter, + ) + + class _LazyAuthAgentOpsExporter(OTLPSpanExporter): + """OTLP/HTTP exporter that mints the AgentOps JWT on its first export. + + ``export`` runs in the ``BatchSpanProcessor`` worker thread, so the + blocking token fetch never touches an event loop. The result is cached + after the first attempt (success or failure) so it runs at most once. + """ + + def __init__(self, *, endpoint: str | None, api_key: str | None) -> None: + super().__init__(endpoint=endpoint) + self._agentops_api_key = api_key + self._auth_resolved = False + + def _ensure_authenticated(self) -> None: + if self._auth_resolved: + return + self._auth_resolved = True + if not self._agentops_api_key: + return + try: + token = _fetch_agentops_jwt(self._agentops_api_key).get("token") + if token: + # ``_session`` is the requests.Session the base exporter + # POSTs through; updating its Authorization header is how the + # minted JWT reaches every subsequent export. + self._session.headers["Authorization"] = f"Bearer {token}" + except Exception as e: + verbose_logger.debug("AgentOps JWT fetch failed: %s", e) + + def export(self, spans: Any) -> Any: + self._ensure_authenticated() + return super().export(spans) + + options = spec.options or {} + return _LazyAuthAgentOpsExporter( + endpoint=spec.endpoint, api_key=options.get("api_key") + ) + + +def _fetch_agentops_jwt(api_key: str) -> dict[str, Any]: + # Own a short-lived client rather than ``_get_httpx_client()``: that returns + # a process-wide cached ``HTTPHandler`` whose connection pool is shared by + # every caller, so closing it here would break concurrent/subsequent + # requests. This one-shot auth call gets its own client to close. + with httpx.Client(timeout=10) as client: + response = client.post( + url=_AGENTOPS_AUTH_ENDPOINT, + headers={"Content-Type": "application/json", "Connection": "keep-alive"}, + json={"api_key": api_key}, + ) + if response.status_code != 200: + raise RuntimeError(f"Failed to fetch AgentOps token: {response.text}") + return response.json() + + +register_exporter_factory(_AGENTOPS_EXPORTER_KIND, _build_agentops_exporter) diff --git a/litellm/integrations/otel/presets/arize.py b/litellm/integrations/otel/presets/arize.py new file mode 100644 index 00000000000..4df15125f5a --- /dev/null +++ b/litellm/integrations/otel/presets/arize.py @@ -0,0 +1,75 @@ +"""Arize preset — OTLP exporter to Arize + OpenInference vocabulary.""" + +from pydantic import Field +from pydantic_settings import BaseSettings, SettingsConfigDict + +from litellm.integrations.arize.arize import ArizeLogger as _V1ArizeLogger +from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config +from litellm.integrations.otel.presets.utils import ensure_mappers +from litellm.types.utils import StandardCallbackDynamicParams + + +class _ArizeSettings(BaseSettings): + model_config = SettingsConfigDict(case_sensitive=False, extra="ignore") + + # Standard OTLP headers env var, used as the fallback when no Arize + # credentials are configured. + otlp_traces_headers: str | None = Field( + default=None, validation_alias="OTEL_EXPORTER_OTLP_TRACES_HEADERS" + ) + + +def arize_preset( + *, + config_overrides: OpenTelemetryV2Config | None = None, +) -> OpenTelemetryV2Config: + arize_cfg = _V1ArizeLogger.get_arize_config() + headers = _arize_headers(arize_cfg) + base = config_overrides or OpenTelemetryV2Config() + return base.model_copy( + update={ + "exporters": [ + *base.exporters, + ExporterSpec( + kind=arize_cfg.protocol or "otlp_grpc", + endpoint=arize_cfg.endpoint or "https://otlp.arize.com/v1", + headers=headers, + ), + ], + "mapper_names": ensure_mappers(base.mapper_names, "openinference"), + "resource_attributes": { + **base.resource_attributes, + **( + {"model_id": arize_cfg.project_name} + if arize_cfg.project_name + else {} + ), + }, + } + ) + + +def _arize_headers(arize_cfg) -> str | None: + pieces = [] + if arize_cfg.space_id or arize_cfg.space_key: + pieces.append(f"space_id={arize_cfg.space_id or arize_cfg.space_key}") + if arize_cfg.api_key: + pieces.append(f"api_key={arize_cfg.api_key}") + if not pieces: + # Fall back to the standard OTLP headers env var when no Arize + # credentials are configured. + return _ArizeSettings().otlp_traces_headers + return ",".join(pieces) + + +def arize_dynamic_headers(params: StandardCallbackDynamicParams) -> dict[str, str]: + """Per-request Arize OTLP headers from team/key dynamic params.""" + headers: dict[str, str] = {} + # ``arize_space_key`` is the suggested param and wins over ``arize_space_id``. + space = params.get("arize_space_key") or params.get("arize_space_id") + if space: + headers["arize-space-id"] = space + api_key = params.get("arize_api_key") + if api_key: + headers["api_key"] = api_key + return headers diff --git a/litellm/integrations/otel/presets/base.py b/litellm/integrations/otel/presets/base.py new file mode 100644 index 00000000000..b50908e7652 --- /dev/null +++ b/litellm/integrations/otel/presets/base.py @@ -0,0 +1,25 @@ +"""Preset interface. + +A preset is a callable that reads its integration's env vars and produces an +:class:`OpenTelemetryV2Config` (exporter list + mapper-name list + resource +attributes). This ``Protocol`` pins that contract so ``PRESET_BY_CALLBACK`` and +the factory in ``litellm_logging`` are type-checked structurally against it, +matching the ``AttributeMapper`` protocol the mappers use. +""" + +from typing import Protocol, runtime_checkable + +from litellm.integrations.otel.model.config import OpenTelemetryV2Config + + +@runtime_checkable +class Preset(Protocol): + """Reads an integration's env config and returns an ``OpenTelemetryV2Config``. + + ``config_overrides`` lets one preset layer onto another's config (or onto + test-supplied defaults); the factory calls presets with no arguments. + """ + + def __call__( + self, *, config_overrides: OpenTelemetryV2Config | None = None + ) -> OpenTelemetryV2Config: ... diff --git a/litellm/integrations/otel/presets/langfuse.py b/litellm/integrations/otel/presets/langfuse.py new file mode 100644 index 00000000000..011545384b9 --- /dev/null +++ b/litellm/integrations/otel/presets/langfuse.py @@ -0,0 +1,43 @@ +"""Langfuse-OTEL preset.""" + +from litellm.integrations.langfuse.langfuse_otel import ( + LangfuseOtelLogger as _V1Langfuse, +) +from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config +from litellm.integrations.otel.presets.utils import ensure_mappers +from litellm.types.utils import StandardCallbackDynamicParams + + +def langfuse_preset( + *, + config_overrides: OpenTelemetryV2Config | None = None, +) -> OpenTelemetryV2Config: + cfg = _V1Langfuse.get_langfuse_otel_config() + kind = cfg.exporter if isinstance(cfg.exporter, str) else "otlp_http" + base = config_overrides or OpenTelemetryV2Config() + return base.model_copy( + update={ + "exporters": [ + *base.exporters, + ExporterSpec( + kind=kind, + endpoint=cfg.endpoint, + headers=cfg.headers, + ), + ], + "mapper_names": ensure_mappers(base.mapper_names, "langfuse"), + } + ) + + +def langfuse_dynamic_headers(params: StandardCallbackDynamicParams) -> dict[str, str]: + """Per-request Langfuse OTLP headers from team/key dynamic params.""" + public_key = params.get("langfuse_public_key") + secret_key = params.get("langfuse_secret_key") + if public_key and secret_key: + return { + "Authorization": _V1Langfuse._get_langfuse_authorization_header( + public_key=public_key, secret_key=secret_key + ) + } + return {} diff --git a/litellm/integrations/otel/presets/langtrace.py b/litellm/integrations/otel/presets/langtrace.py new file mode 100644 index 00000000000..acdbaf870d3 --- /dev/null +++ b/litellm/integrations/otel/presets/langtrace.py @@ -0,0 +1,22 @@ +"""Langtrace preset — Langtrace consumes generic OTLP + a vendor mapper.""" + +from litellm.integrations.otel.model.config import OpenTelemetryV2Config +from litellm.integrations.otel.presets.utils import ensure_mappers + + +def langtrace_preset( + *, + config_overrides: OpenTelemetryV2Config | None = None, +) -> OpenTelemetryV2Config: + """Compose the Langtrace mapper on top of the customer's OTLP destination. + + Unlike Arize / Phoenix / Langfuse, Langtrace doesn't ship its own endpoint + — users point their existing OTLP collector at Langtrace and just + need the vendor attribute schema applied to outgoing spans. + """ + base = config_overrides or OpenTelemetryV2Config() + return base.model_copy( + update={ + "mapper_names": ensure_mappers(base.mapper_names, "langtrace"), + } + ) diff --git a/litellm/integrations/otel/presets/levo.py b/litellm/integrations/otel/presets/levo.py new file mode 100644 index 00000000000..4c4cba982a4 --- /dev/null +++ b/litellm/integrations/otel/presets/levo.py @@ -0,0 +1,24 @@ +"""Levo preset — OTLP/HTTP to a Levo collector with org+workspace headers.""" + +from litellm.integrations.levo.levo import LevoLogger as _V1Levo +from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config + + +def levo_preset( + *, + config_overrides: OpenTelemetryV2Config | None = None, +) -> OpenTelemetryV2Config: + cfg = _V1Levo.get_levo_config() + base = config_overrides or OpenTelemetryV2Config() + return base.model_copy( + update={ + "exporters": [ + *base.exporters, + ExporterSpec( + kind="otlp_http", + endpoint=cfg.endpoint, + headers=cfg.otlp_auth_headers, + ), + ], + } + ) diff --git a/litellm/integrations/otel/presets/phoenix.py b/litellm/integrations/otel/presets/phoenix.py new file mode 100644 index 00000000000..4c2b165ffca --- /dev/null +++ b/litellm/integrations/otel/presets/phoenix.py @@ -0,0 +1,48 @@ +"""Arize-Phoenix preset.""" + +from pydantic import AliasChoices, Field +from pydantic_settings import BaseSettings, SettingsConfigDict + +from litellm.integrations.arize.arize_phoenix import ( + ArizePhoenixLogger as _V1Phoenix, +) +from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config +from litellm.integrations.otel.presets.utils import ensure_mappers + + +class _PhoenixSettings(BaseSettings): + model_config = SettingsConfigDict(case_sensitive=False, extra="ignore") + + project_name: str = Field( + default="default", + validation_alias=AliasChoices( + "PHOENIX_PROJECT_NAME", "PHOENIX_COLLECTOR_PROJECT_NAME" + ), + ) + + +def phoenix_preset( + *, + config_overrides: OpenTelemetryV2Config | None = None, +) -> OpenTelemetryV2Config: + cfg = _V1Phoenix.get_arize_phoenix_config() + headers = cfg.otlp_auth_headers if hasattr(cfg, "otlp_auth_headers") else None + project_name = _PhoenixSettings().project_name + base = config_overrides or OpenTelemetryV2Config() + return base.model_copy( + update={ + "exporters": [ + *base.exporters, + ExporterSpec( + kind=cfg.protocol if hasattr(cfg, "protocol") else "otlp_http", + endpoint=cfg.endpoint, + headers=headers, + ), + ], + "mapper_names": ensure_mappers(base.mapper_names, "openinference"), + "resource_attributes": { + **base.resource_attributes, + "openinference.project.name": project_name, + }, + } + ) diff --git a/litellm/integrations/otel/presets/utils.py b/litellm/integrations/otel/presets/utils.py new file mode 100644 index 00000000000..fdf8184441d --- /dev/null +++ b/litellm/integrations/otel/presets/utils.py @@ -0,0 +1,16 @@ +"""Shared helpers for the integration presets.""" + +from typing import Iterable + + +def ensure_mappers(mapper_names: Iterable[str], *names: str) -> list[str]: + """Return ``mapper_names`` with each of ``names`` appended if not already present. + + Order is preserved and duplicates are skipped, so composing several presets + (or re-applying one) never double-adds a vocabulary. + """ + result = list(mapper_names) + for name in names: + if name not in result: + result.append(name) + return result diff --git a/litellm/integrations/otel/presets/weave.py b/litellm/integrations/otel/presets/weave.py new file mode 100644 index 00000000000..9fc03c84a6d --- /dev/null +++ b/litellm/integrations/otel/presets/weave.py @@ -0,0 +1,43 @@ +"""Weave (W&B) preset.""" + +from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config +from litellm.integrations.otel.presets.utils import ensure_mappers +from litellm.integrations.weave.weave_otel import ( + _get_weave_authorization_header, + get_weave_otel_config, +) +from litellm.types.utils import StandardCallbackDynamicParams + + +def weave_preset( + *, + config_overrides: OpenTelemetryV2Config | None = None, +) -> OpenTelemetryV2Config: + weave_cfg = get_weave_otel_config() + base = config_overrides or OpenTelemetryV2Config() + return base.model_copy( + update={ + "exporters": [ + *base.exporters, + ExporterSpec( + kind=weave_cfg.protocol or "otlp_http", + endpoint=weave_cfg.endpoint, + headers=weave_cfg.otlp_auth_headers, + ), + ], + # Weave consumes OpenInference + a small Weave-specific overlay. + "mapper_names": ensure_mappers(base.mapper_names, "openinference", "weave"), + } + ) + + +def weave_dynamic_headers(params: StandardCallbackDynamicParams) -> dict[str, str]: + """Per-request Weave OTLP headers from team/key dynamic params.""" + headers: dict[str, str] = {} + api_key = params.get("wandb_api_key") + if api_key: + headers["Authorization"] = _get_weave_authorization_header(api_key=api_key) + project_id = params.get("weave_project_id") + if project_id: + headers["project_id"] = project_id + return headers diff --git a/litellm/integrations/otel/runtime.py b/litellm/integrations/otel/runtime.py new file mode 100644 index 00000000000..ac3b991c971 --- /dev/null +++ b/litellm/integrations/otel/runtime.py @@ -0,0 +1,38 @@ +"""SDK-free entrypoints for proxy-core call sites (auth, …). + +Proxy code may run without the OpenTelemetry SDK installed, so it must not import +``litellm.integrations.otel.logger`` (which imports the SDK at module scope) at +module load. These wrappers import it lazily and no-op when the SDK is absent or +V2 is not the active logger — so a call site can wrap a request phase or seed +identity unconditionally. +""" + +from contextlib import contextmanager +from typing import Any, Iterator + + +@contextmanager +def phase_span(name: str) -> "Iterator[Any]": + """Run a request phase inside a live active span so its DB/service calls nest. + + Yields ``None`` (a plain no-op) when the OTel SDK is unavailable or V2 is not + the active logger. + """ + try: + from litellm.integrations.otel.logger import phase_span as _phase_span + except Exception: + yield None + return + with _phase_span(name) as span: + yield span + + +def seed_request_identity(user_api_key_dict: Any, model: Any = None) -> None: + """Seed request-identity Baggage at the auth boundary (no-op without V2).""" + try: + from litellm.integrations.otel.logger import ( + seed_request_identity as _seed_request_identity, + ) + except Exception: + return + _seed_request_identity(user_api_key_dict, model=model) diff --git a/litellm/litellm_core_utils/get_litellm_params.py b/litellm/litellm_core_utils/get_litellm_params.py index ad9538ac171..b32803b5dfc 100644 --- a/litellm/litellm_core_utils/get_litellm_params.py +++ b/litellm/litellm_core_utils/get_litellm_params.py @@ -1,5 +1,7 @@ from typing import Optional +from litellm.llms.openai.data_residency import infer_openai_data_residency + # Pre-define optional kwargs keys as frozenset for O(1) lookups # These are extracted from kwargs only if present, avoiding unnecessary .get() calls _OPTIONAL_KWARGS_KEYS = frozenset( @@ -103,6 +105,10 @@ def get_litellm_params( if litellm_trace_id is None: litellm_trace_id = _meta.get("trace_id") or _meta.get("session_id") + data_residency: Optional[str] = infer_openai_data_residency( + custom_llm_provider, api_base + ) + # Build base dict with explicit parameters (always included) litellm_params = { "acompletion": acompletion, @@ -112,6 +118,7 @@ def get_litellm_params( "verbose": verbose, "custom_llm_provider": custom_llm_provider, "api_base": api_base, + "data_residency": data_residency, "litellm_call_id": litellm_call_id, "model_alias_map": model_alias_map, "completion_call_id": completion_call_id, diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 63fa0e64695..c127b3873a7 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -1546,6 +1546,11 @@ class Logging(LiteLLMLoggingBaseClass): if self.optional_params else None ), + "data_residency": ( + self.litellm_params.get("data_residency") + if hasattr(self, "litellm_params") and self.litellm_params + else None + ), } except Exception as e: # error creating kwargs for cost calculation debug_info = StandardLoggingModelCostFailureDebugInformation( @@ -1607,6 +1612,90 @@ class Logging(LiteLLMLoggingBaseClass): ) -> Optional[float]: return self._response_cost_calculator(result=result, cache_hit=cache_hit) + @staticmethod + def _is_sync_litellm_request(litellm_params: dict) -> bool: + """True for sync SDK entrypoints (``completion``), false for async (``acompletion``, etc.).""" + return ( + litellm_params.get(CallTypes.acompletion.value, False) is not True + and litellm_params.get(CallTypes.aresponses.value, False) is not True + and litellm_params.get(CallTypes.aembedding.value, False) is not True + and litellm_params.get(CallTypes.aimage_generation.value, False) is not True + and litellm_params.get(CallTypes.atranscription.value, False) is not True + ) + + def _is_assembled_stream_success(self, result=None) -> bool: + """Final assembled stream export (not a per-chunk success call). + + Per-chunk callers pass a ``ModelResponseStream`` (or ``None``); the + final assembled response is any other non-``None`` value (typically a + ``ModelResponse``). Treating a chunk as the assembled response would + prematurely set the ``has_dispatched_final_stream_success`` dedup + guard and silently suppress the real final stream log. + """ + if self.stream is not True: + return False + if result is not None and not isinstance(result, ModelResponseStream): + return True + return ( + "async_complete_streaming_response" in self.model_call_details + or self.model_call_details.get("complete_streaming_response") is not None + ) + + async def dispatch_success_handlers( + self, + result=None, + start_time=None, + end_time=None, + cache_hit=None, + prefer_async_handlers: bool = False, + **kwargs, + ) -> None: + """Route success logging to async and/or sync handlers for this request. + + ``prefer_async_handlers`` only bypasses the sync-SDK-only shortcut (e.g. + ``async for`` on a stream from ``completion()``). Legacy string callbacks + still run via ``executor.submit(success_handler)`` when configured. + """ + from litellm.litellm_core_utils.thread_pool_executor import executor + + if self._is_assembled_stream_success(result): + if self.model_call_details.get("has_dispatched_final_stream_success"): + return + self.model_call_details["has_dispatched_final_stream_success"] = True + + litellm_params = self.model_call_details.get("litellm_params", {}) or {} + sync_sdk = self._is_sync_litellm_request(litellm_params) + passthrough = self.call_type == CallTypes.pass_through.value + if sync_sdk and not prefer_async_handlers and not passthrough: + self.success_handler( + result, + start_time=start_time, + end_time=end_time, + cache_hit=cache_hit, + **kwargs, + ) + return + + await self.async_success_handler( + result, + start_time=start_time, + end_time=end_time, + cache_hit=cache_hit, + **kwargs, + ) + + if not self._should_run_sync_callbacks_for_async_calls(): + return + + executor.submit( + self.success_handler, + result, + start_time=start_time, + end_time=end_time, + cache_hit=cache_hit, + **kwargs, + ) + def should_run_logging( self, event_type: Literal[ @@ -2029,13 +2118,7 @@ class Logging(LiteLLMLoggingBaseClass): standard_logging_object=kwargs.get("standard_logging_object", None), ) litellm_params = self.model_call_details.get("litellm_params", {}) - is_sync_request = ( - litellm_params.get(CallTypes.acompletion.value, False) is not True - and litellm_params.get(CallTypes.aresponses.value, False) is not True - and litellm_params.get(CallTypes.aembedding.value, False) is not True - and litellm_params.get(CallTypes.aimage_generation.value, False) is not True - and litellm_params.get(CallTypes.atranscription.value, False) is not True - ) + is_sync_request = self._is_sync_litellm_request(litellm_params) try: ## BUILD COMPLETE STREAMED RESPONSE complete_streaming_response: Optional[ @@ -2491,9 +2574,11 @@ class Logging(LiteLLMLoggingBaseClass): print_verbose( "Logging Details LiteLLM-Async Success Call, cache_hit={}".format(cache_hit) ) - if not self.should_run_logging( + if not self._is_assembled_stream_success( + result + ) and not self.should_run_logging( event_type="async_success" - ): # prevent double logging + ): # prevent double logging (non-streaming) return ## CALCULATE COST FOR BATCH JOBS @@ -2943,13 +3028,7 @@ class Logging(LiteLLMLoggingBaseClass): ): # prevent double logging return litellm_params = self.model_call_details.get("litellm_params", {}) - is_sync_request = ( - litellm_params.get(CallTypes.acompletion.value, False) is not True - and litellm_params.get(CallTypes.aresponses.value, False) is not True - and litellm_params.get(CallTypes.aembedding.value, False) is not True - and litellm_params.get(CallTypes.aimage_generation.value, False) is not True - and litellm_params.get(CallTypes.atranscription.value, False) is not True - ) + is_sync_request = self._is_sync_litellm_request(litellm_params) try: start_time, end_time = self._failure_handler_helper_fn( @@ -3713,6 +3792,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 try: custom_logger_init_args = custom_logger_init_args or {} if logging_integration == "agentops": # Add AgentOps initialization + _v2 = _maybe_construct_otel_v2("agentops", _in_memory_loggers) + if _v2 is not None: + return _v2 # type: ignore for callback in _in_memory_loggers: if isinstance(callback, AgentOps): return callback # type: ignore @@ -3865,6 +3947,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _in_memory_loggers.append(_opik_logger) return _opik_logger # type: ignore elif logging_integration == "arize": + _v2 = _maybe_construct_otel_v2("arize", _in_memory_loggers) + if _v2 is not None: + return _v2 # type: ignore from litellm.integrations.opentelemetry import ( OpenTelemetry, OpenTelemetryConfig, @@ -3894,6 +3979,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _in_memory_loggers.append(_arize_otel_logger) return _arize_otel_logger # type: ignore elif logging_integration == "arize_phoenix": + _v2 = _maybe_construct_otel_v2("arize_phoenix", _in_memory_loggers) + if _v2 is not None: + return _v2 # type: ignore from litellm.integrations.opentelemetry import ( OpenTelemetry, OpenTelemetryConfig, @@ -3905,31 +3993,6 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 endpoint=arize_phoenix_config.endpoint, headers=arize_phoenix_config.otlp_auth_headers, ) - if arize_phoenix_config.project_name: - existing_attrs = os.environ.get("OTEL_RESOURCE_ATTRIBUTES", "") - # Add openinference.project.name attribute - if existing_attrs: - os.environ["OTEL_RESOURCE_ATTRIBUTES"] = ( - f"{existing_attrs},openinference.project.name={arize_phoenix_config.project_name}" - ) - else: - os.environ["OTEL_RESOURCE_ATTRIBUTES"] = ( - f"openinference.project.name={arize_phoenix_config.project_name}" - ) - - # Set Phoenix project name from environment variable - phoenix_project_name = os.environ.get("PHOENIX_PROJECT_NAME", None) - if phoenix_project_name: - existing_attrs = os.environ.get("OTEL_RESOURCE_ATTRIBUTES", "") - # Add openinference.project.name attribute - if existing_attrs: - os.environ["OTEL_RESOURCE_ATTRIBUTES"] = ( - f"{existing_attrs},openinference.project.name={phoenix_project_name}" - ) - else: - os.environ["OTEL_RESOURCE_ATTRIBUTES"] = ( - f"openinference.project.name={phoenix_project_name}" - ) # auth can be disabled on local deployments of arize phoenix if arize_phoenix_config.otlp_auth_headers is not None: @@ -3949,6 +4012,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _in_memory_loggers.append(_arize_phoenix_otel_logger) return _arize_phoenix_otel_logger # type: ignore elif logging_integration == "levo": + _v2 = _maybe_construct_otel_v2("levo", _in_memory_loggers) + if _v2 is not None: + return _v2 # type: ignore from litellm.integrations.levo.levo import LevoLogger from litellm.integrations.opentelemetry import ( OpenTelemetry, @@ -3974,6 +4040,28 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _in_memory_loggers.append(_levo_otel_logger) return _levo_otel_logger # type: ignore elif logging_integration == "otel": + # Gate the new typed V2 adapter behind LITELLM_OTEL_V2. When off, + # the legacy 3,227-line god-class is used unchanged. The two are + # never registered simultaneously — the dedup loop below treats + # any module under ``litellm.integrations.otel`` or + # ``litellm.integrations.opentelemetry`` as "the OTel callback". + from litellm.integrations.otel.model.config import is_otel_v2_enabled + + if is_otel_v2_enabled(): + from litellm.integrations.otel.logger import OpenTelemetryV2 + + for callback in _in_memory_loggers: + if type(callback) is OpenTelemetryV2: + return callback # type: ignore + otel_logger_v2 = OpenTelemetryV2( + **_get_custom_logger_settings_from_proxy_server( + callback_name=logging_integration + ) + ) + _in_memory_loggers.append(otel_logger_v2) + _maybe_auto_initialize_arize_phoenix(_in_memory_loggers) + return otel_logger_v2 # type: ignore + from litellm.integrations.opentelemetry import OpenTelemetry for callback in _in_memory_loggers: @@ -4112,6 +4200,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 elif logging_integration == "langtrace": if "LANGTRACE_API_KEY" not in os.environ: raise ValueError("LANGTRACE_API_KEY not found in environment variables") + _v2 = _maybe_construct_otel_v2("langtrace", _in_memory_loggers) + if _v2 is not None: + return _v2 # type: ignore from litellm.integrations.opentelemetry import ( OpenTelemetry, @@ -4152,6 +4243,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _in_memory_loggers.append(langfuse_logger) return langfuse_logger # type: ignore elif logging_integration == "langfuse_otel": + _v2 = _maybe_construct_otel_v2("langfuse_otel", _in_memory_loggers) + if _v2 is not None: + return _v2 # type: ignore from litellm.integrations.langfuse.langfuse_otel import LangfuseOtelLogger for callback in _in_memory_loggers: @@ -4168,6 +4262,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _in_memory_loggers.append(_otel_logger) return _otel_logger # type: ignore elif logging_integration == "weave_otel": + _v2 = _maybe_construct_otel_v2("weave_otel", _in_memory_loggers) + if _v2 is not None: + return _v2 # type: ignore from litellm.integrations.opentelemetry import OpenTelemetryConfig from litellm.integrations.weave.weave_otel import ( WeaveOtelLogger, @@ -4316,6 +4413,42 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 return None +def _maybe_construct_otel_v2( + callback_name: str, _in_memory_loggers: list +) -> Optional[Any]: + """If ``LITELLM_OTEL_V2`` is on, build (or reuse) a single ``OpenTelemetryV2`` + instance configured via the preset for ``callback_name``. + + Returns ``None`` when V2 is off OR when there's no preset registered for + ``callback_name`` — callers should then fall through to the legacy path. + """ + from litellm.integrations.otel.model.config import is_otel_v2_enabled + + if not is_otel_v2_enabled(): + return None + from litellm.integrations.otel.logger import OpenTelemetryV2 + from litellm.integrations.otel.presets import PRESET_BY_CALLBACK + + preset_fn = PRESET_BY_CALLBACK.get(callback_name) + if preset_fn is None: + return None + for callback in _in_memory_loggers: + if ( + isinstance(callback, OpenTelemetryV2) + and getattr(callback, "callback_name", None) == callback_name + ): + return callback + try: + config = preset_fn() + except Exception: + # If env vars are missing or the preset raises, defer to the legacy path + # so customers get the same error story they had before V2 landed. + return None + v2_logger = OpenTelemetryV2(config=config, callback_name=callback_name) + _in_memory_loggers.append(v2_logger) + return v2_logger + + def _maybe_auto_initialize_arize_phoenix(_in_memory_loggers: list) -> None: """ Auto-initialize ArizePhoenixLogger when Phoenix env vars are detected. diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 59d0465e6d4..882561ed2e8 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -9,6 +9,7 @@ from litellm.types.utils import ( CacheCreationTokenDetails, CallTypes, CompletionTokensDetailsWrapper, + DataResidency, ImageResponse, ModelInfo, PassthroughCallTypes, @@ -29,6 +30,9 @@ _IMAGE_RESPONSE_CALL_TYPES = frozenset( } ) +# Pre-resolved DataResidency enum values for fast membership checks +_VALID_DATA_RESIDENCIES = frozenset(r.value for r in DataResidency) + def _is_above_128k(tokens: float) -> bool: if tokens > 128000: @@ -617,11 +621,46 @@ def _calculate_input_cost( return prompt_cost +def _get_regional_uplift_multiplier( + model_info: ModelInfo, data_residency: Optional[str] +) -> float: + """ + Resolve the per-model regional-processing uplift multiplier for a given + data-residency region. + + OpenAI applies a flat percentage uplift (e.g. +10%) on all token costs for + requests served from a regionalized hostname (eu./us.api.openai.com). The + multiplier is stored on the model entry as + ``regional_processing_uplift_multiplier_`` (e.g. 1.10). + + Returns 1.0 (no uplift) when ``data_residency`` is ``None`` or when the + model has no multiplier configured for the given region. + """ + if data_residency is None: + return 1.0 + residency = data_residency.lower() + if residency not in _VALID_DATA_RESIDENCIES: + return 1.0 + multiplier = model_info.get(f"regional_processing_uplift_multiplier_{residency}") + if multiplier is None: + return 1.0 + try: + return float(cast(float, multiplier)) + except (TypeError, ValueError): + verbose_logger.exception( + "Invalid regional_processing_uplift_multiplier_%s for model; " + "defaulting to 1.0", + residency, + ) + return 1.0 + + def generic_cost_per_token( # noqa: PLR0915 model: str, usage: Usage, custom_llm_provider: str, service_tier: Optional[str] = None, + data_residency: Optional[str] = None, ) -> Tuple[float, float]: """ Calculates the cost per token for a given model, prompt tokens, and completion tokens. @@ -631,6 +670,8 @@ def generic_cost_per_token( # noqa: PLR0915 Input: - model: str, the model name without provider prefix - usage: LiteLLM Usage block, containing anthropic caching information + - data_residency: optional OpenAI data-residency region (e.g. "eu", "us"), + used to apply the per-model regional-processing uplift multiplier. Returns: Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd @@ -781,6 +822,14 @@ def generic_cost_per_token( # noqa: PLR0915 ) completion_cost += float(image_tokens) * _output_cost_per_image_token + ## REGIONAL DATA-RESIDENCY UPLIFT + # Applied as a flat multiplier across all token costs for the request + # when the upstream is a regionalized OpenAI host (eu./us.api.openai.com). + uplift = _get_regional_uplift_multiplier(model_info, data_residency) + if uplift != 1.0: + prompt_cost *= uplift + completion_cost *= uplift + return prompt_cost, completion_cost diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 32ae61d7f58..b44d21368f8 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -3,6 +3,7 @@ Common utility functions used for translating messages across providers """ import io +import json import mimetypes import re from os import PathLike @@ -132,6 +133,39 @@ def strip_none_values_from_message(message: AllMessageValues) -> AllMessageValue return cast(AllMessageValues, {k: v for k, v in message.items() if v is not None}) +def extract_search_results_text(search_results: object) -> str: + """ + Extract model-visible text from OpenAI tool-message ``search_results``. + + Used by token estimators and TPM limiters so large search result payloads + cannot bypass preflight checks via a small ``content`` field. + + Counts every string field forwarded on Bedrock ``SearchResultBlock``: + ``source``, ``title``, ``content[].text``, and ``citations``. + """ + if not isinstance(search_results, list): + return "" + texts = "" + for result in search_results: + if not isinstance(result, dict): + continue + for key in ("source", "title"): + value = result.get(key) + if isinstance(value, str): + texts += value + content = result.get("content") + if isinstance(content, list): + for block in content: + if isinstance(block, dict): + text = block.get("text") + if isinstance(text, str): + texts += text + citations = result.get("citations") + if citations is not None: + texts += json.dumps(citations, separators=(",", ":")) + return texts + + def convert_content_list_to_str( message: Union[AllMessageValues, ChatCompletionResponseMessage], ) -> str: @@ -152,6 +186,7 @@ def convert_content_list_to_str( elif message_content is not None and isinstance(message_content, str): texts = message_content + texts += extract_search_results_text(message.get("search_results")) return texts diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index f169f86079a..46e9b43a429 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -3658,6 +3658,7 @@ from litellm.types.llms.bedrock import ( ToolInputSchemaBlock as BedrockToolInputSchemaBlock, ) from litellm.types.llms.bedrock import ToolJsonSchemaBlock as BedrockToolJsonSchemaBlock +from litellm.types.llms.bedrock import SearchResultBlock from litellm.types.llms.bedrock import ToolResultBlock as BedrockToolResultBlock from litellm.types.llms.bedrock import ( ToolResultContentBlock as BedrockToolResultContentBlock, @@ -3997,7 +3998,7 @@ def _convert_to_bedrock_tool_call_invoke( for tool in tool_calls: if "function" in tool: tool_id = tool["id"] - name = tool["function"].get("name", "") + name = make_valid_bedrock_tool_name(tool["function"].get("name", "")) arguments = tool["function"].get("arguments", "") if not arguments or not arguments.strip(): @@ -4063,6 +4064,122 @@ def _convert_to_bedrock_tool_call_invoke( ) +def _append_bedrock_tool_result_media_block( + tool_result_content_blocks: List[BedrockToolResultContentBlock], + processed_block: BedrockContentBlock, + content: dict, + content_type: str, +) -> None: + if "image" in processed_block: + tool_result_content_blocks.append( + BedrockToolResultContentBlock(image=processed_block["image"]) + ) + elif "document" in processed_block: + tool_result_content_blocks.append( + BedrockToolResultContentBlock(document=processed_block["document"]) + ) + else: + verbose_logger.warning( + "Bedrock Converse: unrecognized BedrockContentBlock keys " + "%s for %s tool-result block %s; dropping.", + list(processed_block.keys()), + content_type, + content, + ) + + +def _append_bedrock_tool_result_image_url_block( + tool_result_content_blocks: List[BedrockToolResultContentBlock], + content: dict, +) -> None: + format: Optional[str] = None + if isinstance(content["image_url"], dict): + image_url = content["image_url"]["url"] + format = content["image_url"].get("format") + else: + image_url = content["image_url"] + processed_block = BedrockImageProcessor.process_image_sync( + image_url=image_url, + format=format, + ) + _append_bedrock_tool_result_media_block( + tool_result_content_blocks, processed_block, content, "image_url" + ) + + +def _append_bedrock_tool_result_file_block( + tool_result_content_blocks: List[BedrockToolResultContentBlock], + content: dict, +) -> None: + # Match the user-message path (_process_file_message): accept either + # file_data (base64 data URI) or file_id (server-side reference / URL). + file_obj = content.get("file") or {} + file_data = file_obj.get("file_data") + file_id = file_obj.get("file_id") + if file_data is None and file_id is None: + raise litellm.BadRequestError( + message="file_data and file_id cannot both be None. Got={}".format(content), + model="", + llm_provider="bedrock", + ) + processed_block = BedrockImageProcessor.process_image_sync( + image_url=cast(str, file_id or file_data), + format=file_obj.get("format"), + ) + _append_bedrock_tool_result_media_block( + tool_result_content_blocks, processed_block, content, "file" + ) + + +def _parse_bedrock_tool_result_content_list( + content_list: List, +) -> List[BedrockToolResultContentBlock]: + tool_result_content_blocks: List[BedrockToolResultContentBlock] = [] + for content in content_list: + if content["type"] == "text": + tool_result_content_blocks.append( + BedrockToolResultContentBlock(text=content["text"]) + ) + elif content["type"] == "image_url": + _append_bedrock_tool_result_image_url_block( + tool_result_content_blocks, content + ) + elif content["type"] == "file": + _append_bedrock_tool_result_file_block(tool_result_content_blocks, content) + return tool_result_content_blocks + + +def _build_bedrock_tool_result_content_blocks( + message: Union[ChatCompletionToolMessage, ChatCompletionFunctionMessage], +) -> tuple[List[BedrockToolResultContentBlock], bool]: + # Optional OpenAI tool-message extension: + # allow structured Bedrock search results on tool messages and map them + # directly to toolResult.content[].searchResult for Converse API. + # + # If `search_results` is present, we intentionally prefer it over `content` + # to avoid generating mixed text + searchResult blocks. + search_results = message.get("search_results") + if isinstance(search_results, list): + tool_result_content_blocks: List[BedrockToolResultContentBlock] = [] + for result in search_results: + if not isinstance(result, dict): + continue + tool_result_content_blocks.append( + BedrockToolResultContentBlock( + searchResult=cast(SearchResultBlock, result) + ) + ) + if tool_result_content_blocks: + return tool_result_content_blocks, True + + message_content = message["content"] + if isinstance(message_content, str): + return [BedrockToolResultContentBlock(text=message_content)], False + if isinstance(message_content, List): + return _parse_bedrock_tool_result_content_list(message_content), False + return [], False + + def _convert_to_bedrock_tool_call_result( message: Union[ChatCompletionToolMessage, ChatCompletionFunctionMessage], ) -> BedrockContentBlock: @@ -4106,90 +4223,18 @@ def _convert_to_bedrock_tool_call_result( """ - """ - tool_result_content_blocks: List[BedrockToolResultContentBlock] = [] - if isinstance(message["content"], str): - tool_result_content_blocks.append( - BedrockToolResultContentBlock(text=message["content"]) - ) - elif isinstance(message["content"], List): - content_list = message["content"] - for content in content_list: - if content["type"] == "text": - tool_result_content_blocks.append( - BedrockToolResultContentBlock(text=content["text"]) - ) - elif content["type"] == "image_url": - format: Optional[str] = None - if isinstance(content["image_url"], dict): - image_url = content["image_url"]["url"] - format = content["image_url"].get("format") - else: - image_url = content["image_url"] - _block: BedrockContentBlock = BedrockImageProcessor.process_image_sync( - image_url=image_url, - format=format, - ) - if "image" in _block: - tool_result_content_blocks.append( - BedrockToolResultContentBlock(image=_block["image"]) - ) - elif "document" in _block: - tool_result_content_blocks.append( - BedrockToolResultContentBlock(document=_block["document"]) - ) - else: - verbose_logger.warning( - "Bedrock Converse: unrecognized BedrockContentBlock keys " - "%s for image_url tool-result block %s; dropping.", - list(_block.keys()), - content, - ) - elif content["type"] == "file": - # Match the user-message path (_process_file_message): accept - # either file_data (base64 data URI) or file_id (server-side - # reference / URL) and hand off to BedrockImageProcessor. Raise - # BadRequestError on both-None rather than silently dropping. - file_obj = content.get("file") or {} - file_data = file_obj.get("file_data") - file_id = file_obj.get("file_id") - if file_data is None and file_id is None: - raise litellm.BadRequestError( - message="file_data and file_id cannot both be None. Got={}".format( - content - ), - model="", - llm_provider="bedrock", - ) - file_format = file_obj.get("format") - _file_block: BedrockContentBlock = ( - BedrockImageProcessor.process_image_sync( - image_url=cast(str, file_id or file_data), - format=file_format, - ) - ) - if "document" in _file_block: - tool_result_content_blocks.append( - BedrockToolResultContentBlock(document=_file_block["document"]) - ) - elif "image" in _file_block: - tool_result_content_blocks.append( - BedrockToolResultContentBlock(image=_file_block["image"]) - ) - else: - verbose_logger.warning( - "Bedrock Converse: unrecognized BedrockContentBlock keys " - "%s for file tool-result block %s; dropping.", - list(_file_block.keys()), - content, - ) + tool_result_content_blocks, used_search_results = ( + _build_bedrock_tool_result_content_blocks(message) + ) message.get("name", "") id = str(message.get("tool_call_id", str(uuid.uuid4()))) tool_result = BedrockToolResultBlock( - content=tool_result_content_blocks, - toolUseId=id, + content=tool_result_content_blocks, toolUseId=id ) + if used_search_results: + tool_result["status"] = cast(Literal["success"], "success") content_block = BedrockContentBlock(toolResult=tool_result) @@ -5323,16 +5368,10 @@ def _bedrock_converse_messages_pt( # noqa: PLR0915 def make_valid_bedrock_tool_name(input_tool_name: str) -> str: - """ - Replaces any invalid characters in the input tool name with underscores - and ensures the resulting string is a valid identifier for Bedrock tools - """ + """Normalize tool names to Bedrock pattern [a-zA-Z][a-zA-Z0-9_-]*.""" def replace_invalid(char): - """ - Bedrock tool names only supports alpha-numeric characters and underscores - """ - if char.isalnum() or char == "_": + if char.isalnum() or char in ("_", "-"): return char return "_" @@ -5492,7 +5531,7 @@ def _bedrock_tools_pt( raw_name = f"litellm_unnamed_tool_{tool_idx}" # related issue: https://github.com/BerriAI/litellm/issues/5007 - # Bedrock tool names must satisfy regular expression pattern: [a-zA-Z][a-zA-Z0-9_]* ensure this is true + # Bedrock tool names must satisfy pattern: [a-zA-Z][a-zA-Z0-9_-]* name = make_valid_bedrock_tool_name(input_tool_name=raw_name) if _tool_description: # bedrock doesn't accept empty "" or None descriptions description = _tool_description diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index c4528ff74e3..33bb6d7d2ea 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -86,6 +86,12 @@ class RealTimeStreaming: # When a text message is blocked, hold the guardrail reason so the next # response.create can be rewritten to include the failure context. self._pending_guardrail_message: Optional[str] = None + # Track whether session.created has already been sent to the client + # (e.g. synthetic event in deferred setup mode). + self._session_created_sent_to_client: bool = False + # Track whether we have already sent the guardrail turn-detection update + # that disables provider auto-response for transcription guardrails. + self._guardrail_turn_detection_update_sent: bool = False _SESSION_EVENT_TYPES = frozenset(["session.created", "session.updated"]) _AUDIO_FORMAT_MAP: Dict[str, Dict[str, Any]] = { @@ -248,40 +254,82 @@ class RealTimeStreaming: ## SYNC LOGGING executor.submit(self.logging_obj.success_handler(self.messages)) - async def _send_to_backend(self, message: str) -> None: + async def _send_to_backend(self, message: str) -> bool: """Send a message to the backend WebSocket. If a provider_config is set the message is first passed through transform_realtime_request so that provider-specific translation (e.g. dropping session.update for Vertex AI) is applied even for guardrail-injected messages. + + Returns True if at least one message was actually delivered to the + backend, False if the provider transformation produced no output and + the message was effectively dropped. """ if self.provider_config: transformed = self.provider_config.transform_realtime_request( message, self.model, self.session_configuration_request ) + sent = False for msg in transformed: + # Send first; only cache the setup payload once the backend + # has actually accepted it. Caching before send would leave + # ``session_configuration_request`` populated after a failed + # send, causing subsequent client session.update messages to + # be treated as "subsequent" and dropped even though the + # backend never received the original setup. await self.backend_ws.send(msg) # type: ignore[union-attr, attr-defined] - else: - await self.backend_ws.send(message) # type: ignore[union-attr, attr-defined] + self._cache_session_configuration_request(msg) + sent = True + return sent + await self.backend_ws.send(message) # type: ignore[union-attr, attr-defined] + return True + + def _cache_session_configuration_request(self, transformed_message: str) -> None: + """Store setup payload once sent to backend. + + Updates the cached setup on every successful setup send so follow-up + ``session.update`` messages (which produce a merged setup with new + ``generationConfig`` / ``systemInstruction`` / etc.) are reflected in + the cache used by downstream readers (``transform_session_created_event``, + ``return_new_content_delta_events`` modality lookup, ...). + """ + try: + message_obj = json.loads(transformed_message) + if "setup" in message_obj: + self.session_configuration_request = transformed_message + except (json.JSONDecodeError, TypeError): + return def _make_disable_auto_response_message(self) -> str: """Return a session.update that disables VAD auto-response.""" + turn_detection: Dict[str, Any] = { + "type": "server_vad", + "create_response": False, + } if self._backend_uses_beta_protocol: - session: Dict[str, Any] = { - "turn_detection": {"create_response": False}, - } + session: Dict[str, Any] = {"turn_detection": turn_detection} else: session = { "type": "realtime", - "audio": { - "input": { - "turn_detection": {"create_response": False}, - } - }, + "audio": {"input": {"turn_detection": turn_detection}}, } return json.dumps({"type": "session.update", "session": session}) + async def _maybe_send_guardrail_turn_detection_update(self) -> None: + """Disable provider auto-response once when transcription guardrails are enabled.""" + if self._guardrail_turn_detection_update_sent: + return + if not self._has_audio_transcription_guardrails(): + return + sent = await self._send_to_backend(self._make_disable_auto_response_message()) + # Only mark as sent when the provider transformation actually delivered + # the update to the backend. Otherwise (e.g. Gemini drops session.update + # after the initial setup), leave the flag unset so future opportunities + # — such as a duplicate session.created — can retry. + if sent: + self._guardrail_turn_detection_update_sent = True + def _has_realtime_guardrails(self) -> bool: """Return True if any callback is registered for realtime guardrail event types.""" from litellm.integrations.custom_guardrail import CustomGuardrail @@ -320,12 +368,20 @@ class RealTimeStreaming: self, transcript: str, item_id: Optional[str] = None, + pre_block_backend_message: Optional[str] = None, ) -> bool: """ Run registered guardrails on a completed speech transcription. Returns True if blocked (synthetic warning already sent to client). Returns False if clean (caller should send response.create to the backend). + + ``pre_block_backend_message`` (if provided) is sent to the backend + BEFORE any of the guardrail's own backend messages when a block is + triggered. This is needed for protocol contracts that require a + specific message to be sent first — e.g. Gemini Live requires a + matching ``toolResponse`` immediately after a ``toolCall`` before any + other client messages can be accepted. """ from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.types.guardrails import GuardrailEventHooks @@ -385,6 +441,13 @@ class RealTimeStreaming: getattr(callback, "realtime_violation_message", None) or safe_msg ) + # Deliver any caller-supplied backend message FIRST so that + # protocol contracts requiring a specific ordering (e.g. + # Gemini Live's mandatory ``toolResponse`` after a + # ``toolCall``) are honored before the guardrail's own + # clientContent / cancel messages are sent. + if pre_block_backend_message is not None: + await self._send_to_backend(pre_block_backend_message) # Cancel any in-progress LLM response (e.g. VAD auto-response). await self._send_to_backend(json.dumps({"type": "response.cancel"})) # Send the policy violation hint (shows as small gray status text in UI). @@ -480,16 +543,34 @@ class RealTimeStreaming: else [transformed_response] ) for event in events: + is_session_created_event = ( + isinstance(event, dict) and event.get("type") == "session.created" + ) + if is_session_created_event: + if self._session_created_sent_to_client: + # A synthetic session.created (with placeholder defaults) was + # already forwarded to the client when we connected. The + # provider's real session.created (e.g. emitted from Gemini + # `setupComplete`) carries the authoritative modalities/model + # from the client's session.update. Re-emit it as + # `session.updated` so the client learns the corrected + # configuration without seeing two `session.created` events. + event = {**event, "type": "session.updated"} + else: + self._session_created_sent_to_client = True event_str = json.dumps(event) - ## For audio/VAD guardrail path: forward session.created first, then inject. - if ( - isinstance(event, dict) - and event.get("type") == "session.created" - and self._has_audio_transcription_guardrails() - ): + ## For audio/VAD guardrail path: forward the (possibly retyped) + ## session.created first, then invoke the one-time guardrail + ## turn-detection update. ``_maybe_send_guardrail_turn_detection_update`` + ## is idempotent (gated by ``_guardrail_turn_detection_update_sent``), + ## so duplicate session.created events — including those emitted + ## after a synthetic session.created from ``llm_http_handler`` in + ## deferred-setup mode — still get a single chance to inject the + ## update if a prior attempt was dropped by the provider transform. + if is_session_created_event and self._has_audio_transcription_guardrails(): self.store_message(event_str) await self.websocket.send_text(event_str) - await self._send_to_backend(self._make_disable_auto_response_message()) + await self._maybe_send_guardrail_turn_detection_update() continue ## GUARDRAIL: run on transcription events in provider_config path too if ( @@ -564,10 +645,19 @@ class RealTimeStreaming: try: raw_response = await self.backend_ws.recv( # type: ignore[union-attr] decode=False - ) # improves performance + ) except TypeError: raw_response = await self.backend_ws.recv() # type: ignore[union-attr, assignment] + if isinstance(raw_response, bytes): + try: + raw_response = raw_response.decode("utf-8") + except UnicodeDecodeError: + verbose_logger.warning( + "Received non-UTF-8 binary frame from backend, skipping." + ) + continue + if self.provider_config: try: await self._handle_provider_config_message(raw_response) @@ -783,12 +873,13 @@ class RealTimeStreaming: item["content"] = new_content return item - async def client_ack_messages(self): + async def client_ack_messages(self): # noqa: PLR0915 try: while True: message = await self.websocket.receive_text() ## GUARDRAIL: intercept conversation.item.create for text-based injection. + guardrail_turn_detection_injected = False try: msg_obj = json.loads(message) msg_type = msg_obj.get("type") @@ -796,7 +887,68 @@ class RealTimeStreaming: if msg_type == "conversation.item.create": # Check user text messages for prompt injection item = msg_obj.get("item", {}) - if item.get("role") == "user": + # Check function_call_output first so a client cannot + # bypass the tool-result guardrail by also setting + # role="user" on a function_call_output item. + if item.get("type") == "function_call_output": + # Tool results are client-controlled and fed to the + # model; check them with the same guardrail used for + # user text so an attacker cannot smuggle blocked + # content into a function_call_output. + output = item.get("output", "") + output_text = ( + output + if isinstance(output, str) + else json.dumps(output) + ) + if output_text: + # Build the sanitized function_call_output up + # front so we can hand it to the guardrail + # runner as the pre-block message. Providers + # that pair every toolCall with a toolResponse + # (e.g. Gemini/Vertex Live) require the + # toolResponse to arrive BEFORE any other + # client message — otherwise the guardrail's + # own clientContent would violate the + # pending-tool-call protocol contract and the + # backend could close the connection before + # the sanitized response ever lands. Dropping + # the blocked item outright would similarly + # leave such providers waiting indefinitely. + # The sanitized payload carries no blocked + # content — only a generic policy marker. + sanitized_msg = json.dumps( + { + **msg_obj, + "item": { + **item, + "output": json.dumps( + { + "error": "Tool output blocked by content policy", + } + ), + }, + } + ) + blocked = await self.run_realtime_guardrails( + output_text, + pre_block_backend_message=sanitized_msg, + ) + if blocked: + # ``_pending_guardrail_message`` is + # intentionally NOT set here. That flag + # exists to swallow the reflexive + # ``response.create`` an OpenAI client + # sends immediately after a user text + # message. In a tool-calling flow the + # client may not send a ``response.create`` + # at all (e.g. Gemini SDKs auto-respond), + # so leaving the flag set would + # incorrectly drop an unrelated + # ``response.create`` from a later + # interaction turn. + continue + elif item.get("role") == "user": content_list = item.get("content", []) texts = [ c.get("text", "") @@ -824,6 +976,89 @@ class RealTimeStreaming: self._pending_guardrail_message = None continue + ## GUARDRAIL: Inject turn_detection into first session.update + # if needed. Done BEFORE the GA remap so the injected + # ``create_response`` rides along with any client-provided + # turn_detection fields (e.g. silence_duration_ms) into the + # nested ``audio.input.turn_detection`` path produced by the + # remap. Doing this after the remap would create a separate + # minimal root-level ``turn_detection`` and silently drop + # the client's nested settings. + if ( + msg_type == "session.update" + and self.session_configuration_request is None + and not self._guardrail_turn_detection_update_sent + and self._has_audio_transcription_guardrails() + ): + session = msg_obj.setdefault("session", {}) + if isinstance(session, dict): + existing_td = session.get("turn_detection") + if not isinstance(existing_td, dict): + existing_td = {} + existing_td["create_response"] = False + session["turn_detection"] = existing_td + message = json.dumps(msg_obj) + guardrail_turn_detection_injected = True + verbose_logger.debug( + "Injected turn_detection into first session.update for audio transcription guardrails" + ) + + ## GUARDRAIL: Force ``create_response`` to False in any + # client-provided ``turn_detection`` so a later + # ``session.update`` cannot re-enable VAD auto-response + # and bypass the transcription guardrail after the + # initial disable. Covers both the flat beta key and the + # nested GA ``audio.input.turn_detection`` shape, since + # the GA remap below also accepts either form. Skipped + # when the injection block above already ran for this + # message, to avoid redundant double-serialization. + if ( + msg_type == "session.update" + and not guardrail_turn_detection_injected + and self._has_audio_transcription_guardrails() + ): + session = msg_obj.get("session") + if isinstance(session, dict): + td_overridden = False + flat_td = session.get("turn_detection") + flat_td_present = flat_td is not None + if flat_td_present: + if not isinstance(flat_td, dict): + flat_td = {} + if flat_td.get("create_response") is not False: + flat_td["create_response"] = False + session["turn_detection"] = flat_td + td_overridden = True + nested_td_present = False + audio = session.get("audio") + if isinstance(audio, dict): + audio_input = audio.get("input") + if isinstance(audio_input, dict): + nested_td = audio_input.get("turn_detection") + if nested_td is not None: + nested_td_present = True + if not isinstance(nested_td, dict): + nested_td = {} + if ( + nested_td.get("create_response") + is not False + ): + nested_td["create_response"] = False + audio_input["turn_detection"] = nested_td + td_overridden = True + # Symmetric with the first-update injection block: + # if the client omitted turn_detection entirely on + # a subsequent session.update, still inject the + # ``create_response: False`` override so the + # transcription guardrail cannot be re-enabled by + # any downstream merge that drops the original + # disable. + if not flat_td_present and not nested_td_present: + session["turn_detection"] = {"create_response": False} + td_overridden = True + if td_overridden: + message = json.dumps(msg_obj) + # GA compatibility: remap beta-style session fields only when # the upstream is in GA mode. Beta upstreams expect the flat # session shape unchanged. @@ -841,17 +1076,20 @@ class RealTimeStreaming: pass ## LOGGING + # Log after any in-place modifications (GA remap, guardrail + # turn_detection injection) so audit logs reflect what we + # actually forward to the backend. self.store_input(message=message) - ## FORWARD TO BACKEND - if self.provider_config: - message = self.provider_config.transform_realtime_request( - message, self.model - ) - for msg in message: - await self.backend_ws.send(msg) # type: ignore[union-attr] - else: - await self.backend_ws.send(message) # type: ignore[union-attr] + ## FORWARD TO BACKEND + # Only mark the guardrail turn_detection update as sent after the + # backend actually accepted the message. Setting the flag earlier + # would permanently disable the injection if ``_send_to_backend`` + # raised — neither this loop nor + # ``_maybe_send_guardrail_turn_detection_update`` would retry. + sent = await self._send_to_backend(message) + if guardrail_turn_detection_injected and sent: + self._guardrail_turn_detection_update_sent = True except Exception as e: verbose_logger.debug(f"Error in client ack messages: {e}") diff --git a/litellm/litellm_core_utils/sensitive_data_masker.py b/litellm/litellm_core_utils/sensitive_data_masker.py index d7803455b4a..4928dd08386 100644 --- a/litellm/litellm_core_utils/sensitive_data_masker.py +++ b/litellm/litellm_core_utils/sensitive_data_masker.py @@ -146,6 +146,37 @@ class SensitiveDataMasker: return masked_data +_default_masker = SensitiveDataMasker() + + +def mask_sensitive_keys( + data: Dict[str, Any], sensitive_fields: Set[str] +) -> Dict[str, Any]: + """Return a new dict with values masked for keys listed in ``sensitive_fields``. + + Unlike :meth:`SensitiveDataMasker.mask_dict`, this does exact key-name + matching (not segment matching), so callers explicitly enumerate which + fields to mask. Non-string and None values are passed through unchanged. + + Values shorter than ``visible_prefix + visible_suffix`` (8 by default) + fall outside :meth:`SensitiveDataMasker._mask_value`'s partial-reveal + range and are replaced with a fixed-length all-mask string, so a short + credential is never returned verbatim. + """ + masked: Dict[str, Any] = {} + mask_char = _default_masker.mask_char + min_visible = _default_masker.visible_prefix + _default_masker.visible_suffix + for key, value in data.items(): + if value is not None and key in sensitive_fields and isinstance(value, str): + if len(value) < min_visible: + masked[key] = mask_char * len(value) if value else value + else: + masked[key] = _default_masker._mask_value(value) + else: + masked[key] = value + return masked + + # Usage example: """ masker = SensitiveDataMasker() diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index fa7faf3035d..55042a733ed 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -59,6 +59,8 @@ FUNCTION_CALL_ATTRIBUTE = "function_call" _SYNC_ITER_EXHAUSTED = object() +_GCHUNK_FIELDS: frozenset = frozenset(GChunk.__annotations__) + def _next_sync_or_exhausted(it: Any) -> Any: """ @@ -181,6 +183,30 @@ class CustomStreamWrapper: self.created: Optional[int] = None self._last_returned_hidden_params: Optional[dict] = None + _cached_logging_provider = self.logging_obj.model_call_details.get( + "custom_llm_provider", None + ) + self._cached_logging_llm_provider: Optional[str] = _cached_logging_provider + _effective_model = model or "" + if ( + custom_llm_provider == "openai" + and custom_llm_provider != _cached_logging_provider + ): + _effective_model = "{}/{}".format( + _cached_logging_provider, _effective_model + ) + self._cached_model_name: str = _effective_model + + # Snapshot assumes self._hidden_params is populated from litellm_params + # at init and never mutated during the stream. If that ever changes, + # this cache must be removed. + self._base_hidden_params: Dict[str, Any] = { + **self._hidden_params, + "response_cost": None, + } + + self._post_streaming_hooks: Optional[List] = None + def _check_max_streaming_duration(self) -> None: """Raise litellm.Timeout if the stream has exceeded LITELLM_MAX_STREAMING_DURATION_SECONDS.""" from litellm.constants import LITELLM_MAX_STREAMING_DURATION_SECONDS @@ -681,29 +707,16 @@ class CustomStreamWrapper: def model_response_creator( self, chunk: Optional[dict] = None, hidden_params: Optional[dict] = None ): - _model = self.model - _received_llm_provider = self.custom_llm_provider - _logging_obj_llm_provider = self.logging_obj.model_call_details.get("custom_llm_provider", None) # type: ignore - if ( - _received_llm_provider == "openai" - and _received_llm_provider != _logging_obj_llm_provider - ): - _model = "{}/{}".format(_logging_obj_llm_provider, _model) + _model = self._cached_model_name + _logging_obj_llm_provider = self._cached_logging_llm_provider + if chunk is None: - chunk = {} + args: Dict[str, Any] = {"model": _model} else: - # pop model keyword chunk.pop("model", None) - - chunk_dict = {} - for key, value in chunk.items(): - if key != "stream": - chunk_dict[key] = value - - args = { - "model": _model, - **chunk_dict, - } + args = {"model": _model} + if chunk: + args.update({k: v for k, v in chunk.items() if k != "stream"}) model_response = ModelResponseStream(**args) if self.response_id is not None: @@ -717,15 +730,23 @@ class CustomStreamWrapper: model_response.created = self.created else: self.created = model_response.created + + # Spread order is load-bearing: _base_hidden_params (model_id, api_base, ...) + # must win over both caller-supplied hidden_params and the computed + # custom_llm_provider/created_at values, so it comes last. if hidden_params is not None: - model_response._hidden_params = hidden_params - model_response._hidden_params["custom_llm_provider"] = _logging_obj_llm_provider - model_response._hidden_params["created_at"] = time.time() - model_response._hidden_params = { - **model_response._hidden_params, - **self._hidden_params, - "response_cost": None, - } + model_response._hidden_params = { + **hidden_params, + "custom_llm_provider": _logging_obj_llm_provider, + "created_at": time.time(), + **self._base_hidden_params, + } + else: + model_response._hidden_params = { + "custom_llm_provider": _logging_obj_llm_provider, + "created_at": time.time(), + **self._base_hidden_params, + } if ( len(model_response.choices) > 0 @@ -1627,7 +1648,17 @@ class CustomStreamWrapper: from litellm.integrations.custom_logger import CustomLogger from litellm.types.utils import CallTypes - # Get request kwargs from logging object + if self._post_streaming_hooks is None: + self._post_streaming_hooks = [ + cb + for cb in litellm.callbacks + if isinstance(cb, CustomLogger) + and hasattr(cb, "async_post_call_streaming_deployment_hook") + ] + + if not self._post_streaming_hooks: + return chunk + request_data = self.logging_obj.model_call_details call_type_str = self.logging_obj.call_type @@ -1636,18 +1667,14 @@ class CustomStreamWrapper: except ValueError: typed_call_type = None - # Call hooks for all callbacks - for callback in litellm.callbacks: - if isinstance(callback, CustomLogger) and hasattr( - callback, "async_post_call_streaming_deployment_hook" - ): - result = await callback.async_post_call_streaming_deployment_hook( - request_data=request_data, - response_chunk=chunk, - call_type=typed_call_type, - ) - if result is not None: - chunk = result + for callback in self._post_streaming_hooks: + result = await callback.async_post_call_streaming_deployment_hook( + request_data=request_data, + response_chunk=chunk, + call_type=typed_call_type, + ) + if result is not None: + chunk = result return chunk except Exception as e: @@ -1808,8 +1835,10 @@ class CustomStreamWrapper: processed_chunk, None, None, cache_hit ) ) - ## SYNC LOGGING - self.logging_obj.success_handler(processed_chunk, None, None, cache_hit) + ## SYNC LOGGING — only for sync SDK entrypoints; async proxy paths export via async_success_handler + litellm_params = self.logging_obj.model_call_details.get("litellm_params", {}) + if self.logging_obj._is_sync_litellm_request(litellm_params): + self.logging_obj.success_handler(processed_chunk, None, None, cache_hit) def finish_reason_handler(self): model_response = self.model_response_creator() @@ -1888,17 +1917,15 @@ class CustomStreamWrapper: response = self._add_mcp_list_tools_to_first_chunk(response) self.sent_first_chunk = True - if hasattr( - response, "usage" - ): # remove usage from chunk, only send on final chunk - # Convert the object to a dictionary + # ModelResponseStream declares `usage` as a field, so + # hasattr(response, "usage") is always True — must check + # `is not None` to avoid running this path on every chunk. + if getattr(response, "usage", None) is not None: obj_dict = response.model_dump() - # Remove an attribute (e.g., 'attr2') if "usage" in obj_dict: del obj_dict["usage"] - # Create a new object without the removed attribute response = self.model_response_creator( chunk=obj_dict, hidden_params=response._hidden_params ) @@ -2206,23 +2233,19 @@ class CustomStreamWrapper: cache_hit, ) else: + # prefer_async_handlers routes CustomLogger to async_success_handler + # when consumers use ``async for`` on sync-SDK streams. Legacy string + # callbacks still run via executor.submit inside dispatch_success_handlers. asyncio.create_task( - self.logging_obj.async_success_handler( + self.logging_obj.dispatch_success_handlers( complete_streaming_response, cache_hit=cache_hit, start_time=None, end_time=None, + prefer_async_handlers=True, ) ) - executor.submit( - self.logging_obj.success_handler, - complete_streaming_response, - cache_hit=cache_hit, - start_time=None, - end_time=None, - ) - raise StopAsyncIteration # Re-raise StopIteration else: self.sent_last_chunk = True @@ -2398,10 +2421,7 @@ def generic_chunk_has_all_required_fields(chunk: dict) -> bool: :param chunk: The dictionary to check. :return: True if all required fields are present, False otherwise. """ - _all_fields = GChunk.__annotations__ - - decision = all(key in _all_fields for key in chunk) - return decision + return all(key in _GCHUNK_FIELDS for key in chunk) def convert_generic_chunk_to_model_response_stream( diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index e6a68de07e9..74b41062174 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -486,6 +486,14 @@ def _count_messages( use_default_image_token_count, default_token_count, ) + elif key == "search_results" and isinstance(value, list): + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + extract_search_results_text, + ) + + search_results_text = extract_search_results_text(value) + if search_results_text: + num_tokens += params.count_function(search_results_text) else: # Skip unsupported keys instead of raising an error continue @@ -764,11 +772,29 @@ def _format_function_definitions(tools): lines.append("namespace functions {") lines.append("") for tool in tools: + if not isinstance(tool, dict): + continue function = tool.get("function") + if not isinstance(function, dict): + # Anthropic tool shape → OpenAI function dict for token counting. + params = tool.get("input_schema") or tool.get("parameters") or {} + if not isinstance(params, dict): + params = {} + function = { + "name": tool.get("name"), + "description": tool.get("description"), + "parameters": params, + } + function_name = function.get("name") + if not function_name: + # Skip malformed tools missing a name to avoid emitting + # ``type None = ...`` which would produce inaccurate token counts. + continue if function_description := function.get("description"): lines.append(f"// {function_description}") - function_name = function.get("name") - parameters = function.get("parameters", {}) + parameters = function.get("parameters") or {} + if not isinstance(parameters, dict): + parameters = {} properties = parameters.get("properties") if properties and properties.keys(): lines.append(f"type {function_name} = (_: {{") diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 0b56eb86d9c..57609cfcd26 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -337,13 +337,12 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ) @staticmethod - def _supports_effort_level(model: str, level: str) -> bool: - """Check ``supports_{level}_reasoning_effort`` in the model map. + def _supports_model_capability(model: str, key: str) -> bool: + """Check a boolean capability ``key`` in the model map. Strips bedrock/vertex prefixes so a provider-routed Claude still resolves to the Anthropic model-map entry. """ - key = f"supports_{level}_reasoning_effort" try: if _supports_factory( model=model, @@ -372,8 +371,6 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): except Exception: pass try: - import litellm - for cand in candidates: if cand in litellm.model_cost and ( litellm.model_cost[cand].get(key) is True @@ -383,6 +380,13 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): pass return False + @staticmethod + def _supports_effort_level(model: str, level: str) -> bool: + """Check ``supports_{level}_reasoning_effort`` in the model map.""" + return AnthropicConfig._supports_model_capability( + model, f"supports_{level}_reasoning_effort" + ) + @staticmethod def _validate_effort_for_model(model: str, effort: Optional[str]) -> Optional[str]: """Return ``None`` if ``effort`` is allowed on ``model``, else an error message.""" @@ -400,7 +404,15 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): @staticmethod def _model_supports_effort_param(model: str) -> bool: - """Whether the model accepts ``output_config.effort`` at all.""" + """Whether the model accepts ``output_config.effort`` at all. + + A model qualifies if its map entry advertises ``supports_output_config`` + or any ``supports_*_reasoning_effort`` flag. The two are independent + signals: e.g. Claude Opus 4.5 supports ``output_config`` without + advertising a non-default (max/xhigh) effort level. + """ + if AnthropicConfig._supports_model_capability(model, "supports_output_config"): + return True return any( AnthropicConfig._supports_effort_level(model, level) for level in ("low", "minimal", "medium", "high", "xhigh", "max") @@ -1793,7 +1805,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): self._ensure_context_management_beta_header( headers, optional_params["context_management"] ) - if optional_params.get("output_format") is not None: + output_config = optional_params.get("output_config") + if optional_params.get("output_format") is not None or ( + isinstance(output_config, dict) and output_config.get("format") is not None + ): self._ensure_beta_header( headers, ANTHROPIC_BETA_HEADER_VALUES.STRUCTURED_OUTPUT_2025_09_25.value ) diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py b/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py index 8ed6126d2eb..efb913f709a 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py @@ -4,6 +4,7 @@ from typing import ( AsyncIterator, Coroutine, Dict, + Iterator, List, Optional, Tuple, @@ -12,9 +13,16 @@ from typing import ( ) import litellm +from litellm._logging import verbose_logger +from litellm.litellm_core_utils.asyncify import run_async_function from litellm.llms.anthropic.experimental_pass_through.adapters.transformation import ( AnthropicAdapter, ) +from litellm.llms.anthropic.experimental_pass_through.context_management import ( + AnthropicContextManagementError, + PolyfillResult, + apply_context_management, +) from litellm.llms.anthropic.experimental_pass_through.utils import ( is_reasoning_auto_summary_enabled, ) @@ -28,15 +36,266 @@ if TYPE_CHECKING: pass -# Anthropic-only fields that the translator above already maps into the -# OpenAI-format completion_kwargs (output_config → reasoning_effort / -# response_format, etc.). They must be filtered out of the raw -# extra_kwargs re-merge below or non-Anthropic backends reject the call -# with 400 "Extra inputs are not permitted". Add new entries here when -# extending AnthropicMessagesRequestOptionalParams with another Anthropic- -# specific key. +# Anthropic-only keys already mapped by the translator; strip on extra_kwargs re-merge. ANTHROPIC_ONLY_REQUEST_KEYS: frozenset[str] = frozenset({"output_config"}) + +def _messages_have_compaction_block(messages: List[Dict]) -> bool: + """Return True when any message carries a ``compaction`` content block.""" + for msg in messages: + content = msg.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "compaction": + return True + return False + + +def _extract_proxy_litellm_metadata(kwargs: Dict[str, Any]) -> Optional[Dict[str, Any]]: + """Return ``kwargs["litellm_metadata"]`` when it's a dict; ``None`` otherwise. + + The proxy attaches its auth/spend-attribution fields (``user_api_key``, + ``user_api_key_team_id``, ``litellm_call_id``, the full ``UserAPIKeyAuth`` + object under ``user_api_key_auth``, ...) to ``data["litellm_metadata"]`` + for ``/v1/messages`` (see + ``LiteLLMProxyRequestSetup.add_user_api_key_auth_to_request_metadata`` and + ``LITELLM_METADATA_ROUTES``). The Anthropic-shape ``metadata`` arg only + carries ``user_id`` and must not be conflated. Returns ``None`` for SDK + callers that bypass the proxy entirely. + """ + litellm_metadata = kwargs.get("litellm_metadata") + if not isinstance(litellm_metadata, dict): + return None + return litellm_metadata + + +async def _prepare_context_managed_request( + *, + model: str, + messages: List[Dict], + tools: Optional[List[Dict]], + system: Optional[Any], + context_management_spec: Any, + litellm_metadata: Optional[Dict], + drop_params: Optional[bool], + llm_router: Any, + user_api_key_auth: Any = None, +) -> Optional[PolyfillResult]: + """Apply client compaction history, then optional context_management polyfill.""" + from litellm.llms.anthropic.experimental_pass_through.context_management.editors.compact import ( + apply_client_compaction_block_history, + ) + + # Skip the client-history pre-processing when a ``compact_20260112`` + # polyfill spec will run: that editor already slices around any client-sent + # compaction block in its Phase A (and uses the full post-compaction tail + # for its token-threshold check). Pre-collapsing to just the latest user + # question here would starve the polyfill of conversation context and + # silently drop intermediate turns. + polyfill_will_run = _polyfill_will_run( + context_management_spec=context_management_spec, + drop_params=drop_params, + ) + + if polyfill_will_run: + history_result: Optional[PolyfillResult] = None + working_messages: List[Dict] = messages + working_system: Optional[Any] = system + else: + history_result = apply_client_compaction_block_history( + messages=cast(List[Dict[str, Any]], messages), + system=system, + ) + working_messages = ( + history_result.messages if history_result is not None else messages + ) + working_system = history_result.system if history_result is not None else system + + polyfill_result = await _run_polyfill_if_enabled( + model=model, + messages=working_messages, + tools=tools, + system=working_system, + context_management_spec=context_management_spec, + litellm_metadata=litellm_metadata, + drop_params=drop_params, + llm_router=llm_router, + user_api_key_auth=user_api_key_auth, + ) + + if polyfill_result is not None: + return polyfill_result + + # Safety net: if we skipped client-history pre-processing because a + # ``compact_20260112`` polyfill was expected to handle the compaction + # block itself but the polyfill ultimately did not produce a result + # (e.g. it crashed and was best-effort swallowed in + # ``_run_polyfill_if_enabled``), apply the slice-only fallback now so + # Anthropic-specific ``compaction`` content blocks don't leak through + # to non-Anthropic backends that would reject them. + if polyfill_will_run and history_result is None: + history_result = apply_client_compaction_block_history( + messages=cast(List[Dict[str, Any]], messages), + system=system, + ) + return history_result + + +def _polyfill_will_run( + *, + context_management_spec: Any, + drop_params: Optional[bool], +) -> bool: + """Return True when ``compact_20260112`` will run via the polyfill dispatcher. + + Mirrors the gating in ``_run_polyfill_if_enabled``: an empty spec or + effective ``drop_params`` short-circuits the polyfill. The pre-processing + skip only applies when the dispatcher will actually invoke + ``apply_compact_20260112`` (which has its own compaction-block slicing). + """ + edits = _normalize_spec_edits( + context_management_spec=context_management_spec, + drop_params=drop_params, + ) + if edits is None: + return False + + from litellm.llms.anthropic.experimental_pass_through.context_management.constants import ( + COMPACT_EDIT_TYPE, + ) + + return any( + isinstance(edit, dict) and edit.get("type") == COMPACT_EDIT_TYPE + for edit in edits + ) + + +def _spec_has_non_compact_edits( + *, + context_management_spec: Any, + drop_params: Optional[bool], +) -> bool: + """Return True when the spec includes edits other than ``compact_20260112``. + + Used to decide whether a polyfill failure can be silently swallowed + (compact-only specs have a safe compaction-block slicing fallback) or + must be surfaced (other editors like ``clear_tool_uses_20250919`` have + no slice-only fallback and would otherwise be dropped without notice). + """ + edits = _normalize_spec_edits( + context_management_spec=context_management_spec, + drop_params=drop_params, + ) + if edits is None: + return False + + from litellm.llms.anthropic.experimental_pass_through.context_management.constants import ( + COMPACT_EDIT_TYPE, + ) + + return any( + isinstance(edit, dict) + and isinstance(edit.get("type"), str) + and edit.get("type") != COMPACT_EDIT_TYPE + for edit in edits + ) + + +def _normalize_spec_edits( + *, + context_management_spec: Any, + drop_params: Optional[bool], +) -> Optional[List[Dict[str, Any]]]: + """Return the normalized ``edits`` list, or ``None`` if the polyfill won't run. + + Delegates spec-shape normalization to the dispatcher's ``_normalize_spec`` + so the prediction here can't drift from what the dispatcher actually does. + """ + if not context_management_spec: + return None + + effective_drop_params = ( + drop_params if drop_params is not None else litellm.drop_params + ) + if effective_drop_params: + return None + + from litellm.llms.anthropic.experimental_pass_through.context_management.dispatcher import ( + _normalize_spec, + ) + + try: + return _normalize_spec(context_management_spec) + except Exception: + return None + + +async def _run_polyfill_if_enabled( + *, + model: str, + messages: List[Dict], + tools: Optional[List[Dict]], + system: Optional[Any], + context_management_spec: Any, + litellm_metadata: Optional[Dict], + drop_params: Optional[bool], + llm_router: Any, + user_api_key_auth: Any = None, +) -> Optional[PolyfillResult]: + """Run the async context_management polyfill if a spec is present. + + Returns ``None`` when the spec is empty or drop_params is on. Raises + ``AnthropicContextManagementError`` so the /v1/messages endpoint can + emit an Anthropic-format 400. All other exceptions are best-effort + swallowed (matches v0 behavior). + """ + if not context_management_spec: + return None + + effective_drop_params = ( + drop_params if drop_params is not None else litellm.drop_params + ) + if effective_drop_params: + return None + + try: + return await apply_context_management( + model=model, + messages=messages, + tools=tools, + system=system, + context_management_spec=context_management_spec, + litellm_metadata=litellm_metadata, + llm_router=llm_router, + user_api_key_auth=user_api_key_auth, + ) + except AnthropicContextManagementError: + # Surface validation errors so the endpoint can emit an Anthropic-format + # 400. Other exception types fall into the best-effort branch below. + raise + except Exception as e: + verbose_logger.exception( + "context_management polyfill: skipping edits due to error: %s", e + ) + # Best-effort swallow is only safe for compact-only specs, where the + # caller's compaction-block-slicing safety net produces a correct + # (if degraded) result. When the spec also requested non-compact + # edits (e.g. ``clear_tool_uses_20250919``), the safety net does + # NOT re-run those editors, so silently returning ``None`` would + # drop them with no error surface. Raise instead so the endpoint + # emits an Anthropic-format error. + if _spec_has_non_compact_edits( + context_management_spec=context_management_spec, + drop_params=drop_params, + ): + raise AnthropicContextManagementError( + status_code=500, + message=f"context_management polyfill failed: {e}", + ) from e + return None + + ######################################################## # init adapter ANTHROPIC_ADAPTER = AnthropicAdapter() @@ -163,7 +422,7 @@ class LiteLLMMessagesToCompletionTransformationHandler: metadata: Optional[Dict] = None, stop_sequences: Optional[List[str]] = None, stream: Optional[bool] = False, - system: Optional[str] = None, + system: Optional[Union[str, List[Dict[str, Any]]]] = None, temperature: Optional[float] = None, thinking: Optional[Dict] = None, tool_choice: Optional[Dict] = None, @@ -307,19 +566,56 @@ class LiteLLMMessagesToCompletionTransformationHandler: top_p: Optional[float] = None, output_format: Optional[Dict] = None, **kwargs, - ) -> Union[AnthropicMessagesResponse, AsyncIterator]: + ) -> Union[AnthropicMessagesResponse, AsyncIterator[Any], Iterator[bytes]]: """Handle non-Anthropic models asynchronously using the adapter""" + context_management = kwargs.pop("context_management", None) + drop_params: Optional[bool] = kwargs.get("drop_params", None) + litellm_router = kwargs.pop("litellm_router", None) + if litellm_router is None: + try: + from litellm.proxy.proxy_server import llm_router as _proxy_router + + litellm_router = _proxy_router + except Exception: + pass + + proxy_litellm_metadata = _extract_proxy_litellm_metadata(kwargs) + user_api_key_auth = ( + proxy_litellm_metadata.get("user_api_key_auth") + if proxy_litellm_metadata is not None + else None + ) + + polyfill_result = await _prepare_context_managed_request( + model=model, + messages=messages, + tools=tools, + system=system, + context_management_spec=context_management, + litellm_metadata=proxy_litellm_metadata, + drop_params=drop_params, + llm_router=litellm_router, + user_api_key_auth=user_api_key_auth, + ) + + effective_messages = ( + polyfill_result.messages if polyfill_result is not None else messages + ) + effective_system = ( + polyfill_result.system if polyfill_result is not None else system + ) + ( completion_kwargs, tool_name_mapping, ) = LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs( max_tokens=max_tokens, - messages=messages, + messages=effective_messages, model=model, metadata=metadata, stop_sequences=stop_sequences, stream=stream, - system=system, + system=effective_system, temperature=temperature, thinking=thinking, tool_choice=tool_choice, @@ -338,6 +634,8 @@ class LiteLLMMessagesToCompletionTransformationHandler: completion_response, model=model, tool_name_mapping=tool_name_mapping, + polyfill_result=polyfill_result, + is_async=True, ) ) if transformed_stream is not None: @@ -347,6 +645,7 @@ class LiteLLMMessagesToCompletionTransformationHandler: anthropic_response = ANTHROPIC_ADAPTER.translate_completion_output_params( cast(ModelResponse, completion_response), tool_name_mapping=tool_name_mapping, + polyfill_result=polyfill_result, ) if anthropic_response is not None: return anthropic_response @@ -372,8 +671,13 @@ class LiteLLMMessagesToCompletionTransformationHandler: **kwargs, ) -> Union[ AnthropicMessagesResponse, + Iterator[bytes], AsyncIterator[Any], - Coroutine[Any, Any, Union[AnthropicMessagesResponse, AsyncIterator[Any]]], + Coroutine[ + Any, + Any, + Union[AnthropicMessagesResponse, AsyncIterator[Any], Iterator[bytes]], + ], ]: """Handle non-Anthropic models using the adapter.""" if _is_async is True: @@ -395,17 +699,72 @@ class LiteLLMMessagesToCompletionTransformationHandler: **kwargs, ) + # Run the context_management polyfill on the sync path too so that + # ``litellm.messages.create()`` callers don't silently lose edits like + # ``clear_tool_uses_20250919``. The dispatcher is async (so the + # ``compact_20260112`` editor can ``await`` the summarization model); + # bridge to it via ``run_async_function``. + context_management = kwargs.pop("context_management", None) + drop_params: Optional[bool] = kwargs.get("drop_params", None) + # Deliberately do NOT auto-attach the proxy ``llm_router`` here: + # ``run_async_function`` spawns a new event loop in a worker thread + # to bridge to the async dispatcher, but the proxy router's httpx + # ``AsyncClient`` instances are bound to the proxy's main event loop. + # Reusing them from the new thread's loop violates httpx's single-loop + # invariant and can raise ``RuntimeError: Event loop is closed`` or + # produce stalled connections. The summary editor falls back to + # ``litellm.acompletion`` (which creates a fresh client per call) when + # ``llm_router`` is ``None``, which is safe to call from the bridged + # loop. The async ``async_anthropic_messages_handler`` path is + # unaffected because it ``await``s within the original event loop. + litellm_router = kwargs.pop("litellm_router", None) + + # Skip the async bridge entirely when there is nothing for either the + # polyfill or the client-history slice-only fallback to do. The vast + # majority of sync ``litellm.messages.create()`` requests carry no + # ``context_management`` spec and no client-sent ``compaction`` block, + # and bridging through a worker-thread event loop just to discover + # there is no work is pure overhead. + if context_management is None and not _messages_have_compaction_block(messages): + polyfill_result: Optional[PolyfillResult] = None + else: + proxy_litellm_metadata = _extract_proxy_litellm_metadata(kwargs) + user_api_key_auth = ( + proxy_litellm_metadata.get("user_api_key_auth") + if proxy_litellm_metadata is not None + else None + ) + polyfill_result = run_async_function( + _prepare_context_managed_request, + model=model, + messages=messages, + tools=tools, + system=system, + context_management_spec=context_management, + litellm_metadata=proxy_litellm_metadata, + drop_params=drop_params, + llm_router=litellm_router, + user_api_key_auth=user_api_key_auth, + ) + + effective_messages = ( + polyfill_result.messages if polyfill_result is not None else messages + ) + effective_system = ( + polyfill_result.system if polyfill_result is not None else system + ) + ( completion_kwargs, tool_name_mapping, ) = LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs( max_tokens=max_tokens, - messages=messages, + messages=effective_messages, model=model, metadata=metadata, stop_sequences=stop_sequences, stream=stream, - system=system, + system=effective_system, temperature=temperature, thinking=thinking, tool_choice=tool_choice, @@ -424,6 +783,8 @@ class LiteLLMMessagesToCompletionTransformationHandler: completion_response, model=model, tool_name_mapping=tool_name_mapping, + polyfill_result=polyfill_result, + is_async=False, ) ) if transformed_stream is not None: @@ -433,6 +794,7 @@ class LiteLLMMessagesToCompletionTransformationHandler: anthropic_response = ANTHROPIC_ADAPTER.translate_completion_output_params( cast(ModelResponse, completion_response), tool_name_mapping=tool_name_mapping, + polyfill_result=polyfill_result, ) if anthropic_response is not None: return anthropic_response diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py index c65dfb22730..bacb9f8ddf6 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py @@ -3,11 +3,26 @@ import json import traceback from collections import deque -from typing import TYPE_CHECKING, Any, AsyncIterator, Dict, Iterator, Literal, Optional +from typing import ( + TYPE_CHECKING, + Any, + AsyncIterator, + Dict, + Iterator, + List, + Literal, + Optional, +) -from litellm import verbose_logger +from litellm._logging import verbose_logger from litellm._uuid import uuid -from litellm.types.llms.anthropic import UsageDelta +from litellm.types.llms.anthropic import ( + AppliedEdit, + CompactionBlock, + ContextManagementResponse, + UsageDelta, + UsageIteration, +) from litellm.types.utils import AdapterCompletionStreamWrapper if TYPE_CHECKING: @@ -37,22 +52,208 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): holding_stop_reason_chunk: Optional[Any] = None queued_usage_chunk: bool = False current_content_block_index: int = 0 - current_content_block_start: ContentBlockContentBlockDict = TextBlock( - type="text", - text="", - ) - chunk_queue: deque = deque() # Queue for buffering multiple chunks def __init__( self, completion_stream: Any, model: str, tool_name_mapping: Optional[Dict[str, str]] = None, + applied_edits: Optional[List[AppliedEdit]] = None, + compaction_block: Optional[CompactionBlock] = None, + iterations_usage: Optional[List[UsageIteration]] = None, ): super().__init__(completion_stream) self.model = model # Mapping of truncated tool names to original names (for OpenAI's 64-char limit) self.tool_name_mapping = tool_name_mapping or {} + # Polyfill applied_edits on final message_delta. + self.applied_edits: List[AppliedEdit] = list(applied_edits or []) + # Synthesized compaction block from compact_20260112 polyfill (streaming). + self.compaction_block = compaction_block + self.iterations_usage = iterations_usage + self.sent_compaction_block: bool = False + # Per-phase flags so the compaction block's start/delta/stop events + # are emitted (and the public state machine is advanced) in + # lock-step with the caller actually consuming each event. Pre- + # queuing all three would set ``sent_content_block_finish=True`` + # before the client received ``content_block_stop``, leaving the + # observable state inconsistent during the drain window. + self.sent_compaction_block_start: bool = False + self.sent_compaction_block_delta: bool = False + # Per-instance queue for buffering multiple chunks. Must be initialized + # here (not at class level) so concurrent streams don't share the same + # deque and corrupt each other's SSE event order. + self.chunk_queue: deque = deque() + # Per-instance default content block. Must be initialized here (not at + # class level) so concurrent streams don't share the same mutable dict + # — `_should_start_new_content_block` mutates `tool_block["name"]` in + # place, which would otherwise leak across streams. + self.current_content_block_start: ( + "AnthropicStreamWrapper.ContentBlockContentBlockDict" + ) = self.TextBlock( + type="text", + text="", + ) + + def _merge_usage_into_held_stop_reason_chunk(self, chunk: Any) -> Dict[str, Any]: + """Merge usage data from ``chunk`` into the held ``message_delta`` chunk. + + Shared by both the sync ``__next__`` and async ``__anext__`` paths so + the subtle hold-and-merge logic (cache tokens, ``context_management`` + attachment, ``UsageDelta`` shape) lives in exactly one place. + + Caller is responsible for managing ``self.holding_stop_reason_chunk`` + and ``self.queued_usage_chunk`` state and for queuing the returned + merged chunk. + """ + assert self.holding_stop_reason_chunk is not None + merged_chunk = self.holding_stop_reason_chunk.copy() + if "delta" not in merged_chunk: + merged_chunk["delta"] = {} + + uncached_input_tokens = chunk.usage.prompt_tokens or 0 + if ( + hasattr(chunk.usage, "prompt_tokens_details") + and chunk.usage.prompt_tokens_details + ): + cached_tokens = ( + getattr(chunk.usage.prompt_tokens_details, "cached_tokens", 0) or 0 + ) + uncached_input_tokens -= cached_tokens + + usage_dict: UsageDelta = { + "input_tokens": uncached_input_tokens, + "output_tokens": chunk.usage.completion_tokens or 0, + } + if ( + hasattr(chunk.usage, "_cache_creation_input_tokens") + and chunk.usage._cache_creation_input_tokens > 0 + ): + usage_dict["cache_creation_input_tokens"] = ( + chunk.usage._cache_creation_input_tokens + ) + if ( + hasattr(chunk.usage, "_cache_read_input_tokens") + and chunk.usage._cache_read_input_tokens > 0 + ): + usage_dict["cache_read_input_tokens"] = chunk.usage._cache_read_input_tokens + merged_chunk["usage"] = usage_dict + if self.applied_edits and "context_management" not in merged_chunk: + merged_chunk["context_management"] = ContextManagementResponse( + applied_edits=list(self.applied_edits) + ) + return self._augment_message_delta_usage(merged_chunk) + + def _ensure_context_management_attached( + self, message_delta_chunk: Dict[str, Any] + ) -> Dict[str, Any]: + """Attach ``context_management`` to a ``message_delta`` chunk if + ``self.applied_edits`` is non-empty and the chunk does not already + carry it. Returns the (possibly new) chunk dict. + + Centralizing this guard ensures every ``message_delta`` emission + path (merge-with-usage and direct-flush-of-held) consistently + surfaces ``applied_edits`` to the client. + """ + if not self.applied_edits or "context_management" in message_delta_chunk: + return message_delta_chunk + augmented = message_delta_chunk.copy() + augmented["context_management"] = ContextManagementResponse( + applied_edits=list(self.applied_edits) + ) + return augmented + + def _augment_message_delta_usage( + self, message_delta_chunk: Dict[str, Any] + ) -> Dict[str, Any]: + """Attach polyfill compaction iteration usage to the final message_delta. + + Also defensively re-attaches ``context_management`` so the direct + held-chunk flush path stays in sync with the merge path's guarantee + when ``self.applied_edits`` is non-empty. + """ + message_delta_chunk = self._ensure_context_management_attached( + message_delta_chunk + ) + if self.iterations_usage is None: + return message_delta_chunk + usage = message_delta_chunk.get("usage") + if not isinstance(usage, dict) or "iterations" in usage: + return message_delta_chunk + + input_tokens = usage.get("input_tokens", 0) or 0 + output_tokens = usage.get("output_tokens", 0) or 0 + augmented = message_delta_chunk.copy() + augmented_usage = dict(usage) + iterations: List[UsageIteration] = list(self.iterations_usage) + # Only emit a ``message`` iteration when we have real token data. + # Without a separate usage chunk (e.g. provider sent finish_reason + # alone), the held ``message_delta`` carries placeholder zeros from + # the translate step; reporting a zero-token iteration would be + # misleading and inconsistent with the non-streaming path. + if input_tokens > 0 or output_tokens > 0: + message_iteration: UsageIteration = { + "type": "message", + "input_tokens": input_tokens, + "output_tokens": output_tokens, + } + iterations.append(message_iteration) + augmented_usage["iterations"] = iterations # type: ignore[typeddict-unknown-key] + augmented["usage"] = augmented_usage + return augmented + + def _next_compaction_event(self) -> Optional[Dict[str, Any]]: + """Return the next compaction content-block SSE event, or ``None``. + + Anthropic delivers compaction as a single delta (no token-by-token + streaming), but we still surface it as a proper + start → delta → stop trio. Each call returns exactly one event so + the state machine (``sent_content_block_finish``, + ``current_content_block_index``) is advanced *only* when the + terminal stop event is actually handed back to the caller. This + prevents an observable window where the flags claim the block is + finished while the stop event is still buffered. + """ + if self.compaction_block is None or self.sent_compaction_block: + return None + + compaction_index = self.current_content_block_index + + if not self.sent_compaction_block_start: + self.sent_compaction_block_start = True + return { + "type": "content_block_start", + "index": compaction_index, + # Mirror the text-block shape ({"type": "text", "text": ""}): + # send an empty ``content`` field so clients that introspect + # ``content_block_start`` see the full block schema. The + # actual summary text arrives via the ``content_block_delta`` + # below. + "content_block": {"type": "compaction", "content": ""}, + } + + if not self.sent_compaction_block_delta: + self.sent_compaction_block_delta = True + summary_content = self.compaction_block.get("content") or "" + return { + "type": "content_block_delta", + "index": compaction_index, + "delta": {"type": "compaction_delta", "content": summary_content}, + } + + stop_event = { + "type": "content_block_stop", + "index": compaction_index, + } + # Don't touch ``sent_content_block_finish`` here: that flag is the + # state machine for the regular text/tool_use/thinking block and is + # independent of the synthetic compaction block lifecycle. Conflating + # them would let outside observers (subclass overrides, introspection + # hooks, exception paths) see ``sent_content_block_finish=True`` + # without any regular content block ever having started. + self._increment_content_block_index() + self.sent_compaction_block = True + return stop_event def _create_initial_usage_delta(self) -> UsageDelta: """ @@ -75,7 +276,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): cache_read_input_tokens=0, ) - def __next__(self): + def __next__(self): # noqa: PLR0915 from .transformation import LiteLLMAnthropicMessagesAdapter try: @@ -103,8 +304,17 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): ) return self.chunk_queue.popleft() + if ( + self.sent_compaction_block is False + and self.compaction_block is not None + ): + compaction_event = self._next_compaction_event() + if compaction_event is not None: + return compaction_event + if self.sent_content_block_start is False: self.sent_content_block_start = True + self.sent_content_block_finish = False self.chunk_queue.append( { "type": "content_block_start", @@ -122,11 +332,45 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): if should_start_new_block: self._increment_content_block_index() + # applied_edits only needs to flow to the final message_delta + # (when finish_reason is set); skip threading it through every + # intermediate chunk. For the hold-and-merge path below, + # context_management is attached directly to the merged chunk, + # so the translated ``processed_chunk`` would be discarded — + # skip the applied_edits attachment in that case to avoid + # allocating a throwaway ``MessageBlockDelta``. + will_merge_into_held = ( + self.holding_stop_reason_chunk is not None + and getattr(chunk, "usage", None) is not None + ) + is_final_chunk = chunk.choices[0].finish_reason is not None processed_chunk = LiteLLMAnthropicMessagesAdapter().translate_streaming_openai_response_to_anthropic( response=chunk, current_content_block_index=self.current_content_block_index, + applied_edits=( + self.applied_edits + if is_final_chunk and not will_merge_into_held + else None + ), ) + # Check if this is a usage chunk and we have a held stop_reason chunk + if will_merge_into_held: + merged_chunk = self._merge_usage_into_held_stop_reason_chunk(chunk) + self.chunk_queue.append(merged_chunk) + self.queued_usage_chunk = True + self.holding_stop_reason_chunk = None + return self.chunk_queue.popleft() + + if self.queued_usage_chunk: + # Usage has already been merged + emitted. Any trailing + # provider events would violate Anthropic SSE ordering + # (no chunks may follow the final ``message_delta``), so + # silently drop them — matches the async ``__anext__`` + # behavior where the block-handling logic is gated on + # ``not self.queued_usage_chunk``. + continue + if should_start_new_block and not self.sent_content_block_finish: # Queue the sequence: content_block_stop -> content_block_start # For text blocks the trigger chunk is not emitted as a separate @@ -178,20 +422,64 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): } ) self.sent_content_block_finish = True - self.chunk_queue.append(processed_chunk) + if processed_chunk.get("delta", {}).get("stop_reason") is not None: + self.holding_stop_reason_chunk = processed_chunk + else: + processed_chunk = self._augment_message_delta_usage( + processed_chunk + ) + self.chunk_queue.append(processed_chunk) return self.chunk_queue.popleft() elif self.holding_chunk is not None: self.chunk_queue.append(self.holding_chunk) + if processed_chunk.get("type") == "message_delta": + processed_chunk = self._augment_message_delta_usage( + processed_chunk + ) self.chunk_queue.append(processed_chunk) self.holding_chunk = None return self.chunk_queue.popleft() else: + if processed_chunk.get("type") == "message_delta": + processed_chunk = self._augment_message_delta_usage( + processed_chunk + ) self.chunk_queue.append(processed_chunk) return self.chunk_queue.popleft() - # Handle any remaining held chunks after stream ends - if self.holding_chunk is not None: - self.chunk_queue.append(self.holding_chunk) + # Handle any remaining held chunks after stream ends. The + # buffered ``holding_chunk`` (a ``content_block_delta``) must + # precede the final ``message_delta`` so Anthropic SSE event + # ordering is preserved. When ``queued_usage_chunk`` is True, + # the final ``message_delta`` has already been emitted; any + # buffered content delta is dropped rather than emitted after + # ``message_delta`` (which would violate SSE ordering and may + # confuse strict Anthropic SDK clients). + if not self.queued_usage_chunk: + if self.holding_chunk is not None: + self.chunk_queue.append(self.holding_chunk) + self.holding_chunk = None + if self.holding_stop_reason_chunk is not None: + # A final ``message_delta`` must be preceded by + # ``content_block_stop`` so the emitted SSE stays in + # valid Anthropic order (... -> content_block_stop -> + # message_delta). Emit ``content_block_stop`` here if + # the active content block was not already closed. + if not self.sent_content_block_finish: + self.chunk_queue.append( + { + "type": "content_block_stop", + "index": self.current_content_block_index, + } + ) + self.sent_content_block_finish = True + self.chunk_queue.append( + self._augment_message_delta_usage( + self.holding_stop_reason_chunk + ) + ) + self.holding_stop_reason_chunk = None + else: self.holding_chunk = None if not self.sent_last_message: @@ -205,6 +493,26 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): except StopIteration: if self.chunk_queue: return self.chunk_queue.popleft() + # Handle any held stop_reason chunk. Emit ``content_block_stop`` + # first if the active content block was not already closed, so + # Anthropic SSE ordering is preserved (content_block_stop -> + # message_delta). + if self.holding_stop_reason_chunk is not None: + if not self.sent_content_block_finish: + self.sent_content_block_finish = True + self.chunk_queue.append( + self._augment_message_delta_usage( + self.holding_stop_reason_chunk + ) + ) + self.holding_stop_reason_chunk = None + return { + "type": "content_block_stop", + "index": self.current_content_block_index, + } + held = self._augment_message_delta_usage(self.holding_stop_reason_chunk) + self.holding_stop_reason_chunk = None + return held if self.sent_last_message is False: self.sent_last_message = True return {"type": "message_stop"} @@ -213,7 +521,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): verbose_logger.error( "Anthropic Adapter - {}\n{}".format(e, traceback.format_exc()) ) - raise StopAsyncIteration + raise StopIteration async def __anext__(self): # noqa: PLR0915 from .transformation import LiteLLMAnthropicMessagesAdapter @@ -243,8 +551,17 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): ) return self.chunk_queue.popleft() + if ( + self.sent_compaction_block is False + and self.compaction_block is not None + ): + compaction_event = self._next_compaction_event() + if compaction_event is not None: + return compaction_event + if self.sent_content_block_start is False: self.sent_content_block_start = True + self.sent_content_block_finish = False self.chunk_queue.append( { "type": "content_block_start", @@ -263,57 +580,31 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): if should_start_new_block: self._increment_content_block_index() + # applied_edits only needs to flow to the final message_delta + # (when finish_reason is set); skip threading it through every + # intermediate chunk. For the hold-and-merge path below, + # context_management is attached directly to the merged chunk, + # so the translated ``processed_chunk`` would be discarded — + # skip the applied_edits attachment in that case to avoid + # allocating a throwaway ``MessageBlockDelta``. + will_merge_into_held = ( + self.holding_stop_reason_chunk is not None + and getattr(chunk, "usage", None) is not None + ) + is_final_chunk = chunk.choices[0].finish_reason is not None processed_chunk = LiteLLMAnthropicMessagesAdapter().translate_streaming_openai_response_to_anthropic( response=chunk, current_content_block_index=self.current_content_block_index, + applied_edits=( + self.applied_edits + if is_final_chunk and not will_merge_into_held + else None + ), ) # Check if this is a usage chunk and we have a held stop_reason chunk - if ( - self.holding_stop_reason_chunk is not None - and getattr(chunk, "usage", None) is not None - ): - # Merge usage into the held stop_reason chunk - merged_chunk = self.holding_stop_reason_chunk.copy() - if "delta" not in merged_chunk: - merged_chunk["delta"] = {} - - # Add usage to the held chunk - uncached_input_tokens = chunk.usage.prompt_tokens or 0 - if ( - hasattr(chunk.usage, "prompt_tokens_details") - and chunk.usage.prompt_tokens_details - ): - cached_tokens = ( - getattr( - chunk.usage.prompt_tokens_details, "cached_tokens", 0 - ) - or 0 - ) - uncached_input_tokens -= cached_tokens - - usage_dict: UsageDelta = { - "input_tokens": uncached_input_tokens, - "output_tokens": chunk.usage.completion_tokens or 0, - } - # Add cache tokens if available (for prompt caching support) - if ( - hasattr(chunk.usage, "_cache_creation_input_tokens") - and chunk.usage._cache_creation_input_tokens > 0 - ): - usage_dict["cache_creation_input_tokens"] = ( - chunk.usage._cache_creation_input_tokens - ) - if ( - hasattr(chunk.usage, "_cache_read_input_tokens") - and chunk.usage._cache_read_input_tokens > 0 - ): - usage_dict["cache_read_input_tokens"] = ( - chunk.usage._cache_read_input_tokens - ) - merged_chunk["usage"] = usage_dict - - # Queue the merged chunk and reset + if will_merge_into_held: + merged_chunk = self._merge_usage_into_held_stop_reason_chunk(chunk) self.chunk_queue.append(merged_chunk) self.queued_usage_chunk = True self.holding_stop_reason_chunk = None @@ -379,28 +670,63 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): ): self.holding_stop_reason_chunk = processed_chunk else: + processed_chunk = self._augment_message_delta_usage( + processed_chunk + ) self.chunk_queue.append(processed_chunk) return self.chunk_queue.popleft() elif self.holding_chunk is not None: # Queue both chunks self.chunk_queue.append(self.holding_chunk) + if processed_chunk.get("type") == "message_delta": + processed_chunk = self._augment_message_delta_usage( + processed_chunk + ) self.chunk_queue.append(processed_chunk) self.holding_chunk = None return self.chunk_queue.popleft() else: - # Queue the current chunk + if processed_chunk.get("type") == "message_delta": + processed_chunk = self._augment_message_delta_usage( + processed_chunk + ) self.chunk_queue.append(processed_chunk) return self.chunk_queue.popleft() - # Handle any remaining held chunks after stream ends + # Handle any remaining held chunks after stream ends. The + # buffered ``holding_chunk`` (a ``content_block_delta``) must + # precede the final ``message_delta`` so Anthropic SSE event + # ordering is preserved. When ``queued_usage_chunk`` is True, + # the final ``message_delta`` has already been emitted; any + # buffered content delta is dropped rather than emitted after + # ``message_delta`` (which would violate SSE ordering and may + # confuse strict Anthropic SDK clients). if not self.queued_usage_chunk: - if self.holding_stop_reason_chunk is not None: - self.chunk_queue.append(self.holding_stop_reason_chunk) - self.holding_stop_reason_chunk = None - if self.holding_chunk is not None: self.chunk_queue.append(self.holding_chunk) self.holding_chunk = None + if self.holding_stop_reason_chunk is not None: + # A final ``message_delta`` must be preceded by + # ``content_block_stop`` so the emitted SSE stays in + # valid Anthropic order (... -> content_block_stop -> + # message_delta). Emit ``content_block_stop`` here if + # the active content block was not already closed. + if not self.sent_content_block_finish: + self.chunk_queue.append( + { + "type": "content_block_stop", + "index": self.current_content_block_index, + } + ) + self.sent_content_block_finish = True + self.chunk_queue.append( + self._augment_message_delta_usage( + self.holding_stop_reason_chunk + ) + ) + self.holding_stop_reason_chunk = None + else: + self.holding_chunk = None if not self.sent_last_message: self.sent_last_message = True @@ -416,9 +742,28 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): # Handle any remaining queued chunks before stopping if self.chunk_queue: return self.chunk_queue.popleft() - # Handle any held stop_reason chunk + # Handle any held stop_reason chunk — clear after capturing so a + # subsequent ``__anext__`` call doesn't re-emit the same chunk + # (matches the sync ``__next__`` path). Emit ``content_block_stop`` + # first if the active content block was not already closed, so + # Anthropic SSE ordering is preserved (content_block_stop -> + # message_delta). if self.holding_stop_reason_chunk is not None: - return self.holding_stop_reason_chunk + if not self.sent_content_block_finish: + self.sent_content_block_finish = True + self.chunk_queue.append( + self._augment_message_delta_usage( + self.holding_stop_reason_chunk + ) + ) + self.holding_stop_reason_chunk = None + return { + "type": "content_block_stop", + "index": self.current_content_block_index, + } + held = self._augment_message_delta_usage(self.holding_stop_reason_chunk) + self.holding_stop_reason_chunk = None + return held if not self.sent_last_message: self.sent_last_message = True return {"type": "message_stop"} diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index 51a1e739a0f..02e0c562654 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -6,6 +6,7 @@ from typing import ( Any, AsyncIterator, Dict, + Iterator, List, Literal, Optional, @@ -75,6 +76,9 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( from litellm.litellm_core_utils.prompt_templates.factory import ( THOUGHT_SIGNATURE_SEPARATOR, ) +from litellm.llms.anthropic.experimental_pass_through.context_management import ( + PolyfillResult, +) from litellm.types.llms.anthropic import ( ANTHROPIC_HOSTED_TOOLS, AllAnthropicToolsValues, @@ -87,14 +91,17 @@ from litellm.types.llms.anthropic import ( AnthropicResponseContentBlockText, AnthropicResponseContentBlockThinking, AnthropicResponseContentBlockToolUse, + AppliedEdit, ContentBlockDelta, ContentJsonBlockDelta, ContentTextBlockDelta, ContentThinkingBlockDelta, ContentThinkingSignatureBlockDelta, + ContextManagementResponse, MessageBlockDelta, MessageDelta, UsageDelta, + UsageIteration, ) from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, @@ -195,6 +202,7 @@ class AnthropicAdapter: self, response: ModelResponse, tool_name_mapping: Optional[Dict[str, str]] = None, + polyfill_result: Optional[PolyfillResult] = None, ) -> Optional[AnthropicMessagesResponse]: """ Translate OpenAI response to Anthropic format. @@ -204,10 +212,12 @@ class AnthropicAdapter: tool_name_mapping: Optional mapping of truncated tool names to original names. Used to restore original names for tools that exceeded OpenAI's 64-char limit. + polyfill_result: PolyfillResult from context_management polyfill. """ return LiteLLMAnthropicMessagesAdapter().translate_openai_response_to_anthropic( response=response, tool_name_mapping=tool_name_mapping, + polyfill_result=polyfill_result, ) def translate_completion_output_params_streaming( @@ -215,7 +225,9 @@ class AnthropicAdapter: completion_stream: Any, model: str, tool_name_mapping: Optional[Dict[str, str]] = None, - ) -> Union[AsyncIterator[bytes], None]: + polyfill_result: Optional[PolyfillResult] = None, + is_async: bool = True, + ) -> Union[AsyncIterator[bytes], Iterator[bytes], None]: """ Translate OpenAI streaming response to Anthropic format. @@ -223,14 +235,35 @@ class AnthropicAdapter: completion_stream: The OpenAI streaming response model: The model name tool_name_mapping: Optional mapping of truncated tool names to original names. + polyfill_result: PolyfillResult from context_management polyfill. + is_async: When ``True`` (default, for back-compat with existing + async callers) returns an ``AsyncIterator[bytes]``. When + ``False`` returns a sync ``Iterator[bytes]`` so sync callers + (e.g. ``litellm.anthropic.messages.create(stream=True)`` via + the sync handler) don't get back an async iterator they + can't iterate without an event loop. """ + applied_edits = ( + polyfill_result.applied_edits_for_response() if polyfill_result else None + ) + compaction_block = ( + polyfill_result.compaction_block if polyfill_result is not None else None + ) + iterations_usage = ( + polyfill_result.iterations_usage if polyfill_result is not None else None + ) anthropic_wrapper = AnthropicStreamWrapper( completion_stream=completion_stream, model=model, tool_name_mapping=tool_name_mapping, + applied_edits=applied_edits, + compaction_block=compaction_block, + iterations_usage=iterations_usage, ) - # Return the SSE-wrapped version for proper event formatting - return anthropic_wrapper.async_anthropic_sse_wrapper() + # Return the SSE-wrapped version for proper event formatting. + if is_async: + return anthropic_wrapper.async_anthropic_sse_wrapper() + return anthropic_wrapper.anthropic_sse_wrapper() class LiteLLMAnthropicMessagesAdapter: @@ -1342,6 +1375,7 @@ class LiteLLMAnthropicMessagesAdapter: self, response: ModelResponse, tool_name_mapping: Optional[Dict[str, str]] = None, + polyfill_result: Optional[PolyfillResult] = None, ) -> AnthropicMessagesResponse: """ Translate OpenAI response to Anthropic format. @@ -1351,12 +1385,17 @@ class LiteLLMAnthropicMessagesAdapter: tool_name_mapping: Optional mapping of truncated tool names to original names. Used to restore original names for tools that exceeded OpenAI's 64-char limit. + polyfill_result: PolyfillResult from context_management polyfill. """ ## translate content block anthropic_content = self._translate_openai_content_to_anthropic( choices=response.choices, # type: ignore tool_name_mapping=tool_name_mapping, ) + + if polyfill_result is not None and polyfill_result.compaction_block is not None: + anthropic_content.insert(0, polyfill_result.compaction_block) # type: ignore[arg-type] + ## extract finish reason anthropic_finish_reason = self._translate_openai_finish_reason_to_anthropic( openai_finish_reason=response.choices[0].finish_reason # type: ignore @@ -1385,6 +1424,14 @@ class LiteLLMAnthropicMessagesAdapter: if cached_tokens > 0: anthropic_usage["cache_read_input_tokens"] = cached_tokens + if polyfill_result is not None and polyfill_result.iterations_usage is not None: + message_iteration: UsageIteration = { + "type": "message", + "input_tokens": uncached_input_tokens, + "output_tokens": usage.completion_tokens or 0, + } + anthropic_usage["iterations"] = list(polyfill_result.iterations_usage) + [message_iteration] # type: ignore[typeddict-unknown-key] + translated_obj = AnthropicMessagesResponse( id=response.id, type="message", @@ -1396,6 +1443,14 @@ class LiteLLMAnthropicMessagesAdapter: stop_reason=anthropic_finish_reason, ) + applied_edits = ( + polyfill_result.applied_edits_for_response() if polyfill_result else None + ) + if applied_edits: + translated_obj["context_management"] = ContextManagementResponse( + applied_edits=list(applied_edits) + ) + return translated_obj def _translate_streaming_openai_chunk_to_anthropic_content_block( @@ -1528,7 +1583,10 @@ class LiteLLMAnthropicMessagesAdapter: return "text_delta", ContentTextBlockDelta(type="text_delta", text=text) def translate_streaming_openai_response_to_anthropic( - self, response: ModelResponse, current_content_block_index: int + self, + response: ModelResponse, + current_content_block_index: int, + applied_edits: Optional[List[AppliedEdit]] = None, ) -> Union[ContentBlockDelta, MessageBlockDelta]: ## base case - final chunk w/ finish reason if response.choices[0].finish_reason is not None: @@ -1578,9 +1636,14 @@ class LiteLLMAnthropicMessagesAdapter: usage_delta["cache_read_input_tokens"] = cached_tokens else: usage_delta = UsageDelta(input_tokens=0, output_tokens=0) - return MessageBlockDelta( + message_block = MessageBlockDelta( type="message_delta", delta=delta, usage=usage_delta # type: ignore ) + if applied_edits: + message_block["context_management"] = ContextManagementResponse( + applied_edits=list(applied_edits) + ) + return message_block ( type_of_content, content_block_delta, diff --git a/litellm/llms/anthropic/experimental_pass_through/context_management/__init__.py b/litellm/llms/anthropic/experimental_pass_through/context_management/__init__.py new file mode 100644 index 00000000000..729b2864524 --- /dev/null +++ b/litellm/llms/anthropic/experimental_pass_through/context_management/__init__.py @@ -0,0 +1,11 @@ +from .constants import CLEARED_TOOL_RESULT_PLACEHOLDER +from .dispatcher import apply_context_management +from .errors import AnthropicContextManagementError +from .result import PolyfillResult + +__all__ = [ + "apply_context_management", + "AnthropicContextManagementError", + "CLEARED_TOOL_RESULT_PLACEHOLDER", + "PolyfillResult", +] diff --git a/litellm/llms/anthropic/experimental_pass_through/context_management/constants.py b/litellm/llms/anthropic/experimental_pass_through/context_management/constants.py new file mode 100644 index 00000000000..ebbc182c427 --- /dev/null +++ b/litellm/llms/anthropic/experimental_pass_through/context_management/constants.py @@ -0,0 +1,45 @@ +"""Constants for the in-gateway context-management polyfill.""" + +CLEAR_TOOL_USES_EDIT_TYPE = "clear_tool_uses_20250919" + +DEFAULT_INPUT_TOKENS_TRIGGER = 100_000 +DEFAULT_KEEP_TOOL_USES = 3 + +CLEARED_TOOL_RESULT_PLACEHOLDER = "[Cleared by context management]" + +# compact_20260112 +COMPACT_EDIT_TYPE = "compact_20260112" +COMPACT_DEFAULT_TRIGGER_TOKENS = 150_000 +COMPACT_MIN_TRIGGER_TOKENS = 50_000 +# Default ``max_tokens`` for the summary call. Required by providers like +# Anthropic that reject requests without it; safely accepted by providers that +# don't strictly require it. Chosen to comfortably fit a long structured +# summary. Operators can override via +# ``general_settings.context_management_summary_max_tokens``. +COMPACT_SUMMARY_MAX_TOKENS = 4096 +COMPACT_SUMMARY_MAX_TOKENS_SETTING_KEY = "context_management_summary_max_tokens" +# Wall-clock bound for the summary sub-call. Without this a slow or +# unresponsive summary model would hang the parent ``/v1/messages`` request +# with no escape hatch; on timeout the editor falls into the standard +# ``summary_call_failed`` path and forwards the request without compaction. +COMPACT_SUMMARY_TIMEOUT_SECONDS = 60.0 +COMPACT_SUMMARY_MODEL_SETTING_KEY = "context_management_summary_model" +COMPACT_SUMMARY_SYSTEM_PREFIX = "Previous conversation summary: " + +# Default summarization prompt from the Anthropic spec. +COMPACT_DEFAULT_INSTRUCTIONS = ( + "You have written a partial transcript for the initial task above. Please " + "write a summary of the transcript. The purpose of this summary is to " + "provide continuity so you can continue to make progress towards solving " + "the task in a future context, where the raw history above may not be " + "accessible and will be replaced with this summary. Write down anything " + "that would be helpful, including the state, next steps, learnings etc. " + "You must wrap your summary in a

block." +) + +# Appended to the default prompt when ``tools`` are present and the caller +# did not supply custom ``instructions``. Matches the guidance in the +# Anthropic docs under "Compaction might fail when tools are defined". +COMPACT_NO_TOOL_CALLS_SUFFIX = ( + " Do not call any tools while writing this summary; respond with text only." +) diff --git a/litellm/llms/anthropic/experimental_pass_through/context_management/dispatcher.py b/litellm/llms/anthropic/experimental_pass_through/context_management/dispatcher.py new file mode 100644 index 00000000000..f7af09ee62a --- /dev/null +++ b/litellm/llms/anthropic/experimental_pass_through/context_management/dispatcher.py @@ -0,0 +1,127 @@ +"""Dispatch ``context_management`` edits to registered polyfill editors.""" + +import inspect +from typing import Any, Awaitable, Callable, Dict, List, Optional, Tuple, Union, cast + +from litellm._logging import verbose_logger +from litellm.types.llms.anthropic import AppliedEdit + +from .constants import CLEAR_TOOL_USES_EDIT_TYPE, COMPACT_EDIT_TYPE +from .editors import apply_clear_tool_uses_20250919, apply_compact_20260112 +from .result import PolyfillResult + +EditorFn = Callable[..., Any] + +_EDITOR_REGISTRY: Dict[str, EditorFn] = { + CLEAR_TOOL_USES_EDIT_TYPE: apply_clear_tool_uses_20250919, + COMPACT_EDIT_TYPE: apply_compact_20260112, +} + + +def _normalize_spec( + spec: Union[Dict[str, Any], List[Dict[str, Any]], None], +) -> Optional[List[Dict[str, Any]]]: + """Accept Anthropic-native dict form or OpenAI list form; return edits list.""" + if isinstance(spec, list): + # Local import to avoid an import cycle at module load. + from litellm.llms.anthropic.chat.transformation import AnthropicConfig + + spec = AnthropicConfig.map_openai_context_management_to_anthropic(spec) + + edits = spec.get("edits") if isinstance(spec, dict) else None + if not edits or not isinstance(edits, list): + return None + return [edit for edit in edits if isinstance(edit, dict)] + + +def _wrap_editor_return(raw: Any, *, fallback_system: Any) -> PolyfillResult: + """Coerce an editor's native return shape into a ``PolyfillResult``. + + v0 sync editors (e.g. ``clear_tool_uses_20250919``) return a 2-tuple + ``(messages, Optional[AppliedEdit])``. The new async ``compact_20260112`` + editor returns a ``PolyfillResult`` directly. + """ + if isinstance(raw, PolyfillResult): + return raw + # Legacy 2-tuple return — sync editors don't mutate ``system``, so + # carry the caller's value forward. + messages, applied = cast(Tuple[List[Dict[str, Any]], Any], raw) + return PolyfillResult( + messages=messages, + system=fallback_system, + applied_edits=[applied] if applied is not None else [], + ) + + +async def apply_context_management( + *, + model: str, + messages: List[Dict[str, Any]], + tools: Optional[List[Dict[str, Any]]], + system: Any, + context_management_spec: Union[Dict[str, Any], List[Dict[str, Any]], None], + litellm_metadata: Optional[Dict[str, Any]] = None, + llm_router: Any = None, + user_api_key_auth: Any = None, +) -> PolyfillResult: + """Run edits in order; return a single ``PolyfillResult``. + + The dispatcher is async so async editors (``compact_20260112``) can + ``await`` the configured summarization model. Sync editors are called + inline — ``inspect.iscoroutinefunction`` decides how each editor is + invoked. + """ + edits = _normalize_spec(context_management_spec) + if not edits: + return PolyfillResult(messages=messages, system=system, applied_edits=[]) + + current_messages = messages + current_system = system + aggregated_applied: List[AppliedEdit] = [] + aggregated_compaction_block = None + aggregated_iterations_usage = None + + for edit_spec in edits: + edit_type = edit_spec.get("type") + editor = _EDITOR_REGISTRY.get(edit_type) if isinstance(edit_type, str) else None + if editor is None: + verbose_logger.debug( + "context_management polyfill: unknown edit type '%s' — skipping", + edit_type, + ) + continue + + kwargs: Dict[str, Any] = { + "model": model, + "messages": current_messages, + "tools": tools, + "system": current_system, + "edit_spec": edit_spec, + } + # Only async editors accept these — passing them to sync v0 editors + # would break their signature. + if inspect.iscoroutinefunction(editor): + kwargs["litellm_metadata"] = litellm_metadata + kwargs["llm_router"] = llm_router + kwargs["user_api_key_auth"] = user_api_key_auth + raw_result = await cast(Callable[..., Awaitable[Any]], editor)(**kwargs) + else: + raw_result = editor(**kwargs) + + result = _wrap_editor_return(raw_result, fallback_system=current_system) + + current_messages = result.messages + current_system = result.system + aggregated_applied.extend(result.applied_edits) + if result.compaction_block is not None: + aggregated_compaction_block = result.compaction_block + if result.iterations_usage is not None: + aggregated_iterations_usage = result.iterations_usage + + return PolyfillResult( + messages=current_messages, + system=current_system, + applied_edits=aggregated_applied, + compaction_block=aggregated_compaction_block, + iterations_usage=aggregated_iterations_usage, + ) diff --git a/litellm/llms/anthropic/experimental_pass_through/context_management/editors/__init__.py b/litellm/llms/anthropic/experimental_pass_through/context_management/editors/__init__.py new file mode 100644 index 00000000000..3e933a9880a --- /dev/null +++ b/litellm/llms/anthropic/experimental_pass_through/context_management/editors/__init__.py @@ -0,0 +1,4 @@ +from .clear_tool_uses import apply_clear_tool_uses_20250919 +from .compact import apply_compact_20260112 + +__all__ = ["apply_clear_tool_uses_20250919", "apply_compact_20260112"] diff --git a/litellm/llms/anthropic/experimental_pass_through/context_management/editors/clear_tool_uses.py b/litellm/llms/anthropic/experimental_pass_through/context_management/editors/clear_tool_uses.py new file mode 100644 index 00000000000..7b1c20ff522 --- /dev/null +++ b/litellm/llms/anthropic/experimental_pass_through/context_management/editors/clear_tool_uses.py @@ -0,0 +1,210 @@ +"""``clear_tool_uses_20250919`` polyfill (v0: ``trigger`` and ``keep`` only).""" + +from typing import Any, Dict, List, Optional, Tuple, cast + +import litellm +from litellm._logging import verbose_logger +from litellm.types.llms.anthropic import AppliedEdit + +from ..constants import ( + CLEAR_TOOL_USES_EDIT_TYPE, + DEFAULT_INPUT_TOKENS_TRIGGER, + DEFAULT_KEEP_TOOL_USES, +) +from ..placeholders import build_cleared_tool_result_content + + +def _count_tool_uses(messages: List[Dict[str, Any]]) -> int: + """Return the number of tool_use content blocks across all messages. + + Only counts blocks with a string ``id`` to stay consistent with + :func:`_collect_tool_use_ids_in_order`, which is the source of truth for + which blocks are clearable. + """ + count = 0 + for msg in messages: + content = msg.get("content") + if isinstance(content, list): + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + if isinstance(block.get("id"), str): + count += 1 + return count + + +def _collect_tool_use_ids_in_order(messages: List[Dict[str, Any]]) -> List[str]: + """Return tool_use ids in the chronological order they appear in messages.""" + ids: List[str] = [] + for msg in messages: + content = msg.get("content") + if isinstance(content, list): + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + block_id = block.get("id") + if isinstance(block_id, str): + ids.append(block_id) + return ids + + +def _trigger_met( + trigger: Dict[str, Any], + model: str, + messages: List[Dict[str, Any]], + tools: Optional[List[Dict[str, Any]]], +) -> Tuple[bool, Optional[int]]: + """Return (trigger_met, input_tokens if counted for reuse).""" + trigger_type = trigger.get("type", "input_tokens") + threshold = trigger.get("value") + + if trigger_type == "tool_uses": + if not isinstance(threshold, int): + return False, None + return _count_tool_uses(messages) > threshold, None + + if not isinstance(threshold, int): + threshold = DEFAULT_INPUT_TOKENS_TRIGGER + current_tokens = litellm.token_counter( + model=model, + messages=messages, + tools=cast(Any, tools), + ) + verbose_logger.debug( + f"context_management polyfill: current_tokens: {current_tokens}" + ) + verbose_logger.debug(f"context_management polyfill: threshold: {threshold}") + return current_tokens > threshold, current_tokens + + +def _resolve_keep_count(keep: Dict[str, Any]) -> int: + keep_type = keep.get("type", "tool_uses") + if keep_type != "tool_uses": + return DEFAULT_KEEP_TOOL_USES + value = keep.get("value") + if not isinstance(value, int) or value < 0: + return DEFAULT_KEEP_TOOL_USES + return value + + +def _last_completed_tool_use_id( + messages: List[Dict[str, Any]], +) -> Optional[str]: + """Latest completed tool_result id; never cleared.""" + last_id: Optional[str] = None + for msg in messages: + content = msg.get("content") + if isinstance(content, list): + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_result": + block_id = block.get("tool_use_id") + if isinstance(block_id, str): + last_id = block_id + return last_id + + +def _clear_tool_results( + messages: List[Dict[str, Any]], ids_to_clear: set +) -> Tuple[List[Dict[str, Any]], int]: + """Clear matching tool_result content; return (messages, cleared_count).""" + cleared = 0 + new_messages: List[Dict[str, Any]] = [] + for msg in messages: + content = msg.get("content") + if not isinstance(content, list): + new_messages.append(msg) + continue + + new_blocks: List[Any] = [] + mutated = False + for block in content: + if ( + isinstance(block, dict) + and block.get("type") == "tool_result" + and block.get("tool_use_id") in ids_to_clear + ): + new_block = { + **block, + "content": build_cleared_tool_result_content(block.get("content")), + } + new_blocks.append(new_block) + mutated = True + cleared += 1 + else: + new_blocks.append(block) + + if mutated: + new_messages.append({**msg, "content": new_blocks}) + else: + new_messages.append(msg) + + return new_messages, cleared + + +def apply_clear_tool_uses_20250919( + *, + model: str, + messages: List[Dict[str, Any]], + tools: Optional[List[Dict[str, Any]]], + system: Any, + edit_spec: Dict[str, Any], +) -> Tuple[List[Dict[str, Any]], Optional[AppliedEdit]]: + """Apply clear_tool_uses; return (messages, AppliedEdit or None).""" + ignored_knobs = [ + knob + for knob in ("clear_at_least", "exclude_tools", "clear_tool_inputs") + if knob in edit_spec + ] + for ignored_knob in ignored_knobs: + verbose_logger.warning( + "context_management polyfill: ignoring '%s' on %s " + "(supported only on Anthropic-family forwarding path in v0)", + ignored_knob, + CLEAR_TOOL_USES_EDIT_TYPE, + ) + + trigger = edit_spec.get("trigger") or { + "type": "input_tokens", + "value": DEFAULT_INPUT_TOKENS_TRIGGER, + } + keep = edit_spec.get("keep") or { + "type": "tool_uses", + "value": DEFAULT_KEEP_TOOL_USES, + } + + met, tokens_before = _trigger_met(trigger, model, messages, tools) + if not met: + return messages, None + + keep_count = _resolve_keep_count(keep) + tool_use_ids = _collect_tool_use_ids_in_order(messages) + if len(tool_use_ids) <= keep_count: + return messages, None + + ids_to_clear = set(tool_use_ids[: len(tool_use_ids) - keep_count]) + + # Never clear the latest completed tool_result (reply context). + last_completed_id = _last_completed_tool_use_id(messages) + if last_completed_id is not None: + ids_to_clear.discard(last_completed_id) + + edited, cleared_count = _clear_tool_results(messages, ids_to_clear) + verbose_logger.debug("context_management polyfill: edited: %s", edited) + if cleared_count == 0: + return messages, None + + if tokens_before is None: + tokens_before = litellm.token_counter( + model=model, messages=messages, tools=cast(Any, tools) + ) + tokens_after = litellm.token_counter( + model=model, messages=edited, tools=cast(Any, tools) + ) + cleared_input_tokens = max(tokens_before - tokens_after, 0) + + applied: AppliedEdit = { + "type": CLEAR_TOOL_USES_EDIT_TYPE, + "cleared_tool_uses": cleared_count, + "cleared_input_tokens": cleared_input_tokens, + } + if ignored_knobs: + applied["warnings"] = [f"{knob}_ignored" for knob in ignored_knobs] + return edited, applied diff --git a/litellm/llms/anthropic/experimental_pass_through/context_management/editors/compact.py b/litellm/llms/anthropic/experimental_pass_through/context_management/editors/compact.py new file mode 100644 index 00000000000..4aae85b17fe --- /dev/null +++ b/litellm/llms/anthropic/experimental_pass_through/context_management/editors/compact.py @@ -0,0 +1,1206 @@ +"""``compact_20260112`` polyfill (server-side context compaction). + +Mirrors Anthropic's native ``compact_20260112`` for non-Anthropic providers: + +- Scans the message history for an existing ``compaction`` block; everything + before it is dropped (slice). +- If still over the configured trigger, calls a separately-configured + summarization model and synthesizes a fresh ``compaction`` block. +- The summary is injected as a system-message prefix on the downstream call + (the user/assistant log carries no ``compaction`` block downstream). +- The synthesized ``compaction`` block is returned via ``PolyfillResult`` so + the response adapter can prepend it to the response ``content`` array. +""" + +import re +from typing import Any, Dict, List, Literal, Optional, Tuple, Union, cast + +import litellm +from litellm._logging import verbose_logger +from litellm.types.llms.anthropic import ( + AppliedEdit, + CompactionBlock, + UsageIteration, +) + +from ..constants import ( + COMPACT_DEFAULT_INSTRUCTIONS, + COMPACT_DEFAULT_TRIGGER_TOKENS, + COMPACT_EDIT_TYPE, + COMPACT_MIN_TRIGGER_TOKENS, + COMPACT_NO_TOOL_CALLS_SUFFIX, + COMPACT_SUMMARY_MAX_TOKENS, + COMPACT_SUMMARY_MAX_TOKENS_SETTING_KEY, + COMPACT_SUMMARY_MODEL_SETTING_KEY, + COMPACT_SUMMARY_SYSTEM_PREFIX, + COMPACT_SUMMARY_TIMEOUT_SECONDS, +) +from ..errors import AnthropicContextManagementError +from ..result import PolyfillResult + +# Auth metadata fields propagated from the parent request to the summary call +# so the summary's spend is attributed to the same scopes. The list mirrors the +# fields populated by +# ``LiteLLMProxyRequestSetup.add_user_api_key_auth_to_request_metadata``. +# ``user_api_key_model_max_budget`` / ``user_api_key_end_user_model_max_budget`` +# are what ``_PROXY_VirtualKeyModelMaxBudgetLimiter`` reads post-call to update +# the per-model spend caches, so without them the summary spend would never +# count against the caller's model budget. ``user_api_key_end_user_id`` / +# ``user_api_key_project_id`` are the scope identifiers the post-call spend hook +# and rate limiter key their counters on, and ``user_api_end_user_max_budget`` +# is the end-user budget the cost callback enforces — without these the summary +# tokens escape the caller's end-user/project budgets and counters. +_PROPAGATED_METADATA_KEYS = ( + "user_api_key", + "user_api_key_alias", + "user_api_key_team_id", + "user_api_key_team_alias", + "user_api_key_user_id", + "user_api_key_user_email", + "user_api_key_org_id", + "user_api_key_project_id", + "user_api_key_end_user_id", + "user_api_end_user_max_budget", + "user_api_key_model_max_budget", + "user_api_key_end_user_model_max_budget", + "litellm_call_id", + "litellm_parent_otel_span", +) + +_SUMMARY_TAG_RE = re.compile(r"(.*?)", re.IGNORECASE | re.DOTALL) + + +def _read_summary_model_setting() -> Optional[str]: + """Look up the configured summarization model from proxy general_settings.""" + try: + from litellm.proxy.proxy_server import general_settings + except Exception: + return None + value = general_settings.get(COMPACT_SUMMARY_MODEL_SETTING_KEY) + return value if isinstance(value, str) and value else None + + +def _read_summary_max_tokens_setting() -> int: + """Look up the configured summary ``max_tokens`` from proxy general_settings. + + Falls back to :data:`COMPACT_SUMMARY_MAX_TOKENS` when the setting is + missing or invalid (non-positive int, wrong type). Operators tune this + when the default doesn't fit their chosen summary model's output budget. + """ + try: + from litellm.proxy.proxy_server import general_settings + except Exception: + return COMPACT_SUMMARY_MAX_TOKENS + value = general_settings.get(COMPACT_SUMMARY_MAX_TOKENS_SETTING_KEY) + if isinstance(value, int) and value > 0: + return value + return COMPACT_SUMMARY_MAX_TOKENS + + +async def _check_summary_model_access( # noqa: PLR0915 + user_api_key_auth: Any, + summary_model: str, + llm_router: Any, +) -> bool: + """Return True when every model-allowlist scope on the parent request is + satisfied for ``summary_model``. + + The summary subrequest does not pass through ``user_api_key_auth`` again, + so without this gate a caller whose configured scope at any of these + levels excludes ``context_management_summary_model`` could still get the + proxy to invoke that model and return its ```` output as a + compaction block. Mirrors the model-scope enforcement that + ``litellm.proxy.auth.common_checks`` runs for the client-requested model: + key, team, user (personal), project, and team-member allowlists. + + Returns True (allow) when ``user_api_key_auth`` is not present — SDK + callers and tests run outside the proxy, where no key/team policy exists. + Returns False when any of the active allowlists denies the summary model + (``ProxyException`` from ``_can_object_call_model`` / ``can_*_model``). + Unexpected errors during an access check fail closed but are logged + separately so operators can distinguish them from a real access-denied + response. DB-lookup failures (object missing from cache or DB) skip the + corresponding scope — matching ``common_checks``, which only enforces a + scope when its backing object can be loaded. + """ + if user_api_key_auth is None: + return True + try: + from litellm.proxy._types import ProxyException + from litellm.proxy.auth.auth_checks import ( + _can_object_call_model, + can_project_access_model, + can_user_call_model, + get_project_object, + get_team_membership, + get_user_object, + ) + from litellm.proxy.proxy_server import ( + prisma_client, + proxy_logging_obj, + user_api_key_cache, + ) + except Exception: + return True + + key_models = list(getattr(user_api_key_auth, "models", None) or []) + team_id = getattr(user_api_key_auth, "team_id", None) + team_model_aliases = getattr(user_api_key_auth, "team_model_aliases", None) + team_models = list(getattr(user_api_key_auth, "team_models", None) or []) + user_id = getattr(user_api_key_auth, "user_id", None) + project_id = getattr(user_api_key_auth, "project_id", None) + + checks: Tuple[Tuple[Literal["key", "team"], List[str]], ...] = ( + ("key", key_models), + ("team", team_models), + ) + for object_type, models in checks: + if not models: + continue + try: + _can_object_call_model( + model=summary_model, + llm_router=llm_router, + models=models, + team_model_aliases=team_model_aliases, + team_id=team_id, + object_type=object_type, + ) + except ProxyException: + return False + except Exception as e: + verbose_logger.warning( + "compact_20260112: unexpected error during %s-level access " + "check for summary_model=%s; denying access: %s", + object_type, + summary_model, + e, + ) + return False + + if user_id is not None and prisma_client is not None: + try: + user_obj = await get_user_object( + user_id=user_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + user_id_upsert=False, + proxy_logging_obj=proxy_logging_obj, + ) + except Exception as e: + verbose_logger.debug( + "compact_20260112: user object lookup failed for " + "summary_model=%s access check; skipping user-level scope: %s", + summary_model, + e, + ) + user_obj = None + if user_obj is not None: + try: + await can_user_call_model( + model=summary_model, + llm_router=llm_router, + user_object=user_obj, + ) + except ProxyException: + return False + except Exception as e: + verbose_logger.warning( + "compact_20260112: unexpected error during user-level " + "access check for summary_model=%s; denying access: %s", + summary_model, + e, + ) + return False + + if project_id is not None and prisma_client is not None: + try: + project_obj = await get_project_object( + project_id=project_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + except Exception as e: + verbose_logger.debug( + "compact_20260112: project object lookup failed for " + "summary_model=%s access check; skipping project-level scope: %s", + summary_model, + e, + ) + project_obj = None + if project_obj is not None and project_obj.models: + try: + can_project_access_model( + model=summary_model, + project_object=project_obj, + llm_router=llm_router, + ) + except ProxyException: + return False + except Exception as e: + verbose_logger.warning( + "compact_20260112: unexpected error during project-level " + "access check for summary_model=%s; denying access: %s", + summary_model, + e, + ) + return False + + if user_id is not None and team_id is not None and prisma_client is not None: + try: + team_membership = await get_team_membership( + user_id=user_id, + team_id=team_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + except Exception as e: + verbose_logger.debug( + "compact_20260112: team membership lookup failed for " + "summary_model=%s access check; skipping member-level scope: %s", + summary_model, + e, + ) + team_membership = None + member_allowed_models = ( + team_membership.litellm_budget_table.allowed_models + if team_membership is not None + and team_membership.litellm_budget_table is not None + else None + ) + if member_allowed_models: + try: + _can_object_call_model( + model=summary_model, + llm_router=llm_router, + models=list(member_allowed_models), + team_model_aliases=team_model_aliases, + team_id=team_id, + object_type="team", + ) + except ProxyException: + return False + except Exception as e: + verbose_logger.warning( + "compact_20260112: unexpected error during member-level " + "access check for summary_model=%s; denying access: %s", + summary_model, + e, + ) + return False + + return True + + +async def _check_summary_model_budget( + user_api_key_auth: Any, + summary_model: str, +) -> bool: + """Return True when the caller is within their per-model budget for + ``summary_model``. + + The summary subrequest never passes back through ``user_api_key_auth``, so + without this gate a caller whose ``model_max_budget`` for + ``context_management_summary_model`` is exhausted could keep consuming that + model via compaction. Mirrors the ``model_max_budget`` / + ``end_user_model_max_budget`` enforcement that ``user_api_key_auth`` runs for + the client-requested model. Returns True outside the proxy or when no + per-model budget is configured. + """ + if user_api_key_auth is None: + return True + try: + from litellm.proxy.proxy_server import model_max_budget_limiter + except Exception: + return True + + model_max_budget = getattr(user_api_key_auth, "model_max_budget", None) + token = getattr(user_api_key_auth, "token", None) + if isinstance(model_max_budget, dict) and model_max_budget and token is not None: + try: + await model_max_budget_limiter.is_key_within_model_budget( + user_api_key_dict=user_api_key_auth, + model=summary_model, + ) + except litellm.BudgetExceededError: + return False + except Exception as e: + verbose_logger.warning( + "compact_20260112: unexpected error during key model-budget " + "check for summary_model=%s; denying: %s", + summary_model, + e, + ) + return False + + end_user_model_max_budget = getattr( + user_api_key_auth, "end_user_model_max_budget", None + ) + end_user_id = getattr(user_api_key_auth, "end_user_id", None) + if ( + isinstance(end_user_model_max_budget, dict) + and end_user_model_max_budget + and end_user_id is not None + ): + try: + await model_max_budget_limiter.is_end_user_within_model_budget( + end_user_id=end_user_id, + end_user_model_max_budget=end_user_model_max_budget, + model=summary_model, + ) + except litellm.BudgetExceededError: + return False + except Exception as e: + verbose_logger.warning( + "compact_20260112: unexpected error during end-user model-budget " + "check for summary_model=%s; denying: %s", + summary_model, + e, + ) + return False + + return True + + +async def _check_summary_model_rate_limit( + user_api_key_auth: Any, + summary_model: str, +) -> bool: + """Return True when the caller is within their configured RPM/TPM limits + for ``summary_model``. + + The summary subrequest never passes back through the proxy's pre-call + rate limiter, so without this gate a caller already at their key / team / + user RPM or TPM could still drive an extra summary-model completion per + allowed ``/v1/messages`` request. This mirrors the read side of + ``_PROXY_MaxParallelRequestsHandler_v3.async_pre_call_hook`` for the + summary model: it builds the same descriptor set and runs the check in + ``read_only`` mode so no counter is reserved or incremented — the summary + call's actual usage is still charged exactly once by the limiter's + post-call success hook (via the propagated ``litellm_metadata``). + + Returns True (allow) outside the proxy, when the active limiter does not + expose the read-only descriptor check (legacy limiter), or when the + descriptor set cannot be built — the only deny signal is a definitive + ``OVER_LIMIT`` response, so an internal error here forwards the request + uncompacted rather than blocking every summary. + """ + if user_api_key_auth is None: + return True + try: + from litellm.proxy.proxy_server import proxy_logging_obj + except Exception: + return True + + limiter = getattr(proxy_logging_obj, "max_parallel_request_limiter", None) + if ( + limiter is None + or not hasattr(limiter, "should_rate_limit") + or not hasattr(limiter, "_create_rate_limit_descriptors") + ): + return True + + try: + metadata = getattr(user_api_key_auth, "metadata", None) or {} + data = {"model": summary_model} + descriptors = limiter._create_rate_limit_descriptors( + user_api_key_dict=user_api_key_auth, + data=data, + rpm_limit_type=metadata.get("rpm_limit_type"), + tpm_limit_type=metadata.get("tpm_limit_type"), + model_has_failures=False, + ) + limiter._add_team_model_rate_limit_descriptor_from_metadata( + user_api_key_dict=user_api_key_auth, + requested_model=summary_model, + descriptors=descriptors, + ) + limiter._add_project_model_rate_limit_descriptor_from_metadata( + user_api_key_dict=user_api_key_auth, + requested_model=summary_model, + descriptors=descriptors, + ) + descriptors.extend( + limiter.create_organization_rate_limit_descriptor( + user_api_key_auth, summary_model + ) + ) + if not descriptors: + return True + response = await limiter.should_rate_limit( + descriptors=descriptors, + parent_otel_span=getattr(user_api_key_auth, "parent_otel_span", None), + read_only=True, + ) + except Exception as e: + verbose_logger.warning( + "compact_20260112: unexpected error during rate-limit check for " + "summary_model=%s; allowing: %s", + summary_model, + e, + ) + return True + return response.get("overall_code") != "OVER_LIMIT" + + +def _find_latest_compaction_index( + messages: List[Dict[str, Any]], +) -> Tuple[Optional[int], Optional[int]]: + """Return (message_index, block_index) of the most recent compaction block. + + ``None, None`` if no compaction block is present. Iterates from the end so + only the latest one is considered. + """ + for msg_idx in range(len(messages) - 1, -1, -1): + content = messages[msg_idx].get("content") + if not isinstance(content, list): + continue + for blk_idx in range(len(content) - 1, -1, -1): + block = content[blk_idx] + if isinstance(block, dict) and block.get("type") == "compaction": + return msg_idx, blk_idx + return None, None + + +def _slice_around_compaction_block( + messages: List[Dict[str, Any]], +) -> Tuple[List[Dict[str, Any]], Optional[Dict[str, Any]]]: + """Apply Anthropic's "drop everything before the compaction block" rule. + + Returns ``(sliced_messages_with_compaction_block, compaction_block_dict)`` + if a block was found, else ``(original_messages, None)``. The sliced result + keeps the compaction block in the assistant turn that originally carried + it (in practice it's the only block in that turn) so callers can still + extract the summary text from it. + """ + msg_idx, blk_idx = _find_latest_compaction_index(messages) + if msg_idx is None or blk_idx is None: + return messages, None + + original_msg = messages[msg_idx] + original_content = original_msg["content"] + compaction_block = cast(Dict[str, Any], original_content[blk_idx]) + + # Per Anthropic's contract everything before the compaction block is + # dropped, including earlier blocks within the same assistant message. + sliced_content = list(original_content[blk_idx:]) + sliced_first_msg = {**original_msg, "content": sliced_content} + + sliced_messages: List[Dict[str, Any]] = [sliced_first_msg] + sliced_messages.extend(messages[msg_idx + 1 :]) + return sliced_messages, compaction_block + + +def _strip_compaction_blocks( + messages: List[Dict[str, Any]], +) -> List[Dict[str, Any]]: + """Drop any ``compaction`` content blocks from messages. + + Used to build the downstream-bound message list — the adapter has no + concept of a compaction block, so it must not see one. + """ + cleaned: List[Dict[str, Any]] = [] + for msg in messages: + content = msg.get("content") + if not isinstance(content, list): + cleaned.append(msg) + continue + filtered = [ + block + for block in content + if not (isinstance(block, dict) and block.get("type") == "compaction") + ] + if not filtered: + # The compaction block was the only content; drop the whole turn. + continue + cleaned.append({**msg, "content": filtered}) + return cleaned + + +def _augment_system_with_summary( + system: Optional[Union[str, List[Dict[str, Any]]]], + summary_text: str, +) -> Union[str, List[Dict[str, Any]]]: + """Prepend a "Previous conversation summary: ..." block to ``system``.""" + prefix = f"{COMPACT_SUMMARY_SYSTEM_PREFIX}{summary_text}\n\n" + if system is None: + return prefix.rstrip() + if isinstance(system, str): + return f"{prefix}{system}" + # List of content blocks: prepend the prefix to the first text block, + # otherwise insert a new text block at the head. + for idx, block in enumerate(system): + if isinstance(block, dict) and block.get("type") == "text": + existing = block.get("text", "") or "" + new_block = {**block, "text": f"{prefix}{existing}"} + return [*system[:idx], new_block, *system[idx + 1 :]] + return [{"type": "text", "text": prefix.rstrip()}, *system] + + +def _resolve_trigger_tokens(edit_spec: Dict[str, Any]) -> Tuple[int, List[str]]: + """Validate and resolve ``trigger.value``. + + Raises ``AnthropicContextManagementError`` if the explicitly-supplied value + is below the 50k minimum. Unknown ``trigger.type`` values fall back to + ``input_tokens`` with a warning. + """ + warnings: List[str] = [] + trigger = edit_spec.get("trigger") or {} + if not isinstance(trigger, dict): + warnings.append("trigger_not_a_dict_using_default") + return COMPACT_DEFAULT_TRIGGER_TOKENS, warnings + + trigger_type = trigger.get("type", "input_tokens") + if trigger_type != "input_tokens": + warnings.append(f"unsupported_trigger_type_{trigger_type}_using_input_tokens") + + value = trigger.get("value") + if value is None: + return COMPACT_DEFAULT_TRIGGER_TOKENS, warnings + if not isinstance(value, int): + warnings.append("trigger_value_not_int_using_default") + return COMPACT_DEFAULT_TRIGGER_TOKENS, warnings + if value < COMPACT_MIN_TRIGGER_TOKENS: + raise AnthropicContextManagementError( + status_code=400, + message=( + f"context_management.compact_20260112.trigger.value must be at " + f"least {COMPACT_MIN_TRIGGER_TOKENS} tokens" + ), + ) + return value, warnings + + +def _build_summary_prompt( + edit_spec: Dict[str, Any], tools: Optional[List[Dict[str, Any]]] +) -> str: + custom = edit_spec.get("instructions") + if isinstance(custom, str) and custom.strip(): + return custom + prompt = COMPACT_DEFAULT_INSTRUCTIONS + if tools: + prompt = f"{prompt}{COMPACT_NO_TOOL_CALLS_SUFFIX}" + return prompt + + +def _propagate_metadata( + parent_litellm_metadata: Optional[Dict[str, Any]], +) -> Dict[str, Any]: + """Extract the parent request's auth/spend-attribution fields for the summary subcall. + + The proxy attaches ``user_api_key``, ``user_api_key_team_id`` etc. to + ``data["litellm_metadata"]`` (see + ``LiteLLMProxyRequestSetup.add_user_api_key_auth_to_request_metadata``). + Without these on the summary subrequest, the router's post-call hooks + cannot attribute summary tokens to the caller's key/team budget. + """ + if not parent_litellm_metadata: + return {} + propagated: Dict[str, Any] = {} + for key in _PROPAGATED_METADATA_KEYS: + if key in parent_litellm_metadata: + propagated[key] = parent_litellm_metadata[key] + return propagated + + +def _count_effective_tokens( + model: str, + effective_messages: List[Dict[str, Any]], + compaction_block: Optional[Dict[str, Any]], + tools: Optional[List[Dict[str, Any]]], + system: Optional[Union[str, List[Dict[str, Any]]]] = None, +) -> int: + """Token-count the conversation as it will appear downstream. + + The compaction block (if any) becomes a system prefix on the downstream + call, so its content still counts even though it isn't in ``messages``. + The system prompt (which may already include a prior compaction summary + prepended via ``_augment_system_with_summary``) is also counted so the + threshold check matches the downstream ``input_tokens`` metric. + """ + # Local import to avoid pulling the adapter at module load time. + from litellm.llms.anthropic.experimental_pass_through.adapters.transformation import ( + LiteLLMAnthropicMessagesAdapter, + ) + + messages_without_compaction = _strip_compaction_blocks(effective_messages) + adapter = LiteLLMAnthropicMessagesAdapter() + try: + openai_shape = adapter.translate_anthropic_messages_to_openai( + messages=cast(Any, messages_without_compaction) + ) + except Exception as e: + verbose_logger.debug( + "compact_20260112: anthropic→openai translation failed during token " + "count, falling back to raw messages: %s", + e, + ) + openai_shape = cast(Any, messages_without_compaction) + + # Translate Anthropic-shaped tools (``input_schema``) to OpenAI-shaped + # tools (``{"type": "function", "function": {...}}``) so ``token_counter`` + # gets a consistent format regardless of which counting path it uses. + # An inaccurate tool token count here could cause the polyfill to skip + # needed compaction or trigger unnecessary summarization. + openai_tools: Optional[List[Dict[str, Any]]] = None + if tools: + try: + translated_tools, _ = adapter.translate_anthropic_tools_to_openai( + tools=cast(Any, tools) + ) + openai_tools = cast(List[Dict[str, Any]], translated_tools) + except Exception as e: + verbose_logger.debug( + "compact_20260112: anthropic→openai tools translation failed " + "during token count, falling back to raw tools: %s", + e, + ) + openai_tools = tools + + total = litellm.token_counter( + model=model, + messages=cast(Any, openai_shape), + tools=cast(Any, openai_tools), + ) + if compaction_block is not None: + content = compaction_block.get("content") or "" + if content: + total += litellm.token_counter(model=model, text=content) + system_text = _system_to_text(system) + if system_text: + total += litellm.token_counter(model=model, text=system_text) + return total + + +def _system_to_text( + system: Optional[Union[str, List[Dict[str, Any]]]], +) -> str: + """Flatten an Anthropic-style ``system`` value into a single string for + token counting. Returns ``""`` when ``system`` carries no text.""" + if system is None: + return "" + if isinstance(system, str): + return system + parts: List[str] = [] + for block in system: + if isinstance(block, dict) and block.get("type") == "text": + text = block.get("text") + if isinstance(text, str) and text: + parts.append(text) + return "\n".join(parts) + + +def _select_last_user_question( + messages: List[Dict[str, Any]], +) -> List[Dict[str, Any]]: + """Pick the most recent ``user`` turn that is a real question. + + Returns a one-element message list with any ``tool_result`` blocks + stripped: after compaction the paired ``tool_use`` assistant turn no + longer exists in the downstream context, so forwarding ``tool_result`` + blocks would translate to orphaned ``role=tool`` messages on + non-Anthropic providers (OpenAI, Gemini, …) and cause a 400 error. + + Falls back to a synthetic continuation prompt if no eligible turn + exists (e.g. the conversation only ever contained ``tool_result`` + turns, or contained no user turns at all). The downstream call always + needs a non-empty user message. + """ + for msg in reversed(messages): + if msg.get("role") != "user": + continue + content = msg.get("content") + if isinstance(content, list): + filtered = [ + blk + for blk in content + if not (isinstance(blk, dict) and blk.get("type") == "tool_result") + ] + if not filtered: + # Purely tool_result — skip and look for an earlier turn. + continue + if len(filtered) < len(content): + return [{**msg, "content": filtered}] + return [msg] + return [ + { + "role": "user", + "content": "Please continue based on the conversation summary above.", + } + ] + + +def _extract_summary_text(raw: Optional[str]) -> Optional[str]: + if not raw: + return None + match = _SUMMARY_TAG_RE.search(raw) + if match is None: + return None + summary = match.group(1).strip() + return summary or None + + +def _system_to_openai_message( + system: Optional[Union[str, List[Dict[str, Any]]]], +) -> Optional[Dict[str, Any]]: + """Translate Anthropic-shaped ``system`` to an OpenAI system message. + + Accepts a bare string or a list of Anthropic content blocks; returns + ``None`` if no usable text is present. Only ``type=="text"`` blocks are + carried over — the summary model has no use for ``cache_control`` or + other non-text metadata. + """ + if isinstance(system, str): + return {"role": "system", "content": system} if system else None + if isinstance(system, list): + parts = [ + block.get("text", "") + for block in system + if isinstance(block, dict) and block.get("type") == "text" + ] + joined = "\n\n".join(part for part in parts if part) + return {"role": "system", "content": joined} if joined else None + return None + + +def _build_summary_messages( + effective_messages: List[Dict[str, Any]], + prompt: str, + system: Optional[Union[str, List[Dict[str, Any]]]] = None, +) -> List[Dict[str, Any]]: + """Build the OpenAI-shape message list for the summary call. + + The caller's ``system`` prompt is prepended (the default summarization + instructions reference "the initial task above", which lives in that + system prompt); the conversation history is translated to OpenAI shape; + the summarization prompt is appended as a final user turn. + """ + from litellm.llms.anthropic.experimental_pass_through.adapters.transformation import ( + LiteLLMAnthropicMessagesAdapter, + ) + + stripped = _strip_compaction_blocks(effective_messages) + try: + openai_messages = ( + LiteLLMAnthropicMessagesAdapter().translate_anthropic_messages_to_openai( + messages=cast(Any, stripped) + ) + ) + except Exception as e: + verbose_logger.warning( + "compact_20260112: anthropic→openai translation failed when " + "building summary call; falling back to raw shape: %s", + e, + ) + openai_messages = cast(Any, stripped) + + summary_messages: List[Dict[str, Any]] = [] + system_message = _system_to_openai_message(system) + if system_message is not None: + summary_messages.append(system_message) + summary_messages.extend(openai_messages) + # If the last turn is already a user message, merge the summarization + # prompt into it. Some providers (and strict OpenAI-compatible endpoints) + # reject two consecutive ``role=user`` messages, which would otherwise + # silently fall into the ``summary_call_failed`` error path. + if summary_messages and _is_user_message(summary_messages[-1]): + last_msg = summary_messages[-1] + summary_messages[-1] = { + **last_msg, + "content": _append_text_to_content(last_msg.get("content"), prompt), + } + else: + summary_messages.append({"role": "user", "content": prompt}) + return summary_messages + + +def _is_user_message(msg: Any) -> bool: + return isinstance(msg, dict) and msg.get("role") == "user" + + +def _append_text_to_content(content: Any, extra_text: str) -> Any: + """Append ``extra_text`` to an OpenAI-shape message ``content`` field. + + Handles the two common shapes: ``str`` and ``list`` of content parts. + For unexpected/empty shapes, fall back so the caller gets a usable value. + """ + if content is None or content == "": + return extra_text + if isinstance(content, str): + return f"{content}\n\n{extra_text}" + if isinstance(content, list): + return [*content, {"type": "text", "text": extra_text}] + return [content, {"type": "text", "text": extra_text}] + + +async def _call_summary_model( + *, + summary_model: str, + summary_messages: List[Dict[str, Any]], + metadata: Dict[str, Any], + llm_router: Any, + allowed_model_region: Optional[str] = None, + max_tokens: int = COMPACT_SUMMARY_MAX_TOKENS, +) -> Any: + """Invoke the configured summary model. + + Prefers ``llm_router.acompletion`` so the model alias resolves against the + proxy's ``model_list``; falls back to ``litellm.acompletion`` if no router + is available (e.g. SDK usage outside the proxy). + """ + # ``max_tokens`` is required by providers like Anthropic and silently + # accepted by providers that don't strictly require it (OpenAI etc.). + # Setting a sensible default here means the feature works regardless of + # which model an admin configures as ``context_management_summary_model``; + # operators can override via ``context_management_summary_max_tokens`` in + # ``general_settings`` when the default doesn't fit the chosen model's + # output budget. + # The propagated proxy auth/spend-attribution fields (``user_api_key`` etc.) + # must travel as ``litellm_metadata`` — that is the parameter the proxy's + # post-call spend hooks read for budget attribution. The provider-level + # ``metadata`` kwarg corresponds to the upstream API request body and would + # not flow into spend tracking. + # ``allowed_model_region`` must travel as a top-level kwarg because the + # router enforces region restrictions by reading ``request_kwargs`` directly + # (see ``Router._common_checks_available_deployment``); without this the + # summary subrequest could be routed to a deployment outside the caller's + # permitted region. + # ``timeout`` bounds how long a slow/unresponsive summary model can stall + # the parent ``/v1/messages`` request. On timeout the caller catches the + # exception and surfaces ``applied_edits[0].error = "summary_call_failed"``, + # forwarding the request without compaction rather than hanging. + call_kwargs: Dict[str, Any] = { + "model": summary_model, + "messages": summary_messages, + "max_tokens": max_tokens, + "timeout": COMPACT_SUMMARY_TIMEOUT_SECONDS, + "litellm_metadata": metadata, + } + # The end-user id must also travel as the top-level ``user`` kwarg: legacy + # limiter hooks and prometheus end-user tracking read it from there rather + # than from ``litellm_metadata``, so without it the summary tokens would not + # debit the caller's end-user counters. + end_user_id = metadata.get("user_api_key_end_user_id") + if end_user_id: + call_kwargs["user"] = end_user_id + if allowed_model_region is not None: + call_kwargs["allowed_model_region"] = allowed_model_region + if llm_router is not None and hasattr(llm_router, "acompletion"): + return await llm_router.acompletion(**call_kwargs) + return await litellm.acompletion(**call_kwargs) + + +def _extract_response_text(response: Any) -> Optional[str]: + try: + choice = response.choices[0] + message = choice.message + content = getattr(message, "content", None) + if isinstance(content, str): + return content + # Some providers return a list of content parts. + if isinstance(content, list): + text_parts = [ + part.get("text", "") + for part in content + if isinstance(part, dict) and part.get("type") == "text" + ] + return "".join(text_parts) or None + except (AttributeError, IndexError, KeyError): + return None + return None + + +def _extract_usage(response: Any) -> Tuple[int, int]: + usage = getattr(response, "usage", None) + if usage is None: + return 0, 0 + return ( + int(getattr(usage, "prompt_tokens", 0) or 0), + int(getattr(usage, "completion_tokens", 0) or 0), + ) + + +def apply_client_compaction_block_history( + *, + messages: List[Dict[str, Any]], + system: Optional[Union[str, List[Dict[str, Any]]]], +) -> Optional[PolyfillResult]: + """Honor client-sent compaction blocks without a ``compact_20260112`` edit. + + When the request omits ``context_management`` but the message history already + contains a ``compaction`` content block (e.g. Claude Code client-side + compaction), apply the same slice-only forwarding as the under-threshold + path: the prior summary is prepended to ``system`` and the post-compaction + tail is forwarded unchanged (with compaction blocks stripped) so recent + turns the summary does not cover are preserved. + """ + effective_messages, prior_compaction_block = _slice_around_compaction_block( + messages + ) + if prior_compaction_block is None: + return None + + verbose_logger.info( + "compact_20260112: client compaction block in message history; " + "applying slice-only forwarding (no context_management edit)" + ) + + prior_summary_text = prior_compaction_block.get("content") or "" + augmented_system: Union[str, List[Dict[str, Any]], None] = system + if isinstance(prior_summary_text, str) and prior_summary_text: + augmented_system = _augment_system_with_summary(system, prior_summary_text) + verbose_logger.info( + "compact_20260112: compaction summary added to main call system prefix (%s chars)", + len(prior_summary_text), + ) + + # Post-compaction turns are recent context the prior summary does not cover, + # so forward them unchanged. Only fall back to the last user question if the + # strip leaves the downstream call with nothing to answer. + downstream_messages = _strip_compaction_blocks(effective_messages) + if not downstream_messages: + downstream_messages = _select_last_user_question(effective_messages) + + return PolyfillResult( + messages=downstream_messages, + system=augmented_system, + applied_edits=[], + ) + + +async def apply_compact_20260112( # noqa: PLR0915 + *, + model: str, + messages: List[Dict[str, Any]], + tools: Optional[List[Dict[str, Any]]], + system: Optional[Union[str, List[Dict[str, Any]]]], + edit_spec: Dict[str, Any], + litellm_metadata: Optional[Dict[str, Any]] = None, + llm_router: Any = None, + user_api_key_auth: Any = None, +) -> PolyfillResult: + """Apply ``compact_20260112``; return a ``PolyfillResult``. + + See module docstring for the algorithm. Errors are best-effort: when the + summary call fails or the response is malformed, the editor returns the + pre-summary state (with ``applied_edits[0].error`` populated) so the + original request still proceeds. + """ + # Validation runs first. Raising AnthropicContextManagementError here is + # the only path on which the polyfill aborts the request. + trigger_tokens, warnings = _resolve_trigger_tokens(edit_spec) + verbose_logger.info( + "compact_20260112: request has compaction trigger (input_tokens threshold=%s)", + trigger_tokens, + ) + if edit_spec.get("pause_after_compaction"): + warnings.append("pause_after_compaction_ignored") + + applied: AppliedEdit = {"type": COMPACT_EDIT_TYPE} + if warnings: + applied["warnings"] = warnings + + # Phase A: slice around any existing compaction block. Runs before the + # opt-in gate below so that even when summarization is disabled we still + # strip Anthropic-only ``compaction`` blocks from messages going to + # non-Anthropic backends (which would reject them). + effective_messages, prior_compaction_block = _slice_around_compaction_block( + messages + ) + prior_summary_text = ( + prior_compaction_block.get("content") if prior_compaction_block else None + ) + augmented_system: Union[str, List[Dict[str, Any]], None] = system + if isinstance(prior_summary_text, str) and prior_summary_text: + augmented_system = _augment_system_with_summary(system, prior_summary_text) + verbose_logger.info( + "compact_20260112: compaction summary added to main call system prefix (%s chars)", + len(prior_summary_text), + ) + + downstream_messages = _strip_compaction_blocks(effective_messages) + + # Opt-in gate: no summary model configured → no-op (but still return the + # Phase A-sliced/stripped messages so compaction blocks don't leak). + summary_model = _read_summary_model_setting() + if summary_model is None: + applied["error"] = "summary_model_not_configured" + # Slice-only forwarding: ``augmented_system`` already carries any prior + # compaction summary, and the post-compaction tail in + # ``downstream_messages`` is recent context the summary does not cover, + # so forward it unchanged. Only fall back to the last user question when + # the strip leaves nothing for the downstream call to answer. + if not downstream_messages: + downstream_messages = _select_last_user_question(effective_messages) + return PolyfillResult( + messages=downstream_messages, + system=augmented_system, + applied_edits=[applied], + ) + + # Phase B: threshold check. + try: + current_tokens = _count_effective_tokens( + model=model, + effective_messages=effective_messages, + # ``augmented_system`` already carries the prior compaction summary + # (prepended via ``_augment_system_with_summary``); pass ``None`` + # here so we don't double-count the summary text. + compaction_block=None, + tools=tools, + system=augmented_system, + ) + except Exception as e: + verbose_logger.warning( + "compact_20260112: token_counter failed; assuming under threshold: %s", e + ) + current_tokens = 0 + + verbose_logger.debug( + "compact_20260112: current_tokens=%s trigger=%s", current_tokens, trigger_tokens + ) + + if current_tokens <= trigger_tokens: + # Slice-only path: the prior compaction summary already lives in + # ``augmented_system``. Post-compaction turns are recent context the + # summary does not cover, so forward ``downstream_messages`` (the + # post-compaction tail with compaction blocks stripped) unchanged. + # Only fall back to the last user question when the strip leaves + # nothing for the downstream call to answer. + if not downstream_messages: + downstream_messages = _select_last_user_question(effective_messages) + return PolyfillResult( + messages=downstream_messages, + system=augmented_system, + applied_edits=[applied], + ) + + # Phase C: summarize. ``augmented_system`` carries any prior compaction + # summary so multi-round compaction does not lose accumulated history — + # ``effective_messages`` only contains turns since the last compaction. + if not await _check_summary_model_access( + user_api_key_auth=user_api_key_auth, + summary_model=summary_model, + llm_router=llm_router, + ): + verbose_logger.warning( + "compact_20260112: caller not authorized for summary_model=%s; " + "skipping summary call", + summary_model, + ) + applied["error"] = "summary_model_access_denied" + return PolyfillResult( + messages=downstream_messages, + system=augmented_system, + applied_edits=[applied], + ) + + if not await _check_summary_model_budget( + user_api_key_auth=user_api_key_auth, + summary_model=summary_model, + ): + verbose_logger.warning( + "compact_20260112: caller over model budget for summary_model=%s; " + "skipping summary call", + summary_model, + ) + applied["error"] = "summary_model_budget_exceeded" + return PolyfillResult( + messages=downstream_messages, + system=augmented_system, + applied_edits=[applied], + ) + + if not await _check_summary_model_rate_limit( + user_api_key_auth=user_api_key_auth, + summary_model=summary_model, + ): + verbose_logger.warning( + "compact_20260112: caller over rate limit for summary_model=%s; " + "skipping summary call", + summary_model, + ) + applied["error"] = "summary_model_rate_limit_exceeded" + return PolyfillResult( + messages=downstream_messages, + system=augmented_system, + applied_edits=[applied], + ) + + prompt = _build_summary_prompt(edit_spec, tools) + summary_messages = _build_summary_messages( + effective_messages, prompt, system=augmented_system + ) + propagated_metadata = _propagate_metadata(litellm_metadata) + allowed_model_region = getattr(user_api_key_auth, "allowed_model_region", None) + + try: + response = await _call_summary_model( + summary_model=summary_model, + summary_messages=summary_messages, + metadata=propagated_metadata, + llm_router=llm_router, + allowed_model_region=allowed_model_region, + max_tokens=_read_summary_max_tokens_setting(), + ) + except Exception as e: + verbose_logger.warning("compact_20260112: summary call failed: %s", e) + applied["error"] = "summary_call_failed" + return PolyfillResult( + messages=downstream_messages, + system=augmented_system, + applied_edits=[applied], + ) + + summary_text = _extract_summary_text(_extract_response_text(response)) + if summary_text is None: + applied["error"] = "summary_extraction_failed" + return PolyfillResult( + messages=downstream_messages, + system=augmented_system, + applied_edits=[applied], + ) + + summary_input_tokens, summary_output_tokens = _extract_usage(response) + applied["summary_input_tokens"] = summary_input_tokens + applied["summary_output_tokens"] = summary_output_tokens + + compaction_block: CompactionBlock = { + "type": "compaction", + "content": summary_text, + } + iterations_usage: List[UsageIteration] = [ + { + "type": "compaction", + "input_tokens": summary_input_tokens, + "output_tokens": summary_output_tokens, + } + ] + + # Per Anthropic's contract, everything before the compaction block is + # dropped. Phase D: the user/assistant log goes empty; the summary lives + # on the system message instead. Anthropic requires a non-empty messages + # array, so keep the most recent original user *question* turn so the + # model has something to answer. Skip ``tool_result``-only user turns: + # in Anthropic's format those are role=user but represent the response + # from a tool, and surfacing one as the sole downstream message would + # produce an orphaned ``tool``-role message on non-Anthropic providers + # with no matching ``tool_calls`` in the prior assistant history. If no + # eligible turn exists, fall back to a synthetic continuation prompt so + # the downstream call still has a non-empty user message. + summarized_system = _augment_system_with_summary(system, summary_text) + verbose_logger.info( + "compact_20260112: compaction summary added to main call system prefix (%s chars)", + len(summary_text), + ) + downstream_messages_after_summary = _select_last_user_question(effective_messages) + + return PolyfillResult( + messages=downstream_messages_after_summary, + system=summarized_system, + applied_edits=[applied], + compaction_block=compaction_block, + iterations_usage=iterations_usage, + ) diff --git a/litellm/llms/anthropic/experimental_pass_through/context_management/errors.py b/litellm/llms/anthropic/experimental_pass_through/context_management/errors.py new file mode 100644 index 00000000000..1b14089a451 --- /dev/null +++ b/litellm/llms/anthropic/experimental_pass_through/context_management/errors.py @@ -0,0 +1,14 @@ +"""Exceptions raised by the context_management polyfill.""" + + +class AnthropicContextManagementError(Exception): + """Validation error from the polyfill, surfaced as an Anthropic-format 4xx. + + The `/v1/messages` endpoint catches this in its exception handler and + emits an Anthropic-shaped error body instead of the default OpenAI shape. + """ + + def __init__(self, *, status_code: int, message: str) -> None: + super().__init__(message) + self.status_code = status_code + self.message = message diff --git a/litellm/llms/anthropic/experimental_pass_through/context_management/placeholders.py b/litellm/llms/anthropic/experimental_pass_through/context_management/placeholders.py new file mode 100644 index 00000000000..f684d970df4 --- /dev/null +++ b/litellm/llms/anthropic/experimental_pass_through/context_management/placeholders.py @@ -0,0 +1,14 @@ +"""Placeholder content for cleared ``tool_result`` blocks (string or block list).""" + +from typing import Any, List, Union + +from .constants import CLEARED_TOOL_RESULT_PLACEHOLDER + + +def build_cleared_tool_result_content( + original_content: Any, +) -> Union[str, List[dict]]: + """Return a string or single text block list, matching ``original_content`` shape.""" + if isinstance(original_content, list): + return [{"type": "text", "text": CLEARED_TOOL_RESULT_PLACEHOLDER}] + return CLEARED_TOOL_RESULT_PLACEHOLDER diff --git a/litellm/llms/anthropic/experimental_pass_through/context_management/result.py b/litellm/llms/anthropic/experimental_pass_through/context_management/result.py new file mode 100644 index 00000000000..36bcde98d0c --- /dev/null +++ b/litellm/llms/anthropic/experimental_pass_through/context_management/result.py @@ -0,0 +1,53 @@ +"""``PolyfillResult`` — the shape returned by the context-management dispatcher. + +Threaded from the dispatcher through ``async_anthropic_messages_handler`` into +the adapter so it can prepend the ``compaction`` block to the response and +attach ``iterations`` to ``usage``. +""" + +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional, Union + +from litellm.types.llms.anthropic import ( + AppliedEdit, + CompactionBlock, + UsageIteration, +) + +from .constants import COMPACT_EDIT_TYPE + + +@dataclass +class PolyfillResult: + messages: List[Dict[str, Any]] + system: Optional[Union[str, List[Dict[str, Any]]]] + applied_edits: List[AppliedEdit] = field(default_factory=list) + compaction_block: Optional[CompactionBlock] = None + iterations_usage: Optional[List[UsageIteration]] = None + + def applied_edits_for_response(self) -> Optional[List[AppliedEdit]]: + """``applied_edits`` to attach on the client-visible response. + + ``compact_20260112`` is included when a new compaction block was + synthesized (success), when the edit carries an ``error`` field + (``summary_model_not_configured``, ``summary_call_failed``, + ``summary_extraction_failed``), or when the edit carries + ``warnings`` (e.g. ``unsupported_trigger_type_X_using_input_tokens``, + ``pause_after_compaction_ignored``) — operators and clients need to + see why compaction was requested but not applied as expected. + Slice-only / under-threshold paths that produced no edit at all + (no block, no error, no warnings) are omitted. Other edit types are + included when the editor returned an ``AppliedEdit``. + """ + visible: List[AppliedEdit] = [] + for edit in self.applied_edits: + if edit.get("type") == COMPACT_EDIT_TYPE: + if ( + self.compaction_block is not None + or edit.get("error") + or edit.get("warnings") + ): + visible.append(edit) + else: + visible.append(edit) + return visible or None diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py index 14e06e047ea..62eced8e6f0 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py @@ -8,7 +8,17 @@ import asyncio import contextvars from functools import partial -from typing import Any, AsyncIterator, Coroutine, Dict, List, Optional, Union, cast +from typing import ( + Any, + AsyncIterator, + Coroutine, + Dict, + Iterator, + List, + Optional, + Union, + cast, +) import litellm from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -189,7 +199,7 @@ async def anthropic_messages( client: Optional[AsyncHTTPHandler] = None, custom_llm_provider: Optional[str] = None, **kwargs, -) -> Union[AnthropicMessagesResponse, AsyncIterator]: +) -> Union[AnthropicMessagesResponse, Iterator[bytes], AsyncIterator[Any]]: """ Async: Make llm api request in Anthropic /messages API spec. @@ -346,8 +356,11 @@ def anthropic_messages_handler( **kwargs, ) -> Union[ AnthropicMessagesResponse, + Iterator[bytes], AsyncIterator[Any], - Coroutine[Any, Any, Union[AnthropicMessagesResponse, AsyncIterator[Any]]], + Coroutine[ + Any, Any, Union[AnthropicMessagesResponse, AsyncIterator[Any], Iterator[bytes]] + ], ]: """ Makes Anthropic `/v1/messages` API calls In the Anthropic API Spec @@ -456,9 +469,14 @@ def anthropic_messages_handler( return LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler( **_shared_kwargs ) + + # The in-gateway context_management polyfill runs inside + # ``async_anthropic_messages_handler`` so it can ``await`` the + # summarization model for ``compact_20260112``. ``context_management`` + # is passed through as a regular kwarg. return ( LiteLLMMessagesToCompletionTransformationHandler.anthropic_messages_handler( - **_shared_kwargs + **_shared_kwargs, ) ) diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py index 15f404d3f53..3a2c09f2183 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py @@ -230,6 +230,8 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): """Translate legacy ``thinking.type=enabled`` to adaptive for 4.6/4.7. Caller-provided ``output_config.effort`` is never overridden. """ + from litellm.llms.anthropic.chat.transformation import AnthropicConfig + if not AnthropicModelInfo._is_adaptive_thinking_model(model): return thinking = optional_params.get("thinking") @@ -237,7 +239,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): return budget = int(thinking.get("budget_tokens") or 0) - if budget >= 24000: + if budget >= 24000 and AnthropicConfig._supports_effort_level(model, "xhigh"): effort = "xhigh" elif budget >= 10000: effort = "high" @@ -427,8 +429,13 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value ) - # Check for structured outputs - if optional_params.get("output_format") is not None: + # Check for structured outputs. Anthropic's newer request shape nests + # the schema under output_config.format; the older top-level + # output_format remains supported for backwards compatibility. + output_config = optional_params.get("output_config") + if optional_params.get("output_format") is not None or ( + isinstance(output_config, dict) and output_config.get("format") is not None + ): beta_values.add( ANTHROPIC_BETA_HEADER_VALUES.STRUCTURED_OUTPUT_2025_09_25.value ) diff --git a/litellm/llms/azure/common_utils.py b/litellm/llms/azure/common_utils.py index 4fc1ae960b8..e1ac1858912 100644 --- a/litellm/llms/azure/common_utils.py +++ b/litellm/llms/azure/common_utils.py @@ -1,3 +1,5 @@ +import asyncio +import hashlib import json import os from typing import Any, Callable, Dict, Literal, NamedTuple, Optional, Union, cast @@ -449,6 +451,25 @@ class BaseAzureLLM(BaseOpenAILLM): ] = None client_initialization_params: dict = locals() client_initialization_params["is_async"] = _is_async + _lp = litellm_params or {} + _ad_provider = _lp.get("azure_ad_token_provider") + _ad_token = _lp.get("azure_ad_token") + _client_secret = _lp.get("client_secret") + _azure_password = _lp.get("azure_password") + client_initialization_params["azure_ad_token"] = ( + hashlib.sha256(_ad_token.encode()).hexdigest() + if isinstance(_ad_token, str) + else None + ) + client_initialization_params["azure_ad_token_provider"] = ( + f"provider_id={id(_ad_provider) if callable(_ad_provider) else None}" + f"|tenant_id={_lp.get('tenant_id')}" + f"|client_id={_lp.get('client_id')}" + f"|client_secret={hashlib.sha256(_client_secret.encode()).hexdigest() if isinstance(_client_secret, str) else None}" + f"|azure_username={_lp.get('azure_username')}" + f"|azure_password={hashlib.sha256(_azure_password.encode()).hexdigest() if isinstance(_azure_password, str) else None}" + f"|azure_scope={_lp.get('azure_scope')}" + ) if client is None: cached_client = self.get_cached_openai_client( client_initialization_params=client_initialization_params, @@ -474,8 +495,29 @@ class BaseAzureLLM(BaseOpenAILLM): if self._is_azure_v1_api_version(api_version): # Extract only params that OpenAI client accepts # Always use /openai/v1/ regardless of whether user passed "v1", "latest", or "preview" - v1_params = { - "api_key": azure_client_params.get("api_key"), + # The OpenAI client accepts a callable for `api_key` and re-invokes it + # on every request (via `_refresh_api_key`), so passing + # `azure_ad_token_provider` directly preserves Azure AD token refresh + # behavior that the regular AzureOpenAI client provides. + v1_api_key: Optional[Union[str, Callable[[], Any]]] = ( + azure_client_params.get("api_key") + or azure_client_params.get("azure_ad_token_provider") + or azure_client_params.get("azure_ad_token") + ) + if _is_async is True and callable(v1_api_key): + # AsyncOpenAI expects an async provider; wrap the sync provider + # returned by azure-identity. Offload to a thread so a token + # refresh (blocking HTTP call to AAD on cache miss) does not + # stall the event loop. + _sync_provider = v1_api_key + + async def _async_v1_api_key() -> str: + return await asyncio.to_thread(_sync_provider) + + v1_api_key = _async_v1_api_key + + v1_params: Dict[str, Any] = { + "api_key": v1_api_key, "base_url": f"{api_base}/openai/v1/", } if "timeout" in azure_client_params: diff --git a/litellm/llms/base_llm/chat/transformation.py b/litellm/llms/base_llm/chat/transformation.py index bec25916c4b..5f35a58ce1f 100644 --- a/litellm/llms/base_llm/chat/transformation.py +++ b/litellm/llms/base_llm/chat/transformation.py @@ -108,10 +108,9 @@ class BaseConfig(ABC): return type_to_response_format_param(response_format=response_format) def is_thinking_enabled(self, non_default_params: dict) -> bool: - return ( - non_default_params.get("thinking", {}).get("type") == "enabled" - or non_default_params.get("reasoning_effort") is not None - ) + return (non_default_params.get("thinking") or {}).get( + "type" + ) == "enabled" or non_default_params.get("reasoning_effort") is not None def is_max_tokens_in_request(self, non_default_params: dict) -> bool: """ diff --git a/litellm/llms/base_llm/managed_resources/__init__.py b/litellm/llms/base_llm/managed_resources/__init__.py index 5eb9b46f89f..a5543e631c0 100644 --- a/litellm/llms/base_llm/managed_resources/__init__.py +++ b/litellm/llms/base_llm/managed_resources/__init__.py @@ -24,10 +24,12 @@ from .utils import ( generate_unified_id_string, is_base64_encoded_unified_id, parse_unified_id, + resolve_passthrough_managed_id_provider, ) __all__ = [ "BaseManagedResource", + "resolve_passthrough_managed_id_provider", "is_base64_encoded_unified_id", "extract_target_model_names_from_unified_id", "extract_resource_type_from_unified_id", diff --git a/litellm/llms/base_llm/managed_resources/utils.py b/litellm/llms/base_llm/managed_resources/utils.py index 59f5ff0d845..e9a6aef689e 100644 --- a/litellm/llms/base_llm/managed_resources/utils.py +++ b/litellm/llms/base_llm/managed_resources/utils.py @@ -7,7 +7,40 @@ different managed resource types (files, vector stores, etc.). import base64 import re -from typing import List, Optional, Union, Literal +from typing import Any, List, Literal, Optional, Union + +PASSTHROUGH_MANAGED_ID_AZURE_PROVIDERS = ("azure", "azure_ai") + + +def resolve_passthrough_managed_id_provider( + custom_llm_provider: Any, +) -> Optional[str]: + """Map a pass-through ``custom_llm_provider`` to the provider scope that + namespaces passthrough managed object IDs, or ``None`` when the route is not + an OpenAI/Azure pass-through and managed IDs must not apply. + + Scoping is keyed on the explicit provider that the pass-through route + forwards (``openai``, ``azure``, ``azure_ai``), not on the upstream URL, so + a third-party OpenAI-compatible endpoint never triggers managed-ID minting. + + ``azure`` and ``azure_ai`` deliberately collapse to one ``"azure"`` scope: + they expose the same Azure OpenAI files/batches surface, so an ID minted + while routing as one must still resolve while routing as the other. + Splitting them would make a managed ID minted on ``azure`` fail to resolve + when replayed on ``azure_ai`` and vice versa. + """ + provider = str( + getattr(custom_llm_provider, "value", custom_llm_provider) or "" + ).lower() + if not provider: + return None + if provider in PASSTHROUGH_MANAGED_ID_AZURE_PROVIDERS or provider.endswith( + (".azure", ".azure_ai") + ): + return "azure" + if provider == "openai" or provider.endswith(".openai"): + return "openai" + return None def is_base64_encoded_unified_id( @@ -177,8 +210,14 @@ def extract_model_id_from_unified_id( if decoded_id: unified_id = decoded_id - # Extract model ID - match = re.search(r"model_id,([^;]+)", unified_id) + # Extract model ID. Anchor to a field boundary (start of string or + # after `;`) so this regex doesn't substring-match the `model_id,` + # inside file_id encodings' `llm_output_file_model_id,` + # field — that would feed the deployment UUID as a model candidate + # into the team-access check and 403 every team-BYOK file attach + # with `Tried to access ` (LIT-3244 patch/1.86.0 second-order + # finding). + match = re.search(r"(?:^|;)model_id,([^;]+)", unified_id) if match: return match.group(1).strip() diff --git a/litellm/llms/base_llm/realtime/transformation.py b/litellm/llms/base_llm/realtime/transformation.py index d5531a532b9..0f239b4ad45 100644 --- a/litellm/llms/base_llm/realtime/transformation.py +++ b/litellm/llms/base_llm/realtime/transformation.py @@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Any, List, Optional, Union import httpx +from litellm.types.llms.openai import OpenAIRealtimeStreamSessionEvents from litellm.types.realtime import ( RealtimeResponseTransformInput, RealtimeResponseTypedDict, @@ -69,6 +70,20 @@ class BaseRealtimeConfig(ABC): ) -> Optional[str]: # message sent to setup the realtime session return None + def transform_session_created_event( + self, + model: str, + logging_session_id: str, + session_configuration_request: Optional[str] = None, + ) -> Optional[Union[dict, OpenAIRealtimeStreamSessionEvents]]: + """ + Optional hook for providers that defer session setup until client `session.update`. + + Return an OpenAI-compatible `session.created` payload when the proxy should + emit a synthetic event immediately after backend websocket connection. + """ + return None + @abstractmethod def transform_realtime_response( self, diff --git a/litellm/llms/base_llm/videos/transformation.py b/litellm/llms/base_llm/videos/transformation.py index 87289ad6a0c..9b4cf777280 100644 --- a/litellm/llms/base_llm/videos/transformation.py +++ b/litellm/llms/base_llm/videos/transformation.py @@ -321,6 +321,23 @@ class BaseVideoConfig(ABC): "video get character is not supported for this provider" ) + def get_video_edit_prefetch_params( + self, + video_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Optional[Tuple[str, Dict]]: + """ + Return (url, body) for a pre-fetch HTTP call that must be made before + transform_video_edit_request, or None if no pre-fetch is required. + + Providers that need to retrieve the source video before constructing the + edit request (e.g. Vertex AI) should override this method. The handler + uses the existing shared httpx client so the call is properly async. + """ + return None + def transform_video_edit_request( self, prompt: str, @@ -329,6 +346,7 @@ class BaseVideoConfig(ABC): litellm_params: GenericLiteLLMParams, headers: dict, extra_body: Optional[Dict[str, Any]] = None, + prefetched_source_data: Optional[Dict[str, Any]] = None, ) -> Tuple[str, Dict]: """ Transform the video edit request into a URL and JSON data. @@ -343,6 +361,7 @@ class BaseVideoConfig(ABC): raw_response: httpx.Response, logging_obj: LiteLLMLoggingObj, custom_llm_provider: Optional[str] = None, + request_data: Optional[Dict] = None, ) -> VideoObject: raise NotImplementedError("video edit is not supported for this provider") diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index efc890d9ee2..90dfa13e938 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -30,6 +30,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( BedrockConverseMessagesProcessor, _bedrock_converse_messages_pt, _bedrock_tools_pt, + make_valid_bedrock_tool_name, ) from litellm.llms.anthropic.chat.transformation import ( DROP_UNSUPPORTED_OUTPUT_CONFIG_WARNING, @@ -40,6 +41,7 @@ from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMExcepti from litellm.types.llms.bedrock import * from litellm.types.llms.openai import ( AllMessageValues, + ChatCompletionAnnotation, ChatCompletionAssistantMessage, ChatCompletionRedactedThinkingBlock, ChatCompletionResponseMessage, @@ -77,6 +79,7 @@ from ..common_utils import ( get_anthropic_beta_from_headers, get_bedrock_tool_name, is_claude_4_5_on_bedrock, + normalize_bedrock_opus_output_config_effort, ) # Computer use tool prefixes supported by Bedrock @@ -447,10 +450,20 @@ class AmazonConverseConfig(BaseConfig): value=reasoning_effort, llm_provider="bedrock_converse", ) + existing_output_config = optional_params.get("output_config") + if not isinstance(existing_output_config, dict): + existing_output_config = {} + existing_output_config.setdefault("effort", mapped_effort) + normalize_bedrock_opus_output_config_effort( + model=model, + output_config=existing_output_config, + ) + mapped_effort = existing_output_config["effort"] self._validate_anthropic_adaptive_effort( model=model, effort=mapped_effort ) - optional_params["output_config"] = {"effort": mapped_effort} + optional_params["output_config"] = existing_output_config + optional_params["_output_config_normalized"] = True @staticmethod def _validate_anthropic_adaptive_effort(model: str, effort: str) -> None: @@ -573,6 +586,9 @@ class AmazonConverseConfig(BaseConfig): ): supported_params.append("thinking") supported_params.append("reasoning_effort") + + if base_model.startswith("anthropic"): + supported_params.append("context_management") return supported_params def map_tool_choice_values( @@ -595,7 +611,9 @@ class AmazonConverseConfig(BaseConfig): elif isinstance(tool_choice, dict): # only supported for anthropic + mistral models - https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ToolChoice.html specific_tool = SpecificToolChoiceBlock( - name=tool_choice.get("function", {}).get("name", "") + name=make_valid_bedrock_tool_name( + tool_choice.get("function", {}).get("name", "") + ) ) return ToolChoiceValuesBlock(tool=specific_tool) else: @@ -932,10 +950,10 @@ class AmazonConverseConfig(BaseConfig): self._handle_reasoning_effort_parameter( model=model, reasoning_effort=value, optional_params=optional_params ) + elif param == "context_management" and isinstance(value, (dict, list)): + self._map_context_management_param(value, optional_params) if param == "requestMetadata": - if value is not None and isinstance(value, dict): - self._validate_request_metadata(value) # type: ignore - optional_params["requestMetadata"] = value + self._map_request_metadata_param(value, optional_params) if param == "service_tier" and isinstance(value, str): self._map_service_tier_param(value, optional_params) @@ -968,6 +986,32 @@ class AmazonConverseConfig(BaseConfig): return optional_params + def _map_request_metadata_param(self, value: Any, optional_params: dict) -> None: + if value is not None and isinstance(value, dict): + self._validate_request_metadata(value) # type: ignore + optional_params["requestMetadata"] = value + + def _map_context_management_param( + self, value: Union[dict, list], optional_params: dict + ) -> None: + # Match the dispatcher's ``_normalize_spec`` behavior: only run the + # OpenAI→Anthropic mapper for list inputs. Dict inputs are already in + # Anthropic-native shape (``{"edits": [...]}``) and should pass + # through unchanged so an Anthropic-format ``context_management`` + # value isn't silently dropped when the mapper can't classify it. + if isinstance(value, list): + mapped = AnthropicConfig.map_openai_context_management_to_anthropic( + cast(Union[dict, list], value) + ) + else: + mapped = value + # Skip when the mapper returned None for malformed input — leaving the + # key out is safer than passing `context_management: null` downstream, + # which Bedrock would reject and which can confuse intermediate checks + # before the final _filter_context_management_for_bedrock_converse step. + if mapped is not None: + optional_params["context_management"] = mapped + def _map_service_tier_param(self, value: str, optional_params: dict) -> None: """Map OpenAI service_tier (string) to Bedrock serviceTier (object). @@ -1198,6 +1242,12 @@ class AmazonConverseConfig(BaseConfig): self, optional_params: dict, model: str ) -> Tuple[dict, dict, dict, Optional[OutputConfigBlock]]: """Prepare and separate request parameters.""" + # Consume the internal ``_output_config_normalized`` marker set by + # ``_handle_reasoning_effort_parameter`` so it does not linger on the + # caller's ``optional_params`` after the transformation returns. + anthropic_output_config_already_normalized = bool( + optional_params.pop("_output_config_normalized", False) + ) # Filter out exception objects before deepcopy to prevent deepcopy failures # Exceptions should not be stored in optional_params (this is a defensive fix) cleaned_params = filter_exceptions_from_params(optional_params) @@ -1216,8 +1266,17 @@ class AmazonConverseConfig(BaseConfig): # Anthropic-only ``output_config`` (snake_case) — re-attached to # ``additionalModelRequestFields`` for Anthropic models below. The - # Bedrock-native ``outputConfig`` (camelCase) is handled separately. + # structured-output ``format`` subfield is consumed into Bedrock's + # native ``outputConfig`` (camelCase), which is handled separately. anthropic_output_config = inference_params.pop("output_config", None) + output_config_format = None + if isinstance(anthropic_output_config, dict): + anthropic_output_config = dict(anthropic_output_config) + candidate_output_config_format = anthropic_output_config.pop("format", None) + if isinstance(candidate_output_config_format, dict): + output_config_format = candidate_output_config_format + if not anthropic_output_config: + anthropic_output_config = None # Extract requestMetadata before processing other parameters request_metadata = inference_params.pop("requestMetadata", None) @@ -1227,6 +1286,30 @@ class AmazonConverseConfig(BaseConfig): output_config: Optional[OutputConfigBlock] = inference_params.pop( "outputConfig", None ) + base_model = BedrockModelInfo.get_base_model(model) + if ( + output_config is None + and output_config_format is not None + and output_config_format.get("type") == "json_schema" + and base_model.startswith("anthropic") + and self._supports_native_structured_outputs( + model, self.custom_llm_provider + ) + ): + output_config = self._create_output_config_for_response_format( + json_schema=output_config_format.get("schema"), + name=output_config_format.get("name"), + description=output_config_format.get("description"), + ) + elif output_config is None and output_config_format is not None: + litellm.verbose_logger.warning( + "Bedrock Converse: dropping `output_config.format` for model=%s — " + "model does not advertise `supports_native_structured_output` in " + "model_prices_and_context_window.json. The schema will not be " + "enforced; pass `response_format` to use the synthetic tool-call " + "fallback.", + model, + ) # keep supported params in 'inference_params', and set all model-specific params in 'additional_request_params' additional_request_params = { @@ -1272,7 +1355,6 @@ class AmazonConverseConfig(BaseConfig): if anthropic_output_config is not None and isinstance( anthropic_output_config, dict ): - base_model = BedrockModelInfo.get_base_model(model) if base_model.startswith("anthropic"): if ( litellm.drop_params is True @@ -1283,6 +1365,11 @@ class AmazonConverseConfig(BaseConfig): model, ) else: + if not anthropic_output_config_already_normalized: + normalize_bedrock_opus_output_config_effort( + model=model, + output_config=anthropic_output_config, + ) effort = anthropic_output_config.get("effort") if effort is not None: self._validate_anthropic_adaptive_effort( @@ -1430,6 +1517,11 @@ class AmazonConverseConfig(BaseConfig): if ANTHROPIC_EFFORT_BETA_HEADER not in anthropic_beta_list: anthropic_beta_list.append(ANTHROPIC_EFFORT_BETA_HEADER) + # Bedrock Converse: compact_20260112 edits only (+ beta header). + AmazonConverseConfig._filter_context_management_for_bedrock_converse( + additional_request_params, anthropic_beta_list + ) + # Set anthropic_beta in additional_request_params if we have any beta features # ONLY apply to Anthropic/Claude models - other models (e.g., Qwen, Llama) don't support this field if anthropic_beta_list and base_model.startswith("anthropic"): @@ -1437,6 +1529,42 @@ class AmazonConverseConfig(BaseConfig): return bedrock_tools, anthropic_beta_list + @staticmethod + def _filter_context_management_for_bedrock_converse( + additional_request_params: dict, + anthropic_beta_list: list, + ) -> None: + """Keep only compact_20260112 edits for Bedrock; add beta header or drop field.""" + from litellm.llms.anthropic.experimental_pass_through.context_management.constants import ( + COMPACT_EDIT_TYPE, + ) + from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES + + cm = additional_request_params.get("context_management") + if not isinstance(cm, dict): + additional_request_params.pop("context_management", None) + return + edits = cm.get("edits") + if not isinstance(edits, list): + additional_request_params.pop("context_management", None) + return + + compact_edits = [ + e + for e in edits + if isinstance(e, dict) and e.get("type") == COMPACT_EDIT_TYPE + ] + if compact_edits: + compact_beta = ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value + if compact_beta not in anthropic_beta_list: + anthropic_beta_list.append(compact_beta) + additional_request_params["context_management"] = { + **cm, + "edits": compact_edits, + } + else: + additional_request_params.pop("context_management", None) + def _transform_request_helper( self, model: str, @@ -1887,6 +2015,75 @@ class AmazonConverseConfig(BaseConfig): return content_str, tools, reasoningContentBlocks, citationsContentBlocks + @staticmethod + def _transform_citations_to_annotations( + citations_content_blocks: Optional[List[CitationsContentBlock]], + ) -> Tuple[Optional[str], Optional[List[ChatCompletionAnnotation]]]: + """ + Convert Bedrock citationsContent blocks into OpenAI-style annotations. + + Returns: + citations_text: concatenated text from citationsContent.content + annotations: OpenAI URL citation annotations + """ + if not citations_content_blocks: + return None, None + + annotations: List[ChatCompletionAnnotation] = [] + citations_text_parts: List[str] = [] + content_offset = 0 + + for citations_block in citations_content_blocks: + block_text = "" + raw_content = citations_block.get("content") + if isinstance(raw_content, list): + for content_part in raw_content: + if isinstance(content_part, dict): + _text = content_part.get("text") + if isinstance(_text, str): + block_text += _text + + block_offset = content_offset + if block_text: + citations_text_parts.append(block_text) + content_offset += len(block_text) + + raw_citations = citations_block.get("citations") + if not isinstance(raw_citations, list): + continue + + for citation in raw_citations: + if not isinstance(citation, dict): + continue + + location = citation.get("location") + if not isinstance(location, dict): + continue + + search_location = location.get("searchResultLocation") + if not isinstance(search_location, dict): + continue + + start = search_location.get("start") + end = search_location.get("end") + if not isinstance(start, int) or not isinstance(end, int): + continue + + annotations.append( + ChatCompletionAnnotation( + type="url_citation", + url_citation={ + "start_index": block_offset + start, + "end_index": block_offset + end, + "title": str(citation.get("title") or ""), + "url": str(citation.get("source") or ""), + }, + ) + ) + + citations_text = "".join(citations_text_parts) if citations_text_parts else None + return citations_text, annotations or None + @staticmethod def _unwrap_bedrock_properties(json_str: str) -> str: """ @@ -2069,6 +2266,24 @@ class AmazonConverseConfig(BaseConfig): provider_specific_fields ) + citations_text, annotations = self._transform_citations_to_annotations( + citationsContentBlocks + ) + citations_included_in_content = False + if citations_text: + stripped_content = content_str.strip() + if not stripped_content: + content_str = citations_text + citations_included_in_content = True + elif not any(char.isalnum() for char in stripped_content): + # Bedrock may emit the cited sentence in citationsContent and only + # punctuation in the text blocks; stitch citations_text in front so + # its annotation span indices stay aligned with the final content. + content_str = citations_text + content_str + citations_included_in_content = True + if annotations and citations_included_in_content: + chat_completion_message["annotations"] = annotations + if reasoningContentBlocks is not None: chat_completion_message["reasoning_content"] = ( self._transform_reasoning_content(reasoningContentBlocks) diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py index d9599b8b9c4..a13336b6c88 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py @@ -16,8 +16,11 @@ from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation AmazonInvokeConfig, ) from litellm.llms.bedrock.common_utils import ( + convert_bedrock_invoke_output_format_to_inline_schema, get_anthropic_beta_from_headers, + normalize_bedrock_opus_output_config_effort, normalize_tool_input_schema_types_for_bedrock_invoke, + pop_bedrock_invoke_output_config_format, remove_custom_field_from_tools, ) from litellm.types.llms.anthropic import ANTHROPIC_TOOL_SEARCH_BETA_HEADER @@ -75,6 +78,17 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): # Use a model name that forces tool-based approach model = "claude-3-sonnet-20240229" + # Clamp ``reasoning_effort`` to the Bedrock effort ceiling before the + # parent mapping converts it to ``output_config.effort`` and the + # downstream effort gate runs. Mirrors the converse path's + # ``_handle_reasoning_effort_parameter`` and the messages path's + # ``_clamp_adaptive_reasoning_effort_for_bedrock`` so adaptive Claude + # requests degrade ``xhigh`` -> ``max`` rather than 400-ing on + # models like Opus 4.6 that don't natively advertise xhigh. + self._clamp_adaptive_reasoning_effort_for_bedrock( + model=original_model, params=non_default_params + ) + optional_params = AnthropicConfig.map_openai_params( self, non_default_params, @@ -88,6 +102,27 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): return optional_params + @staticmethod + def _clamp_adaptive_reasoning_effort_for_bedrock(model: str, params: dict) -> None: + """Lower ``reasoning_effort`` to the Bedrock effort ceiling before mapping. + + Bedrock's adaptive Claude models accept the OpenAI-style + ``reasoning_effort`` tier, but the request validator can reject tiers + the model does not natively advertise (e.g. ``xhigh`` on Opus 4.6). + Clamp the raw tier to the model's + ``bedrock_output_config_effort_ceiling`` so Claude Code "goal mode" + keeps working. Non-adaptive models and models without a ceiling are + left untouched. + """ + if not AnthropicConfig._is_adaptive_thinking_model(model): + return + effort = params.get("reasoning_effort") + if not isinstance(effort, str): + return + clamped = {"effort": effort} + normalize_bedrock_opus_output_config_effort(model=model, output_config=clamped) + params["reasoning_effort"] = clamped["effort"] + def transform_request( self, model: str, @@ -157,6 +192,13 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): for k, v in optional_params.items() if k not in self.aws_authentication_params } + output_config = filtered_params.get("output_config") + if isinstance(output_config, dict): + filtered_params["output_config"] = dict(output_config) + normalize_bedrock_opus_output_config_effort( + model=model, + output_config=filtered_params["output_config"], + ) filtered_params = self._normalize_bedrock_tool_search_tools(filtered_params) anthropic_request = AnthropicConfig.transform_request( @@ -170,7 +212,20 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): anthropic_request.pop("model", None) anthropic_request.pop("stream", None) - anthropic_request.pop("output_format", None) + output_format = anthropic_request.pop("output_format", None) + output_config_format = pop_bedrock_invoke_output_config_format( + anthropic_request + ) + if output_format: + convert_bedrock_invoke_output_format_to_inline_schema( + output_format=output_format, + request_body=anthropic_request, + ) + elif output_config_format: + convert_bedrock_invoke_output_format_to_inline_schema( + output_format=output_config_format, + request_body=anthropic_request, + ) if not ( _supports_factory( model=model, diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 4f4729e4019..bdc5da321c6 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -34,6 +34,15 @@ class BedrockError(BaseLLMException): # Lazy import cache to avoid circular imports and performance impact _get_model_info = None +BedrockOutputConfigEffort = Literal["low", "medium", "high", "max", "xhigh"] +_BEDROCK_OUTPUT_CONFIG_EFFORT_ORDER: Dict[BedrockOutputConfigEffort, int] = { + "low": 0, + "medium": 1, + "high": 2, + "max": 3, + "xhigh": 4, +} + def get_cached_model_info(): """ @@ -51,6 +60,79 @@ def get_cached_model_info(): return _get_model_info +@functools.lru_cache(maxsize=1) +def _get_local_model_cost_map() -> Dict: + from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap + + return GetModelCostMap.load_local_model_cost_map() + + +def pop_bedrock_invoke_output_config_format(request_body: Dict) -> Optional[Dict]: + """ + Remove and return Anthropic's nested ``output_config.format`` field. + + Bedrock Invoke paths convert the schema to inline message text. Any remaining + ``output_config`` keys, such as ``effort``, are left in place. + """ + output_config = request_body.get("output_config") + if not isinstance(output_config, dict): + return None + + output_format = output_config.pop("format", None) + if not output_config: + request_body.pop("output_config", None) + + if isinstance(output_format, dict): + return output_format + return None + + +def convert_bedrock_invoke_output_format_to_inline_schema( + output_format: Dict, + request_body: Dict, +) -> None: + """ + Embed an Anthropic structured-output schema into the last user message. + + Bedrock Invoke does not support ``output_format`` directly, so the schema is + appended to the final user message for prompt-engineered structured output. + The caller's ``messages`` list, message dict, and content list are not + mutated; a fresh ``messages`` list with a copied final user message is + written back to ``request_body``. + """ + schema = output_format.get("schema") + if not schema: + return + + messages = request_body.get("messages") + if not isinstance(messages, list) or not messages: + return + + last_user_idx = None + for i in range(len(messages) - 1, -1, -1): + message = messages[i] + if isinstance(message, dict) and message.get("role") == "user": + last_user_idx = i + break + + if last_user_idx is None: + return + + original = messages[last_user_idx] + content = original.get("content", []) + schema_block = {"type": "text", "text": json.dumps(schema)} + if isinstance(content, str): + new_content = [{"type": "text", "text": content}, schema_block] + elif isinstance(content, list): + new_content = [*content, schema_block] + else: + return + + new_messages = list(messages) + new_messages[last_user_idx] = {**original, "content": new_content} + request_body["messages"] = new_messages + + def remove_custom_field_from_tools(request_body: dict) -> None: """ Remove ``custom`` field from each tool in the request body. @@ -603,6 +685,62 @@ def is_claude_4_5_on_bedrock(model: str) -> bool: return any(pattern in model_lower for pattern in claude_4_5_patterns) +def normalize_bedrock_opus_output_config_effort(model: str, output_config: Any) -> None: + """ + Normalize Anthropic ``output_config.effort`` values for Bedrock Opus ids. + + Bedrock's Claude Opus request validator can accept a narrower effort + vocabulary than Anthropic's compatibility surface. The Bedrock ceiling is + read from ``model_prices_and_context_window.json`` via + ``bedrock_output_config_effort_ceiling``. + + Mutates ``output_config`` in place so callers can accept Claude Code's + ``xhigh`` input without forwarding a provider-invalid value. + """ + if not isinstance(output_config, dict): + return + + effort = output_config.get("effort") + if effort not in _BEDROCK_OUTPUT_CONFIG_EFFORT_ORDER: + return + + ceiling = _get_bedrock_output_config_effort_ceiling(model) + if ceiling is None: + return + + if ( + _BEDROCK_OUTPUT_CONFIG_EFFORT_ORDER[effort] + > _BEDROCK_OUTPUT_CONFIG_EFFORT_ORDER[ceiling] + ): + output_config["effort"] = ceiling + + +def _get_bedrock_output_config_effort_ceiling( + model: str, +) -> Optional[BedrockOutputConfigEffort]: + try: + model_info = get_cached_model_info()( + model=model, + custom_llm_provider="bedrock", + ) + except Exception: + return None + + ceiling = model_info.get("bedrock_output_config_effort_ceiling") + if isinstance(ceiling, str) and ceiling in _BEDROCK_OUTPUT_CONFIG_EFFORT_ORDER: + return ceiling # type: ignore[return-value] + + model_cost_key = model_info.get("key") + if not isinstance(model_cost_key, str): + return None + + local_model_info = _get_local_model_cost_map().get(model_cost_key, {}) + ceiling = local_model_info.get("bedrock_output_config_effort_ceiling") + if isinstance(ceiling, str) and ceiling in _BEDROCK_OUTPUT_CONFIG_EFFORT_ORDER: + return ceiling # type: ignore[return-value] + return None + + # Import after standalone functions to avoid circular imports from litellm.llms.bedrock.count_tokens.bedrock_token_counter import BedrockTokenCounter diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py index 69b61298d33..42c3bd517a9 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py @@ -32,13 +32,19 @@ from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation AmazonInvokeConfig, ) from litellm.llms.bedrock.common_utils import ( + convert_bedrock_invoke_output_format_to_inline_schema, ensure_bedrock_anthropic_messages_tool_names, get_anthropic_beta_from_headers, is_claude_4_5_on_bedrock, + normalize_bedrock_opus_output_config_effort, normalize_tool_input_schema_types_for_bedrock_invoke, + pop_bedrock_invoke_output_config_format, remove_custom_field_from_tools, ) -from litellm.types.llms.anthropic import ANTHROPIC_TOOL_SEARCH_BETA_HEADER +from litellm.types.llms.anthropic import ( + ANTHROPIC_BETA_HEADER_VALUES, + ANTHROPIC_TOOL_SEARCH_BETA_HEADER, +) from litellm.types.llms.bedrock import BedrockInvokeAnthropicMessagesRequest from litellm.types.llms.openai import AllMessageValues from litellm.types.router import GenericLiteLLMParams @@ -442,7 +448,7 @@ class AmazonAnthropicClaudeMessagesConfig( if isinstance(e, dict) and e.get("type") == "compact_20260112" ] if compact_edits: - beta_set.add("compact-2026-01-12") + beta_set.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value) anthropic_messages_request["context_management"] = { **cm, "edits": compact_edits, @@ -450,145 +456,15 @@ class AmazonAnthropicClaudeMessagesConfig( else: anthropic_messages_request.pop("context_management", None) - def _convert_output_format_to_inline_schema( - self, - output_format: Dict, - anthropic_messages_request: Dict, - ) -> None: - """ - Convert Anthropic output_format to inline schema in message content. - - Bedrock Invoke doesn't support the output_format parameter, so we embed - the schema directly into the user message content as text instructions. - - This approach adds the schema to the last user message, instructing the model - to respond in the specified JSON format. - - Args: - output_format: The output_format dict with 'type' and 'schema' - anthropic_messages_request: The request dict to modify in-place - - Ref: https://aws.amazon.com/blogs/machine-learning/structured-data-response-with-amazon-bedrock-prompt-engineering-and-tool-use/ - """ - import json - - # Extract schema from output_format - schema = output_format.get("schema") - if not schema: - return - - # Get messages from the request - messages = anthropic_messages_request.get("messages", []) - if not messages: - return - - # Find the last user message - last_user_message_idx = None - for idx in range(len(messages) - 1, -1, -1): - if messages[idx].get("role") == "user": - last_user_message_idx = idx - break - - if last_user_message_idx is None: - return - - last_user_message = messages[last_user_message_idx] - content = last_user_message.get("content", []) - - # Ensure content is a list - if isinstance(content, str): - content = [{"type": "text", "text": content}] - last_user_message["content"] = content - - # Add schema as text content to the message - schema_text = {"type": "text", "text": json.dumps(schema)} - content.append(schema_text) - - def transform_anthropic_messages_request( + def _get_bedrock_invoke_anthropic_beta_headers( self, model: str, messages: List[Dict], anthropic_messages_optional_request_params: Dict, - litellm_params: GenericLiteLLMParams, headers: dict, - ) -> Dict: - anthropic_messages_request = AnthropicMessagesConfig.transform_anthropic_messages_request( - self=self, - model=model, - messages=messages, - anthropic_messages_optional_request_params=anthropic_messages_optional_request_params, - litellm_params=litellm_params, - headers=headers, - ) - ######################################################### - ############## BEDROCK Invoke SPECIFIC TRANSFORMATION ### - ######################################################### - - # 1. anthropic_version is required for all claude models - if "anthropic_version" not in anthropic_messages_request: - anthropic_messages_request["anthropic_version"] = ( - self.DEFAULT_BEDROCK_ANTHROPIC_API_VERSION - ) - - # 2. `stream` is not allowed in request body for bedrock invoke - if "stream" in anthropic_messages_request: - anthropic_messages_request.pop("stream", None) - - # 3. `model` is not allowed in request body for bedrock invoke - if "model" in anthropic_messages_request: - anthropic_messages_request.pop("model", None) - - injected_thinking_for_clear_thinking = ( - self._ensure_thinking_for_clear_thinking_context_management( - anthropic_messages_request=anthropic_messages_request, - model=model, - ) - ) - - # 4. Remove `ttl` field from cache_control in messages (Bedrock doesn't support it for older models) - self._remove_ttl_from_cache_control( - anthropic_messages_request=anthropic_messages_request, model=model - ) - - # 5. Convert `output_format` to inline schema (Bedrock invoke doesn't support output_format) - output_format = anthropic_messages_request.pop("output_format", None) - if output_format: - self._convert_output_format_to_inline_schema( - output_format=output_format, - anthropic_messages_request=anthropic_messages_request, - ) - - # 5a. Bedrock Invoke supports output_config (effort) for Claude 4.6+ models, - # but older models do not — strip it to avoid request rejection. - # Ref: https://github.com/BerriAI/litellm/issues/22797 - if not ( - _supports_factory( - model=model, - custom_llm_provider="bedrock", - key="supports_output_config", - ) - or AnthropicConfig._model_supports_effort_param(model) - ): - if anthropic_messages_request.pop("output_config", None) is not None: - verbose_logger.warning( - "Bedrock Invoke: stripping unsupported `output_config` for " - "model=%s — neither `supports_output_config` nor any " - "`supports_*_reasoning_effort` flag is set in " - "model_prices_and_context_window.json. Add the capability " - "flag to the model JSON entry if this model accepts " - "`output_config`.", - model, - ) - - # 5b. Remove `custom` field from tools (Bedrock doesn't support it) - # Claude Code sends `custom: {defer_loading: true}` on tool definitions, - # which causes Bedrock to reject the request with "Extra inputs are not permitted" - # Ref: https://github.com/BerriAI/litellm/issues/22847 - remove_custom_field_from_tools(anthropic_messages_request) - normalize_tool_input_schema_types_for_bedrock_invoke(anthropic_messages_request) - ensure_bedrock_anthropic_messages_tool_names(anthropic_messages_request) - - # 6. AUTO-INJECT beta headers based on features used + anthropic_messages_request: Dict, + injected_thinking_for_clear_thinking: bool, + ) -> List[str]: anthropic_model_info = AnthropicModelInfo() tools = anthropic_messages_optional_request_params.get("tools") messages_typed = cast(List[AllMessageValues], messages) @@ -651,6 +527,160 @@ class AmazonAnthropicClaudeMessagesConfig( dropped_user_betas, ) + return filtered_betas + + def _strip_unsupported_bedrock_invoke_fields( + self, + anthropic_messages_request: Dict, + ) -> Dict: + allowed = self.BEDROCK_INVOKE_ALLOWED_TOP_LEVEL_FIELDS + stripped = sorted(k for k in anthropic_messages_request if k not in allowed) + if stripped: + verbose_logger.debug( + "Bedrock Invoke: stripping unsupported top-level request fields: %s", + stripped, + ) + return {k: v for k, v in anthropic_messages_request.items() if k in allowed} + + @staticmethod + def _clamp_adaptive_reasoning_effort_for_bedrock( + model: str, optional_params: Dict + ) -> None: + """Lower ``reasoning_effort`` to the Bedrock effort ceiling before validation. + + The shared ``/v1/messages`` effort gate rejects tiers a model does not + natively support (e.g. ``xhigh`` on Opus 4.6). Bedrock's chat paths instead + clamp the tier to the model's ``bedrock_output_config_effort_ceiling`` so + Claude Code "goal mode" keeps working; mirror that here so the messages + path degrades ``xhigh`` -> ``max`` rather than 400-ing. Non-adaptive models + and models without a ceiling are left untouched. + """ + if not AnthropicModelInfo._is_adaptive_thinking_model(model): + return + effort = optional_params.get("reasoning_effort") + if not isinstance(effort, str): + return + clamped = {"effort": effort} + normalize_bedrock_opus_output_config_effort(model=model, output_config=clamped) + optional_params["reasoning_effort"] = clamped["effort"] + + def transform_anthropic_messages_request( + self, + model: str, + messages: List[Dict], + anthropic_messages_optional_request_params: Dict, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Dict: + self._clamp_adaptive_reasoning_effort_for_bedrock( + model=model, + optional_params=anthropic_messages_optional_request_params, + ) + anthropic_messages_request = AnthropicMessagesConfig.transform_anthropic_messages_request( + self=self, + model=model, + messages=messages, + anthropic_messages_optional_request_params=anthropic_messages_optional_request_params, + litellm_params=litellm_params, + headers=headers, + ) + ######################################################### + ############## BEDROCK Invoke SPECIFIC TRANSFORMATION ### + ######################################################### + + # 1. anthropic_version is required for all claude models + if "anthropic_version" not in anthropic_messages_request: + anthropic_messages_request["anthropic_version"] = ( + self.DEFAULT_BEDROCK_ANTHROPIC_API_VERSION + ) + + # 2. `stream` is not allowed in request body for bedrock invoke + if "stream" in anthropic_messages_request: + anthropic_messages_request.pop("stream", None) + + # 3. `model` is not allowed in request body for bedrock invoke + if "model" in anthropic_messages_request: + anthropic_messages_request.pop("model", None) + + injected_thinking_for_clear_thinking = ( + self._ensure_thinking_for_clear_thinking_context_management( + anthropic_messages_request=anthropic_messages_request, + model=model, + ) + ) + + # 4. Remove `ttl` field from cache_control in messages (Bedrock doesn't support it for older models) + self._remove_ttl_from_cache_control( + anthropic_messages_request=anthropic_messages_request, model=model + ) + + # 5. Convert structured-output params to inline schema. + # Bedrock Invoke doesn't support top-level `output_format`; its + # accepted `output_config` subset is also narrower than Anthropic's, so + # consume the newer `output_config.format` shape here instead of + # forwarding it as an unknown nested key. + existing_output_config = anthropic_messages_request.get("output_config") + if isinstance(existing_output_config, dict): + anthropic_messages_request["output_config"] = dict(existing_output_config) + output_format = anthropic_messages_request.pop("output_format", None) + output_config_format = pop_bedrock_invoke_output_config_format( + anthropic_messages_request + ) + if output_format: + convert_bedrock_invoke_output_format_to_inline_schema( + output_format=output_format, + request_body=anthropic_messages_request, + ) + elif output_config_format: + convert_bedrock_invoke_output_format_to_inline_schema( + output_format=output_config_format, + request_body=anthropic_messages_request, + ) + normalize_bedrock_opus_output_config_effort( + model=model, + output_config=anthropic_messages_request.get("output_config"), + ) + + # 5a. Bedrock Invoke supports output_config (effort) for Claude 4.6+ models, + # but older models do not — strip it to avoid request rejection. + # Ref: https://github.com/BerriAI/litellm/issues/22797 + if not ( + _supports_factory( + model=model, + custom_llm_provider="bedrock", + key="supports_output_config", + ) + or AnthropicConfig._model_supports_effort_param(model) + ): + if anthropic_messages_request.pop("output_config", None) is not None: + verbose_logger.warning( + "Bedrock Invoke: stripping unsupported `output_config` for " + "model=%s — neither `supports_output_config` nor any " + "`supports_*_reasoning_effort` flag is set in " + "model_prices_and_context_window.json. Add the capability " + "flag to the model JSON entry if this model accepts " + "`output_config`.", + model, + ) + + # 5b. Remove `custom` field from tools (Bedrock doesn't support it) + # Claude Code sends `custom: {defer_loading: true}` on tool definitions, + # which causes Bedrock to reject the request with "Extra inputs are not permitted" + # Ref: https://github.com/BerriAI/litellm/issues/22847 + remove_custom_field_from_tools(anthropic_messages_request) + normalize_tool_input_schema_types_for_bedrock_invoke(anthropic_messages_request) + ensure_bedrock_anthropic_messages_tool_names(anthropic_messages_request) + + # 6. AUTO-INJECT beta headers based on features used + filtered_betas = self._get_bedrock_invoke_anthropic_beta_headers( + model=model, + messages=messages, + anthropic_messages_optional_request_params=anthropic_messages_optional_request_params, + headers=headers, + anthropic_messages_request=anthropic_messages_request, + injected_thinking_for_clear_thinking=injected_thinking_for_clear_thinking, + ) + if filtered_betas: anthropic_messages_request["anthropic_beta"] = filtered_betas @@ -669,16 +699,9 @@ class AmazonAnthropicClaudeMessagesConfig( # Catches Anthropic-only extensions (output_config, speed, mcp_servers, ...) # and any future additions Claude Code may start sending. ``context_management`` # has already been pre-filtered to its Bedrock-supported subset above. - allowed = self.BEDROCK_INVOKE_ALLOWED_TOP_LEVEL_FIELDS - stripped = sorted(k for k in anthropic_messages_request if k not in allowed) - if stripped: - verbose_logger.debug( - "Bedrock Invoke: stripping unsupported top-level request fields: %s", - stripped, - ) - anthropic_messages_request = { - k: v for k, v in anthropic_messages_request.items() if k in allowed - } + anthropic_messages_request = self._strip_unsupported_bedrock_invoke_fields( + anthropic_messages_request + ) return anthropic_messages_request diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index c9ab3c648ac..941fe59e825 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -5316,6 +5316,28 @@ class BaseLLMHTTPHandler: ) if _session_config: realtime_streaming.session_configuration_request = _session_config + + # For providers that defer setup until client session.update, optionally + # send synthetic session.created to unblock clients waiting on connect. + if not provider_config.requires_session_configuration(): + synthetic_session = provider_config.transform_session_created_event( + model=model, + logging_session_id=logging_obj.litellm_trace_id, + session_configuration_request=None, + ) + if synthetic_session is not None: + synthetic_session_str = json.dumps(synthetic_session) + # Record before sending so the synthetic session.created is + # captured in the session log alongside provider-driven + # events; without this it would be silently absent from + # success_handler / async_success_handler payloads. + realtime_streaming.store_message(synthetic_session_str) + await websocket.send_text(synthetic_session_str) + realtime_streaming._session_created_sent_to_client = True + verbose_logger.debug( + "Sent synthetic session.created to client to unblock connection" + ) + await realtime_streaming.bidirectional_forward() except websockets.exceptions.InvalidStatusCode as e: # type: ignore @@ -6538,6 +6560,7 @@ class BaseLLMHTTPHandler: api_key=api_key or litellm_params.get("api_key", None), headers=extra_headers or {}, model="", + litellm_params=litellm_params, ) if extra_headers: @@ -6620,6 +6643,7 @@ class BaseLLMHTTPHandler: api_key=api_key or litellm_params.get("api_key", None), headers=extra_headers or {}, model="", + litellm_params=litellm_params, ) if extra_headers: @@ -6712,6 +6736,7 @@ class BaseLLMHTTPHandler: api_key=api_key or litellm_params.get("api_key", None), headers=extra_headers or {}, model="", + litellm_params=litellm_params, ) if extra_headers: headers.update(extra_headers) @@ -6783,6 +6808,7 @@ class BaseLLMHTTPHandler: api_key=api_key or litellm_params.get("api_key", None), headers=extra_headers or {}, model="", + litellm_params=litellm_params, ) if extra_headers: headers.update(extra_headers) @@ -6866,6 +6892,7 @@ class BaseLLMHTTPHandler: api_key=api_key or litellm_params.get("api_key", None), headers=extra_headers or {}, model="", + litellm_params=litellm_params, ) if extra_headers: headers.update(extra_headers) @@ -6923,6 +6950,7 @@ class BaseLLMHTTPHandler: api_key=api_key or litellm_params.get("api_key", None), headers=extra_headers or {}, model="", + litellm_params=litellm_params, ) if extra_headers: headers.update(extra_headers) @@ -6999,6 +7027,7 @@ class BaseLLMHTTPHandler: api_key=api_key or litellm_params.get("api_key", None), headers=extra_headers or {}, model="", + litellm_params=litellm_params, ) if extra_headers: headers.update(extra_headers) @@ -7009,27 +7038,49 @@ class BaseLLMHTTPHandler: litellm_params=dict(litellm_params), ) - url, data = video_provider_config.transform_video_edit_request( - prompt=prompt, + prefetched_source_data = None + prefetch_params = video_provider_config.get_video_edit_prefetch_params( video_id=video_id, api_base=api_base, litellm_params=litellm_params, headers=headers, - extra_body=extra_body, - ) - - logging_obj.pre_call( - input=prompt, - api_key="", - additional_args={ - "complete_input_dict": data, - "api_base": url, - "headers": headers, - "video_id": video_id, - }, ) + if prefetch_params is not None: + prefetch_url, prefetch_body = prefetch_params + try: + prefetch_resp = sync_httpx_client.post( + url=prefetch_url, + headers=headers, + json=prefetch_body, + timeout=timeout, + ) + prefetch_resp.raise_for_status() + except Exception as e: + raise self._handle_error(e=e, provider_config=video_provider_config) + prefetched_source_data = prefetch_resp.json() try: + url, data = video_provider_config.transform_video_edit_request( + prompt=prompt, + video_id=video_id, + api_base=api_base, + litellm_params=litellm_params, + headers=headers, + extra_body=extra_body, + prefetched_source_data=prefetched_source_data, + ) + + logging_obj.pre_call( + input=prompt, + api_key="", + additional_args={ + "complete_input_dict": data, + "api_base": url, + "headers": headers, + "video_id": video_id, + }, + ) + response = sync_httpx_client.post( url=url, headers=headers, @@ -7041,6 +7092,7 @@ class BaseLLMHTTPHandler: raw_response=response, logging_obj=logging_obj, custom_llm_provider=custom_llm_provider, + request_data=data, ) except Exception as e: raise self._handle_error(e=e, provider_config=video_provider_config) @@ -7071,6 +7123,7 @@ class BaseLLMHTTPHandler: api_key=api_key or litellm_params.get("api_key", None), headers=extra_headers or {}, model="", + litellm_params=litellm_params, ) if extra_headers: headers.update(extra_headers) @@ -7081,27 +7134,49 @@ class BaseLLMHTTPHandler: litellm_params=dict(litellm_params), ) - url, data = video_provider_config.transform_video_edit_request( - prompt=prompt, + prefetched_source_data = None + prefetch_params = video_provider_config.get_video_edit_prefetch_params( video_id=video_id, api_base=api_base, litellm_params=litellm_params, headers=headers, - extra_body=extra_body, - ) - - logging_obj.pre_call( - input=prompt, - api_key="", - additional_args={ - "complete_input_dict": data, - "api_base": url, - "headers": headers, - "video_id": video_id, - }, ) + if prefetch_params is not None: + prefetch_url, prefetch_body = prefetch_params + try: + prefetch_resp = await async_httpx_client.post( + url=prefetch_url, + headers=headers, + json=prefetch_body, + timeout=timeout, + ) + prefetch_resp.raise_for_status() + except Exception as e: + raise self._handle_error(e=e, provider_config=video_provider_config) + prefetched_source_data = prefetch_resp.json() try: + url, data = video_provider_config.transform_video_edit_request( + prompt=prompt, + video_id=video_id, + api_base=api_base, + litellm_params=litellm_params, + headers=headers, + extra_body=extra_body, + prefetched_source_data=prefetched_source_data, + ) + + logging_obj.pre_call( + input=prompt, + api_key="", + additional_args={ + "complete_input_dict": data, + "api_base": url, + "headers": headers, + "video_id": video_id, + }, + ) + response = await async_httpx_client.post( url=url, headers=headers, @@ -7113,6 +7188,7 @@ class BaseLLMHTTPHandler: raw_response=response, logging_obj=logging_obj, custom_llm_provider=custom_llm_provider, + request_data=data, ) except Exception as e: raise self._handle_error(e=e, provider_config=video_provider_config) @@ -7160,6 +7236,7 @@ class BaseLLMHTTPHandler: api_key=api_key or litellm_params.get("api_key", None), headers=extra_headers or {}, model="", + litellm_params=litellm_params, ) if extra_headers: headers.update(extra_headers) @@ -7234,6 +7311,7 @@ class BaseLLMHTTPHandler: api_key=api_key or litellm_params.get("api_key", None), headers=extra_headers or {}, model="", + litellm_params=litellm_params, ) if extra_headers: headers.update(extra_headers) @@ -7445,6 +7523,7 @@ class BaseLLMHTTPHandler: api_key=api_key, headers=extra_headers or {}, model="", + litellm_params=litellm_params, ) if extra_headers: diff --git a/litellm/llms/gemini/realtime/transformation.py b/litellm/llms/gemini/realtime/transformation.py index 4378db06358..cf1fc75ef10 100644 --- a/litellm/llms/gemini/realtime/transformation.py +++ b/litellm/llms/gemini/realtime/transformation.py @@ -3,8 +3,10 @@ This file contains the transformation logic for the Gemini realtime API. """ import json +from collections import OrderedDict from typing import Any, Dict, List, Optional, Union, cast +import litellm from litellm import verbose_logger from litellm._uuid import uuid from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -29,6 +31,7 @@ from litellm.types.llms.openai import ( OpenAIRealtimeDoneEvent, OpenAIRealtimeEvents, OpenAIRealtimeEventTypes, + OpenAIRealtimeFunctionCallArgumentsDone, OpenAIRealtimeOutputItemDone, OpenAIRealtimeResponseAudioDone, OpenAIRealtimeResponseContentPartAdded, @@ -36,10 +39,12 @@ from litellm.types.llms.openai import ( OpenAIRealtimeResponseDoneObject, OpenAIRealtimeResponseTextDone, OpenAIRealtimeStreamResponseBaseObject, + OpenAIRealtimeStreamResponseOutputItem, OpenAIRealtimeStreamResponseOutputItemAdded, OpenAIRealtimeStreamSession, OpenAIRealtimeStreamSessionEvents, OpenAIRealtimeTurnDetection, + ResponsesAPIStreamEvents, ) from litellm.types.llms.vertex_ai import ( GeminiResponseModalities, @@ -56,15 +61,43 @@ from litellm.utils import get_empty_usage from ..common_utils import encode_unserializable_types, get_api_key_from_env -MAP_GEMINI_FIELD_TO_OPENAI_EVENT: Dict[str, OpenAIRealtimeEventTypes] = { +MAP_GEMINI_FIELD_TO_OPENAI_EVENT: Dict[ + str, Union[OpenAIRealtimeEventTypes, ResponsesAPIStreamEvents] +] = { "setupComplete": OpenAIRealtimeEventTypes.SESSION_CREATED, "serverContent.generationComplete": OpenAIRealtimeEventTypes.RESPONSE_TEXT_DONE, "serverContent.turnComplete": OpenAIRealtimeEventTypes.RESPONSE_DONE, "serverContent.interrupted": OpenAIRealtimeEventTypes.RESPONSE_DONE, + "toolCall": ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DONE, +} + +# Top-level keys in a Gemini realtime message that map_openai_event knows how +# to handle. Other keys (e.g. ``usageMetadata``) can appear alongside these as +# siblings and must be skipped by the main transform loop — otherwise +# map_openai_event raises ``ValueError`` and the WebSocket session terminates. +_KNOWN_GEMINI_TOP_LEVEL_KEYS: set = { + map_key.split(".", 1)[0] for map_key in MAP_GEMINI_FIELD_TO_OPENAI_EVENT } class GeminiRealtimeConfig(BaseRealtimeConfig): + # Cap the LRU of in-flight tool calls so long sessions with many tool + # calls don't grow the dict without bound. Sized large enough to cover + # bursts of pending tool responses; the oldest entry is evicted when a + # new call beyond the cap arrives. + _TOOL_CALL_ID_TO_NAME_MAX = 256 + + def __init__(self): + super().__init__() + # Store call_id → function_name mapping for tool call round-trip + self._tool_call_id_to_name: "OrderedDict[str, str]" = OrderedDict() + # Buffer ``usageMetadata`` that Gemini Live emits as a standalone + # frame (between turns) so the next ``response.done`` attributes the + # tokens consumed. Without this an authenticated client can drive + # tool-call or normal turns whose token usage is recorded as zero, + # bypassing spend and budget accounting. + self._pending_usage_metadata: Optional[dict] = None + def validate_environment( self, headers: dict, model: str, api_key: Optional[str] = None ) -> dict: @@ -190,10 +223,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): ) vertex_gemini_config = VertexGeminiConfig() - optional_params["generationConfig"]["tools"] = ( - vertex_gemini_config._map_function( - value=value, optional_params=optional_params - ) + # Tools should be at the top level of setup, not inside generationConfig + optional_params["tools"] = vertex_gemini_config._map_function( + value=value, optional_params=optional_params ) elif key == "input_audio_transcription" and value is not None: optional_params["inputAudioTranscription"] = {} @@ -214,6 +246,272 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): optional_params.pop("generationConfig") return optional_params + @staticmethod + def _extract_turn_detection(session: dict) -> Optional[dict]: + """Extract turn_detection from a session.update payload. + + Handles both the flat beta shape (``session.turn_detection``) and the + GA shape (``session.audio.input.turn_detection``). + """ + if not isinstance(session, dict): + return None + td = session.get("turn_detection") + if isinstance(td, dict): + return td + audio = session.get("audio") + if isinstance(audio, dict): + input_cfg = audio.get("input") + if isinstance(input_cfg, dict): + td = input_cfg.get("turn_detection") + if isinstance(td, dict): + return td + return None + + @staticmethod + def _normalize_session_payload_for_mapping(session: dict) -> dict: + """Normalize GA-remapped session fields back to their beta keys. + + ``map_openai_params`` only recognises the flat OpenAI-beta key names + (``modalities``, ``input_audio_transcription``, ``turn_detection``). + For GA clients the upstream shim renames these into the nested GA + schema (``output_modalities``, ``audio.input.transcription``, + ``audio.input.turn_detection``), which would otherwise be silently + dropped here. Surface them back at the top level so the existing + mapping logic picks them up without duplicating provider-specific + knowledge of the GA schema in ``map_openai_params``. + """ + if not isinstance(session, dict): + return session + + normalized = dict(session) + + if "modalities" not in normalized and "output_modalities" in normalized: + normalized["modalities"] = normalized["output_modalities"] + + audio = normalized.get("audio") + if isinstance(audio, dict): + input_cfg = audio.get("input") + if isinstance(input_cfg, dict): + if ( + "input_audio_transcription" not in normalized + and "transcription" in input_cfg + ): + normalized["input_audio_transcription"] = input_cfg["transcription"] + + extracted_turn_detection = GeminiRealtimeConfig._extract_turn_detection( + normalized + ) + if extracted_turn_detection is not None and not isinstance( + normalized.get("turn_detection"), dict + ): + normalized["turn_detection"] = extracted_turn_detection + + return normalized + + def _handle_session_update( + self, + json_message: dict, + model: str, + session_configuration_request: Optional[str], + ) -> List[str]: + """ + Handle session.update by sending setup to Gemini. + + On the FIRST session.update (when session_configuration_request is None), + the full setup with all configuration is sent. + + Subsequent session.update messages are forwarded as a follow-up setup + with the new fields merged into the original setup. Gemini Live treats + a follow-up BidiGenerateContentSetup as a full session replacement + rather than a partial merge, so we carry forward the previous setup + (tools, generationConfig, inputAudioTranscription, systemInstruction, + ...) and overlay the new fields on top. This preserves the old + behavior where clients could refine the session via session.update + (e.g. add tools after the auto-setup on connect), and also keeps the + guardrail-driven turn_detection update working. + """ + session_payload = json_message.get("session") or {} + # Normalize GA-remapped fields (``output_modalities``, + # nested ``audio.input.transcription``, + # ``audio.input.turn_detection``) back to their flat beta keys so + # ``map_openai_params`` picks them up. Without this, GA clients' + # explicit modality / transcription / turn-detection settings + # would be silently dropped because ``map_openai_params`` only + # recognises the flat OpenAI-beta key names. + session_payload = self._normalize_session_payload_for_mapping(session_payload) + new_overrides = self.map_openai_params( + optional_params={}, non_default_params=session_payload + ) + + if session_configuration_request is None: + generation_config = new_overrides.setdefault("generationConfig", {}) + generation_config.setdefault("responseModalities", ["AUDIO"]) + new_overrides.setdefault("inputAudioTranscription", {}) + new_overrides["model"] = f"models/{model}" + verbose_logger.debug( + "Gemini Realtime: Sending initial setup with tools to backend" + ) + return [json.dumps({"setup": new_overrides})] + + if not new_overrides: + verbose_logger.debug( + "Gemini Realtime: Ignoring session.update (no mappable fields)" + ) + return [] + + try: + original_setup = cast( + BidiGenerateContentSetup, + json.loads(session_configuration_request).get("setup", {}), + ) + except (json.JSONDecodeError, AttributeError): + original_setup = {} + + # Deep-merge ``generationConfig`` and ``realtimeInputConfig`` so a + # partial session.update (e.g. only ``temperature`` or only + # ``modalities``) does not silently drop unrelated sub-keys + # (``responseModalities``, ``maxOutputTokens``, ...) from the original + # setup. + follow_up_setup: BidiGenerateContentSetup = { + **original_setup, + **new_overrides, + "model": f"models/{model}", + } + original_generation_config = original_setup.get("generationConfig") + new_generation_config = new_overrides.get("generationConfig") + if isinstance(original_generation_config, dict) and isinstance( + new_generation_config, dict + ): + follow_up_setup["generationConfig"] = { + **original_generation_config, + **new_generation_config, + } + original_realtime_input_config = original_setup.get("realtimeInputConfig") + new_realtime_input_config = new_overrides.get("realtimeInputConfig") + if isinstance(original_realtime_input_config, dict) and isinstance( + new_realtime_input_config, dict + ): + merged_realtime_input_config = { + **original_realtime_input_config, + **new_realtime_input_config, + } + # Deep-merge ``automaticActivityDetection`` so a partial VAD + # update (e.g. the guardrail-injected ``disabled: True`` from + # ``create_response: False``) does not silently drop unrelated + # knobs like ``silenceDurationMs`` / ``prefixPaddingMs`` from + # the original setup. + original_automatic_activity_detection = original_realtime_input_config.get( + "automaticActivityDetection" + ) + new_automatic_activity_detection = new_realtime_input_config.get( + "automaticActivityDetection" + ) + if isinstance(original_automatic_activity_detection, dict) and isinstance( + new_automatic_activity_detection, dict + ): + merged_realtime_input_config["automaticActivityDetection"] = { + **original_automatic_activity_detection, + **new_automatic_activity_detection, + } + follow_up_setup["realtimeInputConfig"] = cast( + BidiGenerateContentRealtimeInputConfig, + merged_realtime_input_config, + ) + verbose_logger.debug( + "Gemini Realtime: Forwarding session.update as follow-up setup" + ) + return [json.dumps({"setup": follow_up_setup})] + + def _handle_conversation_item(self, json_message: dict) -> List[str]: + """ + Handle conversation.item.create for user text or function call output. + + Converts OpenAI format to Gemini's clientContent (for user text) or + toolResponse (for function outputs). + """ + item = json_message.get("item", {}) + item_type = item.get("type") + + # Handle function call output (tool response) + if item_type == "function_call_output": + return self._handle_function_call_output(item) + + # Handle regular text content + return self._handle_user_text_content(item) + + def _handle_function_call_output(self, item: dict) -> List[str]: + """Transform function_call_output to Gemini toolResponse format.""" + call_id = item.get("call_id", "") + output = item.get("output", "{}") + + verbose_logger.debug( + f"Gemini Realtime: Transforming function_call_output for call_id={call_id}" + ) + + # Parse the output to get the result. Gemini's + # functionResponses[].response field is a Struct, so it must be a + # dict; wrap any non-dict (primitives, lists, invalid JSON) under a + # `result` key. + try: + parsed_output = json.loads(output) if isinstance(output, str) else output + except json.JSONDecodeError: + parsed_output = output + output_dict = ( + parsed_output + if isinstance(parsed_output, dict) + else {"result": parsed_output} + ) + + # Look up the function name from stored mapping. Keep the entry so a + # client SDK that retries function_call_output (or sends it twice for + # the same tool call) still produces a Gemini toolResponse with the + # required ``name`` field; refresh the LRU position so an active + # call_id stays warm across long sessions. + function_name = self._tool_call_id_to_name.get(call_id) + if function_name: + self._tool_call_id_to_name.move_to_end(call_id) + else: + verbose_logger.warning( + f"Gemini Realtime: Function name not found for call_id={call_id}. " + "This may cause Gemini to reject the response." + ) + + # Build Gemini toolResponse format + function_response = { + "id": call_id, + "response": output_dict, + } + if function_name: + function_response["name"] = function_name + + tool_response_message = { + "toolResponse": {"functionResponses": [function_response]} + } + + return [json.dumps(tool_response_message)] + + def _handle_user_text_content(self, item: dict) -> List[str]: + """Transform user text content to Gemini clientContent format.""" + content_list = item.get("content", []) + text_parts = [ + c.get("text", "") + for c in content_list + if isinstance(c, dict) and c.get("type") == "input_text" + ] + text = " ".join(filter(None, text_parts)) + if not text: + return [] + + # Build clientContent message with turns (proper Gemini Live API format) + client_content_message = { + "clientContent": { + "turns": [{"role": "user", "parts": [{"text": text}]}], + "turnComplete": True, + } + } + + return [json.dumps(client_content_message)] + def transform_realtime_request( self, message: str, @@ -233,55 +531,42 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): messages: List[str] = [] msg_type = json_message.get("type") - ## HANDLE SESSION UPDATE — translate to Gemini setup; no realtime_input needed ## + ## HANDLE SESSION UPDATE — translate to Gemini setup ## if msg_type == "session.update": - client_session_configuration_request = self.map_openai_params( - optional_params={}, non_default_params=json_message["session"] + return self._handle_session_update( + json_message, model, session_configuration_request ) - client_session_configuration_request["model"] = f"models/{model}" - messages.append(json.dumps({"setup": client_session_configuration_request})) - return messages ## HANDLE response.create — Gemini responds automatically; nothing to forward ## if msg_type == "response.create": return [] - ## HANDLE INPUT AUDIO BUFFER ## + ## HANDLE conversation.item.create — extract user text or function call output ## + if msg_type == "conversation.item.create": + return self._handle_conversation_item(json_message) + + ## HANDLE INPUT AUDIO BUFFER - use realtimeInput for audio streaming ## if msg_type == "input_audio_buffer.append": realtime_input_dict["audio"] = HttpxBlobType( mimeType=self.get_audio_mime_type(), data=json_message["audio"] ) - ## HANDLE conversation.item.create — extract actual user text ## - elif msg_type == "conversation.item.create": - item = json_message.get("item", {}) - content_list = item.get("content", []) - text_parts = [ - c.get("text", "") - for c in content_list - if isinstance(c, dict) and c.get("type") == "input_text" - ] - text = " ".join(filter(None, text_parts)) - if not text: - return [] - realtime_input_dict["text"] = text - else: - # Unknown/unsupported OpenAI event type — drop silently rather than - # forwarding raw JSON as text input to the model. - return [] - if len(realtime_input_dict) != 1: - raise ValueError( - f"Only one argument can be set, got {len(realtime_input_dict)}:" - f" {list(realtime_input_dict.keys())}" + realtime_input_dict = cast( + BidiGenerateContentRealtimeInput, + encode_unserializable_types( + cast(Dict[str, object], realtime_input_dict) + ), ) - realtime_input_dict = cast( - BidiGenerateContentRealtimeInput, - encode_unserializable_types(cast(Dict[str, object], realtime_input_dict)), - ) - - messages.append(json.dumps({"realtime_input": realtime_input_dict})) - return messages + gemini_msg = json.dumps({"realtimeInput": realtime_input_dict}) + verbose_logger.debug( + "Gemini Realtime: Sending audio realtimeInput to backend" + ) + messages.append(gemini_msg) + return messages + # Unknown/unsupported OpenAI event type — drop silently rather than + # forwarding raw JSON as text input to the model. + return [] def transform_session_created_event( self, @@ -300,7 +585,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): generation_config = ( session_configuration_request_dict.get("generationConfig", {}) or {} ) - gemini_modalities = generation_config.get("responseModalities", ["TEXT"]) + gemini_modalities = generation_config.get("responseModalities", ["AUDIO"]) _modalities = [ modality.lower() for modality in cast(List[str], gemini_modalities) ] @@ -352,18 +637,18 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): delta_type: ALL_DELTA_TYPES, session_configuration_request: Optional[str] = None, ) -> List[OpenAIRealtimeEvents]: - if session_configuration_request is None: - raise ValueError( - "session_configuration_request is required for Gemini API calls" - ) - - session_configuration_request_dict: BidiGenerateContentSetup = json.loads( - session_configuration_request - ).get("setup", {}) + session_configuration_request_dict: BidiGenerateContentSetup = {} + if session_configuration_request is not None: + try: + session_configuration_request_dict = json.loads( + session_configuration_request + ).get("setup", {}) + except json.JSONDecodeError: + session_configuration_request_dict = {} generation_config = session_configuration_request_dict.get( "generationConfig", {} ) - gemini_modalities = generation_config.get("responseModalities", ["TEXT"]) + gemini_modalities = generation_config.get("responseModalities", ["AUDIO"]) _modalities = [ modality.lower() for modality in cast(List[str], gemini_modalities) ] @@ -576,6 +861,86 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): returned_items.append(response_output_item_done) return returned_items + def _consume_usage_metadata_for_response_done(self, frame: dict) -> Optional[dict]: + """Return the ``usageMetadata`` to attribute to a ``response.done``. + + Gemini Live emits ``usageMetadata`` either alongside the closing + frame (``serverContent.turnComplete`` / ``toolCall``) or as a + standalone frame between turns. The standalone form would otherwise + be discarded by the no-op branch in ``transform_realtime_response`` + and the consumed tokens silently dropped from spend/budget + accounting. ``_pending_usage_metadata`` buffers any such standalone + frames so the next emitted ``response.done`` carries the deferred + token counts. + + Returns the in-frame ``usageMetadata`` if present (and clears the + buffer since the in-frame counts are the authoritative attribution + for this turn), otherwise returns the buffered counts. ``None`` is + returned when neither is available so the caller can fall back to + ``get_empty_usage()``. + """ + # ``pop`` (rather than ``get``) so a single Gemini frame containing + # multiple closing keys (e.g. both ``toolCall`` and + # ``serverContent.turnComplete``) cannot attribute the same + # ``usageMetadata`` to two ``response.done`` events and double-count + # tokens in spend/budget accounting. + in_frame = frame.pop("usageMetadata", None) if isinstance(frame, dict) else None + if isinstance(in_frame, dict): + self._pending_usage_metadata = None + return in_frame + buffered = self._pending_usage_metadata + self._pending_usage_metadata = None + return buffered + + def transform_tool_call_events( + self, + tool_call_message: dict, + response_id: Optional[str] = None, + output_item_id: Optional[str] = None, + ) -> List[OpenAIRealtimeFunctionCallArgumentsDone]: + """ + Transform Gemini toolCall message to OpenAI function call events. + + Converts Gemini's functionCalls format to OpenAI's response.function_call_arguments.done events. + Also stores call_id → name mapping for later use in function_call_output responses. + """ + function_calls = tool_call_message.get("functionCalls", []) + resolved_response_id = response_id or f"resp_{uuid.uuid4()}" + resolved_output_item_id = output_item_id or f"item_{uuid.uuid4()}" + + verbose_logger.debug( + f"Gemini Realtime: Transforming {len(function_calls)} tool call(s) to OpenAI format" + ) + + events: List[OpenAIRealtimeFunctionCallArgumentsDone] = [] + for idx, fc in enumerate(function_calls): + call_id = fc.get("id", "") + name = fc.get("name", "") + + # Store call_id → name mapping for round-trip. Use an LRU so + # repeated function_call_output lookups (retries) still hit, while + # sessions with many tool calls don't grow the dict unboundedly. + if call_id and name: + self._tool_call_id_to_name[call_id] = name + self._tool_call_id_to_name.move_to_end(call_id) + while len(self._tool_call_id_to_name) > self._TOOL_CALL_ID_TO_NAME_MAX: + self._tool_call_id_to_name.popitem(last=False) + + events.append( + OpenAIRealtimeFunctionCallArgumentsDone( + type="response.function_call_arguments.done", + event_id=f"event_{uuid.uuid4()}", + response_id=resolved_response_id, + item_id=f"{resolved_output_item_id}_tool_{idx}", + output_index=idx, + call_id=call_id, + name=name, + arguments=json.dumps(fc.get("args", {})), + ) + ) + + return events + @staticmethod def get_nested_value(obj: dict, path: str) -> Any: keys = path.split(".") @@ -681,14 +1046,20 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): "generationConfig", {} ) temperature = generation_config.get("temperature") - max_output_tokens = generation_config.get("max_output_tokens") - gemini_modalities = generation_config.get("responseModalities", ["TEXT"]) + max_output_tokens = generation_config.get("maxOutputTokens") + gemini_modalities = generation_config.get("responseModalities", ["AUDIO"]) _modalities = [ modality.lower() for modality in cast(List[str], gemini_modalities) ] - if "usageMetadata" in message: + resolved_usage_metadata = self._consume_usage_metadata_for_response_done( + cast(dict, message) + ) + if resolved_usage_metadata is not None: _chat_completion_usage = VertexGeminiConfig._calculate_usage( - completion_response=message, + completion_response=cast( + BidiGenerateContentServerMessage, + {**cast(dict, message), "usageMetadata": resolved_usage_metadata}, + ), ) else: _chat_completion_usage = get_empty_usage() @@ -716,7 +1087,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): if temperature is not None: response_done_event["response"]["temperature"] = temperature if max_output_tokens is not None: - response_done_event["response"]["max_output_tokens"] = max_output_tokens + response_done_event["response"]["max_output_tokens"] = cast( + int, max_output_tokens + ) return response_done_event @@ -808,13 +1181,18 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): def map_openai_event( self, key: str, - value: dict, + value: Any, current_delta_type: Optional[ALL_DELTA_TYPES], - json_message: dict, - ) -> OpenAIRealtimeEventTypes: - model_turn_event = value.get("modelTurn") - generation_complete_event = value.get("generationComplete") - openai_event: Optional[OpenAIRealtimeEventTypes] = None + ) -> Union[OpenAIRealtimeEventTypes, ResponsesAPIStreamEvents]: + if isinstance(value, dict): + model_turn_event = value.get("modelTurn") + generation_complete_event = value.get("generationComplete") + else: + model_turn_event = None + generation_complete_event = None + openai_event: Optional[ + Union[OpenAIRealtimeEventTypes, ResponsesAPIStreamEvents] + ] = None if model_turn_event: # check if model turn event openai_event = self.map_model_turn_event(model_turn_event) elif generation_complete_event: @@ -822,15 +1200,27 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): delta_type=current_delta_type ) else: - # Check if this key or any nested key matches our mapping - for map_key, openai_event in MAP_GEMINI_FIELD_TO_OPENAI_EVENT.items(): - if map_key == key or ( - "." in map_key - and GeminiRealtimeConfig.get_nested_value(json_message, map_key) - is not None - ): - openai_event = openai_event + # Check if this key or any nested key matches our mapping. Use a + # distinct loop variable so we don't shadow ``openai_event`` and + # leak the last dict value when no entry matches. Scope dotted-key + # lookups to the current ``key``/``value`` pair — checking the + # whole ``json_message`` would let a sibling key (e.g. + # ``serverContent.turnComplete``) misclassify the event currently + # being processed (e.g. ``toolCall``). + for map_key, candidate_event in MAP_GEMINI_FIELD_TO_OPENAI_EVENT.items(): + if map_key == key: + openai_event = candidate_event break + if "." in map_key: + prefix, _, nested_path = map_key.partition(".") + if ( + prefix == key + and isinstance(value, dict) + and GeminiRealtimeConfig.get_nested_value(value, nested_path) + is not None + ): + openai_event = candidate_event + break if openai_event is None: raise ValueError(f"Unknown openai event: {key}, value: {value}") return openai_event @@ -854,6 +1244,15 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): message_str = str(message) raise ValueError(f"Invalid JSON message: {message_str}") + verbose_logger.debug( + "Realtime Response Transform: Gemini frame keys=%s", + ( + sorted(json_message.keys()) + if isinstance(json_message, dict) + else type(json_message).__name__ + ), + ) + logging_session_id = logging_obj.litellm_trace_id current_output_item_id = realtime_response_transform_input[ @@ -913,32 +1312,44 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): ) # If serverContent only contained transcription(s) and no model - # content, return early — the main loop would fail on unknown keys. + # content, mark it as already handled so the main loop skips it + # (map_openai_event would raise on an unknown serverContent + # subkey). Fall through so sibling top-level keys such as + # ``toolCall`` are still processed in the main loop. _model_content_keys = { "modelTurn", "turnComplete", "interrupted", "generationComplete", } - if not any(k in server_content for k in _model_content_keys): - return { - "response": returned_message, - "current_output_item_id": current_output_item_id, - "current_response_id": current_response_id, - "current_delta_chunks": current_delta_chunks, - "current_conversation_id": current_conversation_id, - "current_item_chunks": current_item_chunks, - "current_delta_type": current_delta_type, - "session_configuration_request": session_configuration_request, - } + server_content_handled = not any( + k in server_content for k in _model_content_keys + ) + else: + server_content_handled = False - for key, value in json_message.items(): + tool_call_handled = False + # Snapshot the items so handlers below can safely mutate + # ``json_message`` (e.g. ``_consume_usage_metadata_for_response_done`` + # pops ``usageMetadata`` to prevent a single frame from attributing + # the same token counts to two ``response.done`` events). + for key, value in list(json_message.items()): + # Skip sibling metadata keys (e.g. ``usageMetadata``) that can + # accompany a primary payload like ``toolCall`` or ``serverContent``. + # ``map_openai_event`` raises ValueError on unknown keys, which + # would otherwise terminate the WebSocket session. + if key not in _KNOWN_GEMINI_TOP_LEVEL_KEYS: + continue + # serverContent was a transcription-only payload already emitted + # above; skip it here so map_openai_event doesn't raise on the + # missing model-content subkeys. + if key == "serverContent" and server_content_handled: + continue # Check if this key or any nested key matches our mapping openai_event = self.map_openai_event( key=key, value=value, current_delta_type=current_delta_type, - json_message=json_message, ) if openai_event == OpenAIRealtimeEventTypes.SESSION_CREATED: @@ -947,8 +1358,226 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): logging_session_id, realtime_response_transform_input["session_configuration_request"], ) - session_configuration_request = json.dumps(transformed_message) returned_message.append(transformed_message) + elif openai_event == ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DONE: + # Handle toolCall from Gemini. If the payload has no function + # calls, emit nothing — an orphaned response.created/done pair + # with no output items would confuse OpenAI-compatible clients. + # Mark the key as intentionally consumed (mirroring + # ``server_content_handled``) so any sibling keys in the same + # frame are still processed by the rest of the loop and the + # post-loop guard doesn't treat the no-op as fatal. + if not value.get("functionCalls"): + tool_call_handled = True + continue + + if current_conversation_id is None: + current_conversation_id = f"conv_{uuid.uuid4()}" + + # Extract session-level response metadata once so both + # response.created and response.done can include matching + # modalities/temperature/max_output_tokens fields. + session_setup: BidiGenerateContentSetup = {} + if session_configuration_request is not None: + try: + session_setup = json.loads(session_configuration_request).get( + "setup", {} + ) + except (json.JSONDecodeError, TypeError): + session_setup = {} + tool_call_generation_config = ( + session_setup.get("generationConfig", {}) or {} + ) + tool_call_modalities = [ + modality.lower() + for modality in cast( + List[str], + tool_call_generation_config.get( + "responseModalities", ["AUDIO"] + ), + ) + ] + + # Emit response.created preamble if this is the first event in the response + if current_response_id is None: + current_response_id = f"resp_{uuid.uuid4()}" + current_output_item_id = f"item_{uuid.uuid4()}" + + # Mirror the audio/text path: include modalities, + # temperature, and max_output_tokens on response.created so + # spec-compliant clients see consistent response metadata + # regardless of whether the response starts with content or + # a tool call. + returned_message.append( + { + "type": "response.created", + "event_id": f"event_{uuid.uuid4()}", + "response": { + "object": "realtime.response", + "id": current_response_id, + "status": "in_progress", + "output": [], + "conversation_id": current_conversation_id, + "modalities": tool_call_modalities, + "temperature": tool_call_generation_config.get( + "temperature" + ), + "max_output_tokens": tool_call_generation_config.get( + "maxOutputTokens" + ), + }, + } + ) + + tool_call_events = self.transform_tool_call_events( + value, + response_id=current_response_id, + output_item_id=current_output_item_id, + ) + # Emit output_item.added and conversation.item.created for each function call + for idx, tool_call in enumerate(tool_call_events): + item_id = tool_call["item_id"] + function_call_item: OpenAIRealtimeStreamResponseOutputItem = { + "id": item_id, + "object": "realtime.item", + "type": "function_call", + "status": "completed", + "call_id": tool_call["call_id"], + "name": tool_call["name"], + "arguments": tool_call["arguments"], + } + # response.output_item.added + returned_message.append( + OpenAIRealtimeStreamResponseOutputItemAdded( + type="response.output_item.added", + event_id=f"event_{uuid.uuid4()}", + response_id=current_response_id, + output_index=idx, + item={ + **function_call_item, + "status": "in_progress", + "arguments": "", + }, + ) + ) + # response.function_call_arguments.delta — Gemini delivers + # the full arguments string in a single toolCall frame + # rather than streaming partial chunks, so emit one delta + # carrying the complete payload before the matching + # ``.done`` event. Spec-compliant OpenAI Realtime SDK + # clients accumulate ``delta.delta`` and rely on at least + # one delta before ``.done``. + returned_message.append( + cast( + OpenAIRealtimeEvents, + { + "type": "response.function_call_arguments.delta", + "event_id": f"event_{uuid.uuid4()}", + "response_id": current_response_id, + "item_id": item_id, + "output_index": idx, + "call_id": tool_call["call_id"], + "delta": tool_call["arguments"], + }, + ) + ) + # response.function_call_arguments.done + returned_message.append(tool_call) + # response.output_item.done — pass a fresh copy so + # downstream handlers that mutate the item dict (e.g. the + # beta-protocol translator) don't corrupt the references + # used by sibling events sharing the same function_call_item. + returned_message.append( + OpenAIRealtimeOutputItemDone( + type="response.output_item.done", + event_id=f"event_{uuid.uuid4()}", + response_id=current_response_id, + output_index=idx, + item={**function_call_item}, + ) + ) + # conversation.item.created + returned_message.append( + OpenAIRealtimeConversationItemCreated( + type="conversation.item.created", + event_id=f"event_{uuid.uuid4()}", + item={**function_call_item}, + ) + ) + + # response.done - close the response so clients can submit tool + # results. Mirror the non-tool-call RESPONSE_DONE path: if Gemini + # delivered ``usageMetadata`` alongside this ``toolCall`` frame, + # propagate the real token counts so spend/budget accounting + # records the tokens consumed by the tool-call turn. Standalone + # ``usageMetadata`` frames emitted in a separate WebSocket frame + # are buffered on the instance so the next ``response.done`` + # picks them up (otherwise an authenticated client could drive + # tool-call turns whose token usage is recorded as zero, + # bypassing budgets). Falls back to an empty usage block when + # neither is available (OpenAI-compatible clients expect + # ``usage`` to always be present on response.done). + resolved_tool_call_usage_metadata = ( + self._consume_usage_metadata_for_response_done(json_message) + ) + if resolved_tool_call_usage_metadata is not None: + _tool_call_chat_completion_usage = ( + VertexGeminiConfig._calculate_usage( + completion_response=cast( + BidiGenerateContentServerMessage, + { + **json_message, + "usageMetadata": resolved_tool_call_usage_metadata, + }, + ), + ) + ) + else: + _tool_call_chat_completion_usage = get_empty_usage() + tool_call_responses_api_usage = LiteLLMCompletionResponsesConfig._transform_chat_completion_usage_to_responses_usage( + _tool_call_chat_completion_usage, + ) + tool_call_done_event = OpenAIRealtimeDoneEvent( + type="response.done", + event_id=f"event_{uuid.uuid4()}", + response=OpenAIRealtimeResponseDoneObject( + id=current_response_id, + object="realtime.response", + status="completed", + output=[ + { + "id": te["item_id"], + "object": "realtime.item", + "type": "function_call", + "status": "completed", + "call_id": te["call_id"], + "name": te["name"], + "arguments": te["arguments"], + } + for te in tool_call_events + ], + conversation_id=current_conversation_id, + modalities=tool_call_modalities, + usage=tool_call_responses_api_usage.model_dump(), + ), + ) + tool_call_temperature = tool_call_generation_config.get("temperature") + if tool_call_temperature is not None: + tool_call_done_event["response"][ + "temperature" + ] = tool_call_temperature + tool_call_max_output_tokens = tool_call_generation_config.get( + "maxOutputTokens" + ) + if tool_call_max_output_tokens is not None: + tool_call_done_event["response"]["max_output_tokens"] = cast( + int, tool_call_max_output_tokens + ) + returned_message.append(tool_call_done_event) + # Reset IDs so the next model turn (after tool results) starts a + # fresh response with its own response.created preamble. + current_output_item_id = None + current_response_id = None elif openai_event == OpenAIRealtimeEventTypes.RESPONSE_DONE: transformed_response_done_event = self.transform_response_done_event( message=BidiGenerateContentServerMessage(**json_message), # type: ignore @@ -958,16 +1587,37 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): output_items=None, ) returned_message.append(transformed_response_done_event) + # Reset IDs so a subsequent turn (e.g. a `toolCall` arriving in + # a later WebSocket frame after `turnComplete`) starts a fresh + # response with its own `response.created` preamble instead of + # reusing the just-completed response ID. + current_output_item_id = None + current_response_id = None elif ( openai_event == OpenAIRealtimeEventTypes.RESPONSE_TEXT_DELTA or openai_event == OpenAIRealtimeEventTypes.RESPONSE_TEXT_DONE or openai_event == OpenAIRealtimeEventTypes.RESPONSE_AUDIO_DELTA or openai_event == OpenAIRealtimeEventTypes.RESPONSE_AUDIO_DONE ): + # Pass the locally-updated state (rather than the original + # input snapshot) so that prior iterations of this loop — + # e.g. a tool-call or response.done that just reset + # current_response_id/current_output_item_id to None — are + # honoured by the modality handler. + _modality_input: RealtimeResponseTransformInput = { + **realtime_response_transform_input, + "current_output_item_id": current_output_item_id, + "current_response_id": current_response_id, + "current_conversation_id": current_conversation_id, + "current_delta_chunks": current_delta_chunks, + "current_item_chunks": current_item_chunks, + "current_delta_type": current_delta_type, + "session_configuration_request": session_configuration_request, + } _returned_message = self.handle_openai_modality_event( openai_event, json_message, - realtime_response_transform_input, + _modality_input, delta_type="text" if "text" in openai_event.value else "audio", ) returned_message.extend(_returned_message["returned_message"]) @@ -979,6 +1629,41 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): else: raise ValueError(f"Unknown openai event: {openai_event}") if len(returned_message) == 0: + # A frame whose only top-level keys are sibling metadata (e.g. + # a standalone ``{"usageMetadata": {...}}`` emitted by Gemini + # Live between turns) is not an error — there is just nothing + # to forward to the OpenAI-shaped client. Returning the + # unchanged state keeps the WebSocket alive; raising would + # terminate the session for a benign no-op frame. + # serverContent already consumed by the transcription handler is + # a benign no-op for downstream — treat it like a metadata-only + # key when deciding whether to raise. + unhandled_known_keys = [ + key + for key in json_message + if key in _KNOWN_GEMINI_TOP_LEVEL_KEYS + and not (key == "serverContent" and server_content_handled) + and not (key == "toolCall" and tool_call_handled) + ] + # Buffer standalone usage metadata so the next response.done can + # attribute the token counts. Without this, an authenticated + # client driving turns whose usageMetadata is emitted in a + # separate frame would have those tokens recorded as zero spend, + # bypassing budget enforcement. + standalone_usage_metadata = json_message.get("usageMetadata") + if isinstance(standalone_usage_metadata, dict): + self._pending_usage_metadata = standalone_usage_metadata + if not unhandled_known_keys: + return { + "response": returned_message, + "current_output_item_id": current_output_item_id, + "current_response_id": current_response_id, + "current_delta_chunks": current_delta_chunks, + "current_conversation_id": current_conversation_id, + "current_item_chunks": current_item_chunks, + "current_delta_type": current_delta_type, + "session_configuration_request": session_configuration_request, + } if isinstance(message, bytes): message_str = message.decode("utf-8", errors="replace") else: @@ -993,6 +1678,13 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): transformed_message=returned_message, current_item_chunks=current_item_chunks, ) + + for msg in returned_message: + event_type = msg.get("type") if isinstance(msg, dict) else "unknown" + verbose_logger.debug( + "Realtime Response Transform: OpenAI event=%s", event_type + ) + return { "response": returned_message, "current_output_item_id": current_output_item_id, @@ -1005,7 +1697,10 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): } def requires_session_configuration(self) -> bool: - return True + # Default behavior is backwards-compatible: send setup on connect. + # Opt-in to deferred setup for tool-injection flow via: + # litellm.gemini_live_defer_setup = True + return not litellm.gemini_live_defer_setup def session_configuration_request(self, model: str) -> str: """ diff --git a/litellm/llms/gemini/videos/transformation.py b/litellm/llms/gemini/videos/transformation.py index 9714c8a3923..77a95bfa5ab 100644 --- a/litellm/llms/gemini/videos/transformation.py +++ b/litellm/llms/gemini/videos/transformation.py @@ -581,12 +581,23 @@ class GeminiVideoConfig(BaseVideoConfig): raise NotImplementedError("video get character is not supported for Gemini") def transform_video_edit_request( - self, prompt, video_id, api_base, litellm_params, headers, extra_body=None + self, + prompt, + video_id, + api_base, + litellm_params, + headers, + extra_body=None, + prefetched_source_data=None, ): raise NotImplementedError("video edit is not supported for Gemini") def transform_video_edit_response( - self, raw_response, logging_obj, custom_llm_provider=None + self, + raw_response, + logging_obj, + custom_llm_provider=None, + request_data=None, ): raise NotImplementedError("video edit is not supported for Gemini") diff --git a/litellm/llms/langgraph/chat/transformation.py b/litellm/llms/langgraph/chat/transformation.py index 00cc3a8f516..9808b665b54 100644 --- a/litellm/llms/langgraph/chat/transformation.py +++ b/litellm/llms/langgraph/chat/transformation.py @@ -139,14 +139,16 @@ class LangGraphConfig(BaseConfig): def _convert_messages_to_langgraph_format( self, messages: List[AllMessageValues] - ) -> List[Dict[str, str]]: + ) -> List[Dict[str, Any]]: """ Convert OpenAI-format messages to LangGraph format. OpenAI format: {"role": "user", "content": "..."} LangGraph format: {"role": "human", "content": "..."} + + Preserves per-message ``metadata`` when present (e.g. A2A ``skillId``). """ - langgraph_messages: List[Dict[str, str]] = [] + langgraph_messages: List[Dict[str, Any]] = [] for msg in messages: role = msg.get("role", "user") content = msg.get("content", "") @@ -169,7 +171,15 @@ class LangGraphConfig(BaseConfig): if not isinstance(content, str): content = str(content) - langgraph_messages.append({"role": langgraph_role, "content": content}) + langgraph_message: Dict[str, Any] = { + "role": langgraph_role, + "content": content, + } + message_metadata = msg.get("metadata") + if isinstance(message_metadata, dict) and message_metadata: + langgraph_message["metadata"] = message_metadata + + langgraph_messages.append(langgraph_message) return langgraph_messages diff --git a/litellm/llms/openai/cost_calculation.py b/litellm/llms/openai/cost_calculation.py index 32b71a43afa..6935cafd0d9 100644 --- a/litellm/llms/openai/cost_calculation.py +++ b/litellm/llms/openai/cost_calculation.py @@ -19,7 +19,10 @@ def cost_router(call_type: CallTypes) -> Literal["cost_per_token", "cost_per_sec def cost_per_token( - model: str, usage: Usage, service_tier: Optional[str] = None + model: str, + usage: Usage, + service_tier: Optional[str] = None, + data_residency: Optional[str] = None, ) -> Tuple[float, float]: """ Calculates the cost per token for a given model, prompt tokens, and completion tokens. @@ -27,6 +30,9 @@ def cost_per_token( Input: - model: str, the model name without provider prefix - usage: LiteLLM Usage block, containing anthropic caching information + - data_residency: optional OpenAI data-residency region (e.g. "eu", "us"), + inferred from api_base. Applies the model's regional-processing + uplift multiplier when set. Returns: Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd @@ -37,6 +43,7 @@ def cost_per_token( usage=usage, custom_llm_provider="openai", service_tier=service_tier, + data_residency=data_residency, ) # ### Non-cached text tokens # non_cached_text_tokens = usage.prompt_tokens diff --git a/litellm/llms/openai/data_residency.py b/litellm/llms/openai/data_residency.py new file mode 100644 index 00000000000..7162f70ca5f --- /dev/null +++ b/litellm/llms/openai/data_residency.py @@ -0,0 +1,41 @@ +""" +Helpers for resolving OpenAI data-residency (regional processing) from an +api_base URL. + +OpenAI enforces hostname-per-region for projects with geography restrictions +enabled and rejects requests sent to the wrong host, so the api_base hostname +is the authoritative signal of which region a request was processed in. +""" + +from typing import Dict, Optional +from urllib.parse import urlparse + +# Mapping of OpenAI regional hostnames to the corresponding data-residency +# value used by the cost calculator. See +# https://developers.openai.com/api/docs/pricing for the regional-processing +# uplift these hostnames trigger. +_OPENAI_REGIONAL_HOSTS: Dict[str, str] = { + "eu.api.openai.com": "eu", + "us.api.openai.com": "us", +} + + +def infer_openai_data_residency( + custom_llm_provider: Optional[str], api_base: Optional[str] +) -> Optional[str]: + """ + Derive the OpenAI data-residency region from an api_base URL. + + Returns ``"eu"`` for the EU regional host, ``"us"`` for the US regional + host, and ``None`` for the default global host, any non-OpenAI provider, + or any non-OpenAI URL. + """ + if custom_llm_provider != "openai" or not api_base: + return None + try: + host = urlparse(api_base).hostname + except (TypeError, ValueError): + return None + if not host: + return None + return _OPENAI_REGIONAL_HOSTS.get(host.lower()) diff --git a/litellm/llms/openai/videos/transformation.py b/litellm/llms/openai/videos/transformation.py index 2d165a7d7df..520a42e9dd1 100644 --- a/litellm/llms/openai/videos/transformation.py +++ b/litellm/llms/openai/videos/transformation.py @@ -534,6 +534,7 @@ class OpenAIVideoConfig(BaseVideoConfig): litellm_params: GenericLiteLLMParams, headers: dict, extra_body: Optional[Dict[str, Any]] = None, + prefetched_source_data: Optional[Dict[str, Any]] = None, ) -> Tuple[str, Dict]: original_video_id = extract_original_video_id(video_id) url = f"{api_base.rstrip('/')}/edits" @@ -547,6 +548,7 @@ class OpenAIVideoConfig(BaseVideoConfig): raw_response: httpx.Response, logging_obj: Any, custom_llm_provider: Optional[str] = None, + request_data: Optional[Dict] = None, ) -> VideoObject: video_obj = VideoObject(**raw_response.json()) if custom_llm_provider and video_obj.id: diff --git a/litellm/llms/runwayml/videos/transformation.py b/litellm/llms/runwayml/videos/transformation.py index 4f84816a2bc..b1723f494ec 100644 --- a/litellm/llms/runwayml/videos/transformation.py +++ b/litellm/llms/runwayml/videos/transformation.py @@ -623,12 +623,23 @@ class RunwayMLVideoConfig(BaseVideoConfig): raise NotImplementedError("video get character is not supported for RunwayML") def transform_video_edit_request( - self, prompt, video_id, api_base, litellm_params, headers, extra_body=None + self, + prompt, + video_id, + api_base, + litellm_params, + headers, + extra_body=None, + prefetched_source_data=None, ): raise NotImplementedError("video edit is not supported for RunwayML") def transform_video_edit_response( - self, raw_response, logging_obj, custom_llm_provider=None + self, + raw_response, + logging_obj, + custom_llm_provider=None, + request_data=None, ): raise NotImplementedError("video edit is not supported for RunwayML") diff --git a/litellm/llms/vertex_ai/realtime/transformation.py b/litellm/llms/vertex_ai/realtime/transformation.py index 2b4746b174e..ea4dbccc8c8 100644 --- a/litellm/llms/vertex_ai/realtime/transformation.py +++ b/litellm/llms/vertex_ai/realtime/transformation.py @@ -14,6 +14,7 @@ Auth: OAuth2 Bearer token (not an API key). import json from typing import List, Optional +from litellm import verbose_logger from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig @@ -26,6 +27,7 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig): """ def __init__(self, access_token: str, project: str, location: str) -> None: + super().__init__() self._access_token = access_token self._project = project self._location = location @@ -138,6 +140,62 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig): # Request translation # ------------------------------------------------------------------ + def _vertex_model_path(self, model: str) -> str: + """Return the fully-qualified Vertex AI model resource path.""" + return ( + f"projects/{self._project}" + f"/locations/{self._location}" + f"/publishers/google/models/{model}" + ) + + def _build_vertex_ai_setup_config(self, model: str, session_params: dict) -> dict: + """Build Vertex AI setup configuration with proper model path and defaults.""" + # Normalize GA-remapped fields (``output_modalities``, nested + # ``audio.input.transcription``, ``audio.input.turn_detection``) back to + # their flat beta keys so ``map_openai_params`` picks them up. Without + # this, GA clients' explicit modality / transcription / turn-detection + # settings would be silently dropped because ``map_openai_params`` only + # recognises the flat OpenAI-beta key names. + session_params = self._normalize_session_payload_for_mapping(session_params) + setup_config = self.map_openai_params( + optional_params={}, non_default_params=session_params + ) + + # Use full Vertex AI model path + setup_config["model"] = self._vertex_model_path(model) + + # Add Vertex AI specific defaults if not provided + generation_config = setup_config.setdefault("generationConfig", {}) + generation_config.setdefault("responseModalities", ["AUDIO"]) + + # Ensure Vertex defaults for realtimeInputConfig apply even when + # the client provided a partial ``turn_detection`` (e.g. only + # ``silence_duration_ms``). ``map_automatic_turn_detection`` sets + # ``disabled=True`` whenever ``create_response`` is absent or + # ``False``. Force ``disabled=False`` only when the client did + # not explicitly request ``create_response: False`` — that path + # is how transcription guardrails suppress automatic responses, + # and overriding it here would silently bypass the guardrail. + # Vertex Live has no "VAD on, no auto-response" mode, so callers + # that need that behaviour must accept that VAD is off. + client_turn_detection = session_params.get("turn_detection") + client_disabled_auto_response = ( + isinstance(client_turn_detection, dict) + and client_turn_detection.get("create_response") is False + ) + realtime_input_config = setup_config.setdefault("realtimeInputConfig", {}) + automatic_detection = realtime_input_config.setdefault( + "automaticActivityDetection", {} + ) + if not client_disabled_auto_response: + automatic_detection["disabled"] = False + automatic_detection.setdefault("silenceDurationMs", 800) + + setup_config.setdefault("inputAudioTranscription", {}) + setup_config.setdefault("outputAudioTranscription", {}) + + return setup_config + def transform_realtime_request( self, message: str, @@ -147,16 +205,50 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig): """ Translate OpenAI realtime client messages to Vertex AI format. - ``session.update`` is intentionally ignored (returns []) because - Vertex AI only accepts a single ``setup`` message at the start of - the connection — sending a second one causes a 1007 close error. - The initial setup (sent automatically before bidirectional_forward) - already includes AUDIO modality and server VAD, so there is nothing - more to configure. + On the first ``session.update`` (when no setup has been sent yet) the + full ``BidiGenerateContentSetup`` is built with Vertex AI's model path + and forwarded. Any later ``session.update`` is dropped: Vertex AI + documents ``setup`` as the first-and-only client message, and a second + ``setup`` closes the connection with a 1007 policy error. """ json_message = json.loads(message) - if json_message.get("type") == "session.update": - # Do not forward as a second setup — Vertex AI rejects it. + msg_type = json_message.get("type") + + if msg_type == "session.update": + if session_configuration_request is None: + setup_config = self._build_vertex_ai_setup_config( + model, json_message.get("session") or {} + ) + gemini_setup_msg = json.dumps({"setup": setup_config}) + + verbose_logger.debug( + "Vertex AI Realtime: Sending initial setup with tools to backend" + ) + return [gemini_setup_msg] + + # A follow-up session.update can't be forwarded as a second setup + # (Vertex Live closes the WebSocket with 1007). If this drop is + # silencing the audio-transcription guardrail's create_response + # disable, surface a warning so operators know the model will + # auto-respond before the guardrail can gate it on Vertex AI. + client_turn_detection = GeminiRealtimeConfig._extract_turn_detection( + json_message.get("session") or {} + ) + if ( + isinstance(client_turn_detection, dict) + and client_turn_detection.get("create_response") is False + ): + verbose_logger.warning( + "Vertex AI Realtime: Dropping subsequent session.update " + "(turn_detection.create_response=False) — Vertex Live " + "rejects a second setup message. Audio-transcription " + "guardrails cannot suppress the model's auto-response on " + "Vertex AI in non-deferred mode." + ) + else: + verbose_logger.debug( + "Vertex AI Realtime: Ignoring session.update (setup already sent)" + ) return [] return super().transform_realtime_request( diff --git a/litellm/llms/vertex_ai/videos/transformation.py b/litellm/llms/vertex_ai/videos/transformation.py index ed6176cef05..b84966354b8 100644 --- a/litellm/llms/vertex_ai/videos/transformation.py +++ b/litellm/llms/vertex_ai/videos/transformation.py @@ -40,6 +40,29 @@ else: BaseLLMException = Any +def _build_vertex_video_usage_from_request_data( + request_data: Optional[Dict[str, Any]], +) -> Dict[str, Any]: + """Build usage metadata (duration, resolution) for video cost calculation.""" + usage_data: Dict[str, Any] = {} + if not request_data: + return usage_data + + parameters = request_data.get("parameters", {}) + duration = ( + parameters.get("durationSeconds") or DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS + ) + if duration is not None: + try: + usage_data["duration_seconds"] = float(duration) + except (ValueError, TypeError): + pass + res = parameters.get("resolution") + if res is not None and str(res).strip() != "": + usage_data["video_resolution"] = str(res).strip().lower() + return usage_data + + def _convert_image_to_vertex_format(image_file) -> Dict[str, str]: """ Convert image file to Vertex AI format with base64 encoding and MIME type. @@ -363,23 +386,7 @@ class VertexAIVideoConfig(BaseVideoConfig, VertexBase): id=video_id, object="video", status="processing", model=model ) - usage_data: Dict[str, Any] = {} - if request_data: - parameters = request_data.get("parameters", {}) - duration = ( - parameters.get("durationSeconds") - or DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS - ) - if duration is not None: - try: - usage_data["duration_seconds"] = float(duration) - except (ValueError, TypeError): - pass - res = parameters.get("resolution") - if res is not None and str(res).strip() != "": - usage_data["video_resolution"] = str(res).strip().lower() - - video_obj.usage = usage_data + video_obj.usage = _build_vertex_video_usage_from_request_data(request_data) return video_obj def transform_video_status_retrieve_request( @@ -647,15 +654,123 @@ class VertexAIVideoConfig(BaseVideoConfig, VertexBase): def transform_video_get_character_response(self, raw_response, logging_obj): raise NotImplementedError("video get character is not supported for Vertex AI") + def get_video_edit_prefetch_params( + self, + video_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[str, Dict]: + """Return the fetchPredictOperation URL and body needed to retrieve the source video.""" + return self.transform_video_status_retrieve_request( + video_id=video_id, + api_base=api_base, + litellm_params=litellm_params, + headers=headers, + ) + def transform_video_edit_request( - self, prompt, video_id, api_base, litellm_params, headers, extra_body=None - ): - raise NotImplementedError("video edit is not supported for Vertex AI") + self, + prompt: str, + video_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + extra_body: Optional[Dict[str, Any]] = None, + prefetched_source_data: Optional[Dict[str, Any]] = None, + ) -> Tuple[str, Dict]: + """ + Build a predictLongRunning edit request from the pre-fetched source video. + + The actual fetchPredictOperation HTTP call is hoisted into the handler so + it can use the shared async/sync httpx client instead of blocking the loop. + """ + if prefetched_source_data is None: + raise ValueError( + "prefetched_source_data is required for Vertex AI video edit. " + "Ensure get_video_edit_prefetch_params is called by the handler." + ) + + if not prefetched_source_data.get("done", False): + raise ValueError( + "Source video generation is not complete yet. " + "Check the video status before editing." + ) + + videos = prefetched_source_data.get("response", {}).get("videos", []) + if not videos: + raise ValueError("No videos found in the completed operation. Cannot edit.") + + source_video = videos[0] + video_input: Dict[str, Any] = {} + if "gcsUri" in source_video: + video_input["gcsUri"] = source_video["gcsUri"] + elif "bytesBase64Encoded" in source_video: + video_input["bytesBase64Encoded"] = source_video["bytesBase64Encoded"] + video_input["mimeType"] = source_video.get("mimeType", "video/mp4") + else: + raise ValueError( + "Source video has neither gcsUri nor bytesBase64Encoded. Cannot edit." + ) + + operation_name = extract_original_video_id(video_id) + model = self.extract_model_from_operation_name(operation_name) or "" + + instance_dict: Dict[str, Any] = {"prompt": prompt, "video": video_input} + request_data: Dict[str, Any] = {"instances": [instance_dict]} + + if extra_body: + extra_body_copy = dict(extra_body) + nested_params = extra_body_copy.pop("parameters", None) + vertex_params: Dict[str, Any] = {} + if isinstance(nested_params, dict): + vertex_params.update(nested_params) + vertex_params.update(extra_body_copy) + if vertex_params: + request_data["parameters"] = vertex_params + + edit_url = f"{api_base.rstrip('/')}/{model}:predictLongRunning" + return edit_url, request_data def transform_video_edit_response( - self, raw_response, logging_obj, custom_llm_provider=None - ): - raise NotImplementedError("video edit is not supported for Vertex AI") + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + custom_llm_provider: Optional[str] = None, + request_data: Optional[Dict] = None, + ) -> VideoObject: + """ + Transform the Veo video edit response. + + Veo returns the same operation response as video generation: + {"name": "projects/.../operations/OPERATION_ID"} + + usage includes duration_seconds and optional video_resolution from the + edit request parameters for cost calculation. + """ + response_data = raw_response.json() + + operation_name = response_data.get("name") + if not operation_name: + raise ValueError(f"No operation name in Veo edit response: {response_data}") + + model = self.extract_model_from_operation_name(operation_name) or "" + + if custom_llm_provider: + video_id = encode_video_id_with_provider( + operation_name, custom_llm_provider, model + ) + else: + video_id = operation_name + + video_obj = VideoObject( + id=video_id, + object="video", + status="processing", + model=model, + ) + video_obj.usage = _build_vertex_video_usage_from_request_data(request_data) + return video_obj def transform_video_extension_request( self, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 62e576ea0f7..ce6d4ac824c 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -731,7 +731,6 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true }, "anthropic.claude-haiku-4-5@20251001": { @@ -755,7 +754,6 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_streaming": true, "supports_native_structured_output": true }, @@ -926,8 +924,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "anthropic.claude-opus-4-20250514-v1:0": { "cache_creation_input_token_cost": 1.875e-05, @@ -952,8 +949,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "anthropic.claude-opus-4-5-20251101-v1:0": { "cache_creation_input_token_cost": 6.25e-06, @@ -977,12 +973,12 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_reasoning": true, - "supports_minimal_reasoning_effort": true, "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 159, - "supports_native_structured_output": true + "supports_native_structured_output": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "high" }, "anthropic.claude-opus-4-6-v1": { "cache_creation_input_token_cost": 6.25e-06, @@ -1009,10 +1005,10 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true, + "supports_output_config": true, "supports_max_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "bedrock_output_config_effort_ceiling": "max" }, "global.anthropic.claude-opus-4-6-v1": { "cache_creation_input_token_cost": 6.25e-06, @@ -1039,10 +1035,10 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true, + "supports_output_config": true, "supports_max_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "bedrock_output_config_effort_ceiling": "max" }, "us.anthropic.claude-opus-4-6-v1": { "cache_creation_input_token_cost": 6.875e-06, @@ -1069,10 +1065,10 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true, + "supports_output_config": true, "supports_max_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "bedrock_output_config_effort_ceiling": "max" }, "eu.anthropic.claude-opus-4-6-v1": { "cache_creation_input_token_cost": 6.875e-06, @@ -1098,10 +1094,10 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true, + "supports_output_config": true, "supports_max_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "bedrock_output_config_effort_ceiling": "max" }, "au.anthropic.claude-opus-4-6-v1": { "cache_creation_input_token_cost": 6.875e-06, @@ -1127,10 +1123,10 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true, + "supports_output_config": true, "supports_max_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "bedrock_output_config_effort_ceiling": "max" }, "anthropic.claude-opus-4-7": { "cache_creation_input_token_cost": 6.25e-06, @@ -1158,10 +1154,10 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true, "supports_max_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh" }, "anthropic.claude-mythos-preview": { "input_cost_per_token": 0, @@ -1175,8 +1171,8 @@ "supports_vision": true, "supports_prompt_caching": false, "supports_reasoning": true, - "supports_minimal_reasoning_effort": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_output_config": true }, "global.anthropic.claude-opus-4-7": { "cache_creation_input_token_cost": 6.25e-06, @@ -1204,10 +1200,10 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true, "supports_max_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh" }, "us.anthropic.claude-opus-4-7": { "cache_creation_input_token_cost": 6.875e-06, @@ -1235,10 +1231,10 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true, "supports_max_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh" }, "eu.anthropic.claude-opus-4-7": { "cache_creation_input_token_cost": 6.875e-06, @@ -1265,10 +1261,10 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true, "supports_max_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh" }, "au.anthropic.claude-opus-4-7": { "cache_creation_input_token_cost": 6.875e-06, @@ -1295,10 +1291,165 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true, "supports_max_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh" + }, + "anthropic.claude-opus-4-8": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": true, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh" + }, + "global.anthropic.claude-opus-4-8": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": true, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh" + }, + "us.anthropic.claude-opus-4-8": { + "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_1hr": 1.1e-05, + "cache_read_input_token_cost": 5.5e-07, + "input_cost_per_token": 5.5e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.75e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": true, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh" + }, + "eu.anthropic.claude-opus-4-8": { + "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_1hr": 1.1e-05, + "cache_read_input_token_cost": 5.5e-07, + "input_cost_per_token": 5.5e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.75e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": true, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh" + }, + "au.anthropic.claude-opus-4-8": { + "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_1hr": 1.1e-05, + "cache_read_input_token_cost": 5.5e-07, + "input_cost_per_token": 5.5e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.75e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": true, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh" }, "anthropic.claude-sonnet-4-6": { "cache_creation_input_token_cost": 3.75e-06, @@ -1326,9 +1477,8 @@ "supports_max_reasoning_effort": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true }, "global.anthropic.claude-sonnet-4-6": { "cache_creation_input_token_cost": 3.75e-06, @@ -1356,9 +1506,8 @@ "supports_max_reasoning_effort": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true }, "us.anthropic.claude-sonnet-4-6": { "cache_creation_input_token_cost": 4.125e-06, @@ -1386,9 +1535,8 @@ "supports_max_reasoning_effort": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true }, "eu.anthropic.claude-sonnet-4-6": { "cache_creation_input_token_cost": 4.125e-06, @@ -1415,9 +1563,8 @@ "supports_max_reasoning_effort": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true }, "au.anthropic.claude-sonnet-4-6": { "cache_creation_input_token_cost": 4.125e-06, @@ -1444,9 +1591,8 @@ "supports_max_reasoning_effort": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true }, "jp.anthropic.claude-sonnet-4-6": { "cache_creation_input_token_cost": 4.125e-06, @@ -1473,9 +1619,8 @@ "supports_max_reasoning_effort": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true }, "anthropic.claude-sonnet-4-20250514-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -1504,8 +1649,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -1537,7 +1681,6 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 159, "supports_native_structured_output": true }, "anthropic.claude-v1": { @@ -1788,7 +1931,6 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true }, "apac.anthropic.claude-3-sonnet-20240229-v1:0": { @@ -1834,8 +1976,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "assemblyai/best": { "input_cost_per_second": 3.333e-05, @@ -1877,7 +2018,6 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true }, "azure/ada": { @@ -1965,10 +2105,10 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_reasoning": true, - "supports_minimal_reasoning_effort": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "supports_output_config": true }, "azure_ai/claude-opus-4-6": { "input_cost_per_token": 5e-06, @@ -1995,9 +2135,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 159, - "supports_max_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true, + "supports_max_reasoning_effort": true }, "azure_ai/claude-opus-4-7": { "input_cost_per_token": 5e-06, @@ -2025,9 +2164,35 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "tool_use_system_prompt_tokens": 159, - "supports_max_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_max_reasoning_effort": true + }, + "azure_ai/claude-opus-4-8": { + "input_cost_per_token": 5e-06, + "output_cost_per_token": 2.5e-05, + "litellm_provider": "azure_ai", + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_read_input_token_cost": 5e-07, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true }, "azure_ai/claude-opus-4-1": { "cache_creation_input_token_cost": 1.875e-05, @@ -2092,8 +2257,7 @@ "supports_max_reasoning_effort": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, - "supports_minimal_reasoning_effort": true + "supports_output_config": true }, "azure/computer-use-preview": { "input_cost_per_token": 3e-06, @@ -9467,8 +9631,7 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_web_search": true, - "tool_use_system_prompt_tokens": 159 + "supports_web_search": true }, "claude-3-haiku-20240307": { "cache_creation_input_token_cost": 3e-07, @@ -9486,8 +9649,7 @@ "supports_prompt_caching": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 264 + "supports_vision": true }, "claude-3-opus-20240229": { "cache_creation_input_token_cost": 1.875e-05, @@ -9506,8 +9668,7 @@ "supports_prompt_caching": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 395 + "supports_vision": true }, "claude-4-opus-20250514": { "cache_creation_input_token_cost": 1.875e-05, @@ -9532,8 +9693,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "claude-4-sonnet-20250514": { "cache_creation_input_token_cost": 3.75e-06, @@ -9563,8 +9723,7 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_web_search": true, - "tool_use_system_prompt_tokens": 159 + "supports_web_search": true }, "claude-sonnet-4-5": { "cache_creation_input_token_cost": 3.75e-06, @@ -9593,8 +9752,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 346 + "supports_vision": true }, "claude-sonnet-4-5-20250929": { "cache_creation_input_token_cost": 3.75e-06, @@ -9624,8 +9782,7 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_web_search": true, - "tool_use_system_prompt_tokens": 346 + "supports_web_search": true }, "claude-sonnet-4-6": { "cache_creation_input_token_cost": 3.75e-06, @@ -9653,8 +9810,7 @@ "supports_max_reasoning_effort": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, - "supports_minimal_reasoning_effort": true + "supports_output_config": true }, "claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -9678,8 +9834,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "claude-opus-4-1": { "cache_creation_input_token_cost": 1.875e-05, @@ -9705,8 +9860,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "claude-opus-4-1-20250805": { "cache_creation_input_token_cost": 1.875e-05, @@ -9733,8 +9887,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "claude-opus-4-20250514": { "cache_creation_input_token_cost": 1.875e-05, @@ -9761,8 +9914,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "claude-opus-4-5-20251101": { "cache_creation_input_token_cost": 6.25e-06, @@ -9786,11 +9938,10 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_reasoning": true, - "supports_minimal_reasoning_effort": true, "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_output_config": true }, "claude-opus-4-5": { "cache_creation_input_token_cost": 6.25e-06, @@ -9814,11 +9965,10 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_reasoning": true, - "supports_minimal_reasoning_effort": true, "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_output_config": true }, "claude-opus-4-6": { "cache_creation_input_token_cost": 6.25e-06, @@ -9846,13 +9996,12 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "provider_specific_entry": { "us": 1.1, "fast": 6.0 }, - "supports_max_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true, + "supports_max_reasoning_effort": true }, "claude-opus-4-6-20260205": { "cache_creation_input_token_cost": 6.25e-06, @@ -9880,13 +10029,12 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "provider_specific_entry": { "us": 1.1, "fast": 6.0 }, "supports_max_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true }, "claude-opus-4-7": { "cache_creation_input_token_cost": 6.25e-06, @@ -9916,12 +10064,11 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, - "tool_use_system_prompt_tokens": 346, "provider_specific_entry": { "us": 1.1, "fast": 6.0 }, - "supports_minimal_reasoning_effort": true + "supports_output_config": true }, "claude-opus-4-7-20260416": { "cache_creation_input_token_cost": 6.25e-06, @@ -9951,12 +10098,45 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, - "tool_use_system_prompt_tokens": 346, "provider_specific_entry": { "us": 1.1, "fast": 6.0 }, - "supports_minimal_reasoning_effort": true + "supports_output_config": true + }, + "claude-opus-4-8": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "anthropic", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "provider_specific_entry": { + "us": 1.1, + "fast": 2.0 + }, + "supports_output_config": true }, "claude-sonnet-4-20250514": { "deprecation_date": "2026-05-14", @@ -9987,8 +10167,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "cloudflare/@cf/meta/llama-2-7b-chat-fp16": { "input_cost_per_token": 1.923e-06, @@ -11233,8 +11412,8 @@ "supports_assistant_prefill": true, "supports_function_calling": true, "supports_reasoning": true, - "supports_minimal_reasoning_effort": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_output_config": true }, "databricks/databricks-claude-sonnet-4": { "input_cost_per_token": 2.9999900000000002e-06, @@ -13400,7 +13579,6 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true }, "eu.anthropic.claude-3-5-sonnet-20240620-v1:0": { @@ -13528,8 +13706,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "eu.anthropic.claude-opus-4-20250514-v1:0": { "cache_creation_input_token_cost": 1.875e-05, @@ -13554,8 +13731,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "eu.anthropic.claude-sonnet-4-20250514-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -13584,8 +13760,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "eu.anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.125e-06, @@ -13615,7 +13790,6 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true }, "eu.meta.llama3-2-1b-instruct-v1:0": { @@ -14958,7 +15132,7 @@ "mode": "chat", "output_cost_per_reasoning_token": 1.5e-06, "output_cost_per_token": 1.5e-06, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models", + "source": "https://ai.google.dev/gemini-api/docs/models", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -17901,22 +18075,9 @@ }, "github_copilot/claude-haiku-4.5": { "litellm_provider": "github_copilot", - "max_input_tokens": 128000, - "max_output_tokens": 16000, - "max_tokens": 16000, - "mode": "chat", - "supported_endpoints": [ - "/v1/chat/completions" - ], - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_vision": true - }, - "github_copilot/claude-opus-4.5": { - "litellm_provider": "github_copilot", - "max_input_tokens": 128000, - "max_output_tokens": 16000, - "max_tokens": 16000, + "max_input_tokens": 200000, + "max_output_tokens": 32000, + "max_tokens": 32000, "mode": "chat", "supported_endpoints": [ "/v1/chat/completions" @@ -17924,7 +18085,22 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_vision": true, - "supports_minimal_reasoning_effort": true + "supports_reasoning": true + }, + "github_copilot/claude-opus-4.5": { + "litellm_provider": "github_copilot", + "max_input_tokens": 200000, + "max_output_tokens": 32000, + "max_tokens": 32000, + "mode": "chat", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_output_config": true }, "github_copilot/claude-opus-4.6-fast": { "litellm_provider": "github_copilot", @@ -17939,6 +18115,22 @@ "supports_parallel_function_calling": true, "supports_vision": true }, + "github_copilot/claude-opus-4.7": { + "litellm_provider": "github_copilot", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/messages" + ], + "supports_vision": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_reasoning": true + }, "github_copilot/claude-opus-41": { "litellm_provider": "github_copilot", "max_input_tokens": 80000, @@ -17965,16 +18157,33 @@ }, "github_copilot/claude-sonnet-4.5": { "litellm_provider": "github_copilot", - "max_input_tokens": 128000, - "max_output_tokens": 16000, - "max_tokens": 16000, + "max_input_tokens": 200000, + "max_output_tokens": 32000, + "max_tokens": 32000, "mode": "chat", "supported_endpoints": [ "/v1/chat/completions" ], "supports_function_calling": true, "supports_parallel_function_calling": true, - "supports_vision": true + "supports_vision": true, + "supports_reasoning": true + }, + "github_copilot/claude-sonnet-4.6": { + "litellm_provider": "github_copilot", + "max_input_tokens": 200000, + "max_output_tokens": 32000, + "max_tokens": 32000, + "mode": "chat", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/messages" + ], + "supports_vision": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_reasoning": true }, "github_copilot/gemini-2.5-pro": { "litellm_provider": "github_copilot", @@ -17984,7 +18193,25 @@ "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true, - "supports_vision": true + "supports_vision": true, + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_reasoning": true + }, + "github_copilot/gemini-3-flash-preview": { + "litellm_provider": "github_copilot", + "max_input_tokens": 128000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_vision": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_reasoning": true }, "github_copilot/gemini-3-pro-preview": { "litellm_provider": "github_copilot", @@ -17996,13 +18223,30 @@ "supports_parallel_function_calling": true, "supports_vision": true }, + "github_copilot/gemini-3.1-pro-preview": { + "litellm_provider": "github_copilot", + "max_input_tokens": 128000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_vision": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_reasoning": true + }, "github_copilot/gpt-3.5-turbo": { "litellm_provider": "github_copilot", "max_input_tokens": 16384, "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", - "supports_function_calling": true + "supports_function_calling": true, + "supported_endpoints": [ + "/v1/chat/completions" + ] }, "github_copilot/gpt-3.5-turbo-0613": { "litellm_provider": "github_copilot", @@ -18010,7 +18254,10 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", - "supports_function_calling": true + "supports_function_calling": true, + "supported_endpoints": [ + "/v1/chat/completions" + ] }, "github_copilot/gpt-4": { "litellm_provider": "github_copilot", @@ -18018,7 +18265,22 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", - "supports_function_calling": true + "supports_function_calling": true, + "supported_endpoints": [ + "/v1/chat/completions" + ] + }, + "github_copilot/gpt-4-0125-preview": { + "litellm_provider": "github_copilot", + "max_input_tokens": 128000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "chat", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": true }, "github_copilot/gpt-4-0613": { "litellm_provider": "github_copilot", @@ -18026,16 +18288,22 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", - "supports_function_calling": true + "supports_function_calling": true, + "supported_endpoints": [ + "/v1/chat/completions" + ] }, "github_copilot/gpt-4-o-preview": { "litellm_provider": "github_copilot", - "max_input_tokens": 64000, + "max_input_tokens": 128000, "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", "supports_function_calling": true, - "supports_parallel_function_calling": true + "supports_parallel_function_calling": true, + "supported_endpoints": [ + "/v1/chat/completions" + ] }, "github_copilot/gpt-4.1": { "litellm_provider": "github_copilot", @@ -18046,7 +18314,10 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supported_endpoints": [ + "/v1/chat/completions" + ] }, "github_copilot/gpt-4.1-2025-04-14": { "litellm_provider": "github_copilot", @@ -18057,68 +18328,89 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supported_endpoints": [ + "/v1/chat/completions" + ] }, "github_copilot/gpt-41-copilot": { "litellm_provider": "github_copilot", - "mode": "completion" + "mode": "chat" }, "github_copilot/gpt-4o": { "litellm_provider": "github_copilot", - "max_input_tokens": 64000, + "max_input_tokens": 128000, "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true, - "supports_vision": true + "supports_vision": true, + "supported_endpoints": [ + "/v1/chat/completions" + ] }, "github_copilot/gpt-4o-2024-05-13": { "litellm_provider": "github_copilot", - "max_input_tokens": 64000, + "max_input_tokens": 128000, "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true, - "supports_vision": true + "supports_vision": true, + "supported_endpoints": [ + "/v1/chat/completions" + ] }, "github_copilot/gpt-4o-2024-08-06": { "litellm_provider": "github_copilot", - "max_input_tokens": 64000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "supports_function_calling": true, - "supports_parallel_function_calling": true - }, - "github_copilot/gpt-4o-2024-11-20": { - "litellm_provider": "github_copilot", - "max_input_tokens": 64000, + "max_input_tokens": 128000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true, - "supports_vision": true + "supported_endpoints": [ + "/v1/chat/completions" + ] + }, + "github_copilot/gpt-4o-2024-11-20": { + "litellm_provider": "github_copilot", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": true, + "supported_endpoints": [ + "/v1/chat/completions" + ] }, "github_copilot/gpt-4o-mini": { "litellm_provider": "github_copilot", - "max_input_tokens": 64000, + "max_input_tokens": 128000, "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", "supports_function_calling": true, - "supports_parallel_function_calling": true + "supports_parallel_function_calling": true, + "supported_endpoints": [ + "/v1/chat/completions" + ] }, "github_copilot/gpt-4o-mini-2024-07-18": { "litellm_provider": "github_copilot", - "max_input_tokens": 64000, + "max_input_tokens": 128000, "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", "supports_function_calling": true, - "supports_parallel_function_calling": true + "supports_parallel_function_calling": true, + "supported_endpoints": [ + "/v1/chat/completions" + ] }, "github_copilot/gpt-5": { "litellm_provider": "github_copilot", @@ -18137,14 +18429,19 @@ }, "github_copilot/gpt-5-mini": { "litellm_provider": "github_copilot", - "max_input_tokens": 128000, + "max_input_tokens": 264000, "max_output_tokens": 64000, "max_tokens": 64000, "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supports_reasoning": true }, "github_copilot/gpt-5.1": { "litellm_provider": "github_copilot", @@ -18177,7 +18474,7 @@ }, "github_copilot/gpt-5.2": { "litellm_provider": "github_copilot", - "max_input_tokens": 128000, + "max_input_tokens": 264000, "max_output_tokens": 64000, "max_tokens": 64000, "mode": "chat", @@ -18188,11 +18485,27 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_reasoning": true + }, + "github_copilot/gpt-5.2-codex": { + "litellm_provider": "github_copilot", + "max_input_tokens": 400000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "supported_endpoints": [ + "/v1/responses" + ], + "supports_vision": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_reasoning": true }, "github_copilot/gpt-5.3-codex": { "litellm_provider": "github_copilot", - "max_input_tokens": 128000, + "max_input_tokens": 400000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", @@ -18202,25 +18515,96 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_reasoning": true + }, + "github_copilot/gpt-5.4": { + "litellm_provider": "github_copilot", + "max_input_tokens": 400000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supports_vision": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_reasoning": true + }, + "github_copilot/gpt-5.4-mini": { + "litellm_provider": "github_copilot", + "max_input_tokens": 400000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "supported_endpoints": [ + "/v1/responses" + ], + "supports_vision": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_reasoning": true + }, + "github_copilot/gpt-5.5": { + "litellm_provider": "github_copilot", + "max_input_tokens": 400000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "supported_endpoints": [ + "/v1/responses" + ], + "supports_vision": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_reasoning": true + }, + "github_copilot/oswe-vscode-prime": { + "litellm_provider": "github_copilot", + "max_input_tokens": 264000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supports_vision": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true }, "github_copilot/text-embedding-3-small": { "litellm_provider": "github_copilot", "max_input_tokens": 8191, "max_tokens": 8191, - "mode": "embedding" + "mode": "embedding", + "supported_endpoints": [ + "/v1/embeddings" + ] }, "github_copilot/text-embedding-3-small-inference": { "litellm_provider": "github_copilot", "max_input_tokens": 8191, "max_tokens": 8191, - "mode": "embedding" + "mode": "embedding", + "supported_endpoints": [ + "/v1/embeddings" + ] }, "github_copilot/text-embedding-ada-002": { "litellm_provider": "github_copilot", "max_input_tokens": 8191, "max_tokens": 8191, - "mode": "embedding" + "mode": "embedding", + "supported_endpoints": [ + "/v1/embeddings" + ] }, "chatgpt/gpt-5.4": { "litellm_provider": "chatgpt", @@ -18438,7 +18822,7 @@ "output_cost_per_token": 2.5e-05, "supports_function_calling": true, "supports_vision": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true }, "gmi/anthropic/claude-sonnet-4.5": { "input_cost_per_token": 3e-06, @@ -18738,7 +19122,6 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true }, "global.anthropic.claude-sonnet-4-20250514-v1:0": { @@ -18768,8 +19151,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "global.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.25e-06, @@ -18792,7 +19174,6 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true }, "global.amazon.nova-2-lite-v1:0": { @@ -19014,6 +19395,8 @@ "output_cost_per_token": 8e-06, "output_cost_per_token_batches": 4e-06, "output_cost_per_token_priority": 1.4e-05, + "regional_processing_uplift_multiplier_eu": 1.10, + "regional_processing_uplift_multiplier_us": 1.10, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -19087,6 +19470,8 @@ "output_cost_per_token": 1.6e-06, "output_cost_per_token_batches": 8e-07, "output_cost_per_token_priority": 2.8e-06, + "regional_processing_uplift_multiplier_eu": 1.10, + "regional_processing_uplift_multiplier_us": 1.10, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -19160,6 +19545,8 @@ "output_cost_per_token": 4e-07, "output_cost_per_token_batches": 2e-07, "output_cost_per_token_priority": 8e-07, + "regional_processing_uplift_multiplier_eu": 1.10, + "regional_processing_uplift_multiplier_us": 1.10, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -19231,6 +19618,8 @@ "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, "output_cost_per_token_priority": 1.7e-05, + "regional_processing_uplift_multiplier_eu": 1.10, + "regional_processing_uplift_multiplier_us": 1.10, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, @@ -19272,6 +19661,8 @@ "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, + "regional_processing_uplift_multiplier_eu": 1.10, + "regional_processing_uplift_multiplier_us": 1.10, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, @@ -19293,6 +19684,8 @@ "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, + "regional_processing_uplift_multiplier_eu": 1.10, + "regional_processing_uplift_multiplier_us": 1.10, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, @@ -19581,6 +19974,8 @@ "output_cost_per_token": 6e-07, "output_cost_per_token_batches": 3e-07, "output_cost_per_token_priority": 1e-06, + "regional_processing_uplift_multiplier_eu": 1.10, + "regional_processing_uplift_multiplier_us": 1.10, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, @@ -20284,6 +20679,8 @@ "output_cost_per_token": 1e-05, "output_cost_per_token_flex": 5e-06, "output_cost_per_token_priority": 2e-05, + "regional_processing_uplift_multiplier_eu": 1.10, + "regional_processing_uplift_multiplier_us": 1.10, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -21206,6 +21603,8 @@ "mode": "responses", "output_cost_per_token": 0.00012, "output_cost_per_token_batches": 6e-05, + "regional_processing_uplift_multiplier_eu": 1.10, + "regional_processing_uplift_multiplier_us": 1.10, "supported_endpoints": [ "/v1/batch", "/v1/responses" @@ -21612,6 +22011,8 @@ "output_cost_per_token": 2e-06, "output_cost_per_token_flex": 1e-06, "output_cost_per_token_priority": 3.6e-06, + "regional_processing_uplift_multiplier_eu": 1.10, + "regional_processing_uplift_multiplier_us": 1.10, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -21693,6 +22094,8 @@ "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, + "regional_processing_uplift_multiplier_eu": 1.10, + "regional_processing_uplift_multiplier_us": 1.10, "mode": "chat", "output_cost_per_token": 4e-07, "output_cost_per_token_flex": 2e-07, @@ -22878,7 +23281,6 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true }, "jp.anthropic.claude-haiku-4-5-20251001-v1:0": { @@ -22901,7 +23303,6 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true }, "crusoe/deepseek-ai/DeepSeek-R1-0528": { @@ -26843,8 +27244,7 @@ "supports_computer_use": true, "supports_function_calling": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "openrouter/anthropic/claude-3.7-sonnet": { "input_cost_per_image": 0.0048, @@ -26860,8 +27260,7 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "openrouter/anthropic/claude-opus-4": { "input_cost_per_image": 0.0048, @@ -26880,8 +27279,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "openrouter/anthropic/claude-opus-4.1": { "input_cost_per_image": 0.0048, @@ -26901,8 +27299,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "openrouter/anthropic/claude-sonnet-4": { "input_cost_per_image": 0.0048, @@ -26925,8 +27322,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "openrouter/anthropic/claude-sonnet-4.6": { "cache_creation_input_token_cost": 3.75e-06, @@ -26950,9 +27346,7 @@ "supports_reasoning": true, "supports_max_reasoning_effort": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159, - "supports_minimal_reasoning_effort": true + "supports_vision": true }, "openrouter/anthropic/claude-opus-4.5": { "cache_creation_input_token_cost": 6.25e-06, @@ -26967,12 +27361,11 @@ "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, - "supports_minimal_reasoning_effort": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_output_config": true }, "openrouter/anthropic/claude-opus-4.6": { "cache_creation_input_token_cost": 6.25e-06, @@ -26991,9 +27384,7 @@ "supports_reasoning": true, "supports_max_reasoning_effort": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 346, - "supports_minimal_reasoning_effort": true + "supports_vision": true }, "openrouter/anthropic/claude-sonnet-4.5": { "input_cost_per_image": 0.0048, @@ -27016,8 +27407,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "openrouter/anthropic/claude-haiku-4.5": { "cache_creation_input_token_cost": 1.25e-06, @@ -27035,8 +27425,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 346 + "supports_vision": true }, "openrouter/anthropic/claude-opus-4.7": { "cache_creation_input_token_cost": 6.25e-06, @@ -27058,8 +27447,7 @@ "supports_max_reasoning_effort": true, "supports_tool_choice": true, "supports_vision": true, - "supports_xhigh_reasoning_effort": true, - "tool_use_system_prompt_tokens": 346 + "supports_xhigh_reasoning_effort": true }, "openrouter/bytedance/ui-tars-1.5-7b": { "input_cost_per_token": 1e-07, @@ -28243,10 +28631,10 @@ "supports_tool_choice": true }, "openrouter/xiaomi/mimo-v2-flash": { - "input_cost_per_token": 9e-08, - "output_cost_per_token": 2.9e-07, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 3e-07, "cache_creation_input_token_cost": 0.0, - "cache_read_input_token_cost": 0.0, + "cache_read_input_token_cost": 1e-08, "litellm_provider": "openrouter", "max_input_tokens": 262144, "max_output_tokens": 16384, @@ -28256,7 +28644,43 @@ "supports_tool_choice": true, "supports_reasoning": true, "supports_vision": false, - "supports_prompt_caching": false + "supports_prompt_caching": true + }, + "openrouter/xiaomi/mimo-v2.5-pro": { + "input_cost_per_token": 1e-06, + "output_cost_per_token": 3e-06, + "cache_creation_input_token_cost": 0.0, + "cache_read_input_token_cost": 2e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 1048576, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_vision": false, + "supports_response_schema": true, + "supports_prompt_caching": true + }, + "openrouter/xiaomi/mimo-v2.5": { + "input_cost_per_token": 4e-07, + "output_cost_per_token": 2e-06, + "cache_creation_input_token_cost": 0.0, + "cache_read_input_token_cost": 8e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_vision": true, + "supports_audio_input": true, + "supports_video_input": true, + "supports_response_schema": true, + "supports_prompt_caching": true }, "openrouter/z-ai/glm-4.7": { "input_cost_per_token": 4e-07, @@ -28987,14 +29411,16 @@ "mode": "responses", "supports_web_search": true, "supports_reasoning": false, - "supports_function_calling": true + "supports_function_calling": true, + "supports_output_config": true }, "perplexity/anthropic/claude-opus-4-7": { "litellm_provider": "perplexity", "mode": "responses", "supports_web_search": true, "supports_reasoning": false, - "supports_function_calling": true + "supports_function_calling": true, + "supports_output_config": true }, "perplexity/anthropic/claude-opus-4-5": { "litellm_provider": "perplexity", @@ -29002,7 +29428,7 @@ "supports_web_search": true, "supports_reasoning": false, "supports_function_calling": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true }, "perplexity/anthropic/claude-sonnet-4-5": { "litellm_provider": "perplexity", @@ -31255,7 +31681,6 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true }, "us.anthropic.claude-3-5-sonnet-20240620-v1:0": { @@ -31383,8 +31808,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "us.anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.125e-06, @@ -31416,7 +31840,6 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true }, "us-gov.anthropic.claude-sonnet-4-5-20250929-v1:0": { @@ -31442,7 +31865,6 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true }, "au.anthropic.claude-haiku-4-5-20251001-v1:0": { @@ -31464,7 +31886,6 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true }, "us.anthropic.claude-opus-4-20250514-v1:0": { @@ -31490,8 +31911,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "us.anthropic.claude-opus-4-5-20251101-v1:0": { "cache_creation_input_token_cost": 6.875e-06, @@ -31512,15 +31932,15 @@ "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, - "supports_minimal_reasoning_effort": true, "supports_pdf_input": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 159, - "supports_native_structured_output": true + "supports_native_structured_output": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "high" }, "global.anthropic.claude-opus-4-5-20251101-v1:0": { "cache_creation_input_token_cost": 6.25e-06, @@ -31541,15 +31961,15 @@ "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, - "supports_minimal_reasoning_effort": true, "supports_pdf_input": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 159, - "supports_native_structured_output": true + "supports_native_structured_output": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "high" }, "eu.anthropic.claude-opus-4-5-20251101-v1:0": { "cache_creation_input_token_cost": 6.25e-06, @@ -31569,15 +31989,15 @@ "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, - "supports_minimal_reasoning_effort": true, "supports_pdf_input": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 159, - "supports_native_structured_output": true + "supports_native_structured_output": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "high" }, "us.anthropic.claude-sonnet-4-20250514-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -31606,8 +32026,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "us.deepseek.r1-v1:0": { "input_cost_per_token": 1.35e-06, @@ -32150,13 +32569,13 @@ "output_cost_per_token": 2.5e-05, "supports_assistant_prefill": true, "supports_computer_use": true, - "supports_minimal_reasoning_effort": true, "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "supports_output_config": true }, "vercel_ai_gateway/anthropic/claude-opus-4.6": { "cache_creation_input_token_cost": 6.25e-06, @@ -32176,7 +32595,7 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true }, "vercel_ai_gateway/anthropic/claude-sonnet-4": { "cache_creation_input_token_cost": 3.75e-06, @@ -33184,8 +33603,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "vertex_ai/claude-3-haiku": { "input_cost_per_token": 2.5e-07, @@ -33288,8 +33706,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "vertex_ai/claude-opus-4-1": { "cache_creation_input_token_cost": 1.875e-05, @@ -33343,14 +33760,13 @@ "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, - "supports_minimal_reasoning_effort": true, "supports_pdf_input": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_output_config": true }, "vertex_ai/claude-opus-4-5@20251101": { "cache_creation_input_token_cost": 6.25e-06, @@ -33370,15 +33786,14 @@ "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, - "supports_minimal_reasoning_effort": true, "supports_pdf_input": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 159, - "supports_native_streaming": true + "supports_native_streaming": true, + "supports_output_config": true }, "vertex_ai/claude-opus-4-6": { "cache_creation_input_token_cost": 6.25e-06, @@ -33404,9 +33819,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, - "supports_max_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true, + "supports_max_reasoning_effort": true }, "vertex_ai/claude-opus-4-6@default": { "cache_creation_input_token_cost": 6.25e-06, @@ -33432,9 +33846,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, - "supports_max_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_output_config": true, + "supports_max_reasoning_effort": true }, "vertex_ai/claude-opus-4-7": { "cache_creation_input_token_cost": 6.25e-06, @@ -33461,9 +33874,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "tool_use_system_prompt_tokens": 346, - "supports_max_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_max_reasoning_effort": true }, "vertex_ai/claude-opus-4-7@default": { "cache_creation_input_token_cost": 6.25e-06, @@ -33490,9 +33901,63 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "tool_use_system_prompt_tokens": 346, - "supports_max_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_max_reasoning_effort": true + }, + "vertex_ai/claude-opus-4-8": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true + }, + "vertex_ai/claude-opus-4-8@default": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true }, "vertex_ai/claude-sonnet-4-5": { "cache_creation_input_token_cost": 3.75e-06, @@ -33540,13 +34005,12 @@ "supports_max_reasoning_effort": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_minimal_reasoning_effort": true + "supports_output_config": true }, "vertex_ai/claude-sonnet-4-5@20250929": { "cache_creation_input_token_cost": 3.75e-06, @@ -33598,8 +34062,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "vertex_ai/claude-sonnet-4": { "cache_creation_input_token_cost": 3.75e-06, @@ -33628,8 +34091,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "vertex_ai/claude-sonnet-4@20250514": { "cache_creation_input_token_cost": 3.75e-06, @@ -33658,8 +34120,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "supports_vision": true }, "vertex_ai/mistralai/codestral-2@001": { "input_cost_per_token": 3e-07, @@ -40652,13 +41113,12 @@ "supports_max_reasoning_effort": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_minimal_reasoning_effort": true + "supports_output_config": true }, "duckduckgo/search": { "litellm_provider": "duckduckgo", @@ -40974,7 +41434,6 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true, "supports_pdf_input": true }, @@ -40997,7 +41456,6 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true, "supports_pdf_input": true } diff --git a/litellm/passthrough/utils.py b/litellm/passthrough/utils.py index d39a0dda152..9484922833a 100644 --- a/litellm/passthrough/utils.py +++ b/litellm/passthrough/utils.py @@ -71,6 +71,11 @@ class BasePassthroughUtils: request_headers.pop("content-length", None) request_headers.pop("host", None) + custom_header_names = {header_name.lower() for header_name in headers} + for header_name in list(request_headers.keys()): + if header_name.lower() in custom_header_names: + request_headers.pop(header_name, None) + # Combine request headers with custom headers headers = {**request_headers, **headers} diff --git a/litellm/proxy/_experimental/mcp_server/CLAUDE.md b/litellm/proxy/_experimental/mcp_server/CLAUDE.md new file mode 100644 index 00000000000..0ba8f73315f --- /dev/null +++ b/litellm/proxy/_experimental/mcp_server/CLAUDE.md @@ -0,0 +1 @@ +MCP note: **`available_on_public_internet: false` with `delegate_auth_to_upstream: true` (oauth2, interactive - not `client_credentials`)** - LiteLLM still allows the anonymous upstream PKCE path (no proxy API key for `/authorize` and matching MCP routes). The internal-only flag mainly affects other surfaces (e.g. IP-based discovery). Rely on the upstream IdP and network policy; the dashboard shows a warning when both are set, and the proxy logs a warning when the server is loaded from config or the database diff --git a/litellm/proxy/_experimental/mcp_server/auth/litellm_auth_handler.py b/litellm/proxy/_experimental/mcp_server/auth/litellm_auth_handler.py index 75b75d3ba44..7122c64ec64 100644 --- a/litellm/proxy/_experimental/mcp_server/auth/litellm_auth_handler.py +++ b/litellm/proxy/_experimental/mcp_server/auth/litellm_auth_handler.py @@ -20,7 +20,7 @@ class MCPAuthenticatedUser(AuthenticatedUser): def __init__( self, - user_api_key_auth: UserAPIKeyAuth, + user_api_key_auth: Optional[UserAPIKeyAuth], mcp_auth_header: Optional[str] = None, mcp_servers: Optional[List[str]] = None, mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]] = None, diff --git a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py index 708ec7f1176..2aacab80f57 100644 --- a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py +++ b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py @@ -118,15 +118,19 @@ class MCPRequestHandler: return b"{}" request.body = mock_body # type: ignore + # Inline import — auth_utils participates in a proxy import cycle. + from litellm.proxy.auth.auth_utils import ( # noqa: PLC0415 + get_request_route, + ) + + request_route = get_request_route(request) # Only OAuth metadata routes registered under /.well-known/ are public. - # Match on request.url.path (path-only, exact prefix) so the substring - # cannot be smuggled via query string, hostname, or a deeper URL segment. - if request.url.path.startswith("/.well-known/"): + if request_route.startswith("/.well-known/"): validated_user_api_key_auth = UserAPIKeyAuth() elif ( not litellm_api_key and MCPRequestHandler._target_servers_delegate_auth_to_upstream( # noqa: E501 - path=request.url.path, mcp_servers=mcp_servers + path=request_route, mcp_servers=mcp_servers ) ): # Operator opted this oauth2 server into upstream-delegated auth @@ -174,7 +178,7 @@ class MCPRequestHandler: "401", "403", ) and MCPRequestHandler._target_servers_use_oauth2( - path=request.url.path, mcp_servers=mcp_servers + path=request_route, mcp_servers=mcp_servers ): verbose_logger.debug( "MCP OAuth2: target server is OAuth2-mode, treating " @@ -562,25 +566,33 @@ class MCPRequestHandler: ) ) + key_access_group_grants = ( + await MCPRequestHandler._get_key_access_group_mcp_server_extras( + user_api_key_auth + ) + ) + ######################################################### # Calculate key/team allowed servers using inheritance and intersection logic ######################################################### - allowed_mcp_servers: List[str] = [] - has_lower_level_mcp_restrictions = ( - len(allowed_mcp_servers_for_key) > 0 - or len(allowed_mcp_servers_for_team) > 0 - ) - if len(allowed_mcp_servers_for_team) > 0: - if len(allowed_mcp_servers_for_key) > 0: - # Key has its own MCP permissions - use intersection with team permissions - for _mcp_server in allowed_mcp_servers_for_key: - if _mcp_server in allowed_mcp_servers_for_team: - allowed_mcp_servers.append(_mcp_server) - else: - # Key has no MCP permissions - inherit from team - allowed_mcp_servers = allowed_mcp_servers_for_team + key_set = set(allowed_mcp_servers_for_key) + team_set = set(allowed_mcp_servers_for_team) + grants_set = set(key_access_group_grants) + + has_lower_level_mcp_restrictions = bool(key_set or team_set or grants_set) + + # 1. Key/team ceiling. An empty set means "this level does not restrict". + if not team_set: + base = key_set # no team restriction + elif not key_set: + base = team_set # key has no own perms → inherits team else: - allowed_mcp_servers = allowed_mcp_servers_for_key + base = key_set & team_set # both restrict → intersect + + # 2. Add the key's access-group grants on top. These are additive: + # attaching a group to the key grants its servers regardless of the + # team ceiling. + allowed_mcp_servers: List[str] = list(base | grants_set) ######################################################### # Check end_user permissions if end_user_id is set @@ -873,43 +885,98 @@ class MCPRequestHandler: return True return False + @staticmethod + async def _get_key_access_group_mcp_server_extras( + user_api_key_auth: Optional[UserAPIKeyAuth] = None, + ) -> List[str]: + """ + Resolve the key's unified `access_group_ids` (LiteLLM_AccessGroupTable) to + MCP server IDs as additive grants: a group attached to the key extends the + key's allowed servers on top of the key/team ceiling rather than being + capped by the team. Attaching the group to the key is itself the grant — + no `assigned_key_ids` / `assigned_team_ids` re-check. Tag-style + `mcp_access_groups` (per-server tags) live in the key's object_permission + scope, not here. + """ + if user_api_key_auth is None: + return [] + try: + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + from litellm.proxy.auth.auth_checks import ( + _get_mcp_server_ids_from_access_groups, + ) + from litellm.proxy.proxy_server import ( + prisma_client, + proxy_logging_obj, + user_api_key_cache, + ) + + raw_server_ids = await _get_mcp_server_ids_from_access_groups( + access_group_ids=user_api_key_auth.access_group_ids or [], + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + if not raw_server_ids: + return [] + # Permission entries may be server_ids OR names/aliases — expand to ids. + return global_mcp_server_manager.expand_permission_list(raw_server_ids) + except Exception as e: + verbose_logger.warning( + f"Failed to get key access group MCP server grants: {str(e)}" + ) + return [] + @staticmethod async def _get_allowed_mcp_servers_for_key( user_api_key_auth: Optional[UserAPIKeyAuth] = None, ) -> List[str]: + """ + Get the key's own MCP ceiling from its object_permission + (mcp_servers, tag-style mcp_access_groups, mcp_tool_permissions). + + Unified key.access_group_ids are NOT resolved here — they are additive + grants handled by _get_key_access_group_mcp_server_extras and unioned on + top of the key/team ceiling, so they must not enter this scope (which is + intersected against the team). + """ + if user_api_key_auth is None: + return [] try: + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + from litellm.proxy.auth.auth_checks import ( + get_object_permission, + ) + from litellm.proxy.proxy_server import ( + prisma_client, + proxy_logging_obj, + user_api_key_cache, + ) + # Get key object permission (already loaded in main auth flow, or fetch from DB) key_object_permission = MCPRequestHandler._get_key_object_permission( user_api_key_auth ) if ( key_object_permission is None - and user_api_key_auth and user_api_key_auth.object_permission_id + and prisma_client is not None ): - from litellm.proxy.auth.auth_checks import get_object_permission - from litellm.proxy.proxy_server import ( - prisma_client, - proxy_logging_obj, - user_api_key_cache, + key_object_permission = await get_object_permission( + object_permission_id=user_api_key_auth.object_permission_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=user_api_key_auth.parent_otel_span, + proxy_logging_obj=proxy_logging_obj, ) - - if prisma_client is not None: - key_object_permission = await get_object_permission( - object_permission_id=user_api_key_auth.object_permission_id, - prisma_client=prisma_client, - user_api_key_cache=user_api_key_cache, - parent_otel_span=user_api_key_auth.parent_otel_span, - proxy_logging_obj=proxy_logging_obj, - ) if key_object_permission is None: return [] # Permission entries may be server_ids OR names/aliases — expand to ids. - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( - global_mcp_server_manager, - ) - direct_mcp_servers = global_mcp_server_manager.expand_permission_list( key_object_permission.mcp_servers or [] ) @@ -944,42 +1011,78 @@ class MCPRequestHandler: """ Get allowed MCP servers for a team. - Note: object_permission is automatically loaded by get_team_object() in main auth flow. + Unions two sources: + - Legacy team.object_permission (mcp_servers, mcp_access_groups, + mcp_tool_permissions). + - Unified team.access_group_ids → access_group.access_mcp_server_ids. + Mirrors the model-side pattern in can_team_access_model — the group + is already attached to the team, so the team relationship is itself + the gate (no assigned_team_ids check needed here). """ try: - # Get team object permission (already loaded in main auth flow) - object_permissions = await MCPRequestHandler._get_team_object_permission( - user_api_key_auth - ) - - if object_permissions is None: - return [] - - # Permission entries may be server_ids OR names/aliases — expand to ids. from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) + from litellm.proxy.auth.auth_checks import ( + _get_mcp_server_ids_from_access_groups, + get_team_object, + ) + from litellm.proxy.proxy_server import ( + prisma_client, + proxy_logging_obj, + user_api_key_cache, + ) + + if ( + user_api_key_auth is None + or not user_api_key_auth.team_id + or prisma_client is None + ): + return [] + + team_obj: Optional[LiteLLM_TeamTable] = await get_team_object( + team_id=user_api_key_auth.team_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=user_api_key_auth.parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + ) + if team_obj is None: + return [] + + team_access_group_servers = await _get_mcp_server_ids_from_access_groups( + access_group_ids=team_obj.access_group_ids or [], + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + + object_permissions = team_obj.object_permission + if object_permissions is None: + return list(set(team_access_group_servers)) direct_mcp_servers = global_mcp_server_manager.expand_permission_list( object_permissions.mcp_servers or [] ) - # Get MCP servers from access groups - access_group_servers = ( + legacy_access_group_servers = ( await MCPRequestHandler._get_mcp_servers_from_access_groups( object_permissions.mcp_access_groups or [] ) ) - # servers referenced in tool permissions should also be accessible tool_perm_servers = list( global_mcp_server_manager.expand_tool_permissions( object_permissions.mcp_tool_permissions ).keys() ) - # Combine all lists - all_servers = direct_mcp_servers + access_group_servers + tool_perm_servers + all_servers = ( + direct_mcp_servers + + legacy_access_group_servers + + tool_perm_servers + + team_access_group_servers + ) return list(set(all_servers)) except Exception as e: verbose_logger.warning( diff --git a/litellm/proxy/_experimental/mcp_server/db.py b/litellm/proxy/_experimental/mcp_server/db.py index a6f0d145e9b..e30667776c1 100644 --- a/litellm/proxy/_experimental/mcp_server/db.py +++ b/litellm/proxy/_experimental/mcp_server/db.py @@ -30,6 +30,8 @@ from litellm.types.mcp import MCPCredentials def _prepare_mcp_server_data( data: Union[NewMCPServerRequest, UpdateMCPServerRequest], + exclude_unset: bool = False, + fields_set: Optional[Set[str]] = None, ) -> Dict[str, Any]: """ Helper function to prepare MCP server data for database operations. @@ -37,17 +39,39 @@ def _prepare_mcp_server_data( Args: data: NewMCPServerRequest or UpdateMCPServerRequest object + exclude_unset: When True, only fields the caller explicitly provided are + included. Used for partial updates (PUT /v1/mcp/server) so omitted + fields keep their existing DB value instead of being silently reset + to a Pydantic schema default. ``exclude_none`` is not enough here: + non-Optional fields (e.g. ``transport=MCPTransport.sse``, + ``mcp_access_groups=[]``, ``allow_all_keys=False``) are backfilled + with their default when omitted, and a non-None default survives the + ``exclude_none`` filter and overwrites the row. Returns: Dict with properly serialized JSON fields """ from litellm.litellm_core_utils.safe_json_dumps import safe_dumps - # Convert model to dict - data_dict = data.model_dump(exclude_none=True) - # Ensure alias is always present in the dict (even if None) - if "alias" not in data_dict: - data_dict["alias"] = getattr(data, "alias", None) + # Convert model to dict. + # - Partial update (exclude_unset): only caller-provided keys are emitted, so + # omitted fields are never written and keep their existing DB value. + # - Create (exclude_none): drop None-valued fields and let DB defaults apply. + if exclude_unset: + if fields_set is None: + fields_set = data.fields_set() + data_dict = data.model_dump(exclude_unset=True) + # ``validate_and_normalize_mcp_server_payload`` always assigns ``alias`` + # on the payload, which marks it as set even when the caller omitted it. + # Drop it only when the original request omitted alias; an explicit + # ``alias=None`` is a valid request to clear the stored alias. + if data_dict.get("alias") is None and "alias" not in fields_set: + data_dict.pop("alias", None) + else: + data_dict = data.model_dump(exclude_none=True) + # Ensure alias is always present in the dict (even if None) + if "alias" not in data_dict: + data_dict["alias"] = getattr(data, "alias", None) # Handle credentials serialization credentials = data_dict.get("credentials") @@ -57,33 +81,33 @@ def _prepare_mcp_server_data( ) data_dict["credentials"] = safe_dumps(data_dict["credentials"]) - # Handle static_headers serialization - if data.static_headers is not None: - data_dict["static_headers"] = safe_dumps(data.static_headers) + # Serialize JSON fields from ``data_dict`` (not ``data``) so the + # exclude_unset filter is respected. Reading back from ``data`` would + # reintroduce defaults (e.g. ``env={}``) for fields the caller never set. + if data_dict.get("static_headers") is not None: + data_dict["static_headers"] = safe_dumps(data_dict["static_headers"]) - # Handle mcp_info serialization - if data.mcp_info is not None: - data_dict["mcp_info"] = safe_dumps(data.mcp_info) + if data_dict.get("mcp_info") is not None: + data_dict["mcp_info"] = safe_dumps(data_dict["mcp_info"]) - # Handle env serialization - if data.env is not None: - data_dict["env"] = safe_dumps(data.env) + if data_dict.get("env") is not None: + data_dict["env"] = safe_dumps(data_dict["env"]) - # Handle tool name override serialization - if data.tool_name_to_display_name is not None: + if data_dict.get("tool_name_to_display_name") is not None: data_dict["tool_name_to_display_name"] = safe_dumps( - data.tool_name_to_display_name + data_dict["tool_name_to_display_name"] ) - if data.tool_name_to_description is not None: + if data_dict.get("tool_name_to_description") is not None: data_dict["tool_name_to_description"] = safe_dumps( - data.tool_name_to_description + data_dict["tool_name_to_description"] ) # mcp_access_groups is already List[str], no serialization needed - # Force include is_byok even when False (exclude_none=True would not drop it, - # but be explicit to ensure a False value is always written to the DB). - data_dict["is_byok"] = getattr(data, "is_byok", False) + # On create, force is_byok so a False value is always written to the DB. On + # partial update, only write it when the caller explicitly provided it. + if not exclude_unset: + data_dict["is_byok"] = getattr(data, "is_byok", False) return data_dict @@ -398,7 +422,10 @@ async def create_mcp_server( async def update_mcp_server( - prisma_client: PrismaClient, data: UpdateMCPServerRequest, touched_by: str + prisma_client: PrismaClient, + data: UpdateMCPServerRequest, + touched_by: str, + fields_set: Optional[Set[str]] = None, ) -> LiteLLM_MCPServerTable: """ Update a new mcp server record in the db @@ -407,8 +434,13 @@ async def update_mcp_server( from litellm.litellm_core_utils.safe_json_dumps import safe_dumps - # Use helper to prepare data with proper JSON serialization - data_dict = _prepare_mcp_server_data(data) + # Use helper to prepare data with proper JSON serialization. + # exclude_unset=True makes this a true partial update: fields the caller did + # not provide are not written, so they keep their existing DB value instead + # of being reset to a schema default (transport=sse, allow_all_keys=False...). + data_dict = _prepare_mcp_server_data( + data, exclude_unset=True, fields_set=fields_set + ) # Pre-fetch existing record once if we need it for auth_type or credential logic existing = None diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index 652e284ed49..8324ba641a4 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -1,3 +1,4 @@ +import html as _html import json from typing import Any, Dict, Optional from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse @@ -618,8 +619,105 @@ async def token_endpoint( ) +# Per RFC 6749 §4.1.2.1, an IdP that rejects an OAuth authorization request +# redirects back to the configured redirect URI with ``error`` / +# ``error_description`` / ``error_uri`` query params and no ``code``. The MCP +# loopback flow funnels that response through this /callback endpoint, so +# the endpoint must accept either a successful (``code``+``state``) or an +# error response. Declaring ``code``/``state`` as required would cause +# FastAPI to reject the error response with a 422 before the handler runs, +# which strands the MCP client waiting on the loopback (see LIT-2750). + + +def _render_oauth_error_html(error: str, description: Optional[str]) -> HTMLResponse: + """Render an actionable HTML page for an IdP-reported OAuth error. + + Used when we cannot propagate the error back to the registered + ``redirect_uri`` (state missing or undecryptable). Returned with a 400 + status so the failure is observable to operators while still being a + human-readable page for the end user. + """ + safe_error = _html.escape(error or "unknown_error") + safe_description = _html.escape(description) if description else "" + description_html = f"

{safe_description}

" if safe_description else "" + body = ( + "" + "

Authentication failed

" + f"

Error: {safe_error}

" + f"{description_html}" + "

You can close this window and try again.

" + "" + ) + return HTMLResponse(body, status_code=400) + + @router.get("/callback") -async def callback(request: Request, code: str, state: str): +async def callback( + request: Request, + code: Optional[str] = None, + state: Optional[str] = None, + error: Optional[str] = None, + error_description: Optional[str] = None, + error_uri: Optional[str] = None, +): + """OAuth 2.0 authorization response handler for MCP loopback clients. + + Accepts either: + + - A successful authorization response (``code`` + ``state``), which is + forwarded back to the validated client ``redirect_uri`` with the + original (un-wrapped) ``state``. + - An error response (``error``[+``error_description``/``error_uri``]), per + RFC 6749 §4.1.2.1. When ``state`` is present and decodes to a trusted + ``redirect_uri``, the error params are propagated back to the client so + its OAuth library can surface them. Otherwise we render an HTML error + page so the user is not left on an opaque 422 / blank screen. + """ + # 1. IdP-reported error path (e.g. ``?error=access_denied``). + if error: + verbose_logger.info( + "MCP /callback received IdP error: error=%s, error_description=%s", + error, + error_description, + ) + if state: + try: + state_data = decode_state_hash(state) + original_state = state_data.get("original_state") + redirect_uri = _get_validated_client_redirect_uri(request, state_data) + except HTTPException: + # Untrusted/invalid client redirect_uri — surface inline rather + # than blindly forwarding the error to an attacker-controlled URL. + return _render_oauth_error_html(error, error_description) + except Exception: + # State could not be decrypted (expired key, tampered, etc.). + return _render_oauth_error_html(error, error_description) + + params: Dict[str, str] = {"error": error} + if error_description: + params["error_description"] = error_description + if error_uri: + params["error_uri"] = error_uri + if original_state is not None: + params["state"] = original_state + complete_returned_url = _append_query_params(redirect_uri, params) + return RedirectResponse(url=complete_returned_url, status_code=302) + + # No state — nothing to round-trip to. Show the user the error. + return _render_oauth_error_html(error, error_description) + + # 2. Neither success nor error parameters present — most likely a stray + # GET / dropped SSO redirect chain. Surface a 400 instead of 422. + if not code or not state: + missing = [ + name for name, value in (("code", code), ("state", state)) if not value + ] + return _render_oauth_error_html( + "invalid_request", + f"Missing authorization {' and '.join(repr(m) for m in missing)} parameter(s).", + ) + + # 3. Successful authorization response. try: state_data = decode_state_hash(state) original_state = state_data["original_state"] diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index d0e9ad7b2a4..f35aa30a7c9 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -892,6 +892,7 @@ class MCPServerManager: is_byok=bool(getattr(mcp_server, "is_byok", False)), byok_description=getattr(mcp_server, "byok_description", None) or [], byok_api_key_help_url=getattr(mcp_server, "byok_api_key_help_url", None), + source_url=getattr(mcp_server, "source_url", None), # AWS SigV4 fields aws_access_key_id=aws_creds.get("aws_access_key_id"), aws_secret_access_key=aws_creds.get("aws_secret_access_key"), @@ -3750,6 +3751,7 @@ class MCPServerManager: is_byok=server.is_byok, byok_description=server.byok_description, byok_api_key_help_url=server.byok_api_key_help_url, + source_url=server.source_url, instructions=server.instructions, ) diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index f31005be0cb..a05ce3f7417 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -6,6 +6,8 @@ LiteLLM MCP Server Routes import asyncio import contextlib +import hashlib +import json import time import types import traceback @@ -28,7 +30,7 @@ from fastapi import FastAPI, HTTPException from pydantic import AnyUrl, ConfigDict from starlette.requests import Request as StarletteRequest from starlette.responses import JSONResponse -from starlette.types import Receive, Scope, Send +from starlette.types import Message, Receive, Scope, Send from litellm._logging import verbose_logger from litellm.constants import MAXIMUM_TRACEBACK_LINES_TO_LOG @@ -74,6 +76,19 @@ from litellm.utils import Rules, client, function_setup _byok_cred_cache: Dict[Tuple[str, str], Tuple[Optional[str], float]] = {} _BYOK_CRED_CACHE_TTL = 60 # seconds _BYOK_CRED_CACHE_MAX_SIZE = 4096 # cap to prevent unbounded growth +_STATEFUL_SESSION_IDLE_TIMEOUT_SECONDS = 30 * 60 +# Upper bound on concurrent stateful sessions a single caller may hold. Each +# `initialize` creates a session that survives until the idle timeout, so +# without a cap an authenticated client could spam `initialize` and exhaust +# memory. The caller's own oldest idle sessions are evicted to make room; if +# the cap is still hit (every session in flight), the new `initialize` is +# rejected with 429. +_MAX_STATEFUL_SESSIONS_PER_OWNER = 100 +# Maximum bytes to peek when sniffing the JSON-RPC method on a POST. +# An `initialize` envelope is a few hundred bytes; capping the peek +# prevents an authenticated client from forcing the proxy to buffer an +# arbitrarily large body just to make a routing decision. +_MCP_ROUTING_PEEK_MAX_BYTES = 4096 def _invalidate_byok_cred_cache(user_id: str, server_id: str) -> None: @@ -242,13 +257,45 @@ if MCP_AVAILABLE: sse: SseServerTransport = SseServerTransport("/mcp/sse/messages") # Create session managers - session_manager = StreamableHTTPSessionManager( + session_manager_stateless = StreamableHTTPSessionManager( app=server, event_store=None, json_response=False, # enables SSE streaming stateless=True, ) + session_manager_stateful = StreamableHTTPSessionManager( + app=server, + event_store=None, # TODO: Add EventStore for reconnection/event replay if needed + json_response=False, # enables SSE streaming + stateless=False, + ) + _stateful_session_auth_contexts: Dict[str, MCPAuthenticatedUser] = {} + _stateful_session_auth_context_last_seen: Dict[str, float] = {} + # Maps session_id -> owner identifier (hashed API key/token) so we can + # reject requests that supply a session_id created by a different caller. + # Without this, a leaked mcp-session-id could be driven (or terminated) + # by any other authenticated proxy user. + _stateful_session_owners: Dict[str, str] = {} + # Per-session lock that serializes ``handle_request`` for the same + # mcp-session-id. The stored ``MCPAuthenticatedUser`` is mutated in place + # by ``_update_auth_context`` each request; without this lock, two + # concurrent requests on the same session would clobber each other's + # auth headers / mcp_servers / oauth state while in-flight callbacks are + # still reading the shared object. + _stateful_session_locks: Dict[str, asyncio.Lock] = {} + _stateful_session_active_request_counts: Dict[str, int] = {} + + def _remove_stateful_session_tracking(session_id: str) -> None: + _stateful_session_auth_contexts.pop(session_id, None) + _stateful_session_auth_context_last_seen.pop(session_id, None) + _stateful_session_owners.pop(session_id, None) + _stateful_session_locks.pop(session_id, None) + _stateful_session_active_request_counts.pop(session_id, None) + + # Keep this alias so existing references to session_manager still work + session_manager = session_manager_stateless + # Create SSE session manager sse_session_manager = StreamableHTTPSessionManager( app=server, @@ -259,11 +306,100 @@ if MCP_AVAILABLE: # Context managers for proper lifecycle management _session_manager_cm = None + _session_manager_stateful_cm = None _sse_session_manager_cm = None + _stateful_auth_context_cleanup_task: Optional[asyncio.Task] = None + + async def _purge_expired_stateful_session_auth_contexts( + now: Optional[float] = None, + ) -> None: + """Terminate expired stateful sessions and drop their auth contexts.""" + now = time.monotonic() if now is None else now + server_instances = getattr(session_manager_stateful, "_server_instances", {}) + expired_session_ids = [] + for session_id, last_seen in _stateful_session_auth_context_last_seen.items(): + if _stateful_session_active_request_counts.get(session_id, 0) > 0: + continue + if ( + now - last_seen >= _STATEFUL_SESSION_IDLE_TIMEOUT_SECONDS + or session_id not in server_instances + ): + expired_session_ids.append(session_id) + + for session_id in expired_session_ids: + # Re-check the active-request count immediately before tearing + # the session down. ``await transport.terminate()`` yields to + # the event loop, so a request that started after the first + # collection pass could otherwise observe its transport being + # ripped out from under it mid-flight. + if _stateful_session_active_request_counts.get(session_id, 0) > 0: + continue + # Pop transport + terminate BEFORE removing owner/auth tracking. + # Reversing the order avoids a window where ``_stateful_session_owners`` + # is empty but ``server_instances`` still serves the session — a + # concurrent request in that window would observe ``expected_owner + # is None`` and bypass the owner-binding check. + transport = server_instances.pop(session_id, None) + if transport is not None: + await transport.terminate() + _remove_stateful_session_tracking(session_id) + + for session_id in list(_stateful_session_auth_context_last_seen): + if session_id not in _stateful_session_auth_contexts: + _remove_stateful_session_tracking(session_id) + + async def _enforce_stateful_session_cap_for_owner(owner: str) -> bool: + """ + Bound the number of concurrent stateful sessions a single caller holds + before routing a new ``initialize`` to the stateful manager. + + Evicts the caller's *own* oldest idle sessions (no in-flight requests) + to make room, so a busy-but-legitimate client keeps its newest sessions + and other callers are never affected. Returns ``True`` if the new + session may proceed, or ``False`` when the caller is already at the cap + with every session in flight (the new ``initialize`` should be rejected). + """ + server_instances = getattr(session_manager_stateful, "_server_instances", {}) + + def _owned_live_session_ids() -> List[str]: + return [ + session_id + for session_id, session_owner in _stateful_session_owners.items() + if session_owner == owner and session_id in server_instances + ] + + owned = _owned_live_session_ids() + if len(owned) < _MAX_STATEFUL_SESSIONS_PER_OWNER: + return True + + for session_id in sorted( + owned, + key=lambda sid: _stateful_session_auth_context_last_seen.get(sid, 0.0), + ): + if len(_owned_live_session_ids()) < _MAX_STATEFUL_SESSIONS_PER_OWNER: + break + if _stateful_session_active_request_counts.get(session_id, 0) > 0: + continue + transport = server_instances.pop(session_id, None) + if transport is not None: + await transport.terminate() + _remove_stateful_session_tracking(session_id) + + return len(_owned_live_session_ids()) < _MAX_STATEFUL_SESSIONS_PER_OWNER + + async def _cleanup_expired_stateful_session_auth_contexts() -> None: + while True: + await asyncio.sleep(_STATEFUL_SESSION_IDLE_TIMEOUT_SECONDS) + try: + await _purge_expired_stateful_session_auth_contexts() + except Exception as e: + verbose_logger.exception( + f"Error cleaning up expired MCP stateful sessions: {e}" + ) async def initialize_session_managers(): """Initialize the session managers. Can be called from main app lifespan.""" - global _SESSION_MANAGERS_INITIALIZED, _session_manager_cm, _sse_session_manager_cm + global _SESSION_MANAGERS_INITIALIZED, _session_manager_cm, _session_manager_stateful_cm, _sse_session_manager_cm, _stateful_auth_context_cleanup_task # Use async lock to prevent concurrent initialization async with _INITIALIZATION_LOCK: @@ -273,12 +409,17 @@ if MCP_AVAILABLE: verbose_logger.info("Initializing MCP session managers...") # Start the session managers with context managers - _session_manager_cm = session_manager.run() + _session_manager_cm = session_manager_stateless.run() + _session_manager_stateful_cm = session_manager_stateful.run() _sse_session_manager_cm = sse_session_manager.run() # Enter the context managers await _session_manager_cm.__aenter__() + await _session_manager_stateful_cm.__aenter__() await _sse_session_manager_cm.__aenter__() + _stateful_auth_context_cleanup_task = asyncio.create_task( + _cleanup_expired_stateful_session_auth_contexts() + ) _SESSION_MANAGERS_INITIALIZED = True verbose_logger.info( @@ -287,21 +428,29 @@ if MCP_AVAILABLE: async def shutdown_session_managers(): """Shutdown the session managers.""" - global _SESSION_MANAGERS_INITIALIZED, _session_manager_cm, _sse_session_manager_cm + global _SESSION_MANAGERS_INITIALIZED, _session_manager_cm, _session_manager_stateful_cm, _sse_session_manager_cm, _stateful_auth_context_cleanup_task if _SESSION_MANAGERS_INITIALIZED: verbose_logger.info("Shutting down MCP session managers...") try: + if _stateful_auth_context_cleanup_task: + _stateful_auth_context_cleanup_task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await _stateful_auth_context_cleanup_task if _session_manager_cm: await _session_manager_cm.__aexit__(None, None, None) + if _session_manager_stateful_cm: + await _session_manager_stateful_cm.__aexit__(None, None, None) if _sse_session_manager_cm: await _sse_session_manager_cm.__aexit__(None, None, None) except Exception as e: verbose_logger.exception(f"Error during session manager shutdown: {e}") _session_manager_cm = None + _session_manager_stateful_cm = None _sse_session_manager_cm = None + _stateful_auth_context_cleanup_task = None _SESSION_MANAGERS_INITIALIZED = False @contextlib.asynccontextmanager @@ -366,7 +515,7 @@ if MCP_AVAILABLE: @server.call_tool() async def mcp_server_tool_call( - name: str, arguments: Dict[str, Any] | None + name: str, arguments: Optional[Dict[str, Any]] ) -> CallToolResult: """ Call a specific tool with the provided arguments @@ -409,7 +558,7 @@ if MCP_AVAILABLE: if host_token and hasattr(host_ctx, "session") and host_ctx.session: host_session = host_ctx.session - async def forward_progress(progress: float, total: float | None): + async def forward_progress(progress: float, total: Optional[float]): """Forward progress notifications from external MCP to Host""" try: await host_session.send_progress_notification( @@ -551,7 +700,7 @@ if MCP_AVAILABLE: @server.get_prompt() async def get_prompt( - name: str, arguments: dict[str, str] | None + name: str, arguments: Optional[Dict[str, str]] ) -> GetPromptResult: """ Get a specific prompt with the provided arguments @@ -2697,6 +2846,144 @@ if MCP_AVAILABLE: raw_headers, ) + def _get_session_id_from_scope(scope: Scope) -> Optional[str]: + """ + Extract mcp-session-id from ASGI scope headers. + Returns None if not present. + """ + for header_name, header_value in scope.get("headers", []): + name = ( + header_name if isinstance(header_name, bytes) else header_name.encode() + ) + if name.lower() == b"mcp-session-id": + return ( + header_value.decode() + if isinstance(header_value, bytes) + else str(header_value) + ) + return None + + def _owner_fingerprint_for( + user_api_key_auth: Optional[UserAPIKeyAuth], + oauth2_headers: Optional[Dict[str, str]] = None, + client_ip: Optional[str] = None, + ) -> str: + """ + Stable, non-reversible identifier for the caller used to bind an + mcp-session-id to its creator. Hash the resolved credential before + using it so custom key formats are never stored in cleartext. + + For OAuth2 passthrough (``UserAPIKeyAuth()`` with no key/user_id), + the caller's identity is the upstream OAuth bearer; hash it so two + OAuth callers with different tokens don't both fingerprint to + ``anonymous`` and end up sharing a session. + + When no caller-identifying credentials are available at all + (e.g. proxy running without master key, or an unauthenticated + passthrough path), fall back to the client IP so two unrelated + anonymous callers from different sources do not collapse to a + single ``anonymous`` owner and end up able to drive each other's + stateful sessions. Note: when even client IP is unavailable + (exotic deployments without trusted X-Forwarded-For and direct + socket info), the fingerprint degrades to the ``anonymous`` + sentinel and cannot meaningfully protect against another + unauthenticated caller who learns the session id — owner-binding + is best-effort in that mode. + """ + + def _bytes_for_hash(value: Any) -> Optional[bytes]: + """Only hash str/bytes secrets; skip mocks and other unexpected types.""" + if value is None: + return None + if isinstance(value, (bytes, bytearray)): + return bytes(value) + if isinstance(value, str): + return value.encode("utf-8") + return None + + if user_api_key_auth is not None: + key_material = _bytes_for_hash(getattr(user_api_key_auth, "api_key", None)) + if key_material: + api_key_hash = hashlib.sha256(key_material).hexdigest() + return f"key:{api_key_hash}" + uid_material = _bytes_for_hash(getattr(user_api_key_auth, "user_id", None)) + if uid_material: + user_id_hash = hashlib.sha256(uid_material).hexdigest() + return f"user:{user_id_hash}" + if oauth2_headers: + authz = oauth2_headers.get("Authorization") or oauth2_headers.get( + "authorization" + ) + authz_bytes = _bytes_for_hash(authz) + if authz_bytes: + return f"oauth:{hashlib.sha256(authz_bytes).hexdigest()}" + if client_ip and isinstance(client_ip, str): + return f"ip:{hashlib.sha256(client_ip.encode('utf-8')).hexdigest()}" + return "anonymous" + + def _is_initialize_request(body: bytes) -> bool: + """ + Check if the request body is a JSON-RPC initialize method. + Returns True if method is "initialize", False otherwise or on parse error. + """ + if not body: + return False + try: + data = json.loads(body) + return isinstance(data, dict) and data.get("method") == "initialize" + except (json.JSONDecodeError, TypeError): + return False + + async def _read_request_body_for_routing( + receive: Receive, + ) -> Tuple[List[Message], bytes]: + """ + Read just enough of the request body to decide whether this is a + JSON-RPC ``initialize`` call. Returns the consumed ASGI messages so + the caller can replay them faithfully to the downstream handler, and + the peeked body bytes (capped at ``_MCP_ROUTING_PEEK_MAX_BYTES``). + + Stops reading from the wire as soon as either (a) we have peeked + ``_MCP_ROUTING_PEEK_MAX_BYTES`` of body, or (b) the body is complete. + The remainder of an oversized body is streamed lazily through + ``wrapped_receive`` in the caller — so an authenticated client cannot + force the proxy to buffer an arbitrarily large payload just to make a + routing decision. + """ + consumed_messages: List[Message] = [] + body_chunks: List[bytes] = [] + peeked_bytes = 0 + + while True: + message = await receive() + consumed_messages.append(message) + + if message.get("type") != "http.request": + break + + body = message.get("body", b"") or b"" + if body: + # Only retain up to the remaining peek budget for sniffing. + # The full ``message`` is already in memory (delivered by + # the ASGI server) and must round-trip to the downstream + # handler via ``consumed_messages``, but ``body_chunks`` is + # purely for the JSON-RPC method check — there is no reason + # to copy a large body frame into a second buffer. + remaining = _MCP_ROUTING_PEEK_MAX_BYTES - peeked_bytes + if remaining > 0: + body_chunks.append(body[:remaining]) + peeked_bytes += min(len(body), remaining) + + if not message.get("more_body", False): + break + + if peeked_bytes >= _MCP_ROUTING_PEEK_MAX_BYTES: + # Stop draining; downstream replay will pull remaining chunks + # directly from the original `receive` via wrapped_receive. + break + + return consumed_messages, b"".join(body_chunks) + async def _handle_stale_mcp_session( scope: Scope, receive: Receive, @@ -2760,6 +3047,7 @@ if MCP_AVAILABLE: method = scope.get("method", "").upper() if method == "DELETE": + _remove_stateful_session_tracking(_session_id) verbose_logger.info( "DELETE request for non-existent MCP session '%s'. " "Returning success (idempotent DELETE).", @@ -2993,7 +3281,7 @@ if MCP_AVAILABLE: detail="Forbidden", ) - async def handle_streamable_http_mcp( + async def handle_streamable_http_mcp( # noqa: PLR0915 scope: Scope, receive: Receive, send: Send ) -> None: """Handle MCP requests through StreamableHTTP.""" @@ -3086,38 +3374,215 @@ if MCP_AVAILABLE: if _debug_headers: send = MCPDebug.wrap_send_with_debug_headers(send, _debug_headers) - # Set the auth context variable for easy access in MCP functions - set_auth_context( - user_api_key_auth=user_api_key_auth, - mcp_auth_header=mcp_auth_header, - mcp_servers=mcp_servers, - mcp_server_auth_headers=mcp_server_auth_headers, - oauth2_headers=oauth2_headers, - raw_headers=raw_headers, - client_ip=_client_ip, - ) - # Ensure session managers are initialized if not _SESSION_MANAGERS_INITIALIZED: await initialize_session_managers() # Give it a moment to start up await asyncio.sleep(0.1) - # Handle stale session IDs - either strip them for reconnection - # or return success for idempotent DELETE operations - handled = await _handle_stale_mcp_session( - scope, receive, send, session_manager - ) - if handled: - # Request was fully handled (e.g., DELETE on non-existent session) - return + # Route based on mcp-session-id and request method: + # - Has session ID → stateful (Claude Code, Cursor, VSCode) + # - No session ID + initialize → stateful (so client gets mcp-session-id) + # - No session ID + other → stateless (curl, Inspector, Notion) + session_id = _get_session_id_from_scope(scope) + is_initialize = False + consumed_messages: List[Message] = [] - async with _gateway_initialize_instructions_request_scope( - user_api_key_auth, - mcp_servers, - _client_ip, - ): - await session_manager.handle_request(scope, receive, send) + # Owner-binding: a live stateful session may only be driven by the + # caller that created it. Reject mismatches with 403 so a leaked + # mcp-session-id cannot be hijacked by another authenticated user. + # + # Run before ``_handle_stale_mcp_session`` so a non-owner cannot + # force-clean another caller's residual tracking entries via a + # stale DELETE, and before peeking the request body so the 403 + # response sees a pristine ``receive`` channel. + if session_id: + expected_owner = _stateful_session_owners.get(session_id) + request_owner = _owner_fingerprint_for( + user_api_key_auth, oauth2_headers, _client_ip + ) + if expected_owner is not None and expected_owner != request_owner: + verbose_logger.warning( + "Rejecting MCP request: session '%s' owner mismatch.", + session_id, + ) + forbidden_response = JSONResponse( + status_code=403, + content={ + "error": "Forbidden", + "details": "mcp-session-id is bound to a different caller.", + }, + ) + await forbidden_response(scope, receive, send) + return + + # Handle stale session IDs before choosing a target manager. Stale + # non-DELETE requests have their session header stripped and should + # be routed as no-session requests. + if session_id: + handled = await _handle_stale_mcp_session( + scope, receive, send, session_manager_stateful + ) + if handled: + # Request was fully handled (e.g., DELETE on non-existent session) + return + session_id = _get_session_id_from_scope(scope) + + if scope.get("method") == "POST": + consumed_messages, body = await _read_request_body_for_routing(receive) + is_initialize = _is_initialize_request(body) + + use_stateful = bool(session_id or is_initialize) + target_manager = ( + session_manager_stateful if use_stateful else session_manager_stateless + ) + + verbose_logger.debug( + f"MCP routing to {'stateful' if use_stateful else 'stateless'} manager" + + (f" (session={session_id[:8]}...)" if session_id else "") + + (" (initialize)" if is_initialize else "") + ) + + # A new `initialize` (no session id) is about to create a stateful + # session. Cap how many a single caller can hold so an authenticated + # client cannot spam `initialize` and exhaust memory. + if is_initialize and not session_id: + request_owner = _owner_fingerprint_for( + user_api_key_auth, oauth2_headers, _client_ip + ) + if not await _enforce_stateful_session_cap_for_owner(request_owner): + verbose_logger.warning( + "Rejecting MCP initialize: caller already holds the maximum " + "number of active stateful sessions." + ) + too_many_response = JSONResponse( + status_code=429, + content={ + "error": "Too Many Requests", + "details": "Too many active MCP sessions for this caller.", + }, + ) + await too_many_response(scope, receive, send) + return + + # Replay body messages if we consumed them for peeking + original_receive = receive + if consumed_messages: + + async def wrapped_receive(): + if consumed_messages: + return consumed_messages.pop(0) + return await original_receive() + + receive = wrapped_receive + + # Serialize requests on the same stateful session so concurrent + # callers don't clobber each other's auth context mid-flight. + # + # Skip the lock for streaming GETs (SSE channels held open for the + # life of the session): holding a per-session lock for a long-lived + # stream would block every subsequent POST on the same session. + # POST/DELETE are the methods that actually mutate the shared + # auth context, so serializing those is sufficient for the + # clobbering race between concurrent JSON-RPC calls. + session_lock: Optional[asyncio.Lock] = None + request_method = (scope.get("method") or "").upper() + if use_stateful and session_id and request_method in ("POST", "DELETE"): + session_lock = _stateful_session_locks.setdefault( + session_id, asyncio.Lock() + ) + + active_request_session_ids: List[str] = [] + + def _increment_active_request_session(session_id_to_track: str) -> None: + if session_id_to_track in active_request_session_ids: + return + active_request_session_ids.append(session_id_to_track) + _stateful_session_active_request_counts[session_id_to_track] = ( + _stateful_session_active_request_counts.get(session_id_to_track, 0) + + 1 + ) + + if use_stateful and session_id: + _increment_active_request_session(session_id) + + def _track_initialized_stateful_session( + initialized_session_id: str, + ) -> None: + _increment_active_request_session(initialized_session_id) + + async def _dispatch() -> None: + auth_user = _set_or_update_auth_context( + user_api_key_auth=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_servers=mcp_servers, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + client_ip=_client_ip, + session_id=session_id if use_stateful else None, + touch_last_seen=(scope.get("method") or "").upper() != "DELETE", + copy_existing_session_auth_context=is_initialize, + ) + local_send = send + if use_stateful and is_initialize: + local_send = _wrap_send_with_stateful_session_auth_context( + local_send, + auth_user, + _owner_fingerprint_for( + user_api_key_auth, oauth2_headers, _client_ip + ), + _track_initialized_stateful_session, + ) + + async with _gateway_initialize_instructions_request_scope( + user_api_key_auth, + mcp_servers, + _client_ip, + ): + await target_manager.handle_request(scope, receive, local_send) + if use_stateful and session_id and scope.get("method") == "DELETE": + _remove_stateful_session_tracking(session_id) + + try: + if session_lock is not None: + async with session_lock: + await _dispatch() + else: + await _dispatch() + finally: + for active_request_session_id in active_request_session_ids: + active_request_count = ( + _stateful_session_active_request_counts.get( + active_request_session_id, 0 + ) + - 1 + ) + if active_request_count > 0: + _stateful_session_active_request_counts[ + active_request_session_id + ] = active_request_count + else: + _stateful_session_active_request_counts.pop( + active_request_session_id, None + ) + + if ( + scope.get("method") != "DELETE" + and active_request_session_id in _stateful_session_auth_contexts + ): + _stateful_session_auth_context_last_seen[ + active_request_session_id + ] = time.monotonic() + + # Periodic cleanup iterates _stateful_session_auth_context_last_seen, + # so locks for untracked sessions must be dropped here. + if ( + active_request_count <= 0 + and active_request_session_id + not in _stateful_session_auth_contexts + ): + _stateful_session_locks.pop(active_request_session_id, None) except HTTPException: # Re-raise HTTP exceptions to preserve status codes and details raise @@ -3125,7 +3590,6 @@ if MCP_AVAILABLE: verbose_logger.exception(f"Error handling MCP request: {e}") # Try to send a graceful error response for non-HTTP exceptions try: - from starlette.responses import JSONResponse from starlette.status import HTTP_500_INTERNAL_SERVER_ERROR error_response = JSONResponse( @@ -3231,8 +3695,9 @@ if MCP_AVAILABLE: ############ Auth Context Functions #################### ######################################################## - def set_auth_context( - user_api_key_auth: UserAPIKeyAuth, + def _update_auth_context( + auth_user: MCPAuthenticatedUser, + user_api_key_auth: Optional[UserAPIKeyAuth], mcp_auth_header: Optional[str] = None, mcp_servers: Optional[List[str]] = None, mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]] = None, @@ -3240,6 +3705,23 @@ if MCP_AVAILABLE: raw_headers: Optional[Dict[str, str]] = None, client_ip: Optional[str] = None, ) -> None: + auth_user.user_api_key_auth = user_api_key_auth + auth_user.mcp_auth_header = mcp_auth_header + auth_user.mcp_servers = mcp_servers + auth_user.mcp_server_auth_headers = mcp_server_auth_headers or {} + auth_user.oauth2_headers = oauth2_headers + auth_user.raw_headers = raw_headers + auth_user.client_ip = client_ip + + def set_auth_context( + user_api_key_auth: Optional[UserAPIKeyAuth], + mcp_auth_header: Optional[str] = None, + mcp_servers: Optional[List[str]] = None, + mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]] = None, + oauth2_headers: Optional[Dict[str, str]] = None, + raw_headers: Optional[Dict[str, str]] = None, + client_ip: Optional[str] = None, + ) -> MCPAuthenticatedUser: """ Set the UserAPIKeyAuth in the auth context variable. @@ -3260,6 +3742,84 @@ if MCP_AVAILABLE: client_ip=client_ip, ) auth_context_var.set(auth_user) + return auth_user + + def _set_or_update_auth_context( + user_api_key_auth: Optional[UserAPIKeyAuth], + mcp_auth_header: Optional[str] = None, + mcp_servers: Optional[List[str]] = None, + mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]] = None, + oauth2_headers: Optional[Dict[str, str]] = None, + raw_headers: Optional[Dict[str, str]] = None, + client_ip: Optional[str] = None, + session_id: Optional[str] = None, + touch_last_seen: bool = True, + copy_existing_session_auth_context: bool = False, + ) -> MCPAuthenticatedUser: + auth_user = ( + _stateful_session_auth_contexts.get(session_id) if session_id else None + ) + if auth_user is not None and session_id is not None: + if touch_last_seen: + _stateful_session_auth_context_last_seen[session_id] = time.monotonic() + if copy_existing_session_auth_context: + return set_auth_context( + user_api_key_auth=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_servers=mcp_servers, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + client_ip=client_ip, + ) + _update_auth_context( + auth_user=auth_user, + user_api_key_auth=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_servers=mcp_servers, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + client_ip=client_ip, + ) + auth_context_var.set(auth_user) + return auth_user + return set_auth_context( + user_api_key_auth=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_servers=mcp_servers, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + client_ip=client_ip, + ) + + def _wrap_send_with_stateful_session_auth_context( + send: Send, + auth_user: MCPAuthenticatedUser, + owner_fingerprint: str, + on_session_registered: Optional[Callable[[str], None]] = None, + ) -> Send: + async def wrapped_send(message: Message) -> None: + if message.get("type") == "http.response.start": + for key, value in message.get("headers", []): + header_name = key if isinstance(key, bytes) else str(key).encode() + if header_name.lower() == b"mcp-session-id": + session_id = ( + value.decode() if isinstance(value, bytes) else str(value) + ) + if on_session_registered is not None: + on_session_registered(session_id) + auth_context_var.set(auth_user) + _stateful_session_auth_contexts[session_id] = auth_user + _stateful_session_auth_context_last_seen[session_id] = ( + time.monotonic() + ) + _stateful_session_owners[session_id] = owner_fingerprint + break + await send(message) + + return wrapped_send def get_auth_context() -> Tuple[ Optional[UserAPIKeyAuth], diff --git a/litellm/proxy/_experimental/out/404.html b/litellm/proxy/_experimental/out/404.html index 38a2c3bd836..f27612ff54e 100644 --- a/litellm/proxy/_experimental/out/404.html +++ b/litellm/proxy/_experimental/out/404.html @@ -1 +1 @@ -404: This page could not be found.LiteLLM Dashboard

404

This page could not be found.

\ No newline at end of file +404: This page could not be found.LiteLLM Dashboard

404

This page could not be found.

\ No newline at end of file diff --git a/litellm/proxy/_experimental/out/404/index.html b/litellm/proxy/_experimental/out/404/index.html index fe08df1d0fb..f27612ff54e 100644 --- a/litellm/proxy/_experimental/out/404/index.html +++ b/litellm/proxy/_experimental/out/404/index.html @@ -1 +1 @@ -404: This page could not be found.LiteLLM Dashboard

404

This page could not be found.

\ No newline at end of file +404: This page could not be found.LiteLLM Dashboard

404

This page could not be found.

\ No newline at end of file diff --git a/litellm/proxy/_experimental/out/__next.__PAGE__.txt b/litellm/proxy/_experimental/out/__next.__PAGE__.txt index 18bda7f1065..c024136e8dc 100644 --- a/litellm/proxy/_experimental/out/__next.__PAGE__.txt +++ b/litellm/proxy/_experimental/out/__next.__PAGE__.txt @@ -1,30 +1,10 @@ 1:"$Sreact.fragment" 2:I[347257,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"ClientPageRoot"] -3:I[952683,["/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","/litellm-asset-prefix/_next/static/chunks/0493aafc4891dd29.js","/litellm-asset-prefix/_next/static/chunks/f7e1d08418645368.js","/litellm-asset-prefix/_next/static/chunks/b3d198d6c56a21b8.js","/litellm-asset-prefix/_next/static/chunks/c847ecdf8c790b0b.js","/litellm-asset-prefix/_next/static/chunks/ee5f9a39a526e423.js","/litellm-asset-prefix/_next/static/chunks/adb8beb738574863.js","/litellm-asset-prefix/_next/static/chunks/0549bc9afa7d4888.js","/litellm-asset-prefix/_next/static/chunks/0b470ffc60999bf4.js","/litellm-asset-prefix/_next/static/chunks/0b3d09ff6c6e4335.js","/litellm-asset-prefix/_next/static/chunks/e099566e8bd4ee4e.js","/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","/litellm-asset-prefix/_next/static/chunks/b1c98cc932a0ab19.js","/litellm-asset-prefix/_next/static/chunks/4e17b625d75327a7.js","/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","/litellm-asset-prefix/_next/static/chunks/a06cc76a774dd182.js","/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","/litellm-asset-prefix/_next/static/chunks/0974abc09c5e7ada.js","/litellm-asset-prefix/_next/static/chunks/ae625aa52246581e.js","/litellm-asset-prefix/_next/static/chunks/7a9066dcd4a390ff.js","/litellm-asset-prefix/_next/static/chunks/baadbd26839e7b66.js","/litellm-asset-prefix/_next/static/chunks/2971c4658f1bcd7d.js","/litellm-asset-prefix/_next/static/chunks/134f728fa7099e3e.js","/litellm-asset-prefix/_next/static/chunks/679dbd657c8b5aef.js","/litellm-asset-prefix/_next/static/chunks/88001a7ecaf7b1af.js","/litellm-asset-prefix/_next/static/chunks/cbc99c8fae110c02.js","/litellm-asset-prefix/_next/static/chunks/4e06277331e725da.js","/litellm-asset-prefix/_next/static/chunks/3b30ab8eaa03bc21.js","/litellm-asset-prefix/_next/static/chunks/1c881baaaa68b7a5.js","/litellm-asset-prefix/_next/static/chunks/9955c118354ef6cc.js","/litellm-asset-prefix/_next/static/chunks/496b84010c33cf69.js","/litellm-asset-prefix/_next/static/chunks/a09028cd611c08ef.js","/litellm-asset-prefix/_next/static/chunks/99cf9cf99df5ccfc.js","/litellm-asset-prefix/_next/static/chunks/ad02f56c287539eb.js","/litellm-asset-prefix/_next/static/chunks/5181a28310842d3d.js","/litellm-asset-prefix/_next/static/chunks/0a65da2cd24e2ab6.js","/litellm-asset-prefix/_next/static/chunks/16a1651c0b3e7c8e.js","/litellm-asset-prefix/_next/static/chunks/a8f7c8c5eeb6e042.js","/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","/litellm-asset-prefix/_next/static/chunks/d3ac82723ec9e30d.js","/litellm-asset-prefix/_next/static/chunks/631b1874cba557c9.js","/litellm-asset-prefix/_next/static/chunks/1bc2898be56acd1b.js","/litellm-asset-prefix/_next/static/chunks/908828a91f602d8b.js","/litellm-asset-prefix/_next/static/chunks/878832edb30e99a4.js","/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","/litellm-asset-prefix/_next/static/chunks/003f1ffc5817ab83.js","/litellm-asset-prefix/_next/static/chunks/e1f23fd814ac3500.js","/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","/litellm-asset-prefix/_next/static/chunks/23bfdf9b0544f0b1.js","/litellm-asset-prefix/_next/static/chunks/ca5fbafaf3826374.js","/litellm-asset-prefix/_next/static/chunks/726bebeef472c6cb.js","/litellm-asset-prefix/_next/static/chunks/8f3bf592254c6c3b.js","/litellm-asset-prefix/_next/static/chunks/659ce28f2cb74401.js","/litellm-asset-prefix/_next/static/chunks/16c0e58809eaf2b5.js"],"default"] -1a:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"OutletBoundary"] -1b:"$Sreact.suspense" +3:I[952683,["/litellm-asset-prefix/_next/static/chunks/aefca6f40ea185cd.js","/litellm-asset-prefix/_next/static/chunks/632b4c8e836bd956.js","/litellm-asset-prefix/_next/static/chunks/52c4ecc57f72065e.js","/litellm-asset-prefix/_next/static/chunks/4bb663ff806dc32f.js","/litellm-asset-prefix/_next/static/chunks/ee8f89c672745c59.js","/litellm-asset-prefix/_next/static/chunks/2954392b7a60a6a1.js","/litellm-asset-prefix/_next/static/chunks/04711b0f8ffa7bbd.js","/litellm-asset-prefix/_next/static/chunks/9710770c6333a72f.js","/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","/litellm-asset-prefix/_next/static/chunks/d734cb3d5659b0da.js","/litellm-asset-prefix/_next/static/chunks/a1ef280b7ad5ae6a.js","/litellm-asset-prefix/_next/static/chunks/09c1f51da7e82268.js","/litellm-asset-prefix/_next/static/chunks/542a1a209eb732c6.js","/litellm-asset-prefix/_next/static/chunks/d028f8c28935d281.js","/litellm-asset-prefix/_next/static/chunks/61aa637257592262.js","/litellm-asset-prefix/_next/static/chunks/bf962cd5264be987.js","/litellm-asset-prefix/_next/static/chunks/37f229ef9335f8c3.js","/litellm-asset-prefix/_next/static/chunks/36d1c027ba991a4f.js","/litellm-asset-prefix/_next/static/chunks/786e88f4abdd5c58.js","/litellm-asset-prefix/_next/static/chunks/43c3db1352241a8b.js","/litellm-asset-prefix/_next/static/chunks/31275eb5c6f6332f.js","/litellm-asset-prefix/_next/static/chunks/0ac09b227f50edb4.js","/litellm-asset-prefix/_next/static/chunks/4c848b12d4ecda3d.js","/litellm-asset-prefix/_next/static/chunks/91c828abd7c0aff5.js","/litellm-asset-prefix/_next/static/chunks/08c348f8e09a5cb0.js","/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","/litellm-asset-prefix/_next/static/chunks/b323e0ef008e6348.js","/litellm-asset-prefix/_next/static/chunks/4ac3235460262f36.js","/litellm-asset-prefix/_next/static/chunks/51494a4a4b6fc437.js","/litellm-asset-prefix/_next/static/chunks/878832edb30e99a4.js","/litellm-asset-prefix/_next/static/chunks/9f1486622270556b.js"],"default"] +6:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"OutletBoundary"] +7:"$Sreact.suspense" :HL["/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","style"] -0:{"buildId":"wnL6e5S6xaG1UdkxtYrTo","rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0493aafc4891dd29.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/f7e1d08418645368.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/b3d198d6c56a21b8.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/c847ecdf8c790b0b.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/ee5f9a39a526e423.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/adb8beb738574863.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/0549bc9afa7d4888.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/0b470ffc60999bf4.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/0b3d09ff6c6e4335.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/e099566e8bd4ee4e.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/b1c98cc932a0ab19.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/4e17b625d75327a7.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/a06cc76a774dd182.js","async":true}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","async":true}],["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/0974abc09c5e7ada.js","async":true}],["$","script","script-17",{"src":"/litellm-asset-prefix/_next/static/chunks/ae625aa52246581e.js","async":true}],["$","script","script-18",{"src":"/litellm-asset-prefix/_next/static/chunks/7a9066dcd4a390ff.js","async":true}],["$","script","script-19",{"src":"/litellm-asset-prefix/_next/static/chunks/baadbd26839e7b66.js","async":true}],["$","script","script-20",{"src":"/litellm-asset-prefix/_next/static/chunks/2971c4658f1bcd7d.js","async":true}],["$","script","script-21",{"src":"/litellm-asset-prefix/_next/static/chunks/134f728fa7099e3e.js","async":true}],["$","script","script-22",{"src":"/litellm-asset-prefix/_next/static/chunks/679dbd657c8b5aef.js","async":true}],["$","script","script-23",{"src":"/litellm-asset-prefix/_next/static/chunks/88001a7ecaf7b1af.js","async":true}],["$","script","script-24",{"src":"/litellm-asset-prefix/_next/static/chunks/cbc99c8fae110c02.js","async":true}],["$","script","script-25",{"src":"/litellm-asset-prefix/_next/static/chunks/4e06277331e725da.js","async":true}],["$","script","script-26",{"src":"/litellm-asset-prefix/_next/static/chunks/3b30ab8eaa03bc21.js","async":true}],["$","script","script-27",{"src":"/litellm-asset-prefix/_next/static/chunks/1c881baaaa68b7a5.js","async":true}],["$","script","script-28",{"src":"/litellm-asset-prefix/_next/static/chunks/9955c118354ef6cc.js","async":true}],["$","script","script-29",{"src":"/litellm-asset-prefix/_next/static/chunks/496b84010c33cf69.js","async":true}],["$","script","script-30",{"src":"/litellm-asset-prefix/_next/static/chunks/a09028cd611c08ef.js","async":true}],["$","script","script-31",{"src":"/litellm-asset-prefix/_next/static/chunks/99cf9cf99df5ccfc.js","async":true}],["$","script","script-32",{"src":"/litellm-asset-prefix/_next/static/chunks/ad02f56c287539eb.js","async":true}],["$","script","script-33",{"src":"/litellm-asset-prefix/_next/static/chunks/5181a28310842d3d.js","async":true}],"$L6","$L7","$L8","$L9","$La","$Lb","$Lc","$Ld","$Le","$Lf","$L10","$L11","$L12","$L13","$L14","$L15","$L16","$L17","$L18"],"$L19"]}],"loading":null,"isPartial":false} +0:{"buildId":"TrcGiQpTupSbDYFFfkFHY","rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/52c4ecc57f72065e.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/4bb663ff806dc32f.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/ee8f89c672745c59.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/2954392b7a60a6a1.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/04711b0f8ffa7bbd.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/9710770c6333a72f.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/d734cb3d5659b0da.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/a1ef280b7ad5ae6a.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/09c1f51da7e82268.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/542a1a209eb732c6.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/d028f8c28935d281.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/61aa637257592262.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/bf962cd5264be987.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/37f229ef9335f8c3.js","async":true}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/36d1c027ba991a4f.js","async":true}],["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/786e88f4abdd5c58.js","async":true}],["$","script","script-17",{"src":"/litellm-asset-prefix/_next/static/chunks/43c3db1352241a8b.js","async":true}],["$","script","script-18",{"src":"/litellm-asset-prefix/_next/static/chunks/31275eb5c6f6332f.js","async":true}],["$","script","script-19",{"src":"/litellm-asset-prefix/_next/static/chunks/0ac09b227f50edb4.js","async":true}],["$","script","script-20",{"src":"/litellm-asset-prefix/_next/static/chunks/4c848b12d4ecda3d.js","async":true}],["$","script","script-21",{"src":"/litellm-asset-prefix/_next/static/chunks/91c828abd7c0aff5.js","async":true}],["$","script","script-22",{"src":"/litellm-asset-prefix/_next/static/chunks/08c348f8e09a5cb0.js","async":true}],["$","script","script-23",{"src":"/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","async":true}],["$","script","script-24",{"src":"/litellm-asset-prefix/_next/static/chunks/b323e0ef008e6348.js","async":true}],["$","script","script-25",{"src":"/litellm-asset-prefix/_next/static/chunks/4ac3235460262f36.js","async":true}],["$","script","script-26",{"src":"/litellm-asset-prefix/_next/static/chunks/51494a4a4b6fc437.js","async":true}],["$","script","script-27",{"src":"/litellm-asset-prefix/_next/static/chunks/878832edb30e99a4.js","async":true}],["$","script","script-28",{"src":"/litellm-asset-prefix/_next/static/chunks/9f1486622270556b.js","async":true}]],["$","$L6",null,{"children":["$","$7",null,{"name":"Next.MetadataOutlet","children":"$@8"}]}]]}],"loading":null,"isPartial":false} 4:{} 5:"$0:rsc:props:children:0:props:serverProvidedParams:params" -6:["$","script","script-34",{"src":"/litellm-asset-prefix/_next/static/chunks/0a65da2cd24e2ab6.js","async":true}] -7:["$","script","script-35",{"src":"/litellm-asset-prefix/_next/static/chunks/16a1651c0b3e7c8e.js","async":true}] -8:["$","script","script-36",{"src":"/litellm-asset-prefix/_next/static/chunks/a8f7c8c5eeb6e042.js","async":true}] -9:["$","script","script-37",{"src":"/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","async":true}] -a:["$","script","script-38",{"src":"/litellm-asset-prefix/_next/static/chunks/d3ac82723ec9e30d.js","async":true}] -b:["$","script","script-39",{"src":"/litellm-asset-prefix/_next/static/chunks/631b1874cba557c9.js","async":true}] -c:["$","script","script-40",{"src":"/litellm-asset-prefix/_next/static/chunks/1bc2898be56acd1b.js","async":true}] -d:["$","script","script-41",{"src":"/litellm-asset-prefix/_next/static/chunks/908828a91f602d8b.js","async":true}] -e:["$","script","script-42",{"src":"/litellm-asset-prefix/_next/static/chunks/878832edb30e99a4.js","async":true}] -f:["$","script","script-43",{"src":"/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","async":true}] -10:["$","script","script-44",{"src":"/litellm-asset-prefix/_next/static/chunks/003f1ffc5817ab83.js","async":true}] -11:["$","script","script-45",{"src":"/litellm-asset-prefix/_next/static/chunks/e1f23fd814ac3500.js","async":true}] -12:["$","script","script-46",{"src":"/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","async":true}] -13:["$","script","script-47",{"src":"/litellm-asset-prefix/_next/static/chunks/23bfdf9b0544f0b1.js","async":true}] -14:["$","script","script-48",{"src":"/litellm-asset-prefix/_next/static/chunks/ca5fbafaf3826374.js","async":true}] -15:["$","script","script-49",{"src":"/litellm-asset-prefix/_next/static/chunks/726bebeef472c6cb.js","async":true}] -16:["$","script","script-50",{"src":"/litellm-asset-prefix/_next/static/chunks/8f3bf592254c6c3b.js","async":true}] -17:["$","script","script-51",{"src":"/litellm-asset-prefix/_next/static/chunks/659ce28f2cb74401.js","async":true}] -18:["$","script","script-52",{"src":"/litellm-asset-prefix/_next/static/chunks/16c0e58809eaf2b5.js","async":true}] -19:["$","$L1a",null,{"children":["$","$1b",null,{"name":"Next.MetadataOutlet","children":"$@1c"}]}] -1c:null +8:null diff --git a/litellm/proxy/_experimental/out/__next._full.txt b/litellm/proxy/_experimental/out/__next._full.txt index 78377d64de1..0c119086b9e 100644 --- a/litellm/proxy/_experimental/out/__next._full.txt +++ b/litellm/proxy/_experimental/out/__next._full.txt @@ -1,123 +1,39 @@ 1:"$Sreact.fragment" -<<<<<<< HEAD -2:I[867271,["/my-custom-path/_next/static/chunks/06gao0u0xy9th.js","/my-custom-path/_next/static/chunks/0gszmhudlm6lw.js","/my-custom-path/_next/static/chunks/0pidya1qvuvx8.js"],"default"] -3:I[71195,["/my-custom-path/_next/static/chunks/06gao0u0xy9th.js","/my-custom-path/_next/static/chunks/0gszmhudlm6lw.js","/my-custom-path/_next/static/chunks/0pidya1qvuvx8.js"],"default"] -4:I[339756,["/my-custom-path/_next/static/chunks/06gao0u0xy9th.js","/my-custom-path/_next/static/chunks/0gszmhudlm6lw.js","/my-custom-path/_next/static/chunks/0pidya1qvuvx8.js"],"default"] -5:I[837457,["/my-custom-path/_next/static/chunks/06gao0u0xy9th.js","/my-custom-path/_next/static/chunks/0gszmhudlm6lw.js","/my-custom-path/_next/static/chunks/0pidya1qvuvx8.js"],"default"] -6:I[347257,["/my-custom-path/_next/static/chunks/06gao0u0xy9th.js","/my-custom-path/_next/static/chunks/0gszmhudlm6lw.js","/my-custom-path/_next/static/chunks/0pidya1qvuvx8.js"],"ClientPageRoot"] -7:I[952683,["/my-custom-path/_next/static/chunks/06gao0u0xy9th.js","/my-custom-path/_next/static/chunks/0gszmhudlm6lw.js","/my-custom-path/_next/static/chunks/0pidya1qvuvx8.js","/my-custom-path/_next/static/chunks/0niabq1py.bpc.js","/my-custom-path/_next/static/chunks/0je63_yy.8i72.js","/my-custom-path/_next/static/chunks/0qikgxv.u0j9d.js","/my-custom-path/_next/static/chunks/0fpcaw3chkoi1.js","/my-custom-path/_next/static/chunks/0zmz_tmhm2hsa.js","/my-custom-path/_next/static/chunks/01n1bg17oxp8l.js","/my-custom-path/_next/static/chunks/181y.ii57q7gs.js","/my-custom-path/_next/static/chunks/0gqqvjn11_pp3.js","/my-custom-path/_next/static/chunks/09n64dqzn.le~.js","/my-custom-path/_next/static/chunks/01g5ls2__cy9q.js","/my-custom-path/_next/static/chunks/00dq_sa6qt2eg.js","/my-custom-path/_next/static/chunks/0_ex-stk7ibf_.js","/my-custom-path/_next/static/chunks/0_p625as00xbq.js","/my-custom-path/_next/static/chunks/0tz7rml6rolh7.js","/my-custom-path/_next/static/chunks/09.lm5dax~l7e.js","/my-custom-path/_next/static/chunks/0~2_b290gsxjj.js","/my-custom-path/_next/static/chunks/0jhg37m0hy.uz.js","/my-custom-path/_next/static/chunks/0v1c3mu~xn105.js","/my-custom-path/_next/static/chunks/08401f_hwndsf.js","/my-custom-path/_next/static/chunks/0bkk~1l186k4g.js","/my-custom-path/_next/static/chunks/166fc~7u39du3.js","/my-custom-path/_next/static/chunks/14ut9b-iwzfct.js","/my-custom-path/_next/static/chunks/01u0i~l49qpdz.js","/my-custom-path/_next/static/chunks/02v412-qgvyfr.js","/my-custom-path/_next/static/chunks/0ssorln~eooo0.js","/my-custom-path/_next/static/chunks/0sleuh3e9q.ww.js","/my-custom-path/_next/static/chunks/00q4mtjboprhm.js","/my-custom-path/_next/static/chunks/0noiuetq16.dz.js","/my-custom-path/_next/static/chunks/0c4pfjjue0uc-.js","/my-custom-path/_next/static/chunks/0ys10755n8os_.js","/my-custom-path/_next/static/chunks/0dt4fjjwbrov2.js","/my-custom-path/_next/static/chunks/06oag4x9~8vif.js","/my-custom-path/_next/static/chunks/0-ne4el5qxs1z.js","/my-custom-path/_next/static/chunks/036wlkuzplhfz.js","/my-custom-path/_next/static/chunks/01xm1xt.gmrff.js","/my-custom-path/_next/static/chunks/0cbiddnyyt-22.js","/my-custom-path/_next/static/chunks/01eoe.0n-q6.q.js","/my-custom-path/_next/static/chunks/0nnx~7-7e5t~1.js","/my-custom-path/_next/static/chunks/0mb3erwqomzal.js","/my-custom-path/_next/static/chunks/00n_262-goqx6.js","/my-custom-path/_next/static/chunks/0gs8.sc1ds7.w.js","/my-custom-path/_next/static/chunks/11ibt3khr2hk3.js","/my-custom-path/_next/static/chunks/0aivi0og~sj6k.js","/my-custom-path/_next/static/chunks/08cbskerpmujn.js","/my-custom-path/_next/static/chunks/0ar6956xz0njn.js","/my-custom-path/_next/static/chunks/022.sz94ycw4x.js","/my-custom-path/_next/static/chunks/0ylz5md8p6x7z.js","/my-custom-path/_next/static/chunks/0g7ys.5g150n1.js","/my-custom-path/_next/static/chunks/0ttl0__m~5i8a.js","/my-custom-path/_next/static/chunks/0fns6vsccj10h.js","/my-custom-path/_next/static/chunks/015yfy38oh-h8.js","/my-custom-path/_next/static/chunks/0m6zdocif1gl4.js","/my-custom-path/_next/static/chunks/0ybvtcd~xp-mx.js","/my-custom-path/_next/static/chunks/0j2g7af6k4x7m.js"],"default"] -32:I[168027,["/my-custom-path/_next/static/chunks/06gao0u0xy9th.js","/my-custom-path/_next/static/chunks/0gszmhudlm6lw.js","/my-custom-path/_next/static/chunks/0pidya1qvuvx8.js"],"default",1] -:HL["/my-custom-path/_next/static/chunks/05qmwjqau64bz.css","style"] -:HL["/my-custom-path/_next/static/chunks/04o4lz~y91s1r.css","style"] -:HL["/my-custom-path/_next/static/media/83afe278b6a6bb3c-s.p.0q-301v4kxxnr.woff2","font",{"crossOrigin":"","type":"font/woff2"}] -:HL["/my-custom-path/_next/static/chunks/0jib1e4hgitwz.css","style"] -0:{"P":null,"c":["",""],"q":"","i":false,"f":[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",16],[["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/my-custom-path/_next/static/chunks/05qmwjqau64bz.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/my-custom-path/_next/static/chunks/04o4lz~y91s1r.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","script","script-0",{"src":"/my-custom-path/_next/static/chunks/06gao0u0xy9th.js","async":true,"nonce":"$undefined"}],["$","script","script-1",{"src":"/my-custom-path/_next/static/chunks/0gszmhudlm6lw.js","async":true,"nonce":"$undefined"}],["$","script","script-2",{"src":"/my-custom-path/_next/static/chunks/0pidya1qvuvx8.js","async":true,"nonce":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]],"forbidden":"$undefined","unauthorized":"$undefined"}]}]}]}]}]]}],{"children":[["$","$1","c",{"children":[["$","$L6",null,{"Component":"$7","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@8","$@9"]}}],[["$","link","0",{"rel":"stylesheet","href":"/my-custom-path/_next/static/chunks/0jib1e4hgitwz.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","script","script-0",{"src":"/my-custom-path/_next/static/chunks/0niabq1py.bpc.js","async":true,"nonce":"$undefined"}],["$","script","script-1",{"src":"/my-custom-path/_next/static/chunks/0je63_yy.8i72.js","async":true,"nonce":"$undefined"}],["$","script","script-2",{"src":"/my-custom-path/_next/static/chunks/0qikgxv.u0j9d.js","async":true,"nonce":"$undefined"}],["$","script","script-3",{"src":"/my-custom-path/_next/static/chunks/0fpcaw3chkoi1.js","async":true,"nonce":"$undefined"}],["$","script","script-4",{"src":"/my-custom-path/_next/static/chunks/0zmz_tmhm2hsa.js","async":true,"nonce":"$undefined"}],["$","script","script-5",{"src":"/my-custom-path/_next/static/chunks/01n1bg17oxp8l.js","async":true,"nonce":"$undefined"}],["$","script","script-6",{"src":"/my-custom-path/_next/static/chunks/181y.ii57q7gs.js","async":true,"nonce":"$undefined"}],["$","script","script-7",{"src":"/my-custom-path/_next/static/chunks/0gqqvjn11_pp3.js","async":true,"nonce":"$undefined"}],["$","script","script-8",{"src":"/my-custom-path/_next/static/chunks/09n64dqzn.le~.js","async":true,"nonce":"$undefined"}],["$","script","script-9",{"src":"/my-custom-path/_next/static/chunks/01g5ls2__cy9q.js","async":true,"nonce":"$undefined"}],["$","script","script-10",{"src":"/my-custom-path/_next/static/chunks/00dq_sa6qt2eg.js","async":true,"nonce":"$undefined"}],["$","script","script-11",{"src":"/my-custom-path/_next/static/chunks/0_ex-stk7ibf_.js","async":true,"nonce":"$undefined"}],["$","script","script-12",{"src":"/my-custom-path/_next/static/chunks/0_p625as00xbq.js","async":true,"nonce":"$undefined"}],["$","script","script-13",{"src":"/my-custom-path/_next/static/chunks/0tz7rml6rolh7.js","async":true,"nonce":"$undefined"}],["$","script","script-14",{"src":"/my-custom-path/_next/static/chunks/09.lm5dax~l7e.js","async":true,"nonce":"$undefined"}],["$","script","script-15",{"src":"/my-custom-path/_next/static/chunks/0~2_b290gsxjj.js","async":true,"nonce":"$undefined"}],"$La","$Lb","$Lc","$Ld","$Le","$Lf","$L10","$L11","$L12","$L13","$L14","$L15","$L16","$L17","$L18","$L19","$L1a","$L1b","$L1c","$L1d","$L1e","$L1f","$L20","$L21","$L22","$L23","$L24","$L25","$L26","$L27","$L28","$L29","$L2a","$L2b","$L2c","$L2d","$L2e","$L2f"],"$L30"]}],{},null,false,null]},null,false,null],"$L31",false]],"m":"$undefined","G":["$32",["$L33","$L34"]],"S":true,"h":null,"s":"$undefined","l":"$undefined","p":"$undefined","d":"$undefined","b":"s_hWyEFymvwtKpcaZsNX4"} -35:I[897367,["/my-custom-path/_next/static/chunks/06gao0u0xy9th.js","/my-custom-path/_next/static/chunks/0gszmhudlm6lw.js","/my-custom-path/_next/static/chunks/0pidya1qvuvx8.js"],"OutletBoundary"] -36:"$Sreact.suspense" -38:I[897367,["/my-custom-path/_next/static/chunks/06gao0u0xy9th.js","/my-custom-path/_next/static/chunks/0gszmhudlm6lw.js","/my-custom-path/_next/static/chunks/0pidya1qvuvx8.js"],"ViewportBoundary"] -3a:I[897367,["/my-custom-path/_next/static/chunks/06gao0u0xy9th.js","/my-custom-path/_next/static/chunks/0gszmhudlm6lw.js","/my-custom-path/_next/static/chunks/0pidya1qvuvx8.js"],"MetadataBoundary"] -a:["$","script","script-16",{"src":"/my-custom-path/_next/static/chunks/0jhg37m0hy.uz.js","async":true,"nonce":"$undefined"}] -b:["$","script","script-17",{"src":"/my-custom-path/_next/static/chunks/0v1c3mu~xn105.js","async":true,"nonce":"$undefined"}] -c:["$","script","script-18",{"src":"/my-custom-path/_next/static/chunks/08401f_hwndsf.js","async":true,"nonce":"$undefined"}] -d:["$","script","script-19",{"src":"/my-custom-path/_next/static/chunks/0bkk~1l186k4g.js","async":true,"nonce":"$undefined"}] -e:["$","script","script-20",{"src":"/my-custom-path/_next/static/chunks/166fc~7u39du3.js","async":true,"nonce":"$undefined"}] -f:["$","script","script-21",{"src":"/my-custom-path/_next/static/chunks/14ut9b-iwzfct.js","async":true,"nonce":"$undefined"}] -10:["$","script","script-22",{"src":"/my-custom-path/_next/static/chunks/01u0i~l49qpdz.js","async":true,"nonce":"$undefined"}] -11:["$","script","script-23",{"src":"/my-custom-path/_next/static/chunks/02v412-qgvyfr.js","async":true,"nonce":"$undefined"}] -12:["$","script","script-24",{"src":"/my-custom-path/_next/static/chunks/0ssorln~eooo0.js","async":true,"nonce":"$undefined"}] -13:["$","script","script-25",{"src":"/my-custom-path/_next/static/chunks/0sleuh3e9q.ww.js","async":true,"nonce":"$undefined"}] -14:["$","script","script-26",{"src":"/my-custom-path/_next/static/chunks/00q4mtjboprhm.js","async":true,"nonce":"$undefined"}] -15:["$","script","script-27",{"src":"/my-custom-path/_next/static/chunks/0noiuetq16.dz.js","async":true,"nonce":"$undefined"}] -16:["$","script","script-28",{"src":"/my-custom-path/_next/static/chunks/0c4pfjjue0uc-.js","async":true,"nonce":"$undefined"}] -17:["$","script","script-29",{"src":"/my-custom-path/_next/static/chunks/0ys10755n8os_.js","async":true,"nonce":"$undefined"}] -18:["$","script","script-30",{"src":"/my-custom-path/_next/static/chunks/0dt4fjjwbrov2.js","async":true,"nonce":"$undefined"}] -19:["$","script","script-31",{"src":"/my-custom-path/_next/static/chunks/06oag4x9~8vif.js","async":true,"nonce":"$undefined"}] -1a:["$","script","script-32",{"src":"/my-custom-path/_next/static/chunks/0-ne4el5qxs1z.js","async":true,"nonce":"$undefined"}] -1b:["$","script","script-33",{"src":"/my-custom-path/_next/static/chunks/036wlkuzplhfz.js","async":true,"nonce":"$undefined"}] -1c:["$","script","script-34",{"src":"/my-custom-path/_next/static/chunks/01xm1xt.gmrff.js","async":true,"nonce":"$undefined"}] -1d:["$","script","script-35",{"src":"/my-custom-path/_next/static/chunks/0cbiddnyyt-22.js","async":true,"nonce":"$undefined"}] -1e:["$","script","script-36",{"src":"/my-custom-path/_next/static/chunks/01eoe.0n-q6.q.js","async":true,"nonce":"$undefined"}] -1f:["$","script","script-37",{"src":"/my-custom-path/_next/static/chunks/0nnx~7-7e5t~1.js","async":true,"nonce":"$undefined"}] -20:["$","script","script-38",{"src":"/my-custom-path/_next/static/chunks/0mb3erwqomzal.js","async":true,"nonce":"$undefined"}] -21:["$","script","script-39",{"src":"/my-custom-path/_next/static/chunks/00n_262-goqx6.js","async":true,"nonce":"$undefined"}] -22:["$","script","script-40",{"src":"/my-custom-path/_next/static/chunks/0gs8.sc1ds7.w.js","async":true,"nonce":"$undefined"}] -23:["$","script","script-41",{"src":"/my-custom-path/_next/static/chunks/11ibt3khr2hk3.js","async":true,"nonce":"$undefined"}] -24:["$","script","script-42",{"src":"/my-custom-path/_next/static/chunks/0aivi0og~sj6k.js","async":true,"nonce":"$undefined"}] -25:["$","script","script-43",{"src":"/my-custom-path/_next/static/chunks/08cbskerpmujn.js","async":true,"nonce":"$undefined"}] -26:["$","script","script-44",{"src":"/my-custom-path/_next/static/chunks/0ar6956xz0njn.js","async":true,"nonce":"$undefined"}] -27:["$","script","script-45",{"src":"/my-custom-path/_next/static/chunks/022.sz94ycw4x.js","async":true,"nonce":"$undefined"}] -28:["$","script","script-46",{"src":"/my-custom-path/_next/static/chunks/0ylz5md8p6x7z.js","async":true,"nonce":"$undefined"}] -29:["$","script","script-47",{"src":"/my-custom-path/_next/static/chunks/0g7ys.5g150n1.js","async":true,"nonce":"$undefined"}] -2a:["$","script","script-48",{"src":"/my-custom-path/_next/static/chunks/0ttl0__m~5i8a.js","async":true,"nonce":"$undefined"}] -2b:["$","script","script-49",{"src":"/my-custom-path/_next/static/chunks/0fns6vsccj10h.js","async":true,"nonce":"$undefined"}] -2c:["$","script","script-50",{"src":"/my-custom-path/_next/static/chunks/015yfy38oh-h8.js","async":true,"nonce":"$undefined"}] -2d:["$","script","script-51",{"src":"/my-custom-path/_next/static/chunks/0m6zdocif1gl4.js","async":true,"nonce":"$undefined"}] -2e:["$","script","script-52",{"src":"/my-custom-path/_next/static/chunks/0ybvtcd~xp-mx.js","async":true,"nonce":"$undefined"}] -2f:["$","script","script-53",{"src":"/my-custom-path/_next/static/chunks/0j2g7af6k4x7m.js","async":true,"nonce":"$undefined"}] -30:["$","$L35",null,{"children":["$","$36",null,{"name":"Next.MetadataOutlet","children":"$@37"}]}] -31:["$","$1","h",{"children":[null,["$","$L38",null,{"children":"$L39"}],["$","div",null,{"hidden":true,"children":["$","$L3a",null,{"children":["$","$36",null,{"name":"Next.Metadata","children":"$L3b"}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}] -33:["$","link","0",{"rel":"stylesheet","href":"/my-custom-path/_next/static/chunks/05qmwjqau64bz.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}] -34:["$","link","1",{"rel":"stylesheet","href":"/my-custom-path/_next/static/chunks/04o4lz~y91s1r.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}] -======= -2:I[867271,["/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js"],"default"] -3:I[71195,["/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js"],"default"] -4:I[339756,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"] -5:I[837457,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"] -6:I[347257,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"ClientPageRoot"] -7:I[952683,["/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","/litellm-asset-prefix/_next/static/chunks/0493aafc4891dd29.js","/litellm-asset-prefix/_next/static/chunks/f7e1d08418645368.js","/litellm-asset-prefix/_next/static/chunks/b3d198d6c56a21b8.js","/litellm-asset-prefix/_next/static/chunks/c847ecdf8c790b0b.js","/litellm-asset-prefix/_next/static/chunks/ee5f9a39a526e423.js","/litellm-asset-prefix/_next/static/chunks/adb8beb738574863.js","/litellm-asset-prefix/_next/static/chunks/0549bc9afa7d4888.js","/litellm-asset-prefix/_next/static/chunks/0b470ffc60999bf4.js","/litellm-asset-prefix/_next/static/chunks/0b3d09ff6c6e4335.js","/litellm-asset-prefix/_next/static/chunks/e099566e8bd4ee4e.js","/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","/litellm-asset-prefix/_next/static/chunks/b1c98cc932a0ab19.js","/litellm-asset-prefix/_next/static/chunks/4e17b625d75327a7.js","/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","/litellm-asset-prefix/_next/static/chunks/a06cc76a774dd182.js","/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","/litellm-asset-prefix/_next/static/chunks/0974abc09c5e7ada.js","/litellm-asset-prefix/_next/static/chunks/ae625aa52246581e.js","/litellm-asset-prefix/_next/static/chunks/7a9066dcd4a390ff.js","/litellm-asset-prefix/_next/static/chunks/baadbd26839e7b66.js","/litellm-asset-prefix/_next/static/chunks/2971c4658f1bcd7d.js","/litellm-asset-prefix/_next/static/chunks/134f728fa7099e3e.js","/litellm-asset-prefix/_next/static/chunks/679dbd657c8b5aef.js","/litellm-asset-prefix/_next/static/chunks/88001a7ecaf7b1af.js","/litellm-asset-prefix/_next/static/chunks/cbc99c8fae110c02.js","/litellm-asset-prefix/_next/static/chunks/4e06277331e725da.js","/litellm-asset-prefix/_next/static/chunks/3b30ab8eaa03bc21.js","/litellm-asset-prefix/_next/static/chunks/1c881baaaa68b7a5.js","/litellm-asset-prefix/_next/static/chunks/9955c118354ef6cc.js","/litellm-asset-prefix/_next/static/chunks/496b84010c33cf69.js","/litellm-asset-prefix/_next/static/chunks/a09028cd611c08ef.js","/litellm-asset-prefix/_next/static/chunks/99cf9cf99df5ccfc.js","/litellm-asset-prefix/_next/static/chunks/ad02f56c287539eb.js","/litellm-asset-prefix/_next/static/chunks/5181a28310842d3d.js","/litellm-asset-prefix/_next/static/chunks/0a65da2cd24e2ab6.js","/litellm-asset-prefix/_next/static/chunks/16a1651c0b3e7c8e.js","/litellm-asset-prefix/_next/static/chunks/a8f7c8c5eeb6e042.js","/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","/litellm-asset-prefix/_next/static/chunks/d3ac82723ec9e30d.js","/litellm-asset-prefix/_next/static/chunks/631b1874cba557c9.js","/litellm-asset-prefix/_next/static/chunks/1bc2898be56acd1b.js","/litellm-asset-prefix/_next/static/chunks/908828a91f602d8b.js","/litellm-asset-prefix/_next/static/chunks/878832edb30e99a4.js","/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","/litellm-asset-prefix/_next/static/chunks/003f1ffc5817ab83.js","/litellm-asset-prefix/_next/static/chunks/e1f23fd814ac3500.js","/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","/litellm-asset-prefix/_next/static/chunks/23bfdf9b0544f0b1.js","/litellm-asset-prefix/_next/static/chunks/ca5fbafaf3826374.js","/litellm-asset-prefix/_next/static/chunks/726bebeef472c6cb.js","/litellm-asset-prefix/_next/static/chunks/8f3bf592254c6c3b.js","/litellm-asset-prefix/_next/static/chunks/659ce28f2cb74401.js","/litellm-asset-prefix/_next/static/chunks/16c0e58809eaf2b5.js"],"default"] -31:I[168027,[],"default"] +2:I[867271,["/litellm-asset-prefix/_next/static/chunks/aefca6f40ea185cd.js","/litellm-asset-prefix/_next/static/chunks/632b4c8e836bd956.js"],"default"] +3:I[71195,["/litellm-asset-prefix/_next/static/chunks/aefca6f40ea185cd.js","/litellm-asset-prefix/_next/static/chunks/632b4c8e836bd956.js"],"default"] +4:I[557951,["/litellm-asset-prefix/_next/static/chunks/aefca6f40ea185cd.js","/litellm-asset-prefix/_next/static/chunks/632b4c8e836bd956.js"],"AuthProvider"] +5:I[339756,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"] +6:I[837457,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"] +7:I[347257,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"ClientPageRoot"] +8:I[952683,["/litellm-asset-prefix/_next/static/chunks/aefca6f40ea185cd.js","/litellm-asset-prefix/_next/static/chunks/632b4c8e836bd956.js","/litellm-asset-prefix/_next/static/chunks/52c4ecc57f72065e.js","/litellm-asset-prefix/_next/static/chunks/4bb663ff806dc32f.js","/litellm-asset-prefix/_next/static/chunks/ee8f89c672745c59.js","/litellm-asset-prefix/_next/static/chunks/2954392b7a60a6a1.js","/litellm-asset-prefix/_next/static/chunks/04711b0f8ffa7bbd.js","/litellm-asset-prefix/_next/static/chunks/9710770c6333a72f.js","/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","/litellm-asset-prefix/_next/static/chunks/d734cb3d5659b0da.js","/litellm-asset-prefix/_next/static/chunks/a1ef280b7ad5ae6a.js","/litellm-asset-prefix/_next/static/chunks/09c1f51da7e82268.js","/litellm-asset-prefix/_next/static/chunks/542a1a209eb732c6.js","/litellm-asset-prefix/_next/static/chunks/d028f8c28935d281.js","/litellm-asset-prefix/_next/static/chunks/61aa637257592262.js","/litellm-asset-prefix/_next/static/chunks/bf962cd5264be987.js","/litellm-asset-prefix/_next/static/chunks/37f229ef9335f8c3.js","/litellm-asset-prefix/_next/static/chunks/36d1c027ba991a4f.js","/litellm-asset-prefix/_next/static/chunks/786e88f4abdd5c58.js","/litellm-asset-prefix/_next/static/chunks/43c3db1352241a8b.js","/litellm-asset-prefix/_next/static/chunks/31275eb5c6f6332f.js","/litellm-asset-prefix/_next/static/chunks/0ac09b227f50edb4.js","/litellm-asset-prefix/_next/static/chunks/4c848b12d4ecda3d.js","/litellm-asset-prefix/_next/static/chunks/91c828abd7c0aff5.js","/litellm-asset-prefix/_next/static/chunks/08c348f8e09a5cb0.js","/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","/litellm-asset-prefix/_next/static/chunks/b323e0ef008e6348.js","/litellm-asset-prefix/_next/static/chunks/4ac3235460262f36.js","/litellm-asset-prefix/_next/static/chunks/51494a4a4b6fc437.js","/litellm-asset-prefix/_next/static/chunks/878832edb30e99a4.js","/litellm-asset-prefix/_next/static/chunks/9f1486622270556b.js"],"default"] +1a:I[168027,[],"default"] :HL["/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","style"] -:HL["/litellm-asset-prefix/_next/static/chunks/91037395c95e366d.css","style"] +:HL["/litellm-asset-prefix/_next/static/chunks/1bcca3c38c9deb02.css","style"] :HL["/litellm-asset-prefix/_next/static/media/83afe278b6a6bb3c-s.p.3a6ba036.woff2","font",{"crossOrigin":"","type":"font/woff2"}] :HL["/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","style"] -0:{"P":null,"b":"wnL6e5S6xaG1UdkxtYrTo","c":["",""],"q":"","i":false,"f":[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],[["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/91037395c95e366d.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","async":true,"nonce":"$undefined"}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","async":true,"nonce":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]],"forbidden":"$undefined","unauthorized":"$undefined"}]}]}]}]}]]}],{"children":[["$","$1","c",{"children":[["$","$L6",null,{"Component":"$7","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@8","$@9"]}}],[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0493aafc4891dd29.js","async":true,"nonce":"$undefined"}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/f7e1d08418645368.js","async":true,"nonce":"$undefined"}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/b3d198d6c56a21b8.js","async":true,"nonce":"$undefined"}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/c847ecdf8c790b0b.js","async":true,"nonce":"$undefined"}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/ee5f9a39a526e423.js","async":true,"nonce":"$undefined"}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/adb8beb738574863.js","async":true,"nonce":"$undefined"}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/0549bc9afa7d4888.js","async":true,"nonce":"$undefined"}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/0b470ffc60999bf4.js","async":true,"nonce":"$undefined"}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/0b3d09ff6c6e4335.js","async":true,"nonce":"$undefined"}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/e099566e8bd4ee4e.js","async":true,"nonce":"$undefined"}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","async":true,"nonce":"$undefined"}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/b1c98cc932a0ab19.js","async":true,"nonce":"$undefined"}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/4e17b625d75327a7.js","async":true,"nonce":"$undefined"}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","async":true,"nonce":"$undefined"}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/a06cc76a774dd182.js","async":true,"nonce":"$undefined"}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","async":true,"nonce":"$undefined"}],"$La","$Lb","$Lc","$Ld","$Le","$Lf","$L10","$L11","$L12","$L13","$L14","$L15","$L16","$L17","$L18","$L19","$L1a","$L1b","$L1c","$L1d","$L1e","$L1f","$L20","$L21","$L22","$L23","$L24","$L25","$L26","$L27","$L28","$L29","$L2a","$L2b","$L2c","$L2d","$L2e"],"$L2f"]}],{},null,false,false]},null,false,false],"$L30",false]],"m":"$undefined","G":["$31",[]],"S":true} -32:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"OutletBoundary"] -33:"$Sreact.suspense" -35:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"ViewportBoundary"] -37:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"MetadataBoundary"] -a:["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/0974abc09c5e7ada.js","async":true,"nonce":"$undefined"}] -b:["$","script","script-17",{"src":"/litellm-asset-prefix/_next/static/chunks/ae625aa52246581e.js","async":true,"nonce":"$undefined"}] -c:["$","script","script-18",{"src":"/litellm-asset-prefix/_next/static/chunks/7a9066dcd4a390ff.js","async":true,"nonce":"$undefined"}] -d:["$","script","script-19",{"src":"/litellm-asset-prefix/_next/static/chunks/baadbd26839e7b66.js","async":true,"nonce":"$undefined"}] -e:["$","script","script-20",{"src":"/litellm-asset-prefix/_next/static/chunks/2971c4658f1bcd7d.js","async":true,"nonce":"$undefined"}] -f:["$","script","script-21",{"src":"/litellm-asset-prefix/_next/static/chunks/134f728fa7099e3e.js","async":true,"nonce":"$undefined"}] -10:["$","script","script-22",{"src":"/litellm-asset-prefix/_next/static/chunks/679dbd657c8b5aef.js","async":true,"nonce":"$undefined"}] -11:["$","script","script-23",{"src":"/litellm-asset-prefix/_next/static/chunks/88001a7ecaf7b1af.js","async":true,"nonce":"$undefined"}] -12:["$","script","script-24",{"src":"/litellm-asset-prefix/_next/static/chunks/cbc99c8fae110c02.js","async":true,"nonce":"$undefined"}] -13:["$","script","script-25",{"src":"/litellm-asset-prefix/_next/static/chunks/4e06277331e725da.js","async":true,"nonce":"$undefined"}] -14:["$","script","script-26",{"src":"/litellm-asset-prefix/_next/static/chunks/3b30ab8eaa03bc21.js","async":true,"nonce":"$undefined"}] -15:["$","script","script-27",{"src":"/litellm-asset-prefix/_next/static/chunks/1c881baaaa68b7a5.js","async":true,"nonce":"$undefined"}] -16:["$","script","script-28",{"src":"/litellm-asset-prefix/_next/static/chunks/9955c118354ef6cc.js","async":true,"nonce":"$undefined"}] -17:["$","script","script-29",{"src":"/litellm-asset-prefix/_next/static/chunks/496b84010c33cf69.js","async":true,"nonce":"$undefined"}] -18:["$","script","script-30",{"src":"/litellm-asset-prefix/_next/static/chunks/a09028cd611c08ef.js","async":true,"nonce":"$undefined"}] -19:["$","script","script-31",{"src":"/litellm-asset-prefix/_next/static/chunks/99cf9cf99df5ccfc.js","async":true,"nonce":"$undefined"}] -1a:["$","script","script-32",{"src":"/litellm-asset-prefix/_next/static/chunks/ad02f56c287539eb.js","async":true,"nonce":"$undefined"}] -1b:["$","script","script-33",{"src":"/litellm-asset-prefix/_next/static/chunks/5181a28310842d3d.js","async":true,"nonce":"$undefined"}] -1c:["$","script","script-34",{"src":"/litellm-asset-prefix/_next/static/chunks/0a65da2cd24e2ab6.js","async":true,"nonce":"$undefined"}] -1d:["$","script","script-35",{"src":"/litellm-asset-prefix/_next/static/chunks/16a1651c0b3e7c8e.js","async":true,"nonce":"$undefined"}] -1e:["$","script","script-36",{"src":"/litellm-asset-prefix/_next/static/chunks/a8f7c8c5eeb6e042.js","async":true,"nonce":"$undefined"}] -1f:["$","script","script-37",{"src":"/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","async":true,"nonce":"$undefined"}] -20:["$","script","script-38",{"src":"/litellm-asset-prefix/_next/static/chunks/d3ac82723ec9e30d.js","async":true,"nonce":"$undefined"}] -21:["$","script","script-39",{"src":"/litellm-asset-prefix/_next/static/chunks/631b1874cba557c9.js","async":true,"nonce":"$undefined"}] -22:["$","script","script-40",{"src":"/litellm-asset-prefix/_next/static/chunks/1bc2898be56acd1b.js","async":true,"nonce":"$undefined"}] -23:["$","script","script-41",{"src":"/litellm-asset-prefix/_next/static/chunks/908828a91f602d8b.js","async":true,"nonce":"$undefined"}] -24:["$","script","script-42",{"src":"/litellm-asset-prefix/_next/static/chunks/878832edb30e99a4.js","async":true,"nonce":"$undefined"}] -25:["$","script","script-43",{"src":"/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","async":true,"nonce":"$undefined"}] -26:["$","script","script-44",{"src":"/litellm-asset-prefix/_next/static/chunks/003f1ffc5817ab83.js","async":true,"nonce":"$undefined"}] -27:["$","script","script-45",{"src":"/litellm-asset-prefix/_next/static/chunks/e1f23fd814ac3500.js","async":true,"nonce":"$undefined"}] -28:["$","script","script-46",{"src":"/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","async":true,"nonce":"$undefined"}] -29:["$","script","script-47",{"src":"/litellm-asset-prefix/_next/static/chunks/23bfdf9b0544f0b1.js","async":true,"nonce":"$undefined"}] -2a:["$","script","script-48",{"src":"/litellm-asset-prefix/_next/static/chunks/ca5fbafaf3826374.js","async":true,"nonce":"$undefined"}] -2b:["$","script","script-49",{"src":"/litellm-asset-prefix/_next/static/chunks/726bebeef472c6cb.js","async":true,"nonce":"$undefined"}] -2c:["$","script","script-50",{"src":"/litellm-asset-prefix/_next/static/chunks/8f3bf592254c6c3b.js","async":true,"nonce":"$undefined"}] -2d:["$","script","script-51",{"src":"/litellm-asset-prefix/_next/static/chunks/659ce28f2cb74401.js","async":true,"nonce":"$undefined"}] -2e:["$","script","script-52",{"src":"/litellm-asset-prefix/_next/static/chunks/16c0e58809eaf2b5.js","async":true,"nonce":"$undefined"}] -2f:["$","$L32",null,{"children":["$","$33",null,{"name":"Next.MetadataOutlet","children":"$@34"}]}] -30:["$","$1","h",{"children":[null,["$","$L35",null,{"children":"$L36"}],["$","div",null,{"hidden":true,"children":["$","$L37",null,{"children":["$","$33",null,{"name":"Next.Metadata","children":"$L38"}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}] ->>>>>>> a72414a061afdde0b31aab9a253bbce071e837e9 -8:{} -9:"$0:f:0:1:1:children:0:props:children:0:props:serverProvidedParams:params" -39:[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]] -3c:I[27201,["/my-custom-path/_next/static/chunks/06gao0u0xy9th.js","/my-custom-path/_next/static/chunks/0gszmhudlm6lw.js","/my-custom-path/_next/static/chunks/0pidya1qvuvx8.js"],"IconMark"] -37:null -3b:[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.0~dgapwhi~75y.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"./favicon.ico"}],["$","$L3c","4",{}]] +0:{"P":null,"b":"TrcGiQpTupSbDYFFfkFHY","c":["",""],"q":"","i":false,"f":[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],[["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/1bcca3c38c9deb02.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/aefca6f40ea185cd.js","async":true,"nonce":"$undefined"}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/632b4c8e836bd956.js","async":true,"nonce":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"children":["$","$L5",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L6",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]],"forbidden":"$undefined","unauthorized":"$undefined"}]}]}]}]}]}]]}],{"children":[["$","$1","c",{"children":[["$","$L7",null,{"Component":"$8","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@9","$@a"]}}],[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/52c4ecc57f72065e.js","async":true,"nonce":"$undefined"}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/4bb663ff806dc32f.js","async":true,"nonce":"$undefined"}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/ee8f89c672745c59.js","async":true,"nonce":"$undefined"}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/2954392b7a60a6a1.js","async":true,"nonce":"$undefined"}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/04711b0f8ffa7bbd.js","async":true,"nonce":"$undefined"}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/9710770c6333a72f.js","async":true,"nonce":"$undefined"}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","async":true,"nonce":"$undefined"}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/d734cb3d5659b0da.js","async":true,"nonce":"$undefined"}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/a1ef280b7ad5ae6a.js","async":true,"nonce":"$undefined"}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/09c1f51da7e82268.js","async":true,"nonce":"$undefined"}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/542a1a209eb732c6.js","async":true,"nonce":"$undefined"}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/d028f8c28935d281.js","async":true,"nonce":"$undefined"}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/61aa637257592262.js","async":true,"nonce":"$undefined"}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/bf962cd5264be987.js","async":true,"nonce":"$undefined"}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/37f229ef9335f8c3.js","async":true,"nonce":"$undefined"}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/36d1c027ba991a4f.js","async":true,"nonce":"$undefined"}],"$Lb","$Lc","$Ld","$Le","$Lf","$L10","$L11","$L12","$L13","$L14","$L15","$L16","$L17"],"$L18"]}],{},null,false,false]},null,false,false],"$L19",false]],"m":"$undefined","G":["$1a",[]],"S":true} +1b:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"OutletBoundary"] +1c:"$Sreact.suspense" +1e:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"ViewportBoundary"] +20:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"MetadataBoundary"] +b:["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/786e88f4abdd5c58.js","async":true,"nonce":"$undefined"}] +c:["$","script","script-17",{"src":"/litellm-asset-prefix/_next/static/chunks/43c3db1352241a8b.js","async":true,"nonce":"$undefined"}] +d:["$","script","script-18",{"src":"/litellm-asset-prefix/_next/static/chunks/31275eb5c6f6332f.js","async":true,"nonce":"$undefined"}] +e:["$","script","script-19",{"src":"/litellm-asset-prefix/_next/static/chunks/0ac09b227f50edb4.js","async":true,"nonce":"$undefined"}] +f:["$","script","script-20",{"src":"/litellm-asset-prefix/_next/static/chunks/4c848b12d4ecda3d.js","async":true,"nonce":"$undefined"}] +10:["$","script","script-21",{"src":"/litellm-asset-prefix/_next/static/chunks/91c828abd7c0aff5.js","async":true,"nonce":"$undefined"}] +11:["$","script","script-22",{"src":"/litellm-asset-prefix/_next/static/chunks/08c348f8e09a5cb0.js","async":true,"nonce":"$undefined"}] +12:["$","script","script-23",{"src":"/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","async":true,"nonce":"$undefined"}] +13:["$","script","script-24",{"src":"/litellm-asset-prefix/_next/static/chunks/b323e0ef008e6348.js","async":true,"nonce":"$undefined"}] +14:["$","script","script-25",{"src":"/litellm-asset-prefix/_next/static/chunks/4ac3235460262f36.js","async":true,"nonce":"$undefined"}] +15:["$","script","script-26",{"src":"/litellm-asset-prefix/_next/static/chunks/51494a4a4b6fc437.js","async":true,"nonce":"$undefined"}] +16:["$","script","script-27",{"src":"/litellm-asset-prefix/_next/static/chunks/878832edb30e99a4.js","async":true,"nonce":"$undefined"}] +17:["$","script","script-28",{"src":"/litellm-asset-prefix/_next/static/chunks/9f1486622270556b.js","async":true,"nonce":"$undefined"}] +18:["$","$L1b",null,{"children":["$","$1c",null,{"name":"Next.MetadataOutlet","children":"$@1d"}]}] +19:["$","$1","h",{"children":[null,["$","$L1e",null,{"children":"$L1f"}],["$","div",null,{"hidden":true,"children":["$","$L20",null,{"children":["$","$1c",null,{"name":"Next.Metadata","children":"$L21"}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}] +9:{} +a:"$0:f:0:1:1:children:0:props:children:0:props:serverProvidedParams:params" +1f:[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]] +22:I[27201,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"IconMark"] +1d:null +21:[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.1d32c690.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"./favicon.ico"}],["$","$L22","4",{}]] diff --git a/litellm/proxy/_experimental/out/__next._head.txt b/litellm/proxy/_experimental/out/__next._head.txt index 82758aa5c3f..ea6e5095458 100644 --- a/litellm/proxy/_experimental/out/__next._head.txt +++ b/litellm/proxy/_experimental/out/__next._head.txt @@ -3,4 +3,4 @@ 3:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"MetadataBoundary"] 4:"$Sreact.suspense" 5:I[27201,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"IconMark"] -0:{"buildId":"wnL6e5S6xaG1UdkxtYrTo","rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.1d32c690.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"./favicon.ico"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"loading":null,"isPartial":false} +0:{"buildId":"TrcGiQpTupSbDYFFfkFHY","rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.1d32c690.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"./favicon.ico"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"loading":null,"isPartial":false} diff --git a/litellm/proxy/_experimental/out/__next._index.txt b/litellm/proxy/_experimental/out/__next._index.txt index 545ff2e55cc..ebf6d8fec08 100644 --- a/litellm/proxy/_experimental/out/__next._index.txt +++ b/litellm/proxy/_experimental/out/__next._index.txt @@ -1,8 +1,9 @@ 1:"$Sreact.fragment" -2:I[867271,["/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js"],"default"] -3:I[71195,["/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js"],"default"] -4:I[339756,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"] -5:I[837457,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"] +2:I[867271,["/litellm-asset-prefix/_next/static/chunks/aefca6f40ea185cd.js","/litellm-asset-prefix/_next/static/chunks/632b4c8e836bd956.js"],"default"] +3:I[71195,["/litellm-asset-prefix/_next/static/chunks/aefca6f40ea185cd.js","/litellm-asset-prefix/_next/static/chunks/632b4c8e836bd956.js"],"default"] +4:I[557951,["/litellm-asset-prefix/_next/static/chunks/aefca6f40ea185cd.js","/litellm-asset-prefix/_next/static/chunks/632b4c8e836bd956.js"],"AuthProvider"] +5:I[339756,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"] +6:I[837457,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"] :HL["/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","style"] -:HL["/litellm-asset-prefix/_next/static/chunks/91037395c95e366d.css","style"] -0:{"buildId":"wnL6e5S6xaG1UdkxtYrTo","rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/91037395c95e366d.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","async":true}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}]}]}]}]]}],"loading":null,"isPartial":false} +:HL["/litellm-asset-prefix/_next/static/chunks/1bcca3c38c9deb02.css","style"] +0:{"buildId":"TrcGiQpTupSbDYFFfkFHY","rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/1bcca3c38c9deb02.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/aefca6f40ea185cd.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/632b4c8e836bd956.js","async":true}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"children":["$","$L5",null,{"parallelRouterKey":"children","template":["$","$L6",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}]}]}]}]}]]}],"loading":null,"isPartial":false} diff --git a/litellm/proxy/_experimental/out/__next._tree.txt b/litellm/proxy/_experimental/out/__next._tree.txt index 10f5e5c2721..4a08f4f9e11 100644 --- a/litellm/proxy/_experimental/out/__next._tree.txt +++ b/litellm/proxy/_experimental/out/__next._tree.txt @@ -1,5 +1,5 @@ :HL["/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","style"] -:HL["/litellm-asset-prefix/_next/static/chunks/91037395c95e366d.css","style"] +:HL["/litellm-asset-prefix/_next/static/chunks/1bcca3c38c9deb02.css","style"] :HL["/litellm-asset-prefix/_next/static/media/83afe278b6a6bb3c-s.p.3a6ba036.woff2","font",{"crossOrigin":"","type":"font/woff2"}] :HL["/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","style"] -0:{"buildId":"wnL6e5S6xaG1UdkxtYrTo","tree":{"name":"","paramType":null,"paramKey":"","hasRuntimePrefetch":false,"slots":{"children":{"name":"__PAGE__","paramType":null,"paramKey":"__PAGE__","hasRuntimePrefetch":false,"slots":null,"isRootLayout":false}},"isRootLayout":true},"staleTime":300} +0:{"buildId":"TrcGiQpTupSbDYFFfkFHY","tree":{"name":"","paramType":null,"paramKey":"","hasRuntimePrefetch":false,"slots":{"children":{"name":"__PAGE__","paramType":null,"paramKey":"__PAGE__","hasRuntimePrefetch":false,"slots":null,"isRootLayout":false}},"isRootLayout":true},"staleTime":300} diff --git a/litellm/proxy/_experimental/out/_next/static/wnL6e5S6xaG1UdkxtYrTo/_buildManifest.js b/litellm/proxy/_experimental/out/_next/static/TrcGiQpTupSbDYFFfkFHY/_buildManifest.js similarity index 100% rename from litellm/proxy/_experimental/out/_next/static/wnL6e5S6xaG1UdkxtYrTo/_buildManifest.js rename to litellm/proxy/_experimental/out/_next/static/TrcGiQpTupSbDYFFfkFHY/_buildManifest.js diff --git a/litellm/proxy/_experimental/out/_next/static/wnL6e5S6xaG1UdkxtYrTo/_clientMiddlewareManifest.json b/litellm/proxy/_experimental/out/_next/static/TrcGiQpTupSbDYFFfkFHY/_clientMiddlewareManifest.json similarity index 100% rename from litellm/proxy/_experimental/out/_next/static/wnL6e5S6xaG1UdkxtYrTo/_clientMiddlewareManifest.json rename to litellm/proxy/_experimental/out/_next/static/TrcGiQpTupSbDYFFfkFHY/_clientMiddlewareManifest.json diff --git a/litellm/proxy/_experimental/out/_next/static/wnL6e5S6xaG1UdkxtYrTo/_ssgManifest.js b/litellm/proxy/_experimental/out/_next/static/TrcGiQpTupSbDYFFfkFHY/_ssgManifest.js similarity index 100% rename from litellm/proxy/_experimental/out/_next/static/wnL6e5S6xaG1UdkxtYrTo/_ssgManifest.js rename to litellm/proxy/_experimental/out/_next/static/TrcGiQpTupSbDYFFfkFHY/_ssgManifest.js diff --git a/litellm/proxy/_experimental/out/_next/static/chunks/0-2-9tcizw.kv.js b/litellm/proxy/_experimental/out/_next/static/chunks/0-2-9tcizw.kv.js new file mode 100644 index 00000000000..5e8e78991b9 --- /dev/null +++ b/litellm/proxy/_experimental/out/_next/static/chunks/0-2-9tcizw.kv.js @@ -0,0 +1 @@ +(globalThis.TURBOPACK||(globalThis.TURBOPACK=[])).push(["object"==typeof document?document.currentScript:void 0,888288,e=>{"use strict";var t=e.i(271645);e.s(["default",0,(e,r)=>{let n=void 0!==r,[a,s]=(0,t.useState)(e);return[n?r:a,e=>{n||s(e)}]}])},757440,e=>{"use strict";var t=e.i(290571),r=e.i(271645);e.s(["default",0,e=>{var n=(0,t.__rest)(e,[]);return r.default.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",viewBox:"0 0 24 24",fill:"currentColor"},n),r.default.createElement("path",{d:"M11.9999 13.1714L16.9497 8.22168L18.3639 9.63589L11.9999 15.9999L5.63599 9.63589L7.0502 8.22168L11.9999 13.1714Z"}))}])},446428,854056,e=>{"use strict";let t;var r=e.i(290571),n=e.i(271645);e.s(["default",0,e=>{var t=(0,r.__rest)(e,[]);return n.default.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",viewBox:"0 0 24 24",fill:"currentColor"},t),n.default.createElement("path",{d:"M12 22C6.47715 22 2 17.5228 2 12C2 6.47715 6.47715 2 12 2C17.5228 2 22 6.47715 22 12C22 17.5228 17.5228 22 12 22ZM12 10.5858L9.17157 7.75736L7.75736 9.17157L10.5858 12L7.75736 14.8284L9.17157 16.2426L12 13.4142L14.8284 16.2426L16.2426 14.8284L13.4142 12L16.2426 9.17157L14.8284 7.75736L12 10.5858Z"}))}],446428);var a=e.i(746725),s=e.i(914189),i=e.i(553521),l=e.i(835696),o=e.i(941444),u=e.i(178677),c=e.i(294316),d=e.i(83733),m=e.i(233137),f=e.i(732607),h=e.i(397701),v=e.i(700020);function g(e){var t;return!!(e.enter||e.enterFrom||e.enterTo||e.leave||e.leaveFrom||e.leaveTo)||(null!=(t=e.as)?t:j)!==n.Fragment||1===n.default.Children.count(e.children)}let p=(0,n.createContext)(null);p.displayName="TransitionContext";var b=((t=b||{}).Visible="visible",t.Hidden="hidden",t);let x=(0,n.createContext)(null);function y(e){return"children"in e?y(e.children):e.current.filter(({el:e})=>null!==e.current).filter(({state:e})=>"visible"===e).length>0}function w(e,t){let r=(0,o.useLatestValue)(e),l=(0,n.useRef)([]),u=(0,i.useIsMounted)(),c=(0,a.useDisposables)(),d=(0,s.useEvent)((e,t=v.RenderStrategy.Hidden)=>{let n=l.current.findIndex(({el:t})=>t===e);-1!==n&&((0,h.match)(t,{[v.RenderStrategy.Unmount](){l.current.splice(n,1)},[v.RenderStrategy.Hidden](){l.current[n].state="hidden"}}),c.microTask(()=>{var e;!y(l)&&u.current&&(null==(e=r.current)||e.call(r))}))}),m=(0,s.useEvent)(e=>{let t=l.current.find(({el:t})=>t===e);return t?"visible"!==t.state&&(t.state="visible"):l.current.push({el:e,state:"visible"}),()=>d(e,v.RenderStrategy.Unmount)}),f=(0,n.useRef)([]),g=(0,n.useRef)(Promise.resolve()),p=(0,n.useRef)({enter:[],leave:[]}),b=(0,s.useEvent)((e,r,n)=>{f.current.splice(0),t&&(t.chains.current[r]=t.chains.current[r].filter(([t])=>t!==e)),null==t||t.chains.current[r].push([e,new Promise(e=>{f.current.push(e)})]),null==t||t.chains.current[r].push([e,new Promise(e=>{Promise.all(p.current[r].map(([e,t])=>t)).then(()=>e())})]),"enter"===r?g.current=g.current.then(()=>null==t?void 0:t.wait.current).then(()=>n(r)):n(r)}),x=(0,s.useEvent)((e,t,r)=>{Promise.all(p.current[t].splice(0).map(([e,t])=>t)).then(()=>{var e;null==(e=f.current.shift())||e()}).then(()=>r(t))});return(0,n.useMemo)(()=>({children:l,register:m,unregister:d,onStart:b,onStop:x,wait:g,chains:p}),[m,d,l,b,x,p,g])}x.displayName="NestingContext";let j=n.Fragment,M=v.RenderFeatures.RenderStrategy,C=(0,v.forwardRefWithAs)(function(e,t){let{show:r,appear:a=!1,unmount:i=!0,...o}=e,d=(0,n.useRef)(null),f=g(e),h=(0,c.useSyncRefs)(...f?[d,t]:null===t?[]:[t]);(0,u.useServerHandoffComplete)();let b=(0,m.useOpenClosed)();if(void 0===r&&null!==b&&(r=(b&m.State.Open)===m.State.Open),void 0===r)throw Error("A is used but it is missing a `show={true | false}` prop.");let[j,C]=(0,n.useState)(r?"visible":"hidden"),T=w(()=>{r||C("hidden")}),[$,k]=(0,n.useState)(!0),N=(0,n.useRef)([r]);(0,l.useIsoMorphicEffect)(()=>{!1!==$&&N.current[N.current.length-1]!==r&&(N.current.push(r),k(!1))},[N,r]);let E=(0,n.useMemo)(()=>({show:r,appear:a,initial:$}),[r,a,$]);(0,l.useIsoMorphicEffect)(()=>{r?C("visible"):y(T)||null===d.current||C("hidden")},[r,T]);let _={unmount:i},D=(0,s.useEvent)(()=>{var t;$&&k(!1),null==(t=e.beforeEnter)||t.call(e)}),O=(0,s.useEvent)(()=>{var t;$&&k(!1),null==(t=e.beforeLeave)||t.call(e)}),L=(0,v.useRender)();return n.default.createElement(x.Provider,{value:T},n.default.createElement(p.Provider,{value:E},L({ourProps:{..._,as:n.Fragment,children:n.default.createElement(S,{ref:h,..._,...o,beforeEnter:D,beforeLeave:O})},theirProps:{},defaultTag:n.Fragment,features:M,visible:"visible"===j,name:"Transition"})))}),S=(0,v.forwardRefWithAs)(function(e,t){var r,a;let{transition:i=!0,beforeEnter:o,afterEnter:b,beforeLeave:C,afterLeave:S,enter:T,enterFrom:$,enterTo:k,entered:N,leave:E,leaveFrom:_,leaveTo:D,...O}=e,[L,R]=(0,n.useState)(null),z=(0,n.useRef)(null),H=g(e),A=(0,c.useSyncRefs)(...H?[z,t,R]:null===t?[]:[t]),V=null==(r=O.unmount)||r?v.RenderStrategy.Unmount:v.RenderStrategy.Hidden,{show:B,appear:F,initial:P}=function(){let e=(0,n.useContext)(p);if(null===e)throw Error("A is used but it is missing a parent or .");return e}(),[I,W]=(0,n.useState)(B?"visible":"hidden"),Y=function(){let e=(0,n.useContext)(x);if(null===e)throw Error("A is used but it is missing a parent or .");return e}(),{register:U,unregister:Z}=Y;(0,l.useIsoMorphicEffect)(()=>U(z),[U,z]),(0,l.useIsoMorphicEffect)(()=>{if(V===v.RenderStrategy.Hidden&&z.current)return B&&"visible"!==I?void W("visible"):(0,h.match)(I,{hidden:()=>Z(z),visible:()=>U(z)})},[I,z,U,Z,B,V]);let q=(0,u.useServerHandoffComplete)();(0,l.useIsoMorphicEffect)(()=>{if(H&&q&&"visible"===I&&null===z.current)throw Error("Did you forget to passthrough the `ref` to the actual DOM node?")},[z,I,q,H]);let J=P&&!F,K=F&&B&&P,G=(0,n.useRef)(!1),Q=w(()=>{G.current||(W("hidden"),Z(z))},Y),X=(0,s.useEvent)(e=>{G.current=!0,Q.onStart(z,e?"enter":"leave",e=>{"enter"===e?null==o||o():"leave"===e&&(null==C||C())})}),ee=(0,s.useEvent)(e=>{let t=e?"enter":"leave";G.current=!1,Q.onStop(z,t,e=>{"enter"===e?null==b||b():"leave"===e&&(null==S||S())}),"leave"!==t||y(Q)||(W("hidden"),Z(z))});(0,n.useEffect)(()=>{H&&i||(X(B),ee(B))},[B,H,i]);let et=!(!i||!H||!q||J),[,er]=(0,d.useTransition)(et,L,B,{start:X,end:ee}),en=(0,v.compact)({ref:A,className:(null==(a=(0,f.classNames)(O.className,K&&T,K&&$,er.enter&&T,er.enter&&er.closed&&$,er.enter&&!er.closed&&k,er.leave&&E,er.leave&&!er.closed&&_,er.leave&&er.closed&&D,!er.transition&&B&&N))?void 0:a.trim())||void 0,...(0,d.transitionDataAttributes)(er)}),ea=0;"visible"===I&&(ea|=m.State.Open),"hidden"===I&&(ea|=m.State.Closed),er.enter&&(ea|=m.State.Opening),er.leave&&(ea|=m.State.Closing);let es=(0,v.useRender)();return n.default.createElement(x.Provider,{value:Q},n.default.createElement(m.OpenClosedProvider,{value:ea},es({ourProps:en,theirProps:O,defaultTag:j,features:M,visible:"visible"===I,name:"Transition.Child"})))}),T=(0,v.forwardRefWithAs)(function(e,t){let r=null!==(0,n.useContext)(p),a=null!==(0,m.useOpenClosed)();return n.default.createElement(n.default.Fragment,null,!r&&a?n.default.createElement(C,{ref:t,...e}):n.default.createElement(S,{ref:t,...e}))}),$=Object.assign(C,{Child:T,Root:C});e.s(["Transition",0,$],854056)},206929,e=>{"use strict";var t=e.i(290571),r=e.i(757440),n=e.i(271645),a=e.i(446428),s=e.i(444755),i=e.i(673706),l=e.i(103471),o=e.i(495470),u=e.i(854056),c=e.i(888288);let d=(0,i.makeClassName)("Select"),m=n.default.forwardRef((e,i)=>{let{defaultValue:m="",value:f,onValueChange:h,placeholder:v="Select...",disabled:g=!1,icon:p,enableClear:b=!1,required:x,children:y,name:w,error:j=!1,errorMessage:M,className:C,id:S}=e,T=(0,t.__rest)(e,["defaultValue","value","onValueChange","placeholder","disabled","icon","enableClear","required","children","name","error","errorMessage","className","id"]),$=(0,n.useRef)(null),k=n.Children.toArray(y),[N,E]=(0,c.default)(m,f),_=(0,n.useMemo)(()=>{let e=n.default.Children.toArray(y).filter(n.isValidElement);return(0,l.constructValueToNameMapping)(e)},[y]);return n.default.createElement("div",{className:(0,s.tremorTwMerge)("w-full min-w-[10rem] text-tremor-default",C)},n.default.createElement("div",{className:"relative"},n.default.createElement("select",{title:"select-hidden",required:x,className:(0,s.tremorTwMerge)("h-full w-full absolute left-0 top-0 -z-10 opacity-0"),value:N,onChange:e=>{e.preventDefault()},name:w,disabled:g,id:S,onFocus:()=>{let e=$.current;e&&e.focus()}},n.default.createElement("option",{className:"hidden",value:"",disabled:!0,hidden:!0},v),k.map(e=>{let t=e.props.value,r=e.props.children;return n.default.createElement("option",{className:"hidden",key:t,value:t},r)})),n.default.createElement(o.Listbox,Object.assign({as:"div",ref:i,defaultValue:N,value:N,onChange:e=>{null==h||h(e),E(e)},disabled:g,id:S},T),({value:e})=>{var t;return n.default.createElement(n.default.Fragment,null,n.default.createElement(o.ListboxButton,{ref:$,className:(0,s.tremorTwMerge)("w-full outline-none text-left whitespace-nowrap truncate rounded-tremor-default focus:ring-2 transition duration-100 border pr-8 py-2","border-tremor-border shadow-tremor-input focus:border-tremor-brand-subtle focus:ring-tremor-brand-muted","dark:border-dark-tremor-border dark:shadow-dark-tremor-input dark:focus:border-dark-tremor-brand-subtle dark:focus:ring-dark-tremor-brand-muted",p?"pl-10":"pl-3",(0,l.getSelectButtonColors)((0,l.hasValue)(e),g,j))},p&&n.default.createElement("span",{className:(0,s.tremorTwMerge)("absolute inset-y-0 left-0 flex items-center ml-px pl-2.5")},n.default.createElement(p,{className:(0,s.tremorTwMerge)(d("Icon"),"flex-none h-5 w-5","text-tremor-content-subtle","dark:text-dark-tremor-content-subtle")})),n.default.createElement("span",{className:"w-[90%] block truncate"},e&&null!=(t=_.get(e))?t:v),n.default.createElement("span",{className:(0,s.tremorTwMerge)("absolute inset-y-0 right-0 flex items-center mr-3")},n.default.createElement(r.default,{className:(0,s.tremorTwMerge)(d("arrowDownIcon"),"flex-none h-5 w-5","text-tremor-content-subtle","dark:text-dark-tremor-content-subtle")}))),b&&N?n.default.createElement("button",{type:"button",className:(0,s.tremorTwMerge)("absolute inset-y-0 right-0 flex items-center mr-8"),onClick:e=>{e.preventDefault(),E(""),null==h||h("")}},n.default.createElement(a.default,{className:(0,s.tremorTwMerge)(d("clearIcon"),"flex-none h-4 w-4","text-tremor-content-subtle","dark:text-dark-tremor-content-subtle")})):null,n.default.createElement(u.Transition,{enter:"transition ease duration-100 transform",enterFrom:"opacity-0 -translate-y-4",enterTo:"opacity-100 translate-y-0",leave:"transition ease duration-100 transform",leaveFrom:"opacity-100 translate-y-0",leaveTo:"opacity-0 -translate-y-4"},n.default.createElement(o.ListboxOptions,{anchor:"bottom start",className:(0,s.tremorTwMerge)("z-10 w-[var(--button-width)] divide-y overflow-y-auto outline-none rounded-tremor-default max-h-[228px] border [--anchor-gap:4px]","bg-tremor-background border-tremor-border divide-tremor-border shadow-tremor-dropdown","dark:bg-dark-tremor-background dark:border-dark-tremor-border dark:divide-dark-tremor-border dark:shadow-dark-tremor-dropdown")},y)))})),j&&M?n.default.createElement("p",{className:(0,s.tremorTwMerge)("errorMessage","text-sm text-rose-500 mt-1")},M):null)});m.displayName="Select",e.s(["Select",0,m],206929)},160818,e=>{"use strict";e.i(247167);var t=e.i(931067),r=e.i(271645);let n={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M854.4 800.9c.2-.3.5-.6.7-.9C920.6 722.1 960 621.7 960 512s-39.4-210.1-104.8-288c-.2-.3-.5-.5-.7-.8-1.1-1.3-2.1-2.5-3.2-3.7-.4-.5-.8-.9-1.2-1.4l-4.1-4.7-.1-.1c-1.5-1.7-3.1-3.4-4.6-5.1l-.1-.1c-3.2-3.4-6.4-6.8-9.7-10.1l-.1-.1-4.8-4.8-.3-.3c-1.5-1.5-3-2.9-4.5-4.3-.5-.5-1-1-1.6-1.5-1-1-2-1.9-3-2.8-.3-.3-.7-.6-1-1C736.4 109.2 629.5 64 512 64s-224.4 45.2-304.3 119.2c-.3.3-.7.6-1 1-1 .9-2 1.9-3 2.9-.5.5-1 1-1.6 1.5-1.5 1.4-3 2.9-4.5 4.3l-.3.3-4.8 4.8-.1.1c-3.3 3.3-6.5 6.7-9.7 10.1l-.1.1c-1.6 1.7-3.1 3.4-4.6 5.1l-.1.1c-1.4 1.5-2.8 3.1-4.1 4.7-.4.5-.8.9-1.2 1.4-1.1 1.2-2.1 2.5-3.2 3.7-.2.3-.5.5-.7.8C103.4 301.9 64 402.3 64 512s39.4 210.1 104.8 288c.2.3.5.6.7.9l3.1 3.7c.4.5.8.9 1.2 1.4l4.1 4.7c0 .1.1.1.1.2 1.5 1.7 3 3.4 4.6 5l.1.1c3.2 3.4 6.4 6.8 9.6 10.1l.1.1c1.6 1.6 3.1 3.2 4.7 4.7l.3.3c3.3 3.3 6.7 6.5 10.1 9.6 80.1 74 187 119.2 304.5 119.2s224.4-45.2 304.3-119.2a300 300 0 0010-9.6l.3-.3c1.6-1.6 3.2-3.1 4.7-4.7l.1-.1c3.3-3.3 6.5-6.7 9.6-10.1l.1-.1c1.5-1.7 3.1-3.3 4.6-5 0-.1.1-.1.1-.2 1.4-1.5 2.8-3.1 4.1-4.7.4-.5.8-.9 1.2-1.4a99 99 0 003.3-3.7zm4.1-142.6c-13.8 32.6-32 62.8-54.2 90.2a444.07 444.07 0 00-81.5-55.9c11.6-46.9 18.8-98.4 20.7-152.6H887c-3 40.9-12.6 80.6-28.5 118.3zM887 484H743.5c-1.9-54.2-9.1-105.7-20.7-152.6 29.3-15.6 56.6-34.4 81.5-55.9A373.86 373.86 0 01887 484zM658.3 165.5c39.7 16.8 75.8 40 107.6 69.2a394.72 394.72 0 01-59.4 41.8c-15.7-45-35.8-84.1-59.2-115.4 3.7 1.4 7.4 2.9 11 4.4zm-90.6 700.6c-9.2 7.2-18.4 12.7-27.7 16.4V697a389.1 389.1 0 01115.7 26.2c-8.3 24.6-17.9 47.3-29 67.8-17.4 32.4-37.8 58.3-59 75.1zm59-633.1c11 20.6 20.7 43.3 29 67.8A389.1 389.1 0 01540 327V141.6c9.2 3.7 18.5 9.1 27.7 16.4 21.2 16.7 41.6 42.6 59 75zM540 640.9V540h147.5c-1.6 44.2-7.1 87.1-16.3 127.8l-.3 1.2A445.02 445.02 0 00540 640.9zm0-156.9V383.1c45.8-2.8 89.8-12.5 130.9-28.1l.3 1.2c9.2 40.7 14.7 83.5 16.3 127.8H540zm-56 56v100.9c-45.8 2.8-89.8 12.5-130.9 28.1l-.3-1.2c-9.2-40.7-14.7-83.5-16.3-127.8H484zm-147.5-56c1.6-44.2 7.1-87.1 16.3-127.8l.3-1.2c41.1 15.6 85 25.3 130.9 28.1V484H336.5zM484 697v185.4c-9.2-3.7-18.5-9.1-27.7-16.4-21.2-16.7-41.7-42.7-59.1-75.1-11-20.6-20.7-43.3-29-67.8 37.2-14.6 75.9-23.3 115.8-26.1zm0-370a389.1 389.1 0 01-115.7-26.2c8.3-24.6 17.9-47.3 29-67.8 17.4-32.4 37.8-58.4 59.1-75.1 9.2-7.2 18.4-12.7 27.7-16.4V327zM365.7 165.5c3.7-1.5 7.3-3 11-4.4-23.4 31.3-43.5 70.4-59.2 115.4-21-12-40.9-26-59.4-41.8 31.8-29.2 67.9-52.4 107.6-69.2zM165.5 365.7c13.8-32.6 32-62.8 54.2-90.2 24.9 21.5 52.2 40.3 81.5 55.9-11.6 46.9-18.8 98.4-20.7 152.6H137c3-40.9 12.6-80.6 28.5-118.3zM137 540h143.5c1.9 54.2 9.1 105.7 20.7 152.6a444.07 444.07 0 00-81.5 55.9A373.86 373.86 0 01137 540zm228.7 318.5c-39.7-16.8-75.8-40-107.6-69.2 18.5-15.8 38.4-29.7 59.4-41.8 15.7 45 35.8 84.1 59.2 115.4-3.7-1.4-7.4-2.9-11-4.4zm292.6 0c-3.7 1.5-7.3 3-11 4.4 23.4-31.3 43.5-70.4 59.2-115.4 21 12 40.9 26 59.4 41.8a373.81 373.81 0 01-107.6 69.2z"}}]},name:"global",theme:"outlined"};var a=e.i(9583),s=r.forwardRef(function(e,s){return r.createElement(a.default,(0,t.default)({},e,{ref:s,icon:n}))});e.s(["GlobalOutlined",0,s],160818)},822315,(e,t,r)=>{e.e,t.exports=function(){"use strict";var e="millisecond",t="second",r="minute",n="hour",a="week",s="month",i="quarter",l="year",o="date",u="Invalid Date",c=/^(\d{4})[-/]?(\d{1,2})?[-/]?(\d{0,2})[Tt\s]*(\d{1,2})?:?(\d{1,2})?:?(\d{1,2})?[.:]?(\d+)?$/,d=/\[([^\]]+)]|Y{1,4}|M{1,4}|D{1,2}|d{1,4}|H{1,2}|h{1,2}|a|A|m{1,2}|s{1,2}|Z{1,2}|SSS/g,m=function(e,t,r){var n=String(e);return!n||n.length>=t?e:""+Array(t+1-n.length).join(r)+e},f="en",h={};h[f]={name:"en",weekdays:"Sunday_Monday_Tuesday_Wednesday_Thursday_Friday_Saturday".split("_"),months:"January_February_March_April_May_June_July_August_September_October_November_December".split("_"),ordinal:function(e){var t=["th","st","nd","rd"],r=e%100;return"["+e+(t[(r-20)%10]||t[r]||t[0])+"]"}};var v="$isDayjsObject",g=function(e){return e instanceof y||!(!e||!e[v])},p=function e(t,r,n){var a;if(!t)return f;if("string"==typeof t){var s=t.toLowerCase();h[s]&&(a=s),r&&(h[s]=r,a=s);var i=t.split("-");if(!a&&i.length>1)return e(i[0])}else{var l=t.name;h[l]=t,a=l}return!n&&a&&(f=a),a||!n&&f},b=function(e,t){if(g(e))return e.clone();var r="object"==typeof t?t:{};return r.date=e,r.args=arguments,new y(r)},x={s:m,z:function(e){var t=-e.utcOffset(),r=Math.abs(t);return(t<=0?"+":"-")+m(Math.floor(r/60),2,"0")+":"+m(r%60,2,"0")},m:function e(t,r){if(t.date(){"use strict";var t=e.i(401361);e.s(["EditOutlined",()=>t.default])},564897,e=>{"use strict";e.i(247167);var t=e.i(931067),r=e.i(271645);let n={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M696 480H328c-4.4 0-8 3.6-8 8v48c0 4.4 3.6 8 8 8h368c4.4 0 8-3.6 8-8v-48c0-4.4-3.6-8-8-8z"}},{tag:"path",attrs:{d:"M512 64C264.6 64 64 264.6 64 512s200.6 448 448 448 448-200.6 448-448S759.4 64 512 64zm0 820c-205.4 0-372-166.6-372-372s166.6-372 372-372 372 166.6 372 372-166.6 372-372 372z"}}]},name:"minus-circle",theme:"outlined"};var a=e.i(9583),s=r.forwardRef(function(e,s){return r.createElement(a.default,(0,t.default)({},e,{ref:s,icon:n}))});e.s(["MinusCircleOutlined",0,s],564897)},987432,e=>{"use strict";e.i(247167);var t=e.i(931067),r=e.i(271645);let n={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M893.3 293.3L730.7 130.7c-7.5-7.5-16.7-13-26.7-16V112H144c-17.7 0-32 14.3-32 32v736c0 17.7 14.3 32 32 32h736c17.7 0 32-14.3 32-32V338.5c0-17-6.7-33.2-18.7-45.2zM384 184h256v104H384V184zm456 656H184V184h136v136c0 17.7 14.3 32 32 32h320c17.7 0 32-14.3 32-32V205.8l136 136V840zM512 442c-79.5 0-144 64.5-144 144s64.5 144 144 144 144-64.5 144-144-64.5-144-144-144zm0 224c-44.2 0-80-35.8-80-80s35.8-80 80-80 80 35.8 80 80-35.8 80-80 80z"}}]},name:"save",theme:"outlined"};var a=e.i(9583),s=r.forwardRef(function(e,s){return r.createElement(a.default,(0,t.default)({},e,{ref:s,icon:n}))});e.s(["SaveOutlined",0,s],987432)},995926,e=>{"use strict";var t=e.i(841947);e.s(["XIcon",()=>t.default])},584578,e=>{"use strict";var t=e.i(764205);let r=async(e,r,n,a,s)=>{let i;i="Admin"!=n&&"Admin Viewer"!=n?await (0,t.teamListCall)(e,a?.organization_id||null,r):await (0,t.teamListCall)(e,a?.organization_id||null),console.log(`givenTeams: ${i}`),s(i)};e.s(["fetchTeams",0,r])},468133,e=>{"use strict";var t=e.i(843476),r=e.i(271645),n=e.i(175712),a=e.i(464571),s=e.i(28651),i=e.i(898586),l=e.i(482725),o=e.i(199133),u=e.i(262218),c=e.i(621192),d=e.i(178654),m=e.i(751904),f=e.i(987432),h=e.i(764205),v=e.i(860585),g=e.i(355619),p=e.i(727749),b=e.i(162386);let{Title:x,Text:y}=i.Typography,w=["/key/generate","/key/update","/key/delete","/key/regenerate","/key/service-account/generate","/key/{key_id}/regenerate","/key/block","/key/unblock","/key/bulk_update","/key/{key_id}/reset_spend","/key/info","/key/list","/key/aliases","/team/daily/activity"],j=({label:e,description:r,isEditing:n,viewContent:a,editContent:s})=>(0,t.jsxs)(c.Row,{className:"py-5 border-b border-gray-100 last:border-0",children:[(0,t.jsxs)(d.Col,{span:8,className:"pr-6",children:[(0,t.jsx)("div",{className:"text-sm font-semibold text-gray-900",children:e}),(0,t.jsx)("div",{className:"text-xs text-gray-500 mt-1 leading-relaxed",children:r})]}),(0,t.jsx)(d.Col,{span:16,className:"flex items-center",children:(0,t.jsx)("div",{className:"w-full",children:n?s:a})})]}),M=()=>(0,t.jsx)(y,{className:"text-gray-400 italic",children:"Not set"}),C=(e,r)=>e&&0!==e.length?(0,t.jsx)("div",{className:"flex flex-wrap gap-2",children:e.map(e=>(0,t.jsx)(u.Tag,{color:"blue",children:r?r(e):e},e))}):(0,t.jsx)(M,{}),S={max_budget:null,budget_duration:null,tpm_limit:null,rpm_limit:null,models:[],team_member_permissions:[]};e.s(["default",0,({accessToken:e})=>{let[i,c]=(0,r.useState)(!0),[d,T]=(0,r.useState)(S),[$,k]=(0,r.useState)(!1),[N,E]=(0,r.useState)(S),[_,D]=(0,r.useState)(!1),[O,L]=(0,r.useState)(!1);(0,r.useEffect)(()=>{(async()=>{if(!e)return c(!1);try{let t=await (0,h.getDefaultTeamSettings)(e),r={...S,...t.values||{}};T(r),E(r)}catch(e){console.error("Error fetching team SSO settings:",e),L(!0),p.default.fromBackend("Failed to fetch team settings")}finally{c(!1)}})()},[e]);let R=async()=>{if(e){D(!0);try{let t=await (0,h.updateDefaultTeamSettings)(e,N),r={...S,...t.settings||{}};T(r),E(r),k(!1),p.default.success("Default team settings updated successfully")}catch(e){console.error("Error updating team settings:",e),p.default.fromBackend("Failed to update team settings")}finally{D(!1)}}},z=(e,t)=>{E(r=>({...r,[e]:t}))};return i?(0,t.jsx)("div",{className:"flex justify-center items-center h-64",children:(0,t.jsx)(l.Spin,{size:"large"})}):O?(0,t.jsx)(n.Card,{children:(0,t.jsx)(y,{children:"No team settings available or you do not have permission to view them."})}):(0,t.jsxs)(n.Card,{styles:{body:{padding:32}},children:[(0,t.jsxs)("div",{className:"flex justify-between items-start mb-2",children:[(0,t.jsxs)("div",{children:[(0,t.jsx)(x,{level:3,className:"m-0 text-gray-900",children:"Default Team Settings"}),(0,t.jsx)(y,{className:"text-gray-500 mt-1 block",children:"These settings will be applied by default when creating new teams."})]}),(0,t.jsx)("div",{children:$?(0,t.jsxs)("div",{className:"flex gap-3",children:[(0,t.jsx)(a.Button,{onClick:()=>{k(!1),E(d)},disabled:_,children:"Cancel"}),(0,t.jsx)(a.Button,{type:"primary",onClick:R,loading:_,icon:(0,t.jsx)(f.SaveOutlined,{}),children:"Save Changes"})]}):(0,t.jsx)(a.Button,{onClick:()=>k(!0),icon:(0,t.jsx)(m.EditOutlined,{}),children:"Edit Settings"})})]}),(0,t.jsxs)("div",{className:"mt-8",children:[(0,t.jsxs)("div",{className:"mb-8",children:[(0,t.jsx)("div",{className:"text-xs font-bold text-gray-500 uppercase tracking-wider mb-2",children:"Budget & Rate Limits"}),(0,t.jsxs)("div",{className:"border-t border-gray-100",children:[(0,t.jsx)(j,{label:"Max Budget",description:"Maximum budget (in USD) for new automatically created teams.",isEditing:$,viewContent:null!=d.max_budget?(0,t.jsxs)(y,{children:["$",Number(d.max_budget).toLocaleString()]}):(0,t.jsx)(M,{}),editContent:(0,t.jsx)(s.InputNumber,{className:"w-full",style:{maxWidth:320},value:N.max_budget,onChange:e=>z("max_budget",e),placeholder:"Not set",prefix:"$",min:0})}),(0,t.jsx)(j,{label:"Budget Duration",description:"How frequently the team's budget resets.",isEditing:$,viewContent:d.budget_duration?(0,t.jsx)(y,{children:(0,v.getBudgetDurationLabel)(d.budget_duration)}):(0,t.jsx)(M,{}),editContent:(0,t.jsx)(v.default,{value:N.budget_duration||null,onChange:e=>z("budget_duration",e),style:{maxWidth:320}})}),(0,t.jsx)(j,{label:"TPM Limit",description:"Maximum tokens per minute allowed across all models.",isEditing:$,viewContent:null!=d.tpm_limit?(0,t.jsx)(y,{children:d.tpm_limit.toLocaleString()}):(0,t.jsx)(M,{}),editContent:(0,t.jsx)(s.InputNumber,{className:"w-full",style:{maxWidth:320},value:N.tpm_limit,onChange:e=>z("tpm_limit",e),placeholder:"Not set",min:0})}),(0,t.jsx)(j,{label:"RPM Limit",description:"Maximum requests per minute allowed across all models.",isEditing:$,viewContent:null!=d.rpm_limit?(0,t.jsx)(y,{children:d.rpm_limit.toLocaleString()}):(0,t.jsx)(M,{}),editContent:(0,t.jsx)(s.InputNumber,{className:"w-full",style:{maxWidth:320},value:N.rpm_limit,onChange:e=>z("rpm_limit",e),placeholder:"Not set",min:0})})]})]}),(0,t.jsxs)("div",{className:"mb-8",children:[(0,t.jsx)("div",{className:"text-xs font-bold text-gray-500 uppercase tracking-wider mb-2",children:"Access & Permissions"}),(0,t.jsxs)("div",{className:"border-t border-gray-100",children:[(0,t.jsx)(j,{label:"Models",description:"Default list of models that new teams can access.",isEditing:$,viewContent:C(d.models,g.getModelDisplayName),editContent:(0,t.jsx)(b.ModelSelect,{value:N.models||[],onChange:e=>z("models",e),context:"global",style:{width:"100%"},options:{includeSpecialOptions:!0}})}),(0,t.jsx)(j,{label:"Team Member Permissions",description:"Default permissions granted to members of newly created teams. /key/info and /key/health are always included.",isEditing:$,viewContent:C(d.team_member_permissions),editContent:(0,t.jsx)(o.Select,{mode:"multiple",style:{width:"100%"},value:N.team_member_permissions||[],onChange:e=>z("team_member_permissions",e),placeholder:"Select permissions",tagRender:({label:e,closable:r,onClose:n})=>(0,t.jsx)(u.Tag,{color:"blue",closable:r,onClose:n,className:"mr-1 mt-1 mb-1",children:e}),children:w.map(e=>(0,t.jsx)(o.Select.Option,{value:e,children:e},e))})})]})]})]})]})}])},747871,e=>{"use strict";var t=e.i(843476),r=e.i(271645),n=e.i(269200),a=e.i(942232),s=e.i(977572),i=e.i(427612),l=e.i(64848),o=e.i(496020),u=e.i(304967),c=e.i(994388),d=e.i(599724),m=e.i(389083),f=e.i(764205),h=e.i(727749);e.s(["default",0,({accessToken:e,userID:v})=>{let[g,p]=(0,r.useState)([]);(0,r.useEffect)(()=>{(async()=>{if(e&&v)try{let t=await (0,f.availableTeamListCall)(e);p(t)}catch(e){console.error("Error fetching available teams:",e)}})()},[e,v]);let b=async t=>{if(e&&v)try{await (0,f.teamMemberAddCall)(e,t,{user_id:v,role:"user"}),h.default.success("Successfully joined team"),p(e=>e.filter(e=>e.team_id!==t))}catch(e){console.error("Error joining team:",e),h.default.fromBackend("Failed to join team")}};return(0,t.jsx)(u.Card,{className:"w-full mx-auto flex-auto overflow-y-auto max-h-[50vh]",children:(0,t.jsxs)(n.Table,{children:[(0,t.jsx)(i.TableHead,{children:(0,t.jsxs)(o.TableRow,{children:[(0,t.jsx)(l.TableHeaderCell,{children:"Team Name"}),(0,t.jsx)(l.TableHeaderCell,{children:"Description"}),(0,t.jsx)(l.TableHeaderCell,{children:"Members"}),(0,t.jsx)(l.TableHeaderCell,{children:"Models"}),(0,t.jsx)(l.TableHeaderCell,{children:"Actions"})]})}),(0,t.jsxs)(a.TableBody,{children:[g.map(e=>(0,t.jsxs)(o.TableRow,{children:[(0,t.jsx)(s.TableCell,{children:(0,t.jsx)(d.Text,{children:e.team_alias})}),(0,t.jsx)(s.TableCell,{children:(0,t.jsx)(d.Text,{children:e.description||"No description available"})}),(0,t.jsx)(s.TableCell,{children:(0,t.jsxs)(d.Text,{children:[e.members_with_roles.length," members"]})}),(0,t.jsx)(s.TableCell,{children:(0,t.jsx)("div",{className:"flex flex-col",children:e.models&&0!==e.models.length?e.models.map((e,r)=>(0,t.jsx)(m.Badge,{size:"xs",className:"mb-1",color:"blue",children:(0,t.jsx)(d.Text,{children:e.length>30?`${e.slice(0,30)}...`:e})},r)):(0,t.jsx)(m.Badge,{size:"xs",color:"red",children:(0,t.jsx)(d.Text,{children:"All Proxy Models"})})})}),(0,t.jsx)(s.TableCell,{children:(0,t.jsx)(c.Button,{size:"xs",variant:"secondary",onClick:()=>b(e.team_id),children:"Join Team"})})]},e.team_id)),0===g.length&&(0,t.jsx)(o.TableRow,{children:(0,t.jsx)(s.TableCell,{colSpan:5,className:"text-center",children:(0,t.jsxs)(d.Text,{children:["No available teams to join. See how to set available teams"," ",(0,t.jsx)("a",{href:"https://docs.litellm.ai/docs/proxy/self_serve#all-settings-for-self-serve--sso-flow",target:"_blank",rel:"noopener noreferrer",className:"text-blue-500 hover:text-blue-700 underline",children:"here"}),"."]})})})]})]})})}])}]); \ No newline at end of file diff --git a/litellm/proxy/_experimental/out/_next/static/chunks/0-4-0.z8w6y27.js b/litellm/proxy/_experimental/out/_next/static/chunks/0-4-0.z8w6y27.js new file mode 100644 index 00000000000..1254d125f9a --- /dev/null +++ b/litellm/proxy/_experimental/out/_next/static/chunks/0-4-0.z8w6y27.js @@ -0,0 +1 @@ +(globalThis.TURBOPACK||(globalThis.TURBOPACK=[])).push(["object"==typeof document?document.currentScript:void 0,596239,e=>{"use strict";e.i(247167);var t=e.i(931067),r=e.i(271645);let a={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M574 665.4a8.03 8.03 0 00-11.3 0L446.5 781.6c-53.8 53.8-144.6 59.5-204 0-59.5-59.5-53.8-150.2 0-204l116.2-116.2c3.1-3.1 3.1-8.2 0-11.3l-39.8-39.8a8.03 8.03 0 00-11.3 0L191.4 526.5c-84.6 84.6-84.6 221.5 0 306s221.5 84.6 306 0l116.2-116.2c3.1-3.1 3.1-8.2 0-11.3L574 665.4zm258.6-474c-84.6-84.6-221.5-84.6-306 0L410.3 307.6a8.03 8.03 0 000 11.3l39.7 39.7c3.1 3.1 8.2 3.1 11.3 0l116.2-116.2c53.8-53.8 144.6-59.5 204 0 59.5 59.5 53.8 150.2 0 204L665.3 562.6a8.03 8.03 0 000 11.3l39.8 39.8c3.1 3.1 8.2 3.1 11.3 0l116.2-116.2c84.5-84.6 84.5-221.5 0-306.1zM610.1 372.3a8.03 8.03 0 00-11.3 0L372.3 598.7a8.03 8.03 0 000 11.3l39.6 39.6c3.1 3.1 8.2 3.1 11.3 0l226.4-226.4c3.1-3.1 3.1-8.2 0-11.3l-39.5-39.6z"}}]},name:"link",theme:"outlined"};var l=e.i(9583),o=r.forwardRef(function(e,o){return r.createElement(l.default,(0,t.default)({},e,{ref:o,icon:a}))});e.s(["LinkOutlined",0,o],596239)},362024,e=>{"use strict";var t=e.i(988122);e.s(["Collapse",()=>t.default])},21548,e=>{"use strict";var t=e.i(616303);e.s(["Empty",()=>t.default])},700514,e=>{"use strict";var t=e.i(271645);e.s(["defaultPageSize",0,25,"useBaseUrl",0,()=>{let[e,r]=(0,t.useState)("http://localhost:4000");return(0,t.useEffect)(()=>{{let{protocol:e,host:t}=window.location;r(`${e}//${t}`)}},[]),e}])},918549,e=>{"use strict";let t=(0,e.i(475254).default)("shield",[["path",{d:"M20 13c0 5-3.5 7.5-7.66 8.95a1 1 0 0 1-.67-.01C7.5 20.5 4 18 4 13V6a1 1 0 0 1 1-1c2 0 4.5-1.2 6.24-2.72a1.17 1.17 0 0 1 1.52 0C14.51 3.81 17 5 19 5a1 1 0 0 1 1 1z",key:"oel41y"}]]);e.s(["default",0,t])},98919,e=>{"use strict";var t=e.i(918549);e.s(["Shield",()=>t.default])},727612,e=>{"use strict";let t=(0,e.i(475254).default)("trash-2",[["path",{d:"M3 6h18",key:"d0wm0j"}],["path",{d:"M19 6v14c0 1-1 2-2 2H7c-1 0-2-1-2-2V6",key:"4alrt4"}],["path",{d:"M8 6V4c0-1 1-2 2-2h4c1 0 2 1 2 2v2",key:"v07s0e"}],["line",{x1:"10",x2:"10",y1:"11",y2:"17",key:"1uufr5"}],["line",{x1:"14",x2:"14",y1:"11",y2:"17",key:"xtxkd"}]]);e.s(["Trash2",0,t],727612)},688511,823429,e=>{"use strict";let t=(0,e.i(475254).default)("square-pen",[["path",{d:"M12 3H5a2 2 0 0 0-2 2v14a2 2 0 0 0 2 2h14a2 2 0 0 0 2-2v-7",key:"1m0v6g"}],["path",{d:"M18.375 2.625a1 1 0 0 1 3 3l-9.013 9.014a2 2 0 0 1-.853.505l-2.873.84a.5.5 0 0 1-.62-.62l.84-2.873a2 2 0 0 1 .506-.852z",key:"ohrbg2"}]]);e.s(["default",0,t],823429),e.s(["Edit",0,t],688511)},114600,e=>{"use strict";var t=e.i(290571),r=e.i(444755),a=e.i(673706),l=e.i(271645);let o=(0,a.makeClassName)("Divider"),s=l.default.forwardRef((e,a)=>{let{className:s,children:i}=e,d=(0,t.__rest)(e,["className","children"]);return l.default.createElement("div",Object.assign({ref:a,className:(0,r.tremorTwMerge)(o("root"),"w-full mx-auto my-6 flex justify-between gap-3 items-center text-tremor-default","text-tremor-content","dark:text-dark-tremor-content",s)},d),i?l.default.createElement(l.default.Fragment,null,l.default.createElement("div",{className:(0,r.tremorTwMerge)("w-full h-[1px] bg-tremor-border dark:bg-dark-tremor-border")}),l.default.createElement("div",{className:(0,r.tremorTwMerge)("text-inherit whitespace-nowrap")},i),l.default.createElement("div",{className:(0,r.tremorTwMerge)("w-full h-[1px] bg-tremor-border dark:bg-dark-tremor-border")})):l.default.createElement("div",{className:(0,r.tremorTwMerge)("w-full h-[1px] bg-tremor-border dark:bg-dark-tremor-border")}))});s.displayName="Divider",e.s(["Divider",0,s],114600)},366283,e=>{"use strict";var t=e.i(290571),r=e.i(271645),a=e.i(95779),l=e.i(444755),o=e.i(673706);let s=(0,o.makeClassName)("Callout"),i=r.default.forwardRef((e,i)=>{let{title:d,icon:c,color:n,className:u,children:f}=e,m=(0,t.__rest)(e,["title","icon","color","className","children"]);return r.default.createElement("div",Object.assign({ref:i,className:(0,l.tremorTwMerge)(s("root"),"flex flex-col overflow-hidden rounded-tremor-default text-tremor-default border-l-4 py-3 pr-3 pl-4",n?(0,l.tremorTwMerge)((0,o.getColorClassNames)(n,a.colorPalette.background).bgColor,(0,o.getColorClassNames)(n,a.colorPalette.darkBorder).borderColor,(0,o.getColorClassNames)(n,a.colorPalette.darkText).textColor,"dark:bg-opacity-10 bg-opacity-10"):(0,l.tremorTwMerge)("bg-tremor-brand-faint border-tremor-brand-emphasis text-tremor-brand-emphasis","dark:bg-dark-tremor-brand-muted/70 dark:border-dark-tremor-brand-emphasis dark:text-dark-tremor-brand-emphasis"),u)},m),r.default.createElement("div",{className:(0,l.tremorTwMerge)(s("header"),"flex items-start")},c?r.default.createElement(c,{className:(0,l.tremorTwMerge)(s("icon"),"flex-none h-5 w-5 mr-1.5")}):null,r.default.createElement("h4",{className:(0,l.tremorTwMerge)(s("title"),"font-semibold")},d)),r.default.createElement("p",{className:(0,l.tremorTwMerge)(s("body"),"overflow-y-auto",f?"mt-2":"")},f))});i.displayName="Callout",e.s(["Callout",0,i],366283)},475647,e=>{"use strict";e.i(247167);var t=e.i(931067),r=e.i(271645);let a={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M696 480H544V328c0-4.4-3.6-8-8-8h-48c-4.4 0-8 3.6-8 8v152H328c-4.4 0-8 3.6-8 8v48c0 4.4 3.6 8 8 8h152v152c0 4.4 3.6 8 8 8h48c4.4 0 8-3.6 8-8V544h152c4.4 0 8-3.6 8-8v-48c0-4.4-3.6-8-8-8z"}},{tag:"path",attrs:{d:"M512 64C264.6 64 64 264.6 64 512s200.6 448 448 448 448-200.6 448-448S759.4 64 512 64zm0 820c-205.4 0-372-166.6-372-372s166.6-372 372-372 372 166.6 372 372-166.6 372-372 372z"}}]},name:"plus-circle",theme:"outlined"};var l=e.i(9583),o=r.forwardRef(function(e,o){return r.createElement(l.default,(0,t.default)({},e,{ref:o,icon:a}))});e.s(["PlusCircleOutlined",0,o],475647)},153472,e=>{"use strict";var t,r,a=e.i(266027),l=e.i(954616),o=e.i(912598),s=e.i(243652),i=e.i(135214),d=e.i(764205),c=((t={}).GENERAL_SETTINGS="general_settings",t),n=((r={}).MAXIMUM_SPEND_LOGS_RETENTION_PERIOD="maximum_spend_logs_retention_period",r);let u=async(e,t)=>{try{let r=d.proxyBaseUrl?`${d.proxyBaseUrl}/config/list?config_type=${t}`:`/config/list?config_type=${t}`,a=await fetch(r,{method:"GET",headers:{[(0,d.getGlobalLitellmHeaderName)()]:`Bearer ${e}`,"Content-Type":"application/json"}});if(!a.ok){let e=await a.json(),t=(0,d.deriveErrorMessage)(e);throw(0,d.handleError)(t),Error(t)}return await a.json()}catch(e){throw console.error(`Failed to get proxy config for ${t}:`,e),e}},f=(0,s.createQueryKeys)("proxyConfig"),m=async(e,t)=>{try{let r=d.proxyBaseUrl?`${d.proxyBaseUrl}/config/field/delete`:"/config/field/delete",a=await fetch(r,{method:"POST",headers:{[(0,d.getGlobalLitellmHeaderName)()]:`Bearer ${e}`,"Content-Type":"application/json"},body:JSON.stringify(t)});if(!a.ok){let e=await a.json(),t=(0,d.deriveErrorMessage)(e);throw(0,d.handleError)(t),Error(t)}return await a.json()}catch(e){throw console.error(`Failed to delete proxy config field ${t.field_name}:`,e),e}};e.s(["ConfigType",()=>c,"GeneralSettingsFieldName",()=>n,"proxyConfigKeys",0,f,"useDeleteProxyConfigField",0,()=>{let{accessToken:e}=(0,i.default)(),t=(0,o.useQueryClient)();return(0,l.useMutation)({mutationFn:async t=>{if(!e)throw Error("Access token is required");return await m(e,t)},onSuccess:()=>{t.invalidateQueries({queryKey:f.all})}})},"useProxyConfig",0,e=>{let{accessToken:t}=(0,i.default)();return(0,a.useQuery)({queryKey:f.list({filters:{configType:e}}),queryFn:async()=>await u(t,e),enabled:!!t})}])},286536,77705,e=>{"use strict";var t=e.i(475254);let r=(0,t.default)("eye",[["path",{d:"M2.062 12.348a1 1 0 0 1 0-.696 10.75 10.75 0 0 1 19.876 0 1 1 0 0 1 0 .696 10.75 10.75 0 0 1-19.876 0",key:"1nclc0"}],["circle",{cx:"12",cy:"12",r:"3",key:"1v7zrd"}]]);e.s(["Eye",0,r],286536);let a=(0,t.default)("eye-off",[["path",{d:"M10.733 5.076a10.744 10.744 0 0 1 11.205 6.575 1 1 0 0 1 0 .696 10.747 10.747 0 0 1-1.444 2.49",key:"ct8e1f"}],["path",{d:"M14.084 14.158a3 3 0 0 1-4.242-4.242",key:"151rxh"}],["path",{d:"M17.479 17.499a10.75 10.75 0 0 1-15.417-5.151 1 1 0 0 1 0-.696 10.75 10.75 0 0 1 4.446-5.143",key:"13bj9a"}],["path",{d:"m2 2 20 20",key:"1ooewy"}]]);e.s(["EyeOff",0,a],77705)},514236,e=>{"use strict";var t=e.i(843476),r=e.i(105278);e.s(["default",0,()=>(0,t.jsx)(r.default,{})])}]); \ No newline at end of file diff --git a/litellm/proxy/_experimental/out/_next/static/chunks/0-ne4el5qxs1z.js b/litellm/proxy/_experimental/out/_next/static/chunks/0-ne4el5qxs1z.js new file mode 100644 index 00000000000..a7a6f306c77 --- /dev/null +++ b/litellm/proxy/_experimental/out/_next/static/chunks/0-ne4el5qxs1z.js @@ -0,0 +1,40 @@ +(globalThis.TURBOPACK||(globalThis.TURBOPACK=[])).push(["object"==typeof document?document.currentScript:void 0,921687,e=>{"use strict";var t=e.i(764205);let s=async(e,s)=>{try{let r=s||(0,t.getProxyBaseUrl)(),a=r?`${r}/v1/agents`:"/v1/agents",n=await fetch(a,{method:"GET",headers:{[(0,t.getGlobalLitellmHeaderName)()]:`Bearer ${e}`,"Content-Type":"application/json"}});if(!n.ok){let e=await n.json();throw Error(e.detail||"Failed to fetch agents")}let i=await n.json();return console.log("Fetched agents:",i),i.sort((e,t)=>{let s=e.agent_name||e.agent_id,r=t.agent_name||t.agent_id;return s.localeCompare(r)}),i}catch(e){throw console.error("Error fetching agents:",e),e}},r=async(e,s,r,a)=>{try{let a=await (0,t.modelInfoCall)(e,s,r,1,200),n=a?.data??[],i=(Array.isArray(n)?n:[]).filter(e=>"string"==typeof e?.litellm_params?.model&&e.litellm_params.model.startsWith("litellm_agent/")).map(e=>({model_name:e.model_name??e.model_group??"",litellm_params:{...e.litellm_params,model:e.litellm_params.model,litellm_system_prompt:e.litellm_params?.litellm_system_prompt,tools:Array.isArray(e.litellm_params?.tools)?e.litellm_params.tools:void 0},model_info:e.model_info??null}));return i.sort((e,t)=>e.model_name.localeCompare(t.model_name)),i}catch(e){throw console.error("Error fetching agent models:",e),e}};e.s(["fetchAvailableAgentModels",0,r,"fetchAvailableAgents",0,s])},124608,614677,235267,318059,953860,434788,512882,584976,720762,e=>{"use strict";let t,s,r;e.i(247167);var a,n,i,o,l,c,d,u,h,p,m,g,f,y,x,b,v,w,_,S,j,k,N,E,T,C,A,P,O,I,R,M,$,L,U,D,q,B,W,H,z,F,J,G,V,K,X,Y,Q,Z,ee,et,es,er,ea,en,ei,eo,el,ec,ed,eu,eh,ep,em,eg,ef,ey,ex=e.i(931067),eb=e.i(271645);let ev={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M928 160H96c-17.7 0-32 14.3-32 32v640c0 17.7 14.3 32 32 32h832c17.7 0 32-14.3 32-32V192c0-17.7-14.3-32-32-32zm-40 632H136v-39.9l138.5-164.3 150.1 178L658.1 489 888 761.6V792zm0-129.8L664.2 396.8c-3.2-3.8-9-3.8-12.2 0L424.6 666.4l-144-170.7c-3.2-3.8-9-3.8-12.2 0L136 652.7V232h752v430.2zM304 456a88 88 0 100-176 88 88 0 000 176zm0-116c15.5 0 28 12.5 28 28s-12.5 28-28 28-28-12.5-28-28 12.5-28 28-28z"}}]},name:"picture",theme:"outlined"};var ew=e.i(9583),e_=eb.forwardRef(function(e,t){return eb.createElement(ew.default,(0,ex.default)({},e,{ref:t,icon:ev}))});e.s(["PictureOutlined",0,e_],124608);let eS=new Uint8Array(16),ej=[];for(let e=0;e<256;++e)ej.push((e+256).toString(16).slice(1));let ek=function(e,t,s){return t||e||!crypto.randomUUID?function(e,t,s){let r=(e=e||{}).random??e.rng?.()??crypto.getRandomValues(eS);if(r.length<16)throw Error("Random bytes length must be >= 16");if(r[6]=15&r[6]|64,r[8]=63&r[8]|128,t){if((s=s||0)<0||s+16>t.length)throw RangeError(`UUID byte range ${s}:${s+15} is out of buffer bounds`);for(let e=0;e<16;++e)t[s+e]=r[e];return t}return function(e,t=0){return(ej[e[t+0]]+ej[e[t+1]]+ej[e[t+2]]+ej[e[t+3]]+"-"+ej[e[t+4]]+ej[e[t+5]]+"-"+ej[e[t+6]]+ej[e[t+7]]+"-"+ej[e[t+8]]+ej[e[t+9]]+"-"+ej[e[t+10]]+ej[e[t+11]]+ej[e[t+12]]+ej[e[t+13]]+ej[e[t+14]]+ej[e[t+15]]).toLowerCase()}(r)}(e,t,s):crypto.randomUUID()};e.s(["v4",0,ek],614677);var eN=e.i(843476),eE=e.i(808613),eT=e.i(311451),eC=e.i(28651),eA=e.i(199133),eP=e.i(592968),eO=e.i(827252);function eI(e){if(!e)return[];if(Array.isArray(e))return e.map(e=>eR(e)).filter(e=>void 0!==e);let t=eR(e);return void 0!==t?[t]:[]}function eR(e,t){if(!e)return;let s=void 0!==t?t:e.default;if("object"===e.type){let t="object"!=typeof s||null===s||Array.isArray(s)?{}:{...s};return e.properties&&Object.entries(e.properties).forEach(([e,s])=>{t[e]=eR(s,t[e])}),t}if("array"===e.type){if(Array.isArray(s)){let t=e.items;if(!t)return s;if(0===s.length){let e=eI(t);return e.length?e:s}return Array.isArray(t)?s.map((e,s)=>eR(t[s]??t[t.length-1],e)):s.map(e=>eR(t,e))}return void 0!==s?s:eI(e.items)}if(void 0!==s)return s;switch(e.type){case"integer":case"number":return 0;case"boolean":return!1;default:return""}}let eM=e=>{let t=eR(e);if("object"===e.type||"array"===e.type){let s="array"===e.type?[]:{};return JSON.stringify(t??s,null,2)}return t},e$=(0,eb.forwardRef)(({tool:e,className:t},s)=>{let[r]=eE.Form.useForm(),a=(0,eb.useMemo)(()=>"string"==typeof e.inputSchema?{type:"object",properties:{input:{type:"string",description:"Input for this tool"}},required:["input"]}:e.inputSchema,[e.inputSchema]),n=(0,eb.useMemo)(()=>a.properties?.params?.type==="object"&&a.properties.params.properties?{type:"object",properties:a.properties.params.properties,required:a.properties.params.required||[]}:a,[a]);return((0,eb.useImperativeHandle)(s,()=>({getSubmitValues:async()=>{var e;let t;return e=await r.validateFields(),t={},Object.entries(e).forEach(([e,s])=>{let r=n.properties?.[e];if(r&&null!=s&&""!==s)switch(r.type){case"boolean":t[e]="true"===s||!0===s;break;case"number":case"integer":{let a=Number(s);t[e]=Number.isNaN(a)?s:"integer"===r.type?Math.trunc(a):a;break}case"object":case"array":try{let a="string"==typeof s?JSON.parse(s):s,n="object"===r.type&&null!==a&&"object"==typeof a&&!Array.isArray(a),i="array"===r.type&&Array.isArray(a);"object"===r.type&&n||"array"===r.type&&i?t[e]=a:t[e]=s}catch{t[e]=s}break;case"string":t[e]=String(s);break;default:t[e]=s}else null!=s&&""!==s&&(t[e]=s)}),a.properties?.params?.type==="object"&&a.properties.params.properties?{params:t}:t}})),eb.default.useEffect(()=>{if(r.resetFields(),!n.properties)return;let e={};Object.entries(n.properties).forEach(([t,s])=>{e[t]=eM(s)}),r.setFieldsValue(e)},[r,n,e]),"string"==typeof e.inputSchema)?(0,eN.jsx)(eE.Form,{form:r,layout:"vertical",className:t,children:(0,eN.jsx)(eE.Form.Item,{label:(0,eN.jsxs)("span",{className:"text-sm font-medium text-gray-700",children:["Input ",(0,eN.jsx)("span",{className:"text-red-500",children:"*"})]}),name:"input",rules:[{required:!0,message:"Please enter input for this tool"}],children:(0,eN.jsx)(eT.Input,{placeholder:"Enter input for this tool"})})}):n.properties?(0,eN.jsx)(eE.Form,{form:r,layout:"vertical",className:t,children:Object.entries(n.properties).map(([t,s])=>{let r=eM(s),a=`${e.name}-${t}`;return(0,eN.jsx)(eE.Form.Item,{label:(0,eN.jsxs)("span",{className:"text-sm font-medium text-gray-700 flex items-center",children:[t," ",n.required?.includes(t)&&(0,eN.jsx)("span",{className:"text-red-500",children:"*"}),s.description&&(0,eN.jsx)(eP.Tooltip,{title:s.description,children:(0,eN.jsx)(eO.InfoCircleOutlined,{className:"ml-2 text-gray-400 hover:text-gray-600"})})]}),name:t,initialValue:r,rules:[{required:n.required?.includes(t),message:`Please enter ${t}`},..."object"===s.type||"array"===s.type?[{validator:(e,r)=>{if((null==r||""===r)&&!n.required?.includes(t))return Promise.resolve();try{let e="string"==typeof r?JSON.parse(r):r,t="object"===s.type&&null!==e&&"object"==typeof e&&!Array.isArray(e),a="array"===s.type&&Array.isArray(e);if("object"===s.type&&t||"array"===s.type&&a)return Promise.resolve();return Promise.reject(Error("object"===s.type?"Please enter a JSON object":"Please enter a JSON array"))}catch{return Promise.reject(Error("Invalid JSON"))}}}]:[]],children:"string"===s.type&&s.enum?(0,eN.jsx)(eA.Select,{placeholder:`Select ${t}`,allowClear:!n.required?.includes(t),options:s.enum.map(e=>({value:e,label:e}))}):"string"!==s.type||s.enum?"number"===s.type||"integer"===s.type?(0,eN.jsx)(eC.InputNumber,{step:"integer"===s.type?1:void 0,placeholder:s.description||`Enter ${t}`,className:"w-full",style:{width:"100%"}}):"boolean"===s.type?(0,eN.jsx)(eA.Select,{placeholder:`Select ${t}`,allowClear:!n.required?.includes(t),options:[{value:!0,label:"True"},{value:!1,label:"False"}]}):"object"===s.type||"array"===s.type?(0,eN.jsx)(eT.Input.TextArea,{rows:"object"===s.type?4:3,placeholder:s.description||("object"===s.type?`Enter JSON object for ${t}`:`Enter JSON array for ${t}`),spellCheck:!1,className:"font-mono"}):(0,eN.jsx)(eT.Input,{placeholder:s.description||`Enter ${t}`,allowClear:!0}):(0,eN.jsx)(eT.Input,{placeholder:s.description||`Enter ${t}`,allowClear:!0})},a)})}):(0,eN.jsx)(eE.Form,{form:r,layout:"vertical",className:t,children:(0,eN.jsx)("div",{className:"py-4 text-center text-sm text-gray-500",children:"No parameters required for this tool."})})});e$.displayName="MCPToolArgumentsForm",e.s(["default",0,e$],235267);var eL=e.i(764205);e.s(["default",0,({onChange:e,value:t,className:s,accessToken:r})=>{let[a,n]=(0,eb.useState)([]),[i,o]=(0,eb.useState)(!1);return(0,eb.useEffect)(()=>{(async()=>{if(r)try{let e=await (0,eL.tagListCall)(r);console.log("List tags response:",e),n(Object.values(e))}catch(e){console.error("Error fetching tags:",e)}finally{o(!1)}})()},[r]),(0,eN.jsx)(eA.Select,{mode:"tags",showSearch:!0,placeholder:"Select or create tags",onChange:e,value:t,loading:i,className:s,options:a.map(e=>({label:e.name,value:e.name,title:e.description||e.name})),optionFilterProp:"label",tokenSeparators:[","],maxTagCount:"responsive",allowClear:!0,style:{width:"100%"}})}],318059);let eU=e=>{if(!e)return;let t={};if(e.id&&(t.taskId=e.id),e.contextId&&(t.contextId=e.contextId),e.status&&(t.status={state:e.status.state,timestamp:e.status.timestamp},e.status.message?.parts)){let s=e.status.message.parts.filter(e=>"text"===e.kind&&e.text).map(e=>e.text).join(" ");s&&(t.status.message=s)}return e.metadata&&"object"==typeof e.metadata&&(t.metadata=e.metadata),Object.keys(t).length>0?t:void 0},eD=async(e,t,s,r,a,n,i,o,l,c)=>{let d=l||(0,eL.getProxyBaseUrl)(),u=d?`${d}/a2a/${e}/message/send`:`/a2a/${e}/message/send`,h={jsonrpc:"2.0",id:ek(),method:"message/send",params:{message:{kind:"message",messageId:ek().replace(/-/g,""),role:"user",parts:[{kind:"text",text:t}]}}};c&&c.length>0&&(h.params.metadata={guardrails:c});let p=performance.now();try{let t=await fetch(u,{method:"POST",headers:{[(0,eL.getGlobalLitellmHeaderName)()]:`Bearer ${r}`,"Content-Type":"application/json"},body:JSON.stringify(h),signal:a}),l=performance.now()-p;if(n&&n(l),!t.ok){let e=await t.json();throw Error(e.error?.message||e.detail||`HTTP ${t.status}`)}let c=await t.json(),d=performance.now()-p;if(i&&i(d),c.error)throw Error(c.error.message);let m=c.result;if(m){let t="",r=eU(m);if(r&&o&&o(r),m.artifacts&&Array.isArray(m.artifacts)){for(let e of m.artifacts)if(e.parts&&Array.isArray(e.parts))for(let s of e.parts)"text"===s.kind&&s.text&&(t+=s.text)}else if(m.parts&&Array.isArray(m.parts))for(let e of m.parts)"text"===e.kind&&e.text&&(t+=e.text);else if(m.status?.message?.parts)for(let e of m.status.message.parts)"text"===e.kind&&e.text&&(t+=e.text);t?s(t,`a2a_agent/${e}`):(console.warn("Could not extract text from A2A response, showing raw JSON:",m),s(JSON.stringify(m,null,2),`a2a_agent/${e}`))}}catch(e){if(a?.aborted)return void console.log("A2A request was cancelled");throw console.error("A2A send message error:",e),e}},eq=async(e,t,s,r,a,n,i,o,l)=>{let c,d=l||(0,eL.getProxyBaseUrl)(),u=d?`${d}/a2a/${e}`:`/a2a/${e}`,h=ek(),p=ek().replace(/-/g,""),m=performance.now(),g=!1,f="";try{let l=await fetch(u,{method:"POST",headers:{[(0,eL.getGlobalLitellmHeaderName)()]:`Bearer ${r}`,"Content-Type":"application/json"},body:JSON.stringify({jsonrpc:"2.0",id:h,method:"message/stream",params:{message:{kind:"message",messageId:p,role:"user",parts:[{kind:"text",text:t}]}}}),signal:a});if(!l.ok){let e=await l.json();throw Error(e.error?.message||e.detail||`HTTP ${l.status}`)}let d=l.body?.getReader();if(!d)throw Error("No response body");let y=new TextDecoder,x="",b=!1;for(;!b;){let t=await d.read();b=t.done;let r=t.value;if(b)break;let a=(x+=y.decode(r,{stream:!0})).split("\n");for(let t of(x=a.pop()||"",a))if(t.trim())try{let r=JSON.parse(t);if(!g){g=!0;let e=performance.now()-m;n&&n(e)}let a=r.result;if(a){let t=eU(a);t&&(c={...c,...t});let r=a.kind;if("artifact-update"===r&&a.artifact){let t=a.artifact;if(t.parts&&Array.isArray(t.parts))for(let r of t.parts)"text"===r.kind&&r.text&&(f+=r.text,s(f,`a2a_agent/${e}`))}else if(a.artifacts&&Array.isArray(a.artifacts)){for(let t of a.artifacts)if(t.parts&&Array.isArray(t.parts))for(let r of t.parts)"text"===r.kind&&r.text&&(f+=r.text,s(f,`a2a_agent/${e}`))}else if("status-update"===r);else if(a.parts&&Array.isArray(a.parts))for(let t of a.parts)"text"===t.kind&&t.text&&(f+=t.text,s(f,`a2a_agent/${e}`))}if(r.error){let e=r.error.message||"Unknown A2A error";throw Error(e)}}catch(e){if(e instanceof Error&&e.message&&!e.message.includes("JSON"))throw e;t.trim().length>0&&console.warn("Failed to parse A2A streaming chunk:",t,e)}}let v=performance.now()-m;i&&i(v),c&&o&&o(c)}catch(e){if(a?.aborted)return void console.log("A2A streaming request was cancelled");throw console.error("A2A stream message error:",e),e}};function eB(e,t,s,r,a){if("m"===r)throw TypeError("Private method is not writable");if("a"===r&&!a)throw TypeError("Private accessor was defined without a setter");if("function"==typeof t?e!==t||!a:!t.has(e))throw TypeError("Cannot write private member to an object whose class did not declare it");return"a"===r?a.call(e,s):a?a.value=s:t.set(e,s),s}function eW(e,t,s,r){if("a"===s&&!r)throw TypeError("Private accessor was defined without a getter");if("function"==typeof t?e!==t||!r:!t.has(e))throw TypeError("Cannot read private member from an object whose class did not declare it");return"m"===s?r:"a"===s?r.call(e):r?r.value:t.get(e)}e.s(["makeA2ASendMessageRequest",0,eD,"makeA2AStreamMessageRequest",0,eq],953860);let eH=function(){let{crypto:e}=globalThis;if(e?.randomUUID)return eH=e.randomUUID.bind(e),e.randomUUID();let t=new Uint8Array(1),s=e?()=>e.getRandomValues(t)[0]:()=>255*Math.random()&255;return"10000000-1000-4000-8000-100000000000".replace(/[018]/g,e=>(e^s()&15>>e/4).toString(16))};function ez(e){return"object"==typeof e&&null!==e&&("name"in e&&"AbortError"===e.name||"message"in e&&String(e.message).includes("FetchRequestCanceledException"))}let eF=e=>{if(e instanceof Error)return e;if("object"==typeof e&&null!==e){try{if("[object Error]"===Object.prototype.toString.call(e)){let t=Error(e.message,e.cause?{cause:e.cause}:{});return e.stack&&(t.stack=e.stack),e.cause&&!t.cause&&(t.cause=e.cause),e.name&&(t.name=e.name),t}}catch{}try{return Error(JSON.stringify(e))}catch{}}return Error(e)};class eJ extends Error{}class eG extends eJ{constructor(e,t,s,r,a){super(`${eG.makeMessage(e,t,s)}`),this.status=e,this.headers=r,this.requestID=r?.get("request-id"),this.error=t,this.type=a??null}static makeMessage(e,t,s){let r=t?.message?"string"==typeof t.message?t.message:JSON.stringify(t.message):t?JSON.stringify(t):s;return e&&r?`${e} ${r}`:e?`${e} status code (no body)`:r||"(no status code or body)"}static generate(e,t,s,r){if(!e||!r)return new eK({message:s,cause:eF(t)});let a=t?.error?.type;return 400===e?new eY(e,t,s,r,a):401===e?new eQ(e,t,s,r,a):403===e?new eZ(e,t,s,r,a):404===e?new e0(e,t,s,r,a):409===e?new e1(e,t,s,r,a):422===e?new e2(e,t,s,r,a):429===e?new e4(e,t,s,r,a):e>=500?new e6(e,t,s,r,a):new eG(e,t,s,r,a)}}class eV extends eG{constructor({message:e}={}){super(void 0,void 0,e||"Request was aborted.",void 0)}}class eK extends eG{constructor({message:e,cause:t}){super(void 0,void 0,e||"Connection error.",void 0),t&&(this.cause=t)}}class eX extends eK{constructor({message:e}={}){super({message:e??"Request timed out."})}}class eY extends eG{}class eQ extends eG{}class eZ extends eG{}class e0 extends eG{}class e1 extends eG{}class e2 extends eG{}class e4 extends eG{}class e6 extends eG{}let e3=/^[a-z][a-z0-9+.-]*:/i,e5=e=>(e5=Array.isArray)(e),e8=e5;function e7(e){return"object"!=typeof e?{}:e??{}}function e9(e){if(!e)return!0;for(let t in e)return!1;return!0}let te=e=>{try{return JSON.parse(e)}catch(e){return}},tt="0.92.0",ts=e=>"x32"===e?"x32":"x86_64"===e||"x64"===e?"x64":"arm"===e?"arm":"aarch64"===e||"arm64"===e?"arm64":e?`other:${e}`:"unknown",tr=e=>(e=e.toLowerCase()).includes("ios")?"iOS":"android"===e?"Android":"darwin"===e?"MacOS":"win32"===e?"Windows":"freebsd"===e?"FreeBSD":"openbsd"===e?"OpenBSD":"linux"===e?"Linux":e?`Other:${e}`:"Unknown";function ta(...e){let t=globalThis.ReadableStream;if(void 0===t)throw Error("`ReadableStream` is not defined as a global; You will need to polyfill it, `globalThis.ReadableStream = ReadableStream`");return new t(...e)}function tn(e){let t=Symbol.asyncIterator in e?e[Symbol.asyncIterator]():e[Symbol.iterator]();return ta({start(){},async pull(e){let{done:s,value:r}=await t.next();s?e.close():e.enqueue(r)},async cancel(){await t.return?.()}})}function ti(e){if(e[Symbol.asyncIterator])return e;let t=e.getReader();return{async next(){try{let e=await t.read();return e?.done&&t.releaseLock(),e}catch(e){throw t.releaseLock(),e}},async return(){let e=t.cancel();return t.releaseLock(),await e,{done:!0,value:void 0}},[Symbol.asyncIterator](){return this}}}async function to(e){if(null===e||"object"!=typeof e)return;if(e[Symbol.asyncIterator])return void await e[Symbol.asyncIterator]().return?.();let t=e.getReader(),s=t.cancel();t.releaseLock(),await s}let tl=({headers:e,body:t})=>({bodyHeaders:{"content-type":"application/json"},body:JSON.stringify(t)});function tc(e){let t;return(s??(s=(t=new globalThis.TextEncoder).encode.bind(t)))(e)}function td(e){let t;return(r??(r=(t=new globalThis.TextDecoder).decode.bind(t)))(e)}class tu{constructor(){a.set(this,void 0),n.set(this,void 0),eB(this,a,new Uint8Array,"f"),eB(this,n,null,"f")}decode(e){let t;if(null==e)return[];let s=e instanceof ArrayBuffer?new Uint8Array(e):"string"==typeof e?tc(e):e;eB(this,a,function(e){let t=0;for(let s of e)t+=s.length;let s=new Uint8Array(t),r=0;for(let t of e)s.set(t,r),r+=t.length;return s}([eW(this,a,"f"),s]),"f");let r=[];for(;null!=(t=function(e,t){for(let s=t??0;s{if(e){if(Object.prototype.hasOwnProperty.call(th,e))return e;tx(s).warn(`${t} was set to ${JSON.stringify(e)}, expected one of ${JSON.stringify(Object.keys(th))}`)}};function tm(){}function tg(e,t,s){return!t||th[e]>th[s]?tm:t[e].bind(t)}let tf={error:tm,warn:tm,info:tm,debug:tm},ty=new WeakMap;function tx(e){let t=e.logger,s=e.logLevel??"off";if(!t)return tf;let r=ty.get(t);if(r&&r[0]===s)return r[1];let a={error:tg("error",t,s),warn:tg("warn",t,s),info:tg("info",t,s),debug:tg("debug",t,s)};return ty.set(t,[s,a]),a}let tb=e=>(e.options&&(e.options={...e.options},delete e.options.headers),e.headers&&(e.headers=Object.fromEntries((e.headers instanceof Headers?[...e.headers]:Object.entries(e.headers)).map(([e,t])=>[e,"x-api-key"===e.toLowerCase()||"authorization"===e.toLowerCase()||"cookie"===e.toLowerCase()||"set-cookie"===e.toLowerCase()?"***":t]))),"retryOfRequestLogID"in e&&(e.retryOfRequestLogID&&(e.retryOf=e.retryOfRequestLogID),delete e.retryOfRequestLogID),e);class tv{constructor(e,t,s){this.iterator=e,i.set(this,void 0),this.controller=t,eB(this,i,s,"f")}static fromSSEResponse(e,t,s){let r=!1,a=s?tx(s):console;async function*n(){if(r)throw new eJ("Cannot iterate over a consumed stream, use `.tee()` to split the stream.");r=!0;let s=!1;try{for await(let s of tw(e,t)){if("completion"===s.event)try{yield JSON.parse(s.data)}catch(e){throw a.error("Could not parse message into JSON:",s.data),a.error("From chunk:",s.raw),e}if("message_start"===s.event||"message_delta"===s.event||"message_stop"===s.event||"content_block_start"===s.event||"content_block_delta"===s.event||"content_block_stop"===s.event||"message"===s.event||"user.message"===s.event||"user.interrupt"===s.event||"user.tool_confirmation"===s.event||"user.custom_tool_result"===s.event||"agent.message"===s.event||"agent.thinking"===s.event||"agent.tool_use"===s.event||"agent.tool_result"===s.event||"agent.mcp_tool_use"===s.event||"agent.mcp_tool_result"===s.event||"agent.custom_tool_use"===s.event||"agent.thread_context_compacted"===s.event||"session.status_running"===s.event||"session.status_idle"===s.event||"session.status_rescheduled"===s.event||"session.status_terminated"===s.event||"session.error"===s.event||"session.deleted"===s.event||"span.model_request_start"===s.event||"span.model_request_end"===s.event)try{yield JSON.parse(s.data)}catch(e){throw a.error("Could not parse message into JSON:",s.data),a.error("From chunk:",s.raw),e}if("ping"!==s.event&&"error"===s.event){let t=te(s.data)??s.data,r=t?.error?.type;throw new eG(void 0,t,void 0,e.headers,r)}}s=!0}catch(e){if(ez(e))return;throw e}finally{s||t.abort()}}return new tv(n,t,s)}static fromReadableStream(e,t,s){let r=!1;async function*a(){let t=new tu;for await(let s of ti(e))for(let e of t.decode(s))yield e;for(let e of t.flush())yield e}return new tv(async function*(){if(r)throw new eJ("Cannot iterate over a consumed stream, use `.tee()` to split the stream.");r=!0;let e=!1;try{for await(let t of a())!e&&t&&(yield JSON.parse(t));e=!0}catch(e){if(ez(e))return;throw e}finally{e||t.abort()}},t,s)}[(i=new WeakMap,Symbol.asyncIterator)](){return this.iterator()}tee(){let e=[],t=[],s=this.iterator(),r=r=>({next:()=>{if(0===r.length){let r=s.next();e.push(r),t.push(r)}return r.shift()}});return[new tv(()=>r(e),this.controller,eW(this,i,"f")),new tv(()=>r(t),this.controller,eW(this,i,"f"))]}toReadableStream(){let e,t=this;return ta({async start(){e=t[Symbol.asyncIterator]()},async pull(t){try{let{value:s,done:r}=await e.next();if(r)return t.close();let a=tc(JSON.stringify(s)+"\n");t.enqueue(a)}catch(e){t.error(e)}},async cancel(){await e.return?.()}})}}async function*tw(e,t){if(!e.body){if(t.abort(),void 0!==globalThis.navigator&&"ReactNative"===globalThis.navigator.product)throw new eJ("The default react-native fetch implementation does not support streaming. Please use expo/fetch: https://docs.expo.dev/versions/latest/sdk/expo/#expofetch-api");throw new eJ("Attempted to iterate over a response with no body")}let s=new tS,r=new tu;for await(let t of t_(ti(e.body)))for(let e of r.decode(t)){let t=s.decode(e);t&&(yield t)}for(let e of r.flush()){let t=s.decode(e);t&&(yield t)}}async function*t_(e){let t=new Uint8Array;for await(let s of e){let e;if(null==s)continue;let r=s instanceof ArrayBuffer?new Uint8Array(s):"string"==typeof s?tc(s):s,a=new Uint8Array(t.length+r.length);for(a.set(t),a.set(r,t.length),t=a;-1!==(e=function(e){for(let t=0;t0&&(yield t)}class tS{constructor(){this.event=null,this.data=[],this.chunks=[]}decode(e){var t;let s;if(e.endsWith("\r")&&(e=e.substring(0,e.length-1)),!e){if(!this.event&&!this.data.length)return null;let e={event:this.event,data:this.data.join("\n"),raw:this.chunks};return this.event=null,this.data=[],this.chunks=[],e}if(this.chunks.push(e),e.startsWith(":"))return null;let[r,a,n]=-1!==(s=(t=e).indexOf(":"))?[t.substring(0,s),":",t.substring(s+1)]:[t,"",""];return n.startsWith(" ")&&(n=n.substring(1)),"event"===r?this.event=n:"data"===r&&this.data.push(n),null}}async function tj(e,t){let{response:s,requestLogID:r,retryOfRequestLogID:a,startTime:n}=t,i=await (async()=>{if(t.options.stream)return(tx(e).debug("response",s.status,s.url,s.headers,s.body),t.options.__streamClass)?t.options.__streamClass.fromSSEResponse(s,t.controller):tv.fromSSEResponse(s,t.controller);if(204===s.status)return null;if(t.options.__binaryResponse)return s;let r=s.headers.get("content-type"),a=r?.split(";")[0]?.trim();if(a?.includes("application/json")||a?.endsWith("+json")){if("0"===s.headers.get("content-length"))return;return tk(await s.json(),s)}return await s.text()})();return tx(e).debug(`[${r}] response parsed`,tb({retryOfRequestLogID:a,url:s.url,status:s.status,body:i,durationMs:Date.now()-n})),i}function tk(e,t){return!e||"object"!=typeof e||Array.isArray(e)?e:Object.defineProperty(e,"_request_id",{value:t.headers.get("request-id"),enumerable:!1})}class tN extends Promise{constructor(e,t,s=tj){super(e=>{e(null)}),this.responsePromise=t,this.parseResponse=s,o.set(this,void 0),eB(this,o,e,"f")}_thenUnwrap(e){return new tN(eW(this,o,"f"),this.responsePromise,async(t,s)=>tk(e(await this.parseResponse(t,s),s),s.response))}asResponse(){return this.responsePromise.then(e=>e.response)}async withResponse(){let[e,t]=await Promise.all([this.parse(),this.asResponse()]);return{data:e,response:t,request_id:t.headers.get("request-id")}}parse(){return this.parsedPromise||(this.parsedPromise=this.responsePromise.then(e=>this.parseResponse(eW(this,o,"f"),e))),this.parsedPromise}then(e,t){return this.parse().then(e,t)}catch(e){return this.parse().catch(e)}finally(e){return this.parse().finally(e)}}o=new WeakMap;class tE{constructor(e,t,s,r){l.set(this,void 0),eB(this,l,e,"f"),this.options=r,this.response=t,this.body=s}hasNextPage(){return!!this.getPaginatedItems().length&&null!=this.nextPageRequestOptions()}async getNextPage(){let e=this.nextPageRequestOptions();if(!e)throw new eJ("No next page expected; please check `.hasNextPage()` before calling `.getNextPage()`.");return await eW(this,l,"f").requestAPIList(this.constructor,e)}async *iterPages(){let e=this;for(yield e;e.hasNextPage();)e=await e.getNextPage(),yield e}async *[(l=new WeakMap,Symbol.asyncIterator)](){for await(let e of this.iterPages())for(let t of e.getPaginatedItems())yield t}}class tT extends tN{constructor(e,t,s){super(e,t,async(e,t)=>new s(e,t.response,await tj(e,t),t.options))}async *[Symbol.asyncIterator](){for await(let e of(await this))yield e}}class tC extends tE{constructor(e,t,s,r){super(e,t,s,r),this.data=s.data||[],this.has_more=s.has_more||!1,this.first_id=s.first_id||null,this.last_id=s.last_id||null}getPaginatedItems(){return this.data??[]}hasNextPage(){return!1!==this.has_more&&super.hasNextPage()}nextPageRequestOptions(){if(this.options.query?.before_id){let e=this.first_id;return e?{...this.options,query:{...e7(this.options.query),before_id:e}}:null}let e=this.last_id;return e?{...this.options,query:{...e7(this.options.query),after_id:e}}:null}}class tA extends tE{constructor(e,t,s,r){super(e,t,s,r),this.data=s.data||[],this.next_page=s.next_page||null}getPaginatedItems(){return this.data??[]}nextPageRequestOptions(){let e=this.next_page;return e?{...this.options,query:{...e7(this.options.query),page:e}}:null}}let tP=()=>{if("u"parseInt(e.versions.node.split("."))?" Update to Node 20 LTS or newer, or set `globalThis.File` to `import('node:buffer').File`.":""))}};function tO(e,t,s){return tP(),new File(e,t??"unknown_file",s)}function tI(e,t){let s="object"==typeof e&&null!==e&&("name"in e&&e.name&&String(e.name)||"url"in e&&e.url&&String(e.url)||"filename"in e&&e.filename&&String(e.filename)||"path"in e&&e.path&&String(e.path))||"";return t?s.split(/[\\/]/).pop()||void 0:s}let tR=e=>null!=e&&"object"==typeof e&&"function"==typeof e[Symbol.asyncIterator],tM=async(e,t,s=!0)=>({...e,body:await tL(e.body,t,s)}),t$=new WeakMap,tL=async(e,t,s=!0)=>{if(!await function(e){let t="function"==typeof e?e:e.fetch,s=t$.get(t);if(s)return s;let r=(async()=>{try{let e="Response"in t?t.Response:(await t("data:,")).constructor,s=new FormData;if(s.toString()===await new e(s).text())return!1;return!0}catch{return!0}})();return t$.set(t,r),r}(t))throw TypeError("The provided fetch function does not support file uploads with the current global FormData class.");let r=new FormData;return await Promise.all(Object.entries(e||{}).map(([e,t])=>tU(r,e,t,s))),r},tU=async(e,t,s,r)=>{if(void 0!==s){if(null==s)throw TypeError(`Received null for "${t}"; to pass null in FormData, you must use the string 'null'`);if("string"==typeof s||"number"==typeof s||"boolean"==typeof s)e.append(t,String(s));else if(s instanceof Response){let a={},n=s.headers.get("Content-Type");n&&(a={type:n}),e.append(t,tO([await s.blob()],tI(s,r),a))}else if(tR(s))e.append(t,tO([await new Response(tn(s)).blob()],tI(s,r)));else{let a;if((a=s)instanceof Blob&&"name"in a)e.append(t,tO([s],tI(s,r),{type:s.type}));else if(Array.isArray(s))await Promise.all(s.map(s=>tU(e,t+"[]",s,r)));else if("object"==typeof s)await Promise.all(Object.entries(s).map(([s,a])=>tU(e,`${t}[${s}]`,a,r)));else throw TypeError(`Invalid value given to form, expected a string, number, boolean, object, Array, File or Blob but got ${s} instead`)}}},tD=e=>null!=e&&"object"==typeof e&&"number"==typeof e.size&&"string"==typeof e.type&&"function"==typeof e.text&&"function"==typeof e.slice&&"function"==typeof e.arrayBuffer;async function tq(e,t,s){let r,a;if(tP(),e=await e,t||(t=tI(e,!0)),null!=(r=e)&&"object"==typeof r&&"string"==typeof r.name&&"number"==typeof r.lastModified&&tD(r))return e instanceof File&&null==t&&null==s?e:tO([await e.arrayBuffer()],t??e.name,{type:e.type,lastModified:e.lastModified,...s});if(null!=(a=e)&&"object"==typeof a&&"string"==typeof a.url&&"function"==typeof a.blob){let r=await e.blob();return t||(t=new URL(e.url).pathname.split(/[\\/]/).pop()),tO(await tB(r),t,s)}let n=await tB(e);if(!s?.type){let e=n.find(e=>"object"==typeof e&&"type"in e&&e.type);"string"==typeof e&&(s={...s,type:e})}return tO(n,t,s)}async function tB(e){let t=[];if("string"==typeof e||ArrayBuffer.isView(e)||e instanceof ArrayBuffer)t.push(e);else if(tD(e))t.push(e instanceof Blob?e:await e.arrayBuffer());else if(tR(e))for await(let s of e)t.push(...await tB(s));else{let t=e?.constructor?.name;throw Error(`Unexpected data type: ${typeof e}${t?`; constructor: ${t}`:""}${function(e){if("object"!=typeof e||null===e)return"";let t=Object.getOwnPropertyNames(e);return`; props: [${t.map(e=>`"${e}"`).join(", ")}]`}(e)}`)}return t}class tW{constructor(e){this._client=e}}let tH=Symbol.for("brand.privateNullableHeaders"),tz=e=>{let t=new Headers,s=new Set;for(let r of e){let e=new Set;for(let[a,n]of function*(e){let t;if(!e)return;if(tH in e){let{values:t,nulls:s}=e;for(let e of(yield*t.entries(),s))yield[e,null];return}let s=!1;for(let r of(e instanceof Headers?t=e.entries():e8(e)?t=e:(s=!0,t=Object.entries(e??{})),t)){let e=r[0];if("string"!=typeof e)throw TypeError("expected header name to be a string");let t=e8(r[1])?r[1]:[r[1]],a=!1;for(let r of t)void 0!==r&&(s&&!a&&(a=!0,yield[e,null]),yield[e,r])}}(r)){let r=a.toLowerCase();e.has(r)||(t.delete(a),e.add(r)),null===n?(t.delete(a),s.add(r)):(t.append(a,n),s.delete(r))}}return{[tH]:!0,values:t,nulls:s}};function tF(e){return e.replace(/[^A-Za-z0-9\-._~!$&'()*+,;=:@]+/g,encodeURIComponent)}let tJ=Object.freeze(Object.create(null)),tG=((e=tF)=>function(t,...s){let r;if(1===t.length)return t[0];let a=!1,n=[],i=t.reduce((t,r,i)=>{/[?#]/.test(r)&&(a=!0);let o=s[i],l=(a?encodeURIComponent:e)(""+o);return i!==s.length&&(null==o||"object"==typeof o&&o.toString===Object.getPrototypeOf(Object.getPrototypeOf(o.hasOwnProperty??tJ)??tJ)?.toString)&&(l=o+"",n.push({start:t.length+r.length,length:l.length,error:`Value of type ${Object.prototype.toString.call(o).slice(8,-1)} is not a valid path parameter`})),t+r+(i===s.length?"":l)},""),o=i.split(/[?#]/,1)[0],l=/(?<=^|\/)(?:\.|%2e){1,2}(?=\/|$)/gi;for(;null!==(r=l.exec(o));)n.push({start:r.index,length:r[0].length,error:`Value "${r[0]}" can't be safely passed as a path parameter`});if(n.sort((e,t)=>e.start-t.start),n.length>0){let e=0,t=n.reduce((t,s)=>{let r=" ".repeat(s.start-e),a="^".repeat(s.length);return e=s.start+s.length,t+r+a},"");throw new eJ(`Path parameters result in path with invalid segments: +${n.map(e=>e.error).join("\n")} +${i} +${t}`)}return i})(tF);class tV extends tW{create(e,t){let{betas:s,...r}=e;return this._client.post("/v1/environments?beta=true",{body:r,...t,headers:tz([{"anthropic-beta":[...s??[],"managed-agents-2026-04-01"].toString()},t?.headers])})}retrieve(e,t={},s){let{betas:r}=t??{};return this._client.get(tG`/v1/environments/${e}?beta=true`,{...s,headers:tz([{"anthropic-beta":[...r??[],"managed-agents-2026-04-01"].toString()},s?.headers])})}update(e,t,s){let{betas:r,...a}=t;return this._client.post(tG`/v1/environments/${e}?beta=true`,{body:a,...s,headers:tz([{"anthropic-beta":[...r??[],"managed-agents-2026-04-01"].toString()},s?.headers])})}list(e={},t){let{betas:s,...r}=e??{};return this._client.getAPIList("/v1/environments?beta=true",tA,{query:r,...t,headers:tz([{"anthropic-beta":[...s??[],"managed-agents-2026-04-01"].toString()},t?.headers])})}delete(e,t={},s){let{betas:r}=t??{};return this._client.delete(tG`/v1/environments/${e}?beta=true`,{...s,headers:tz([{"anthropic-beta":[...r??[],"managed-agents-2026-04-01"].toString()},s?.headers])})}archive(e,t={},s){let{betas:r}=t??{};return this._client.post(tG`/v1/environments/${e}/archive?beta=true`,{...s,headers:tz([{"anthropic-beta":[...r??[],"managed-agents-2026-04-01"].toString()},s?.headers])})}}let tK=Symbol("anthropic.sdk.stainlessHelper");function tX(e){return"object"==typeof e&&null!==e&&tK in e}function tY(e,t){let s=new Set;if(e)for(let t of e)tX(t)&&s.add(t[tK]);if(t){for(let e of t)if(tX(e)&&s.add(e[tK]),Array.isArray(e.content))for(let t of e.content)tX(t)&&s.add(t[tK])}return Array.from(s)}function tQ(e,t){let s=tY(e,t);return 0===s.length?{}:{"x-stainless-helper":s.join(", ")}}class tZ extends tW{list(e={},t){let{betas:s,...r}=e??{};return this._client.getAPIList("/v1/files?beta=true",tC,{query:r,...t,headers:tz([{"anthropic-beta":[...s??[],"files-api-2025-04-14"].toString()},t?.headers])})}delete(e,t={},s){let{betas:r}=t??{};return this._client.delete(tG`/v1/files/${e}?beta=true`,{...s,headers:tz([{"anthropic-beta":[...r??[],"files-api-2025-04-14"].toString()},s?.headers])})}download(e,t={},s){let{betas:r}=t??{};return this._client.get(tG`/v1/files/${e}/content?beta=true`,{...s,headers:tz([{"anthropic-beta":[...r??[],"files-api-2025-04-14"].toString(),Accept:"application/binary"},s?.headers]),__binaryResponse:!0})}retrieveMetadata(e,t={},s){let{betas:r}=t??{};return this._client.get(tG`/v1/files/${e}?beta=true`,{...s,headers:tz([{"anthropic-beta":[...r??[],"files-api-2025-04-14"].toString()},s?.headers])})}upload(e,t){var s;let{betas:r,...a}=e;return this._client.post("/v1/files?beta=true",tM({body:a,...t,headers:tz([{"anthropic-beta":[...r??[],"files-api-2025-04-14"].toString()},tX(s=a.file)?{"x-stainless-helper":s[tK]}:{},t?.headers])},this._client))}}class t0 extends tW{retrieve(e,t={},s){let{betas:r}=t??{};return this._client.get(tG`/v1/models/${e}?beta=true`,{...s,headers:tz([{...r?.toString()!=null?{"anthropic-beta":r?.toString()}:void 0},s?.headers])})}list(e={},t){let{betas:s,...r}=e??{};return this._client.getAPIList("/v1/models?beta=true",tC,{query:r,...t,headers:tz([{...s?.toString()!=null?{"anthropic-beta":s?.toString()}:void 0},t?.headers])})}}class t1 extends tW{create(e,t){let{betas:s,...r}=e;return this._client.post("/v1/user_profiles?beta=true",{body:r,...t,headers:tz([{"anthropic-beta":[...s??[],"user-profiles-2026-03-24"].toString()},t?.headers])})}retrieve(e,t={},s){let{betas:r}=t??{};return this._client.get(tG`/v1/user_profiles/${e}?beta=true`,{...s,headers:tz([{"anthropic-beta":[...r??[],"user-profiles-2026-03-24"].toString()},s?.headers])})}update(e,t,s){let{betas:r,...a}=t;return this._client.post(tG`/v1/user_profiles/${e}?beta=true`,{body:a,...s,headers:tz([{"anthropic-beta":[...r??[],"user-profiles-2026-03-24"].toString()},s?.headers])})}list(e={},t){let{betas:s,...r}=e??{};return this._client.getAPIList("/v1/user_profiles?beta=true",tA,{query:r,...t,headers:tz([{"anthropic-beta":[...s??[],"user-profiles-2026-03-24"].toString()},t?.headers])})}createEnrollmentURL(e,t={},s){let{betas:r}=t??{};return this._client.post(tG`/v1/user_profiles/${e}/enrollment_url?beta=true`,{...s,headers:tz([{"anthropic-beta":[...r??[],"user-profiles-2026-03-24"].toString()},s?.headers])})}}class t2 extends tW{list(e,t={},s){let{betas:r,...a}=t??{};return this._client.getAPIList(tG`/v1/agents/${e}/versions?beta=true`,tA,{query:a,...s,headers:tz([{"anthropic-beta":[...r??[],"managed-agents-2026-04-01"].toString()},s?.headers])})}}class t4 extends tW{constructor(){super(...arguments),this.versions=new t2(this._client)}create(e,t){let{betas:s,...r}=e;return this._client.post("/v1/agents?beta=true",{body:r,...t,headers:tz([{"anthropic-beta":[...s??[],"managed-agents-2026-04-01"].toString()},t?.headers])})}retrieve(e,t={},s){let{betas:r,...a}=t??{};return this._client.get(tG`/v1/agents/${e}?beta=true`,{query:a,...s,headers:tz([{"anthropic-beta":[...r??[],"managed-agents-2026-04-01"].toString()},s?.headers])})}update(e,t,s){let{betas:r,...a}=t;return this._client.post(tG`/v1/agents/${e}?beta=true`,{body:a,...s,headers:tz([{"anthropic-beta":[...r??[],"managed-agents-2026-04-01"].toString()},s?.headers])})}list(e={},t){let{betas:s,...r}=e??{};return this._client.getAPIList("/v1/agents?beta=true",tA,{query:r,...t,headers:tz([{"anthropic-beta":[...s??[],"managed-agents-2026-04-01"].toString()},t?.headers])})}archive(e,t={},s){let{betas:r}=t??{};return this._client.post(tG`/v1/agents/${e}/archive?beta=true`,{...s,headers:tz([{"anthropic-beta":[...r??[],"managed-agents-2026-04-01"].toString()},s?.headers])})}}t4.Versions=t2;class t6 extends tW{create(e,t,s){let{view:r,betas:a,...n}=t;return this._client.post(tG`/v1/memory_stores/${e}/memories?beta=true`,{query:{view:r},body:n,...s,headers:tz([{"anthropic-beta":[...a??[],"managed-agents-2026-04-01"].toString()},s?.headers])})}retrieve(e,t,s){let{memory_store_id:r,betas:a,...n}=t;return this._client.get(tG`/v1/memory_stores/${r}/memories/${e}?beta=true`,{query:n,...s,headers:tz([{"anthropic-beta":[...a??[],"managed-agents-2026-04-01"].toString()},s?.headers])})}update(e,t,s){let{memory_store_id:r,view:a,betas:n,...i}=t;return this._client.post(tG`/v1/memory_stores/${r}/memories/${e}?beta=true`,{query:{view:a},body:i,...s,headers:tz([{"anthropic-beta":[...n??[],"managed-agents-2026-04-01"].toString()},s?.headers])})}list(e,t={},s){let{betas:r,...a}=t??{};return this._client.getAPIList(tG`/v1/memory_stores/${e}/memories?beta=true`,tA,{query:a,...s,headers:tz([{"anthropic-beta":[...r??[],"managed-agents-2026-04-01"].toString()},s?.headers])})}delete(e,t,s){let{memory_store_id:r,expected_content_sha256:a,betas:n}=t;return this._client.delete(tG`/v1/memory_stores/${r}/memories/${e}?beta=true`,{query:{expected_content_sha256:a},...s,headers:tz([{"anthropic-beta":[...n??[],"managed-agents-2026-04-01"].toString()},s?.headers])})}}class t3 extends tW{retrieve(e,t,s){let{memory_store_id:r,betas:a,...n}=t;return this._client.get(tG`/v1/memory_stores/${r}/memory_versions/${e}?beta=true`,{query:n,...s,headers:tz([{"anthropic-beta":[...a??[],"managed-agents-2026-04-01"].toString()},s?.headers])})}list(e,t={},s){let{betas:r,...a}=t??{};return this._client.getAPIList(tG`/v1/memory_stores/${e}/memory_versions?beta=true`,tA,{query:a,...s,headers:tz([{"anthropic-beta":[...r??[],"managed-agents-2026-04-01"].toString()},s?.headers])})}redact(e,t,s){let{memory_store_id:r,betas:a}=t;return this._client.post(tG`/v1/memory_stores/${r}/memory_versions/${e}/redact?beta=true`,{...s,headers:tz([{"anthropic-beta":[...a??[],"managed-agents-2026-04-01"].toString()},s?.headers])})}}class t5 extends tW{constructor(){super(...arguments),this.memories=new t6(this._client),this.memoryVersions=new t3(this._client)}create(e,t){let{betas:s,...r}=e;return this._client.post("/v1/memory_stores?beta=true",{body:r,...t,headers:tz([{"anthropic-beta":[...s??[],"managed-agents-2026-04-01"].toString()},t?.headers])})}retrieve(e,t={},s){let{betas:r}=t??{};return this._client.get(tG`/v1/memory_stores/${e}?beta=true`,{...s,headers:tz([{"anthropic-beta":[...r??[],"managed-agents-2026-04-01"].toString()},s?.headers])})}update(e,t,s){let{betas:r,...a}=t;return this._client.post(tG`/v1/memory_stores/${e}?beta=true`,{body:a,...s,headers:tz([{"anthropic-beta":[...r??[],"managed-agents-2026-04-01"].toString()},s?.headers])})}list(e={},t){let{betas:s,...r}=e??{};return this._client.getAPIList("/v1/memory_stores?beta=true",tA,{query:r,...t,headers:tz([{"anthropic-beta":[...s??[],"managed-agents-2026-04-01"].toString()},t?.headers])})}delete(e,t={},s){let{betas:r}=t??{};return this._client.delete(tG`/v1/memory_stores/${e}?beta=true`,{...s,headers:tz([{"anthropic-beta":[...r??[],"managed-agents-2026-04-01"].toString()},s?.headers])})}archive(e,t={},s){let{betas:r}=t??{};return this._client.post(tG`/v1/memory_stores/${e}/archive?beta=true`,{...s,headers:tz([{"anthropic-beta":[...r??[],"managed-agents-2026-04-01"].toString()},s?.headers])})}}t5.Memories=t6,t5.MemoryVersions=t3;class t8{constructor(e,t){this.iterator=e,this.controller=t}async *decoder(){let e=new tu;for await(let t of this.iterator)for(let s of e.decode(t))yield JSON.parse(s);for(let t of e.flush())yield JSON.parse(t)}[Symbol.asyncIterator](){return this.decoder()}static fromResponse(e,t){if(!e.body){if(t.abort(),void 0!==globalThis.navigator&&"ReactNative"===globalThis.navigator.product)throw new eJ("The default react-native fetch implementation does not support streaming. Please use expo/fetch: https://docs.expo.dev/versions/latest/sdk/expo/#expofetch-api");throw new eJ("Attempted to iterate over a response with no body")}return new t8(ti(e.body),t)}}class t7 extends tW{create(e,t){let{betas:s,...r}=e;return this._client.post("/v1/messages/batches?beta=true",{body:r,...t,headers:tz([{"anthropic-beta":[...s??[],"message-batches-2024-09-24"].toString()},t?.headers])})}retrieve(e,t={},s){let{betas:r}=t??{};return this._client.get(tG`/v1/messages/batches/${e}?beta=true`,{...s,headers:tz([{"anthropic-beta":[...r??[],"message-batches-2024-09-24"].toString()},s?.headers])})}list(e={},t){let{betas:s,...r}=e??{};return this._client.getAPIList("/v1/messages/batches?beta=true",tC,{query:r,...t,headers:tz([{"anthropic-beta":[...s??[],"message-batches-2024-09-24"].toString()},t?.headers])})}delete(e,t={},s){let{betas:r}=t??{};return this._client.delete(tG`/v1/messages/batches/${e}?beta=true`,{...s,headers:tz([{"anthropic-beta":[...r??[],"message-batches-2024-09-24"].toString()},s?.headers])})}cancel(e,t={},s){let{betas:r}=t??{};return this._client.post(tG`/v1/messages/batches/${e}/cancel?beta=true`,{...s,headers:tz([{"anthropic-beta":[...r??[],"message-batches-2024-09-24"].toString()},s?.headers])})}async results(e,t={},s){let r=await this.retrieve(e);if(!r.results_url)throw new eJ(`No batch \`results_url\`; Has it finished processing? ${r.processing_status} - ${r.id}`);let{betas:a}=t??{};return this._client.get(r.results_url,{...s,headers:tz([{"anthropic-beta":[...a??[],"message-batches-2024-09-24"].toString(),Accept:"application/binary"},s?.headers]),stream:!0,__binaryResponse:!0})._thenUnwrap((e,t)=>t8.fromResponse(t.response,t.controller))}}let t9={"claude-opus-4-20250514":8192,"claude-opus-4-0":8192,"claude-4-opus-20250514":8192,"anthropic.claude-opus-4-20250514-v1:0":8192,"claude-opus-4@20250514":8192,"claude-opus-4-1-20250805":8192,"anthropic.claude-opus-4-1-20250805-v1:0":8192,"claude-opus-4-1@20250805":8192};function se(e){return e?.output_format??e?.output_config?.format}function st(e,t,s){let r=se(t);return t&&"parse"in(r??{})?ss(e,t,s):{...e,content:e.content.map(e=>"text"===e.type?Object.defineProperty(Object.defineProperty({...e},"parsed_output",{value:null,enumerable:!1}),"parsed",{get:()=>(s.logger.warn("The `parsed` property on `text` blocks is deprecated, please use `parsed_output` instead."),null),enumerable:!1}):e),parsed_output:null}}function ss(e,t,s){let r=null,a=e.content.map(e=>{if("text"===e.type){let a=function(e,t){let s=se(e);if(s?.type!=="json_schema")return null;try{if("parse"in s)return s.parse(t);return JSON.parse(t)}catch(e){throw new eJ(`Failed to parse structured output: ${e}`)}}(t,e.text);return null===r&&(r=a),Object.defineProperty(Object.defineProperty({...e},"parsed_output",{value:a,enumerable:!1}),"parsed",{get:()=>(s.logger.warn("The `parsed` property on `text` blocks is deprecated, please use `parsed_output` instead."),a),enumerable:!1})}return e});return{...e,content:a,parsed_output:r}}let sr=e=>{if(0===e.length)return e;let t=e[e.length-1];switch(t.type){case"separator":return sr(e=e.slice(0,e.length-1));case"number":let s=t.value[t.value.length-1];if("."===s||"-"===s)return sr(e=e.slice(0,e.length-1));case"string":let r=e[e.length-2];if(r?.type==="delimiter"||r?.type==="brace"&&"{"===r.value)return sr(e=e.slice(0,e.length-1));break;case"delimiter":return sr(e=e.slice(0,e.length-1))}return e},sa=e=>{var t;let s,r;return JSON.parse((t=sr((e=>{let t=0,s=[];for(;t{"brace"===e.type&&("{"===e.value?s.push("}"):s.splice(s.lastIndexOf("}"),1)),"paren"===e.type&&("["===e.value?s.push("]"):s.splice(s.lastIndexOf("]"),1))}),s.length>0&&s.reverse().map(e=>{"}"===e?t.push({type:"brace",value:"}"}):"]"===e&&t.push({type:"paren",value:"]"})}),r="",t.map(e=>{"string"===e.type?r+='"'+e.value+'"':r+=e.value}),r))},sn="__json_buf";function si(e){return"tool_use"===e.type||"server_tool_use"===e.type||"mcp_tool_use"===e.type}class so{constructor(e,t){c.add(this),this.messages=[],this.receivedMessages=[],d.set(this,void 0),u.set(this,null),this.controller=new AbortController,h.set(this,void 0),p.set(this,()=>{}),m.set(this,()=>{}),g.set(this,void 0),f.set(this,()=>{}),y.set(this,()=>{}),x.set(this,{}),b.set(this,!1),v.set(this,!1),w.set(this,!1),_.set(this,!1),S.set(this,void 0),j.set(this,void 0),k.set(this,void 0),T.set(this,e=>{if(eB(this,v,!0,"f"),ez(e)&&(e=new eV),e instanceof eV)return eB(this,w,!0,"f"),this._emit("abort",e);if(e instanceof eJ)return this._emit("error",e);if(e instanceof Error){let t=new eJ(e.message);return t.cause=e,this._emit("error",t)}return this._emit("error",new eJ(String(e)))}),eB(this,h,new Promise((e,t)=>{eB(this,p,e,"f"),eB(this,m,t,"f")}),"f"),eB(this,g,new Promise((e,t)=>{eB(this,f,e,"f"),eB(this,y,t,"f")}),"f"),eW(this,h,"f").catch(()=>{}),eW(this,g,"f").catch(()=>{}),eB(this,u,e,"f"),eB(this,k,t?.logger??console,"f")}get response(){return eW(this,S,"f")}get request_id(){return eW(this,j,"f")}async withResponse(){eB(this,_,!0,"f");let e=await eW(this,h,"f");if(!e)throw Error("Could not resolve a `Response` object");return{data:this,response:e,request_id:e.headers.get("request-id")}}static fromReadableStream(e){let t=new so(null);return t._run(()=>t._fromReadableStream(e)),t}static createMessage(e,t,s,{logger:r}={}){let a=new so(t,{logger:r});for(let e of t.messages)a._addMessageParam(e);return eB(a,u,{...t,stream:!0},"f"),a._run(()=>a._createMessage(e,{...t,stream:!0},{...s,headers:{...s?.headers,"X-Stainless-Helper-Method":"stream"}})),a}_run(e){e().then(()=>{this._emitFinal(),this._emit("end")},eW(this,T,"f"))}_addMessageParam(e){this.messages.push(e)}_addMessage(e,t=!0){this.receivedMessages.push(e),t&&this._emit("message",e)}async _createMessage(e,t,s){let r,a=s?.signal;a&&(a.aborted&&this.controller.abort(),r=this.controller.abort.bind(this.controller),a.addEventListener("abort",r));try{eW(this,c,"m",C).call(this);let{response:r,data:a}=await e.create({...t,stream:!0},{...s,signal:this.controller.signal}).withResponse();for await(let e of(this._connected(r),a))eW(this,c,"m",A).call(this,e);if(a.controller.signal?.aborted)throw new eV;eW(this,c,"m",P).call(this)}finally{a&&r&&a.removeEventListener("abort",r)}}_connected(e){this.ended||(eB(this,S,e,"f"),eB(this,j,e?.headers.get("request-id"),"f"),eW(this,p,"f").call(this,e),this._emit("connect"))}get ended(){return eW(this,b,"f")}get errored(){return eW(this,v,"f")}get aborted(){return eW(this,w,"f")}abort(){this.controller.abort()}on(e,t){return(eW(this,x,"f")[e]||(eW(this,x,"f")[e]=[])).push({listener:t}),this}off(e,t){let s=eW(this,x,"f")[e];if(!s)return this;let r=s.findIndex(e=>e.listener===t);return r>=0&&s.splice(r,1),this}once(e,t){return(eW(this,x,"f")[e]||(eW(this,x,"f")[e]=[])).push({listener:t,once:!0}),this}emitted(e){return new Promise((t,s)=>{eB(this,_,!0,"f"),"error"!==e&&this.once("error",s),this.once(e,t)})}async done(){eB(this,_,!0,"f"),await eW(this,g,"f")}get currentMessage(){return eW(this,d,"f")}async finalMessage(){return await this.done(),eW(this,c,"m",N).call(this)}async finalText(){return await this.done(),eW(this,c,"m",E).call(this)}_emit(e,...t){if(eW(this,b,"f"))return;"end"===e&&(eB(this,b,!0,"f"),eW(this,f,"f").call(this));let s=eW(this,x,"f")[e];if(s&&(eW(this,x,"f")[e]=s.filter(e=>!e.once),s.forEach(({listener:e})=>e(...t))),"abort"===e){let e=t[0];eW(this,_,"f")||s?.length||Promise.reject(e),eW(this,m,"f").call(this,e),eW(this,y,"f").call(this,e),this._emit("end");return}if("error"===e){let e=t[0];eW(this,_,"f")||s?.length||Promise.reject(e),eW(this,m,"f").call(this,e),eW(this,y,"f").call(this,e),this._emit("end")}}_emitFinal(){this.receivedMessages.at(-1)&&this._emit("finalMessage",eW(this,c,"m",N).call(this))}async _fromReadableStream(e,t){let s,r=t?.signal;r&&(r.aborted&&this.controller.abort(),s=this.controller.abort.bind(this.controller),r.addEventListener("abort",s));try{eW(this,c,"m",C).call(this),this._connected(null);let t=tv.fromReadableStream(e,this.controller);for await(let e of t)eW(this,c,"m",A).call(this,e);if(t.controller.signal?.aborted)throw new eV;eW(this,c,"m",P).call(this)}finally{r&&s&&r.removeEventListener("abort",s)}}[(d=new WeakMap,u=new WeakMap,h=new WeakMap,p=new WeakMap,m=new WeakMap,g=new WeakMap,f=new WeakMap,y=new WeakMap,x=new WeakMap,b=new WeakMap,v=new WeakMap,w=new WeakMap,_=new WeakMap,S=new WeakMap,j=new WeakMap,k=new WeakMap,T=new WeakMap,c=new WeakSet,N=function(){if(0===this.receivedMessages.length)throw new eJ("stream ended without producing a Message with role=assistant");return this.receivedMessages.at(-1)},E=function(){if(0===this.receivedMessages.length)throw new eJ("stream ended without producing a Message with role=assistant");let e=this.receivedMessages.at(-1).content.filter(e=>"text"===e.type).map(e=>e.text);if(0===e.length)throw new eJ("stream ended without producing a content block with type=text");return e.join(" ")},C=function(){this.ended||eB(this,d,void 0,"f")},A=function(e){if(this.ended)return;let t=eW(this,c,"m",O).call(this,e);switch(this._emit("streamEvent",e,t),e.type){case"content_block_delta":{let s=t.content.at(-1);switch(e.delta.type){case"text_delta":"text"===s.type&&this._emit("text",e.delta.text,s.text||"");break;case"citations_delta":"text"===s.type&&this._emit("citation",e.delta.citation,s.citations??[]);break;case"input_json_delta":si(s)&&s.input&&this._emit("inputJson",e.delta.partial_json,s.input);break;case"thinking_delta":"thinking"===s.type&&this._emit("thinking",e.delta.thinking,s.thinking);break;case"signature_delta":"thinking"===s.type&&this._emit("signature",s.signature);break;case"compaction_delta":"compaction"===s.type&&s.content&&this._emit("compaction",s.content);break;default:sl(e.delta)}break}case"message_stop":this._addMessageParam(t),this._addMessage(st(t,eW(this,u,"f"),{logger:eW(this,k,"f")}),!0);break;case"content_block_stop":this._emit("contentBlock",t.content.at(-1));break;case"message_start":eB(this,d,t,"f")}},P=function(){if(this.ended)throw new eJ("stream has ended, this shouldn't happen");let e=eW(this,d,"f");if(!e)throw new eJ("request ended without sending any chunks");return eB(this,d,void 0,"f"),st(e,eW(this,u,"f"),{logger:eW(this,k,"f")})},O=function(e){let t=eW(this,d,"f");if("message_start"===e.type){if(t)throw new eJ(`Unexpected event order, got ${e.type} before receiving "message_stop"`);return e.message}if(!t)throw new eJ(`Unexpected event order, got ${e.type} before "message_start"`);switch(e.type){case"message_stop":case"content_block_stop":return t;case"message_delta":return t.container=e.delta.container,t.stop_reason=e.delta.stop_reason,t.stop_sequence=e.delta.stop_sequence,t.usage.output_tokens=e.usage.output_tokens,t.context_management=e.context_management,null!=e.usage.input_tokens&&(t.usage.input_tokens=e.usage.input_tokens),null!=e.usage.cache_creation_input_tokens&&(t.usage.cache_creation_input_tokens=e.usage.cache_creation_input_tokens),null!=e.usage.cache_read_input_tokens&&(t.usage.cache_read_input_tokens=e.usage.cache_read_input_tokens),null!=e.usage.server_tool_use&&(t.usage.server_tool_use=e.usage.server_tool_use),null!=e.usage.iterations&&(t.usage.iterations=e.usage.iterations),t;case"content_block_start":return t.content.push(e.content_block),t;case"content_block_delta":{let s=t.content.at(e.index);switch(e.delta.type){case"text_delta":s?.type==="text"&&(t.content[e.index]={...s,text:(s.text||"")+e.delta.text});break;case"citations_delta":s?.type==="text"&&(t.content[e.index]={...s,citations:[...s.citations??[],e.delta.citation]});break;case"input_json_delta":if(s&&si(s)){let r=s[sn]||"";r+=e.delta.partial_json;let a={...s};if(Object.defineProperty(a,sn,{value:r,enumerable:!1,writable:!0}),r)try{a.input=sa(r)}catch(t){let e=new eJ(`Unable to parse tool parameter JSON from model. Please retry your request or adjust your prompt. Error: ${t}. JSON: ${r}`);eW(this,T,"f").call(this,e)}t.content[e.index]=a}break;case"thinking_delta":s?.type==="thinking"&&(t.content[e.index]={...s,thinking:s.thinking+e.delta.thinking});break;case"signature_delta":s?.type==="thinking"&&(t.content[e.index]={...s,signature:e.delta.signature});break;case"compaction_delta":s?.type==="compaction"&&(t.content[e.index]={...s,content:(s.content||"")+e.delta.content});break;default:sl(e.delta)}return t}}},Symbol.asyncIterator)](){let e=[],t=[],s=!1;return this.on("streamEvent",s=>{let r=t.shift();r?r.resolve(s):e.push(s)}),this.on("end",()=>{for(let e of(s=!0,t))e.resolve(void 0);t.length=0}),this.on("abort",e=>{for(let r of(s=!0,t))r.reject(e);t.length=0}),this.on("error",e=>{for(let r of(s=!0,t))r.reject(e);t.length=0}),{next:async()=>e.length?{value:e.shift(),done:!1}:s?{value:void 0,done:!0}:new Promise((e,s)=>t.push({resolve:e,reject:s})).then(e=>e?{value:e,done:!1}:{value:void 0,done:!0}),return:async()=>(this.abort(),{value:void 0,done:!0})}}toReadableStream(){return new tv(this[Symbol.asyncIterator].bind(this),this.controller).toReadableStream()}}function sl(e){}class sc extends Error{constructor(e){super("string"==typeof e?e:e.map(e=>"text"===e.type?e.text:`[${e.type}]`).join(" ")),this.name="ToolError",this.content=e}}let sd=`You have been working on the task described above but have not yet completed it. Write a continuation summary that will allow you (or another instance of yourself) to resume work efficiently in a future context window where the conversation history will be replaced with this summary. Your summary should be structured, concise, and actionable. Include: +1. Task Overview +The user's core request and success criteria +Any clarifications or constraints they specified +2. Current State +What has been completed so far +Files created, modified, or analyzed (with paths if relevant) +Key outputs or artifacts produced +3. Important Discoveries +Technical constraints or requirements uncovered +Decisions made and their rationale +Errors encountered and how they were resolved +What approaches were tried that didn't work (and why) +4. Next Steps +Specific actions needed to complete the task +Any blockers or open questions to resolve +Priority order if multiple steps remain +5. Context to Preserve +User preferences or style requirements +Domain-specific details that aren't obvious +Any promises made to the user +Be concise but complete—err on the side of including information that would prevent duplicate work or repeated mistakes. Write in a way that enables immediate resumption of the task. +Wrap your summary in tags.`;function su(){let e,t;return{promise:new Promise((s,r)=>{e=s,t=r}),resolve:e,reject:t}}class sh{constructor(e,t,s){I.add(this),this.client=e,R.set(this,!1),M.set(this,!1),$.set(this,void 0),L.set(this,void 0),U.set(this,void 0),D.set(this,void 0),q.set(this,void 0),B.set(this,0),eB(this,$,{params:{...t,messages:structuredClone(t.messages)}},"f");const r=["BetaToolRunner",...tY(t.tools,t.messages)].join(", ");eB(this,L,{...s,headers:tz([{"x-stainless-helper":r},s?.headers])},"f"),eB(this,q,su(),"f"),t.compactionControl?.enabled&&console.warn('Anthropic: The `compactionControl` parameter is deprecated and will be removed in a future version. Use server-side compaction instead by passing `edits: [{ type: "compact_20260112" }]` in the params passed to `toolRunner()`. See https://platform.claude.com/docs/en/build-with-claude/compaction')}async *[(R=new WeakMap,M=new WeakMap,$=new WeakMap,L=new WeakMap,U=new WeakMap,D=new WeakMap,q=new WeakMap,B=new WeakMap,I=new WeakSet,W=async function(){let e=eW(this,$,"f").params.compactionControl;if(!e||!e.enabled)return!1;let t=0;if(void 0!==eW(this,U,"f"))try{let e=await eW(this,U,"f");t=e.usage.input_tokens+(e.usage.cache_creation_input_tokens??0)+(e.usage.cache_read_input_tokens??0)+e.usage.output_tokens}catch{return!1}if(t<(e.contextTokenThreshold??1e5))return!1;let s=e.model??eW(this,$,"f").params.model,r=e.summaryPrompt??sd,a=eW(this,$,"f").params.messages;if("assistant"===a[a.length-1].role){let e=a[a.length-1];if(Array.isArray(e.content)){let t=e.content.filter(e=>"tool_use"!==e.type);0===t.length?a.pop():e.content=t}}let n=await this.client.beta.messages.create({model:s,messages:[...a,{role:"user",content:[{type:"text",text:r}]}],max_tokens:eW(this,$,"f").params.max_tokens},{signal:eW(this,L,"f").signal,headers:tz([eW(this,L,"f").headers,{"x-stainless-helper":"compaction"}])});if(n.content[0]?.type!=="text")throw new eJ("Expected text response for compaction");return eW(this,$,"f").params.messages=[{role:"user",content:n.content}],!0},Symbol.asyncIterator)](){var e;if(eW(this,R,"f"))throw new eJ("Cannot iterate over a consumed stream");eB(this,R,!0,"f"),eB(this,M,!0,"f"),eB(this,D,void 0,"f");try{for(;;){let t;try{if(eW(this,$,"f").params.max_iterations&&eW(this,B,"f")>=eW(this,$,"f").params.max_iterations)break;eB(this,M,!1,"f"),eB(this,D,void 0,"f"),eB(this,B,(e=eW(this,B,"f"),++e),"f"),eB(this,U,void 0,"f");let{max_iterations:s,compactionControl:r,...a}=eW(this,$,"f").params;if(a.stream?(t=this.client.beta.messages.stream({...a},eW(this,L,"f")),eB(this,U,t.finalMessage(),"f"),eW(this,U,"f").catch(()=>{}),yield t):(eB(this,U,this.client.beta.messages.create({...a,stream:!1},eW(this,L,"f")),"f"),yield eW(this,U,"f")),!await eW(this,I,"m",W).call(this)){if(!eW(this,M,"f")){let{role:e,content:t}=await eW(this,U,"f");eW(this,$,"f").params.messages.push({role:e,content:t})}let e=await eW(this,I,"m",H).call(this,eW(this,$,"f").params.messages.at(-1));if(e)eW(this,$,"f").params.messages.push(e);else if(!eW(this,M,"f"))break}}finally{t&&t.abort()}}if(!eW(this,U,"f"))throw new eJ("ToolRunner concluded without a message from the server");eW(this,q,"f").resolve(await eW(this,U,"f"))}catch(e){throw eB(this,R,!1,"f"),eW(this,q,"f").promise.catch(()=>{}),eW(this,q,"f").reject(e),eB(this,q,su(),"f"),e}}setMessagesParams(e){"function"==typeof e?eW(this,$,"f").params=e(eW(this,$,"f").params):eW(this,$,"f").params=e,eB(this,M,!0,"f"),eB(this,D,void 0,"f")}setRequestOptions(e){"function"==typeof e?eB(this,L,e(eW(this,L,"f")),"f"):eB(this,L,{...eW(this,L,"f"),...e},"f")}async generateToolResponse(e=eW(this,L,"f").signal){let t=await eW(this,U,"f")??this.params.messages.at(-1);return t?eW(this,I,"m",H).call(this,t,e):null}done(){return eW(this,q,"f").promise}async runUntilDone(){if(!eW(this,R,"f"))for await(let e of this);return this.done()}get params(){return eW(this,$,"f").params}pushMessages(...e){this.setMessagesParams(t=>({...t,messages:[...t.messages,...e]}))}then(e,t){return this.runUntilDone().then(e,t)}}async function sp(e,t=e.messages.at(-1),s){if(!t||"assistant"!==t.role||!t.content||"string"==typeof t.content)return null;let r=t.content.filter(e=>"tool_use"===e.type);return 0===r.length?null:{role:"user",content:await Promise.all(r.map(async t=>{let r=e.tools.find(e=>("name"in e?e.name:e.mcp_server_name)===t.name);if(!r||!("run"in r))return{type:"tool_result",tool_use_id:t.id,content:`Error: Tool '${t.name}' not found`,is_error:!0};try{let e=t.input;"parse"in r&&r.parse&&(e=r.parse(e));let a=await r.run(e,{toolUseBlock:t,signal:s?.signal});return{type:"tool_result",tool_use_id:t.id,content:a}}catch(e){return{type:"tool_result",tool_use_id:t.id,content:e instanceof sc?e.content:`Error: ${e instanceof Error?e.message:String(e)}`,is_error:!0}}}))}}H=async function(e,t=eW(this,L,"f").signal){return void 0!==eW(this,D,"f")||eB(this,D,sp(eW(this,$,"f").params,e,{...eW(this,L,"f"),signal:t}),"f"),eW(this,D,"f")};let sm={"claude-1.3":"November 6th, 2024","claude-1.3-100k":"November 6th, 2024","claude-instant-1.1":"November 6th, 2024","claude-instant-1.1-100k":"November 6th, 2024","claude-instant-1.2":"November 6th, 2024","claude-3-sonnet-20240229":"July 21st, 2025","claude-3-opus-20240229":"January 5th, 2026","claude-2.1":"July 21st, 2025","claude-2.0":"July 21st, 2025","claude-3-7-sonnet-latest":"February 19th, 2026","claude-3-7-sonnet-20250219":"February 19th, 2026"},sg=["claude-mythos-preview","claude-opus-4-6"];class sf extends tW{constructor(){super(...arguments),this.batches=new t7(this._client)}create(e,t){let s=sy(e),{betas:r,...a}=s;a.model in sm&&console.warn(`The model '${a.model}' is deprecated and will reach end-of-life on ${sm[a.model]} +Please migrate to a newer model. Visit https://docs.anthropic.com/en/docs/resources/model-deprecations for more information.`),sg.includes(a.model)&&a.thinking&&"enabled"===a.thinking.type&&console.warn(`Using Claude with ${a.model} and 'thinking.type=enabled' is deprecated. Use 'thinking.type=adaptive' instead which results in better model performance in our testing: https://platform.claude.com/docs/en/build-with-claude/adaptive-thinking`);let n=this._client._options.timeout;if(!a.stream&&null==n){let e=t9[a.model]??void 0;n=this._client.calculateNonstreamingTimeout(a.max_tokens,e)}let i=tQ(a.tools,a.messages);return this._client.post("/v1/messages?beta=true",{body:a,timeout:n??6e5,...t,headers:tz([{...r?.toString()!=null?{"anthropic-beta":r?.toString()}:void 0},i,t?.headers]),stream:s.stream??!1})}parse(e,t){return t={...t,headers:tz([{"anthropic-beta":[...e.betas??[],"structured-outputs-2025-12-15"].toString()},t?.headers])},this.create(e,t).then(t=>ss(t,e,{logger:this._client.logger??console}))}stream(e,t){return so.createMessage(this,e,t)}countTokens(e,t){let{betas:s,...r}=sy(e);return this._client.post("/v1/messages/count_tokens?beta=true",{body:r,...t,headers:tz([{"anthropic-beta":[...s??[],"token-counting-2024-11-01"].toString()},t?.headers])})}toolRunner(e,t){return new sh(this._client,e,t)}}function sy(e){if(!e.output_format)return e;if(e.output_config?.format)throw new eJ("Both output_format and output_config.format were provided. Please use only output_config.format (output_format is deprecated).");let{output_format:t,...s}=e;return{...s,output_config:{...e.output_config,format:t}}}sf.Batches=t7,sf.BetaToolRunner=sh,sf.ToolError=sc;class sx extends tW{list(e,t={},s){let{betas:r,...a}=t??{};return this._client.getAPIList(tG`/v1/sessions/${e}/events?beta=true`,tA,{query:a,...s,headers:tz([{"anthropic-beta":[...r??[],"managed-agents-2026-04-01"].toString()},s?.headers])})}send(e,t,s){let{betas:r,...a}=t;return this._client.post(tG`/v1/sessions/${e}/events?beta=true`,{body:a,...s,headers:tz([{"anthropic-beta":[...r??[],"managed-agents-2026-04-01"].toString()},s?.headers])})}stream(e,t={},s){let{betas:r}=t??{};return this._client.get(tG`/v1/sessions/${e}/events/stream?beta=true`,{...s,headers:tz([{"anthropic-beta":[...r??[],"managed-agents-2026-04-01"].toString()},s?.headers]),stream:!0})}}class sb extends tW{retrieve(e,t,s){let{session_id:r,betas:a}=t;return this._client.get(tG`/v1/sessions/${r}/resources/${e}?beta=true`,{...s,headers:tz([{"anthropic-beta":[...a??[],"managed-agents-2026-04-01"].toString()},s?.headers])})}update(e,t,s){let{session_id:r,betas:a,...n}=t;return this._client.post(tG`/v1/sessions/${r}/resources/${e}?beta=true`,{body:n,...s,headers:tz([{"anthropic-beta":[...a??[],"managed-agents-2026-04-01"].toString()},s?.headers])})}list(e,t={},s){let{betas:r,...a}=t??{};return this._client.getAPIList(tG`/v1/sessions/${e}/resources?beta=true`,tA,{query:a,...s,headers:tz([{"anthropic-beta":[...r??[],"managed-agents-2026-04-01"].toString()},s?.headers])})}delete(e,t,s){let{session_id:r,betas:a}=t;return this._client.delete(tG`/v1/sessions/${r}/resources/${e}?beta=true`,{...s,headers:tz([{"anthropic-beta":[...a??[],"managed-agents-2026-04-01"].toString()},s?.headers])})}add(e,t,s){let{betas:r,...a}=t;return this._client.post(tG`/v1/sessions/${e}/resources?beta=true`,{body:a,...s,headers:tz([{"anthropic-beta":[...r??[],"managed-agents-2026-04-01"].toString()},s?.headers])})}}class sv extends tW{constructor(){super(...arguments),this.events=new sx(this._client),this.resources=new sb(this._client)}create(e,t){let{betas:s,...r}=e;return this._client.post("/v1/sessions?beta=true",{body:r,...t,headers:tz([{"anthropic-beta":[...s??[],"managed-agents-2026-04-01"].toString()},t?.headers])})}retrieve(e,t={},s){let{betas:r}=t??{};return this._client.get(tG`/v1/sessions/${e}?beta=true`,{...s,headers:tz([{"anthropic-beta":[...r??[],"managed-agents-2026-04-01"].toString()},s?.headers])})}update(e,t,s){let{betas:r,...a}=t;return this._client.post(tG`/v1/sessions/${e}?beta=true`,{body:a,...s,headers:tz([{"anthropic-beta":[...r??[],"managed-agents-2026-04-01"].toString()},s?.headers])})}list(e={},t){let{betas:s,...r}=e??{};return this._client.getAPIList("/v1/sessions?beta=true",tA,{query:r,...t,headers:tz([{"anthropic-beta":[...s??[],"managed-agents-2026-04-01"].toString()},t?.headers])})}delete(e,t={},s){let{betas:r}=t??{};return this._client.delete(tG`/v1/sessions/${e}?beta=true`,{...s,headers:tz([{"anthropic-beta":[...r??[],"managed-agents-2026-04-01"].toString()},s?.headers])})}archive(e,t={},s){let{betas:r}=t??{};return this._client.post(tG`/v1/sessions/${e}/archive?beta=true`,{...s,headers:tz([{"anthropic-beta":[...r??[],"managed-agents-2026-04-01"].toString()},s?.headers])})}}sv.Events=sx,sv.Resources=sb;class sw extends tW{create(e,t={},s){let{betas:r,...a}=t??{};return this._client.post(tG`/v1/skills/${e}/versions?beta=true`,tM({body:a,...s,headers:tz([{"anthropic-beta":[...r??[],"skills-2025-10-02"].toString()},s?.headers])},this._client))}retrieve(e,t,s){let{skill_id:r,betas:a}=t;return this._client.get(tG`/v1/skills/${r}/versions/${e}?beta=true`,{...s,headers:tz([{"anthropic-beta":[...a??[],"skills-2025-10-02"].toString()},s?.headers])})}list(e,t={},s){let{betas:r,...a}=t??{};return this._client.getAPIList(tG`/v1/skills/${e}/versions?beta=true`,tA,{query:a,...s,headers:tz([{"anthropic-beta":[...r??[],"skills-2025-10-02"].toString()},s?.headers])})}delete(e,t,s){let{skill_id:r,betas:a}=t;return this._client.delete(tG`/v1/skills/${r}/versions/${e}?beta=true`,{...s,headers:tz([{"anthropic-beta":[...a??[],"skills-2025-10-02"].toString()},s?.headers])})}}class s_ extends tW{constructor(){super(...arguments),this.versions=new sw(this._client)}create(e={},t){let{betas:s,...r}=e??{};return this._client.post("/v1/skills?beta=true",tM({body:r,...t,headers:tz([{"anthropic-beta":[...s??[],"skills-2025-10-02"].toString()},t?.headers])},this._client,!1))}retrieve(e,t={},s){let{betas:r}=t??{};return this._client.get(tG`/v1/skills/${e}?beta=true`,{...s,headers:tz([{"anthropic-beta":[...r??[],"skills-2025-10-02"].toString()},s?.headers])})}list(e={},t){let{betas:s,...r}=e??{};return this._client.getAPIList("/v1/skills?beta=true",tA,{query:r,...t,headers:tz([{"anthropic-beta":[...s??[],"skills-2025-10-02"].toString()},t?.headers])})}delete(e,t={},s){let{betas:r}=t??{};return this._client.delete(tG`/v1/skills/${e}?beta=true`,{...s,headers:tz([{"anthropic-beta":[...r??[],"skills-2025-10-02"].toString()},s?.headers])})}}s_.Versions=sw;class sS extends tW{create(e,t,s){let{betas:r,...a}=t;return this._client.post(tG`/v1/vaults/${e}/credentials?beta=true`,{body:a,...s,headers:tz([{"anthropic-beta":[...r??[],"managed-agents-2026-04-01"].toString()},s?.headers])})}retrieve(e,t,s){let{vault_id:r,betas:a}=t;return this._client.get(tG`/v1/vaults/${r}/credentials/${e}?beta=true`,{...s,headers:tz([{"anthropic-beta":[...a??[],"managed-agents-2026-04-01"].toString()},s?.headers])})}update(e,t,s){let{vault_id:r,betas:a,...n}=t;return this._client.post(tG`/v1/vaults/${r}/credentials/${e}?beta=true`,{body:n,...s,headers:tz([{"anthropic-beta":[...a??[],"managed-agents-2026-04-01"].toString()},s?.headers])})}list(e,t={},s){let{betas:r,...a}=t??{};return this._client.getAPIList(tG`/v1/vaults/${e}/credentials?beta=true`,tA,{query:a,...s,headers:tz([{"anthropic-beta":[...r??[],"managed-agents-2026-04-01"].toString()},s?.headers])})}delete(e,t,s){let{vault_id:r,betas:a}=t;return this._client.delete(tG`/v1/vaults/${r}/credentials/${e}?beta=true`,{...s,headers:tz([{"anthropic-beta":[...a??[],"managed-agents-2026-04-01"].toString()},s?.headers])})}archive(e,t,s){let{vault_id:r,betas:a}=t;return this._client.post(tG`/v1/vaults/${r}/credentials/${e}/archive?beta=true`,{...s,headers:tz([{"anthropic-beta":[...a??[],"managed-agents-2026-04-01"].toString()},s?.headers])})}}class sj extends tW{constructor(){super(...arguments),this.credentials=new sS(this._client)}create(e,t){let{betas:s,...r}=e;return this._client.post("/v1/vaults?beta=true",{body:r,...t,headers:tz([{"anthropic-beta":[...s??[],"managed-agents-2026-04-01"].toString()},t?.headers])})}retrieve(e,t={},s){let{betas:r}=t??{};return this._client.get(tG`/v1/vaults/${e}?beta=true`,{...s,headers:tz([{"anthropic-beta":[...r??[],"managed-agents-2026-04-01"].toString()},s?.headers])})}update(e,t,s){let{betas:r,...a}=t;return this._client.post(tG`/v1/vaults/${e}?beta=true`,{body:a,...s,headers:tz([{"anthropic-beta":[...r??[],"managed-agents-2026-04-01"].toString()},s?.headers])})}list(e={},t){let{betas:s,...r}=e??{};return this._client.getAPIList("/v1/vaults?beta=true",tA,{query:r,...t,headers:tz([{"anthropic-beta":[...s??[],"managed-agents-2026-04-01"].toString()},t?.headers])})}delete(e,t={},s){let{betas:r}=t??{};return this._client.delete(tG`/v1/vaults/${e}?beta=true`,{...s,headers:tz([{"anthropic-beta":[...r??[],"managed-agents-2026-04-01"].toString()},s?.headers])})}archive(e,t={},s){let{betas:r}=t??{};return this._client.post(tG`/v1/vaults/${e}/archive?beta=true`,{...s,headers:tz([{"anthropic-beta":[...r??[],"managed-agents-2026-04-01"].toString()},s?.headers])})}}sj.Credentials=sS;class sk extends tW{constructor(){super(...arguments),this.models=new t0(this._client),this.messages=new sf(this._client),this.agents=new t4(this._client),this.environments=new tV(this._client),this.sessions=new sv(this._client),this.vaults=new sj(this._client),this.memoryStores=new t5(this._client),this.files=new tZ(this._client),this.skills=new s_(this._client),this.userProfiles=new t1(this._client)}}function sN(e){return e?.output_config?.format}function sE(e,t,s){let r=sN(t);return t&&"parse"in(r??{})?sT(e,t,s):{...e,content:e.content.map(e=>"text"===e.type?Object.defineProperty({...e},"parsed_output",{value:null,enumerable:!1}):e),parsed_output:null}}function sT(e,t,s){let r=null,a=e.content.map(e=>{if("text"===e.type){let s=function(e,t){let s=sN(e);if(s?.type!=="json_schema")return null;try{if("parse"in s)return s.parse(t);return JSON.parse(t)}catch(e){throw new eJ(`Failed to parse structured output: ${e}`)}}(t,e.text);return null===r&&(r=s),Object.defineProperty({...e},"parsed_output",{value:s,enumerable:!1})}return e});return{...e,content:a,parsed_output:r}}sk.Models=t0,sk.Messages=sf,sk.Agents=t4,sk.Environments=tV,sk.Sessions=sv,sk.Vaults=sj,sk.MemoryStores=t5,sk.Files=tZ,sk.Skills=s_,sk.UserProfiles=t1;let sC="__json_buf";function sA(e){return"tool_use"===e.type||"server_tool_use"===e.type}class sP{constructor(e,t){z.add(this),this.messages=[],this.receivedMessages=[],F.set(this,void 0),J.set(this,null),this.controller=new AbortController,G.set(this,void 0),V.set(this,()=>{}),K.set(this,()=>{}),X.set(this,void 0),Y.set(this,()=>{}),Q.set(this,()=>{}),Z.set(this,{}),ee.set(this,!1),et.set(this,!1),es.set(this,!1),er.set(this,!1),ea.set(this,void 0),en.set(this,void 0),ei.set(this,void 0),ec.set(this,e=>{if(eB(this,et,!0,"f"),ez(e)&&(e=new eV),e instanceof eV)return eB(this,es,!0,"f"),this._emit("abort",e);if(e instanceof eJ)return this._emit("error",e);if(e instanceof Error){let t=new eJ(e.message);return t.cause=e,this._emit("error",t)}return this._emit("error",new eJ(String(e)))}),eB(this,G,new Promise((e,t)=>{eB(this,V,e,"f"),eB(this,K,t,"f")}),"f"),eB(this,X,new Promise((e,t)=>{eB(this,Y,e,"f"),eB(this,Q,t,"f")}),"f"),eW(this,G,"f").catch(()=>{}),eW(this,X,"f").catch(()=>{}),eB(this,J,e,"f"),eB(this,ei,t?.logger??console,"f")}get response(){return eW(this,ea,"f")}get request_id(){return eW(this,en,"f")}async withResponse(){eB(this,er,!0,"f");let e=await eW(this,G,"f");if(!e)throw Error("Could not resolve a `Response` object");return{data:this,response:e,request_id:e.headers.get("request-id")}}static fromReadableStream(e){let t=new sP(null);return t._run(()=>t._fromReadableStream(e)),t}static createMessage(e,t,s,{logger:r}={}){let a=new sP(t,{logger:r});for(let e of t.messages)a._addMessageParam(e);return eB(a,J,{...t,stream:!0},"f"),a._run(()=>a._createMessage(e,{...t,stream:!0},{...s,headers:{...s?.headers,"X-Stainless-Helper-Method":"stream"}})),a}_run(e){e().then(()=>{this._emitFinal(),this._emit("end")},eW(this,ec,"f"))}_addMessageParam(e){this.messages.push(e)}_addMessage(e,t=!0){this.receivedMessages.push(e),t&&this._emit("message",e)}async _createMessage(e,t,s){let r,a=s?.signal;a&&(a.aborted&&this.controller.abort(),r=this.controller.abort.bind(this.controller),a.addEventListener("abort",r));try{eW(this,z,"m",ed).call(this);let{response:r,data:a}=await e.create({...t,stream:!0},{...s,signal:this.controller.signal}).withResponse();for await(let e of(this._connected(r),a))eW(this,z,"m",eu).call(this,e);if(a.controller.signal?.aborted)throw new eV;eW(this,z,"m",eh).call(this)}finally{a&&r&&a.removeEventListener("abort",r)}}_connected(e){this.ended||(eB(this,ea,e,"f"),eB(this,en,e?.headers.get("request-id"),"f"),eW(this,V,"f").call(this,e),this._emit("connect"))}get ended(){return eW(this,ee,"f")}get errored(){return eW(this,et,"f")}get aborted(){return eW(this,es,"f")}abort(){this.controller.abort()}on(e,t){return(eW(this,Z,"f")[e]||(eW(this,Z,"f")[e]=[])).push({listener:t}),this}off(e,t){let s=eW(this,Z,"f")[e];if(!s)return this;let r=s.findIndex(e=>e.listener===t);return r>=0&&s.splice(r,1),this}once(e,t){return(eW(this,Z,"f")[e]||(eW(this,Z,"f")[e]=[])).push({listener:t,once:!0}),this}emitted(e){return new Promise((t,s)=>{eB(this,er,!0,"f"),"error"!==e&&this.once("error",s),this.once(e,t)})}async done(){eB(this,er,!0,"f"),await eW(this,X,"f")}get currentMessage(){return eW(this,F,"f")}async finalMessage(){return await this.done(),eW(this,z,"m",eo).call(this)}async finalText(){return await this.done(),eW(this,z,"m",el).call(this)}_emit(e,...t){if(eW(this,ee,"f"))return;"end"===e&&(eB(this,ee,!0,"f"),eW(this,Y,"f").call(this));let s=eW(this,Z,"f")[e];if(s&&(eW(this,Z,"f")[e]=s.filter(e=>!e.once),s.forEach(({listener:e})=>e(...t))),"abort"===e){let e=t[0];eW(this,er,"f")||s?.length||Promise.reject(e),eW(this,K,"f").call(this,e),eW(this,Q,"f").call(this,e),this._emit("end");return}if("error"===e){let e=t[0];eW(this,er,"f")||s?.length||Promise.reject(e),eW(this,K,"f").call(this,e),eW(this,Q,"f").call(this,e),this._emit("end")}}_emitFinal(){this.receivedMessages.at(-1)&&this._emit("finalMessage",eW(this,z,"m",eo).call(this))}async _fromReadableStream(e,t){let s,r=t?.signal;r&&(r.aborted&&this.controller.abort(),s=this.controller.abort.bind(this.controller),r.addEventListener("abort",s));try{eW(this,z,"m",ed).call(this),this._connected(null);let t=tv.fromReadableStream(e,this.controller);for await(let e of t)eW(this,z,"m",eu).call(this,e);if(t.controller.signal?.aborted)throw new eV;eW(this,z,"m",eh).call(this)}finally{r&&s&&r.removeEventListener("abort",s)}}[(F=new WeakMap,J=new WeakMap,G=new WeakMap,V=new WeakMap,K=new WeakMap,X=new WeakMap,Y=new WeakMap,Q=new WeakMap,Z=new WeakMap,ee=new WeakMap,et=new WeakMap,es=new WeakMap,er=new WeakMap,ea=new WeakMap,en=new WeakMap,ei=new WeakMap,ec=new WeakMap,z=new WeakSet,eo=function(){if(0===this.receivedMessages.length)throw new eJ("stream ended without producing a Message with role=assistant");return this.receivedMessages.at(-1)},el=function(){if(0===this.receivedMessages.length)throw new eJ("stream ended without producing a Message with role=assistant");let e=this.receivedMessages.at(-1).content.filter(e=>"text"===e.type).map(e=>e.text);if(0===e.length)throw new eJ("stream ended without producing a content block with type=text");return e.join(" ")},ed=function(){this.ended||eB(this,F,void 0,"f")},eu=function(e){if(this.ended)return;let t=eW(this,z,"m",ep).call(this,e);switch(this._emit("streamEvent",e,t),e.type){case"content_block_delta":{let s=t.content.at(-1);switch(e.delta.type){case"text_delta":"text"===s.type&&this._emit("text",e.delta.text,s.text||"");break;case"citations_delta":"text"===s.type&&this._emit("citation",e.delta.citation,s.citations??[]);break;case"input_json_delta":sA(s)&&s.input&&this._emit("inputJson",e.delta.partial_json,s.input);break;case"thinking_delta":"thinking"===s.type&&this._emit("thinking",e.delta.thinking,s.thinking);break;case"signature_delta":"thinking"===s.type&&this._emit("signature",s.signature);break;default:sO(e.delta)}break}case"message_stop":this._addMessageParam(t),this._addMessage(sE(t,eW(this,J,"f"),{logger:eW(this,ei,"f")}),!0);break;case"content_block_stop":this._emit("contentBlock",t.content.at(-1));break;case"message_start":eB(this,F,t,"f")}},eh=function(){if(this.ended)throw new eJ("stream has ended, this shouldn't happen");let e=eW(this,F,"f");if(!e)throw new eJ("request ended without sending any chunks");return eB(this,F,void 0,"f"),sE(e,eW(this,J,"f"),{logger:eW(this,ei,"f")})},ep=function(e){let t=eW(this,F,"f");if("message_start"===e.type){if(t)throw new eJ(`Unexpected event order, got ${e.type} before receiving "message_stop"`);return e.message}if(!t)throw new eJ(`Unexpected event order, got ${e.type} before "message_start"`);switch(e.type){case"message_stop":case"content_block_stop":return t;case"message_delta":return t.stop_reason=e.delta.stop_reason,t.stop_sequence=e.delta.stop_sequence,t.usage.output_tokens=e.usage.output_tokens,null!=e.usage.input_tokens&&(t.usage.input_tokens=e.usage.input_tokens),null!=e.usage.cache_creation_input_tokens&&(t.usage.cache_creation_input_tokens=e.usage.cache_creation_input_tokens),null!=e.usage.cache_read_input_tokens&&(t.usage.cache_read_input_tokens=e.usage.cache_read_input_tokens),null!=e.usage.server_tool_use&&(t.usage.server_tool_use=e.usage.server_tool_use),t;case"content_block_start":return t.content.push({...e.content_block}),t;case"content_block_delta":{let s=t.content.at(e.index);switch(e.delta.type){case"text_delta":s?.type==="text"&&(t.content[e.index]={...s,text:(s.text||"")+e.delta.text});break;case"citations_delta":s?.type==="text"&&(t.content[e.index]={...s,citations:[...s.citations??[],e.delta.citation]});break;case"input_json_delta":if(s&&sA(s)){let r=s[sC]||"";r+=e.delta.partial_json;let a={...s};Object.defineProperty(a,sC,{value:r,enumerable:!1,writable:!0}),r&&(a.input=sa(r)),t.content[e.index]=a}break;case"thinking_delta":s?.type==="thinking"&&(t.content[e.index]={...s,thinking:s.thinking+e.delta.thinking});break;case"signature_delta":s?.type==="thinking"&&(t.content[e.index]={...s,signature:e.delta.signature});break;default:sO(e.delta)}return t}}},Symbol.asyncIterator)](){let e=[],t=[],s=!1;return this.on("streamEvent",s=>{let r=t.shift();r?r.resolve(s):e.push(s)}),this.on("end",()=>{for(let e of(s=!0,t))e.resolve(void 0);t.length=0}),this.on("abort",e=>{for(let r of(s=!0,t))r.reject(e);t.length=0}),this.on("error",e=>{for(let r of(s=!0,t))r.reject(e);t.length=0}),{next:async()=>e.length?{value:e.shift(),done:!1}:s?{value:void 0,done:!0}:new Promise((e,s)=>t.push({resolve:e,reject:s})).then(e=>e?{value:e,done:!1}:{value:void 0,done:!0}),return:async()=>(this.abort(),{value:void 0,done:!0})}}toReadableStream(){return new tv(this[Symbol.asyncIterator].bind(this),this.controller).toReadableStream()}}function sO(e){}class sI extends tW{create(e,t){return this._client.post("/v1/messages/batches",{body:e,...t})}retrieve(e,t){return this._client.get(tG`/v1/messages/batches/${e}`,t)}list(e={},t){return this._client.getAPIList("/v1/messages/batches",tC,{query:e,...t})}delete(e,t){return this._client.delete(tG`/v1/messages/batches/${e}`,t)}cancel(e,t){return this._client.post(tG`/v1/messages/batches/${e}/cancel`,t)}async results(e,t){let s=await this.retrieve(e);if(!s.results_url)throw new eJ(`No batch \`results_url\`; Has it finished processing? ${s.processing_status} - ${s.id}`);return this._client.get(s.results_url,{...t,headers:tz([{Accept:"application/binary"},t?.headers]),stream:!0,__binaryResponse:!0})._thenUnwrap((e,t)=>t8.fromResponse(t.response,t.controller))}}class sR extends tW{constructor(){super(...arguments),this.batches=new sI(this._client)}create(e,t){e.model in sM&&console.warn(`The model '${e.model}' is deprecated and will reach end-of-life on ${sM[e.model]} +Please migrate to a newer model. Visit https://docs.anthropic.com/en/docs/resources/model-deprecations for more information.`),s$.includes(e.model)&&e.thinking&&"enabled"===e.thinking.type&&console.warn(`Using Claude with ${e.model} and 'thinking.type=enabled' is deprecated. Use 'thinking.type=adaptive' instead which results in better model performance in our testing: https://platform.claude.com/docs/en/build-with-claude/adaptive-thinking`);let s=this._client._options.timeout;if(!e.stream&&null==s){let t=t9[e.model]??void 0;s=this._client.calculateNonstreamingTimeout(e.max_tokens,t)}let r=tQ(e.tools,e.messages);return this._client.post("/v1/messages",{body:e,timeout:s??6e5,...t,headers:tz([r,t?.headers]),stream:e.stream??!1})}parse(e,t){return this.create(e,t).then(t=>sT(t,e,{logger:this._client.logger??console}))}stream(e,t){return sP.createMessage(this,e,t,{logger:this._client.logger??console})}countTokens(e,t){return this._client.post("/v1/messages/count_tokens",{body:e,...t})}}let sM={"claude-1.3":"November 6th, 2024","claude-1.3-100k":"November 6th, 2024","claude-instant-1.1":"November 6th, 2024","claude-instant-1.1-100k":"November 6th, 2024","claude-instant-1.2":"November 6th, 2024","claude-3-sonnet-20240229":"July 21st, 2025","claude-3-opus-20240229":"January 5th, 2026","claude-2.1":"July 21st, 2025","claude-2.0":"July 21st, 2025","claude-3-7-sonnet-latest":"February 19th, 2026","claude-3-7-sonnet-20250219":"February 19th, 2026","claude-3-5-haiku-latest":"February 19th, 2026","claude-3-5-haiku-20241022":"February 19th, 2026","claude-opus-4-0":"June 15th, 2026","claude-opus-4-20250514":"June 15th, 2026","claude-sonnet-4-0":"June 15th, 2026","claude-sonnet-4-20250514":"June 15th, 2026"},s$=["claude-mythos-preview","claude-opus-4-6"];sR.Batches=sI;class sL extends tW{retrieve(e,t={},s){let{betas:r}=t??{};return this._client.get(tG`/v1/models/${e}`,{...s,headers:tz([{...r?.toString()!=null?{"anthropic-beta":r?.toString()}:void 0},s?.headers])})}list(e={},t){let{betas:s,...r}=e??{};return this._client.getAPIList("/v1/models",tC,{query:r,...t,headers:tz([{...s?.toString()!=null?{"anthropic-beta":s?.toString()}:void 0},t?.headers])})}}class sU extends tW{create(e,t){let{betas:s,...r}=e;return this._client.post("/v1/complete",{body:r,timeout:this._client._options.timeout??6e5,...t,headers:tz([{...s?.toString()!=null?{"anthropic-beta":s?.toString()}:void 0},t?.headers]),stream:e.stream??!1})}}let sD=e=>void 0!==globalThis.process?globalThis.process.env?.[e]?.trim()||void 0:void 0!==globalThis.Deno&&globalThis.Deno.env?.get?.(e)?.trim()||void 0;class sq{constructor({baseURL:e=sD("ANTHROPIC_BASE_URL"),apiKey:t=sD("ANTHROPIC_API_KEY")??null,authToken:s=sD("ANTHROPIC_AUTH_TOKEN")??null,...r}={}){em.add(this),ef.set(this,void 0);const a={apiKey:t,authToken:s,...r,baseURL:e||"https://api.anthropic.com"};if(!a.dangerouslyAllowBrowser&&"u">typeof window&&void 0!==window.document&&"u">typeof navigator)throw new eJ("It looks like you're running in a browser-like environment.\n\nThis is disabled by default, as it risks exposing your secret API credentials to attackers.\nIf you understand the risks and have appropriate mitigations in place,\nyou can set the `dangerouslyAllowBrowser` option to `true`, e.g.,\n\nnew Anthropic({ apiKey, dangerouslyAllowBrowser: true });\n");this.baseURL=a.baseURL,this.timeout=a.timeout??eg.DEFAULT_TIMEOUT,this.logger=a.logger??console;const n="warn";this.logLevel=n,this.logLevel=tp(a.logLevel,"ClientOptions.logLevel",this)??tp(sD("ANTHROPIC_LOG"),"process.env['ANTHROPIC_LOG']",this)??n,this.fetchOptions=a.fetchOptions,this.maxRetries=a.maxRetries??2,this.fetch=a.fetch??function(){if("u">typeof fetch)return fetch;throw Error("`fetch` is not defined as a global; Either pass `fetch` to the client, `new Anthropic({ fetch })` or polyfill the global, `globalThis.fetch = fetch`")}(),eB(this,ef,tl,"f");const i=sD("ANTHROPIC_CUSTOM_HEADERS");if(i){const e={};for(const t of i.split("\n")){const s=t.indexOf(":");s>=0&&(e[t.substring(0,s).trim()]=t.substring(s+1).trim())}a.defaultHeaders={...e,...a.defaultHeaders}}this._options=a,this.apiKey="string"==typeof t?t:null,this.authToken=s}withOptions(e){return new this.constructor({...this._options,baseURL:this.baseURL,maxRetries:this.maxRetries,timeout:this.timeout,logger:this.logger,logLevel:this.logLevel,fetch:this.fetch,fetchOptions:this.fetchOptions,apiKey:this.apiKey,authToken:this.authToken,...e})}defaultQuery(){return this._options.defaultQuery}validateHeaders({values:e,nulls:t}){if(!(e.get("x-api-key")||e.get("authorization")||this.apiKey&&e.get("x-api-key")||t.has("x-api-key")||this.authToken&&e.get("authorization"))&&!t.has("authorization"))throw Error('Could not resolve authentication method. Expected either apiKey or authToken to be set. Or for one of the "X-Api-Key" or "Authorization" headers to be explicitly omitted')}async authHeaders(e){return tz([await this.apiKeyAuth(e),await this.bearerAuth(e)])}async apiKeyAuth(e){if(null!=this.apiKey)return tz([{"X-Api-Key":this.apiKey}])}async bearerAuth(e){if(null!=this.authToken)return tz([{Authorization:`Bearer ${this.authToken}`}])}stringifyQuery(e){return Object.entries(e).filter(([e,t])=>void 0!==t).map(([e,t])=>{if("string"==typeof t||"number"==typeof t||"boolean"==typeof t)return`${encodeURIComponent(e)}=${encodeURIComponent(t)}`;if(null===t)return`${encodeURIComponent(e)}=`;throw new eJ(`Cannot stringify type ${typeof t}; Expected string, number, boolean, or null. If you need to pass nested query parameters, you can manually encode them, e.g. { query: { 'foo[key1]': value1, 'foo[key2]': value2 } }, and please open a GitHub issue requesting better support for your use case.`)}).join("&")}getUserAgent(){return`${this.constructor.name}/JS ${tt}`}defaultIdempotencyKey(){return`stainless-node-retry-${eH()}`}makeStatusError(e,t,s,r){return eG.generate(e,t,s,r)}buildURL(e,t,s){let r=!eW(this,em,"m",ey).call(this)&&s||this.baseURL,a=new URL(e3.test(e)?e:r+(r.endsWith("/")&&e.startsWith("/")?e.slice(1):e)),n=this.defaultQuery(),i=Object.fromEntries(a.searchParams);return e9(n)&&e9(i)||(t={...i,...n,...t}),"object"==typeof t&&t&&!Array.isArray(t)&&(a.search=this.stringifyQuery(t)),a.toString()}_calculateNonstreamingTimeout(e){if(3600*e/128e3>600)throw new eJ("Streaming is required for operations that may take longer than 10 minutes. See https://github.com/anthropics/anthropic-sdk-typescript#streaming-responses for more details");return 6e5}async prepareOptions(e){}async prepareRequest(e,{url:t,options:s}){}get(e,t){return this.methodRequest("get",e,t)}post(e,t){return this.methodRequest("post",e,t)}patch(e,t){return this.methodRequest("patch",e,t)}put(e,t){return this.methodRequest("put",e,t)}delete(e,t){return this.methodRequest("delete",e,t)}methodRequest(e,t,s){return this.request(Promise.resolve(s).then(s=>({method:e,path:t,...s})))}request(e,t=null){return new tN(this,this.makeRequest(e,t,void 0))}async makeRequest(e,t,s){let r=await e,a=r.maxRetries??this.maxRetries;null==t&&(t=a),await this.prepareOptions(r);let{req:n,url:i,timeout:o}=await this.buildRequest(r,{retryCount:a-t});await this.prepareRequest(n,{url:i,options:r});let l="log_"+(0x1000000*Math.random()|0).toString(16).padStart(6,"0"),c=void 0===s?"":`, retryOf: ${s}`,d=Date.now();if(tx(this).debug(`[${l}] sending request`,tb({retryOfRequestLogID:s,method:r.method,url:i,options:r,headers:n.headers})),r.signal?.aborted)throw new eV;let u=new AbortController,h=await this.fetchWithTimeout(i,n,o,u).catch(eF),p=Date.now();if(h instanceof globalThis.Error){let e=`retrying, ${t} attempts remaining`;if(r.signal?.aborted)throw new eV;let a=ez(h)||/timed? ?out/i.test(String(h)+("cause"in h?String(h.cause):""));if(t)return tx(this).info(`[${l}] connection ${a?"timed out":"failed"} - ${e}`),tx(this).debug(`[${l}] connection ${a?"timed out":"failed"} (${e})`,tb({retryOfRequestLogID:s,url:i,durationMs:p-d,message:h.message})),this.retryRequest(r,t,s??l);if(tx(this).info(`[${l}] connection ${a?"timed out":"failed"} - error; no more retries left`),tx(this).debug(`[${l}] connection ${a?"timed out":"failed"} (error; no more retries left)`,tb({retryOfRequestLogID:s,url:i,durationMs:p-d,message:h.message})),a)throw new eX;throw new eK({cause:h})}let m=[...h.headers.entries()].filter(([e])=>"request-id"===e).map(([e,t])=>", "+e+": "+JSON.stringify(t)).join(""),g=`[${l}${c}${m}] ${n.method} ${i} ${h.ok?"succeeded":"failed"} with status ${h.status} in ${p-d}ms`;if(!h.ok){let e=await this.shouldRetry(h);if(t&&e){let e=`retrying, ${t} attempts remaining`;return await to(h.body),tx(this).info(`${g} - ${e}`),tx(this).debug(`[${l}] response error (${e})`,tb({retryOfRequestLogID:s,url:h.url,status:h.status,headers:h.headers,durationMs:p-d})),this.retryRequest(r,t,s??l,h.headers)}let a=e?"error; no more retries left":"error; not retryable";tx(this).info(`${g} - ${a}`);let n=await h.text().catch(e=>eF(e).message),i=te(n),o=i?void 0:n;throw tx(this).debug(`[${l}] response error (${a})`,tb({retryOfRequestLogID:s,url:h.url,status:h.status,headers:h.headers,message:o,durationMs:Date.now()-d})),this.makeStatusError(h.status,i,o,h.headers)}return tx(this).info(g),tx(this).debug(`[${l}] response start`,tb({retryOfRequestLogID:s,url:h.url,status:h.status,headers:h.headers,durationMs:p-d})),{response:h,options:r,controller:u,requestLogID:l,retryOfRequestLogID:s,startTime:d}}getAPIList(e,t,s){return this.requestAPIList(t,s&&"then"in s?s.then(t=>({method:"get",path:e,...t})):{method:"get",path:e,...s})}requestAPIList(e,t){return new tT(this,this.makeRequest(t,null,void 0),e)}async fetchWithTimeout(e,t,s,r){let{signal:a,method:n,...i}=t||{},o=this._makeAbort(r);a&&a.addEventListener("abort",o,{once:!0});let l=setTimeout(o,s),c=globalThis.ReadableStream&&i.body instanceof globalThis.ReadableStream||"object"==typeof i.body&&null!==i.body&&Symbol.asyncIterator in i.body,d={signal:r.signal,...c?{duplex:"half"}:{},method:"GET",...i};n&&(d.method=n.toUpperCase());try{return await this.fetch.call(void 0,e,d)}finally{clearTimeout(l)}}async shouldRetry(e){let t=e.headers.get("x-should-retry");return"true"===t||"false"!==t&&(408===e.status||409===e.status||429===e.status||!!(e.status>=500))}async retryRequest(e,t,s,r){let a,n,i=r?.get("retry-after-ms");if(i){let e=parseFloat(i);Number.isNaN(e)||(a=e)}let o=r?.get("retry-after");if(o&&!a){let e=parseFloat(o);a=Number.isNaN(e)?Date.parse(o)-Date.now():1e3*e}if(void 0===a){let s=e.maxRetries??this.maxRetries;a=this.calculateDefaultRetryTimeoutMillis(t,s)}return await (n=a,new Promise(e=>setTimeout(e,n))),this.makeRequest(e,t-1,s)}calculateDefaultRetryTimeoutMillis(e,t){return Math.min(.5*Math.pow(2,t-e),8)*(1-.25*Math.random())*1e3}calculateNonstreamingTimeout(e,t){if(36e5*e/128e3>6e5||null!=t&&e>t)throw new eJ("Streaming is required for operations that may take longer than 10 minutes. See https://github.com/anthropics/anthropic-sdk-typescript#long-requests for more details");return 6e5}async buildRequest(e,{retryCount:t=0}={}){let s={...e},{method:r,path:a,query:n,defaultBaseURL:i}=s,o=this.buildURL(a,n,i);"timeout"in s&&((e,t)=>{if("number"!=typeof t||!Number.isInteger(t))throw new eJ(`${e} must be an integer`);if(t<0)throw new eJ(`${e} must be a positive integer`)})("timeout",s.timeout),s.timeout=s.timeout??this.timeout;let{bodyHeaders:l,body:c}=this.buildBody({options:s}),d=await this.buildHeaders({options:e,method:r,bodyHeaders:l,retryCount:t});return{req:{method:r,headers:d,...s.signal&&{signal:s.signal},...globalThis.ReadableStream&&c instanceof globalThis.ReadableStream&&{duplex:"half"},...c&&{body:c},...this.fetchOptions??{},...s.fetchOptions??{}},url:o,timeout:s.timeout}}async buildHeaders({options:e,method:s,bodyHeaders:r,retryCount:a}){let n={};this.idempotencyHeader&&"get"!==s&&(e.idempotencyKey||(e.idempotencyKey=this.defaultIdempotencyKey()),n[this.idempotencyHeader]=e.idempotencyKey);let i=tz([n,{Accept:"application/json","User-Agent":this.getUserAgent(),"X-Stainless-Retry-Count":String(a),...e.timeout?{"X-Stainless-Timeout":String(Math.trunc(e.timeout/1e3))}:{},...t??(t=(()=>{let e="u">typeof Deno&&null!=Deno.build?"deno":"u">typeof EdgeRuntime?"edge":"[object process]"===Object.prototype.toString.call(void 0!==globalThis.process?globalThis.process:0)?"node":"unknown";if("deno"===e)return{"X-Stainless-Lang":"js","X-Stainless-Package-Version":tt,"X-Stainless-OS":tr(Deno.build.os),"X-Stainless-Arch":ts(Deno.build.arch),"X-Stainless-Runtime":"deno","X-Stainless-Runtime-Version":"string"==typeof Deno.version?Deno.version:Deno.version?.deno??"unknown"};if("u">typeof EdgeRuntime)return{"X-Stainless-Lang":"js","X-Stainless-Package-Version":tt,"X-Stainless-OS":"Unknown","X-Stainless-Arch":`other:${EdgeRuntime}`,"X-Stainless-Runtime":"edge","X-Stainless-Runtime-Version":globalThis.process.version};if("node"===e)return{"X-Stainless-Lang":"js","X-Stainless-Package-Version":tt,"X-Stainless-OS":tr(globalThis.process.platform??"unknown"),"X-Stainless-Arch":ts(globalThis.process.arch??"unknown"),"X-Stainless-Runtime":"node","X-Stainless-Runtime-Version":globalThis.process.version??"unknown"};let t=function(){if("u"e.abort()}buildBody({options:{body:e,headers:t}}){if(!e)return{bodyHeaders:void 0,body:void 0};let s=tz([t]);return ArrayBuffer.isView(e)||e instanceof ArrayBuffer||e instanceof DataView||"string"==typeof e&&s.values.has("content-type")||globalThis.Blob&&e instanceof globalThis.Blob||e instanceof FormData||e instanceof URLSearchParams||globalThis.ReadableStream&&e instanceof globalThis.ReadableStream?{bodyHeaders:void 0,body:e}:"object"==typeof e&&(Symbol.asyncIterator in e||Symbol.iterator in e&&"next"in e&&"function"==typeof e.next)?{bodyHeaders:void 0,body:tn(e)}:"object"==typeof e&&"application/x-www-form-urlencoded"===s.values.get("content-type")?{bodyHeaders:{"content-type":"application/x-www-form-urlencoded"},body:this.stringifyQuery(e)}:eW(this,ef,"f").call(this,{body:e,headers:s})}}eg=sq,ef=new WeakMap,em=new WeakSet,ey=function(){return"https://api.anthropic.com"!==this.baseURL},sq.Anthropic=eg,sq.HUMAN_PROMPT="\\n\\nHuman:",sq.AI_PROMPT="\\n\\nAssistant:",sq.DEFAULT_TIMEOUT=6e5,sq.AnthropicError=eJ,sq.APIError=eG,sq.APIConnectionError=eK,sq.APIConnectionTimeoutError=eX,sq.APIUserAbortError=eV,sq.NotFoundError=e0,sq.ConflictError=e1,sq.RateLimitError=e4,sq.BadRequestError=eY,sq.AuthenticationError=eQ,sq.InternalServerError=e6,sq.PermissionDeniedError=eZ,sq.UnprocessableEntityError=e2,sq.toFile=tq;class sB extends sq{constructor(){super(...arguments),this.completions=new sU(this),this.messages=new sR(this),this.models=new sL(this),this.beta=new sk(this)}}sB.Completions=sU,sB.Messages=sR,sB.Models=sL,sB.Beta=sk;var sW=e.i(727749);async function sH(e,t,s,r,a=[],n,i,o,l,c,d,u,h,p,m){if(!r)throw Error("Virtual Key is required");console.log=function(){};let g=m||(0,eL.getProxyBaseUrl)(),f={};a&&a.length>0&&(f["x-litellm-tags"]=a.join(","));let y=new sB({apiKey:r,baseURL:g,dangerouslyAllowBrowser:!0,defaultHeaders:f});try{let r=Date.now(),a=!1,p={model:s,messages:e.map(e=>({role:e.role,content:e.content})),stream:!0,max_tokens:1024,litellm_trace_id:c};for await(let e of(d&&(p.vector_store_ids=d),u&&(p.guardrails=u),h&&(p.policies=h),y.messages.stream(p,{signal:n}))){if(console.log("Stream event:",e),"content_block_delta"===e.type){let n=e.delta;if(!a){a=!0;let e=Date.now()-r;console.log("First token received! Time:",e,"ms"),o&&o(e)}"text_delta"===n.type?t("assistant",n.text,s):"reasoning_delta"===n.type&&i&&i(n.text)}if("message_delta"===e.type&&e.usage&&l){let t=e.usage;console.log("Usage data found:",t);let s={completionTokens:t.output_tokens,promptTokens:t.input_tokens,totalTokens:t.input_tokens+t.output_tokens};l(s)}}}catch(e){throw n?.aborted?console.log("Anthropic messages request was cancelled"):sW.default.fromBackend(`Error occurred while generating model response. Please try again. Error: ${e}`),e}}e.s(["makeAnthropicMessagesRequest",0,sH],434788);var sz=e.i(356449);async function sF(e,t,s,r,a,n,i,o,l,c){console.log=function(){},console.log("isLocal:",!1);let d=c||(0,eL.getProxyBaseUrl)(),u=new sz.default.OpenAI({apiKey:a,baseURL:d,dangerouslyAllowBrowser:!0,defaultHeaders:n&&n.length>0?{"x-litellm-tags":n.join(",")}:void 0});try{let a=await u.audio.speech.create({model:r,input:e,voice:t,...o?{response_format:o}:{},...l?{speed:l}:{}},{signal:i}),n=await a.blob(),c=URL.createObjectURL(n);s(c,r)}catch(e){throw i?.aborted?console.log("Audio speech request was cancelled"):sW.default.fromBackend(`Error occurred while generating speech. Please try again. Error: ${e}`),e}}async function sJ(e,t,s,r,a,n,i,o,l,c,d){console.log=function(){},console.log("isLocal:",!1);let u=d||(0,eL.getProxyBaseUrl)(),h=new sz.default.OpenAI({apiKey:r,baseURL:u,dangerouslyAllowBrowser:!0,defaultHeaders:a&&a.length>0?{"x-litellm-tags":a.join(",")}:void 0});try{console.log("Processing audio file for transcription:",e.name);let r=await h.audio.transcriptions.create({model:s,file:e,...i?{language:i}:{},...o?{prompt:o}:{},...l?{response_format:l}:{},...void 0!==c?{temperature:c}:{}},{signal:n});if(console.log("Transcription response:",r),r&&r.text)t(r.text,s),sW.default.success("Audio transcribed successfully");else throw Error("No transcription text in response")}catch(e){if(console.error("Error making audio transcription request:",e),n?.aborted)console.log("Audio transcription request was cancelled");else{let t="Failed to transcribe audio";e?.error?.message?t=e.error.message:e?.message&&(t=e.message),sW.default.fromBackend(`Audio transcription failed: ${t}`)}throw e}}async function sG(e,t,s,r,a,n){if(!r)throw Error("Virtual Key is required");console.log=function(){};let i=n||(0,eL.getProxyBaseUrl)(),o={};a&&a.length>0&&(o["x-litellm-tags"]=a.join(","));try{let a=i.endsWith("/")?i.slice(0,-1):i,n=`${a}/embeddings`,l=await fetch(n,{method:"POST",headers:{"Content-Type":"application/json",[(0,eL.getGlobalLitellmHeaderName)()]:`Bearer ${r}`,...o},body:JSON.stringify({model:s,input:e})});if(!l.ok){let e=await l.text();throw Error(e||`Request failed with status ${l.status}`)}let c=await l.json(),d=c?.data?.[0]?.embedding;if(!d)throw Error("No embedding returned from server");t(JSON.stringify(d),c?.model??s)}catch(e){throw sW.default.fromBackend(`Error occurred while making embeddings request. Please try again. Error: ${e}`),e}}e.s(["makeOpenAIAudioSpeechRequest",0,sF],512882),e.s(["makeOpenAIAudioTranscriptionRequest",0,sJ],584976),e.s(["makeOpenAIEmbeddingsRequest",0,sG],720762)},488143,(e,t,s)=>{"use strict";function r({widthInt:e,heightInt:t,blurWidth:s,blurHeight:a,blurDataURL:n,objectFit:i}){let o=s?40*s:e,l=a?40*a:t,c=o&&l?`viewBox='0 0 ${o} ${l}'`:"";return`%3Csvg xmlns='http://www.w3.org/2000/svg' ${c}%3E%3Cfilter id='b' color-interpolation-filters='sRGB'%3E%3CfeGaussianBlur stdDeviation='20'/%3E%3CfeColorMatrix values='1 0 0 0 0 0 1 0 0 0 0 0 1 0 0 0 0 0 100 -1' result='s'/%3E%3CfeFlood x='0' y='0' width='100%25' height='100%25'/%3E%3CfeComposite operator='out' in='s'/%3E%3CfeComposite in2='SourceGraphic'/%3E%3CfeGaussianBlur stdDeviation='20'/%3E%3C/filter%3E%3Cimage width='100%25' height='100%25' x='0' y='0' preserveAspectRatio='${c?"none":"contain"===i?"xMidYMid":"cover"===i?"xMidYMid slice":"none"}' style='filter: url(%23b);' href='${n}'/%3E%3C/svg%3E`}Object.defineProperty(s,"__esModule",{value:!0}),Object.defineProperty(s,"getImageBlurSvg",{enumerable:!0,get:function(){return r}})},987690,(e,t,s)=>{"use strict";Object.defineProperty(s,"__esModule",{value:!0});var r={VALID_LOADERS:function(){return n},imageConfigDefault:function(){return i}};for(var a in r)Object.defineProperty(s,a,{enumerable:!0,get:r[a]});let n=["default","imgix","cloudinary","akamai","custom"],i={deviceSizes:[640,750,828,1080,1200,1920,2048,3840],imageSizes:[32,48,64,96,128,256,384],path:"/_next/image",loader:"default",loaderFile:"",domains:[],disableStaticImages:!1,minimumCacheTTL:14400,formats:["image/webp"],maximumDiskCacheSize:void 0,maximumRedirects:3,maximumResponseBody:5e7,dangerouslyAllowLocalIP:!1,dangerouslyAllowSVG:!1,contentSecurityPolicy:"script-src 'none'; frame-src 'none'; sandbox;",contentDispositionType:"attachment",localPatterns:void 0,remotePatterns:[],qualities:[75],unoptimized:!1,customCacheHandler:!1}},908927,(e,t,s)=>{"use strict";Object.defineProperty(s,"__esModule",{value:!0}),Object.defineProperty(s,"getImgProps",{enumerable:!0,get:function(){return c}}),e.r(233525);let r=e.r(543369),a=e.r(488143),n=e.r(987690),i=["-moz-initial","fill","none","scale-down",void 0];function o(e){return void 0!==e.default}function l(e){return void 0===e?e:"number"==typeof e?Number.isFinite(e)?e:NaN:"string"==typeof e&&/^[0-9]+$/.test(e)?parseInt(e,10):NaN}function c({src:e,sizes:t,unoptimized:s=!1,priority:d=!1,preload:u=!1,loading:h,className:p,quality:m,width:g,height:f,fill:y=!1,style:x,overrideSrc:b,onLoad:v,onLoadingComplete:w,placeholder:_="empty",blurDataURL:S,fetchPriority:j,decoding:k="async",layout:N,objectFit:E,objectPosition:T,lazyBoundary:C,lazyRoot:A,...P},O){var I;let R,M,$,{imgConf:L,showAltText:U,blurComplete:D,defaultLoader:q}=O,B=L||n.imageConfigDefault;if("allSizes"in B)R=B;else{let e=[...B.deviceSizes,...B.imageSizes].sort((e,t)=>e-t),t=B.deviceSizes.sort((e,t)=>e-t),s=B.qualities?.sort((e,t)=>e-t);R={...B,allSizes:e,deviceSizes:t,qualities:s}}if(void 0===q)throw Object.defineProperty(Error("images.loaderFile detected but the file is missing default export.\nRead more: https://nextjs.org/docs/messages/invalid-images-config"),"__NEXT_ERROR_CODE",{value:"E163",enumerable:!1,configurable:!0});let W=P.loader||q;delete P.loader,delete P.srcSet;let H="__next_img_default"in W;if(H){if("custom"===R.loader)throw Object.defineProperty(Error(`Image with src "${e}" is missing "loader" prop. +Read more: https://nextjs.org/docs/messages/next-image-missing-loader`),"__NEXT_ERROR_CODE",{value:"E252",enumerable:!1,configurable:!0})}else{let e=W;W=t=>{let{config:s,...r}=t;return e(r)}}if(N){"fill"===N&&(y=!0);let e={intrinsic:{maxWidth:"100%",height:"auto"},responsive:{width:"100%",height:"auto"}}[N];e&&(x={...x,...e});let s={responsive:"100vw",fill:"100vw"}[N];s&&!t&&(t=s)}let z="",F=l(g),J=l(f);if((I=e)&&"object"==typeof I&&(o(I)||void 0!==I.src)){let t=o(e)?e.default:e;if(!t.src)throw Object.defineProperty(Error(`An object should only be passed to the image component src parameter if it comes from a static image import. It must include src. Received ${JSON.stringify(t)}`),"__NEXT_ERROR_CODE",{value:"E460",enumerable:!1,configurable:!0});if(!t.height||!t.width)throw Object.defineProperty(Error(`An object should only be passed to the image component src parameter if it comes from a static image import. It must include height and width. Received ${JSON.stringify(t)}`),"__NEXT_ERROR_CODE",{value:"E48",enumerable:!1,configurable:!0});if(M=t.blurWidth,$=t.blurHeight,S=S||t.blurDataURL,z=t.src,!y)if(F||J){if(F&&!J){let e=F/t.width;J=Math.round(t.height*e)}else if(!F&&J){let e=J/t.height;F=Math.round(t.width*e)}}else F=t.width,J=t.height}let G=!d&&!u&&("lazy"===h||void 0===h);(!(e="string"==typeof e?e:z)||e.startsWith("data:")||e.startsWith("blob:"))&&(s=!0,G=!1),R.unoptimized&&(s=!0),H&&!R.dangerouslyAllowSVG&&e.split("?",1)[0].endsWith(".svg")&&(s=!0);let V=l(m),K=Object.assign(y?{position:"absolute",height:"100%",width:"100%",left:0,top:0,right:0,bottom:0,objectFit:E,objectPosition:T}:{},U?{}:{color:"transparent"},x),X=D||"empty"===_?null:"blur"===_?`url("data:image/svg+xml;charset=utf-8,${(0,a.getImageBlurSvg)({widthInt:F,heightInt:J,blurWidth:M,blurHeight:$,blurDataURL:S||"",objectFit:K.objectFit})}")`:`url("${_}")`,Y=i.includes(K.objectFit)?"fill"===K.objectFit?"100% 100%":"cover":K.objectFit,Q=X?{backgroundSize:Y,backgroundPosition:K.objectPosition||"50% 50%",backgroundRepeat:"no-repeat",backgroundImage:X}:{},Z=function({config:e,src:t,unoptimized:s,width:a,quality:n,sizes:i,loader:o}){if(s){if(t.startsWith("/")&&!t.startsWith("//")){let e=(0,r.getDeploymentId)();if(e){let s=t.indexOf("?");if(-1!==s){let r=new URLSearchParams(t.slice(s+1));r.get("dpl")||(r.append("dpl",e),t=t.slice(0,s)+"?"+r.toString())}else t+=`?dpl=${e}`}}return{src:t,srcSet:void 0,sizes:void 0}}let{widths:l,kind:c}=function({deviceSizes:e,allSizes:t},s,r){if(r){let s=/(^|\s)(1?\d?\d)vw/g,a=[];for(let e;e=s.exec(r);)a.push(parseInt(e[2]));if(a.length){let s=.01*Math.min(...a);return{widths:t.filter(t=>t>=e[0]*s),kind:"w"}}return{widths:t,kind:"w"}}return"number"!=typeof s?{widths:e,kind:"w"}:{widths:[...new Set([s,2*s].map(e=>t.find(t=>t>=e)||t[t.length-1]))],kind:"x"}}(e,a,i),d=l.length-1;return{sizes:i||"w"!==c?i:"100vw",srcSet:l.map((s,r)=>`${o({config:e,src:t,quality:n,width:s})} ${"w"===c?s:r+1}${c}`).join(", "),src:o({config:e,src:t,quality:n,width:l[d]})}}({config:R,src:e,unoptimized:s,width:F,quality:V,sizes:t,loader:W}),ee=G?"lazy":h;return{props:{...P,loading:ee,fetchPriority:j,width:F,height:J,decoding:k,className:p,style:{...K,...Q},sizes:Z.sizes,srcSet:Z.srcSet,src:b||Z.src},meta:{unoptimized:s,preload:u||d,placeholder:_,fill:y}}}},898879,(e,t,s)=>{"use strict";Object.defineProperty(s,"__esModule",{value:!0}),Object.defineProperty(s,"default",{enumerable:!0,get:function(){return o}});let r=e.r(271645),a="u"{}:r.useLayoutEffect,i=a?()=>{}:r.useEffect;function o(e){let{headManager:t,reduceComponentsToState:s}=e;function o(){if(t&&t.mountedInstances){let e=r.Children.toArray(Array.from(t.mountedInstances).filter(Boolean));t.updateHead(s(e))}}return a&&(t?.mountedInstances?.add(e.children),o()),n(()=>(t?.mountedInstances?.add(e.children),()=>{t?.mountedInstances?.delete(e.children)})),n(()=>(t&&(t._pendingUpdate=o),()=>{t&&(t._pendingUpdate=o)})),i(()=>(t&&t._pendingUpdate&&(t._pendingUpdate(),t._pendingUpdate=null),()=>{t&&t._pendingUpdate&&(t._pendingUpdate(),t._pendingUpdate=null)})),null}},325633,(e,t,s)=>{"use strict";Object.defineProperty(s,"__esModule",{value:!0});var r={default:function(){return g},defaultHead:function(){return u}};for(var a in r)Object.defineProperty(s,a,{enumerable:!0,get:r[a]});let n=e.r(563141),i=e.r(151836),o=e.r(843476),l=i._(e.r(271645)),c=n._(e.r(898879)),d=e.r(742732);function u(){return[(0,o.jsx)("meta",{charSet:"utf-8"},"charset"),(0,o.jsx)("meta",{name:"viewport",content:"width=device-width"},"viewport")]}function h(e,t){return"string"==typeof t||"number"==typeof t?e:t.type===l.default.Fragment?e.concat(l.default.Children.toArray(t.props.children).reduce((e,t)=>"string"==typeof t||"number"==typeof t?e:e.concat(t),[])):e.concat(t)}e.r(233525);let p=["name","httpEquiv","charSet","itemProp"];function m(e){let t,s,r,a;return e.reduce(h,[]).reverse().concat(u().reverse()).filter((t=new Set,s=new Set,r=new Set,a={},e=>{let n=!0,i=!1;if(e.key&&"number"!=typeof e.key&&e.key.indexOf("$")>0){i=!0;let s=e.key.slice(e.key.indexOf("$")+1);t.has(s)?n=!1:t.add(s)}switch(e.type){case"title":case"base":s.has(e.type)?n=!1:s.add(e.type);break;case"meta":for(let t=0,s=p.length;t{let s=e.key||t;return l.default.cloneElement(e,{key:s})})}let g=function({children:e}){let t=(0,l.useContext)(d.HeadManagerContext);return(0,o.jsx)(c.default,{reduceComponentsToState:m,headManager:t,children:e})};("function"==typeof s.default||"object"==typeof s.default&&null!==s.default)&&void 0===s.default.__esModule&&(Object.defineProperty(s.default,"__esModule",{value:!0}),Object.assign(s.default,s),t.exports=s.default)},918556,(e,t,s)=>{"use strict";Object.defineProperty(s,"__esModule",{value:!0}),Object.defineProperty(s,"ImageConfigContext",{enumerable:!0,get:function(){return n}});let r=e.r(563141)._(e.r(271645)),a=e.r(987690),n=r.default.createContext(a.imageConfigDefault)},65856,(e,t,s)=>{"use strict";Object.defineProperty(s,"__esModule",{value:!0}),Object.defineProperty(s,"RouterContext",{enumerable:!0,get:function(){return r}});let r=e.r(563141)._(e.r(271645)).default.createContext(null)},670965,(e,t,s)=>{"use strict";function r(e,t){let s=e||75;return t?.qualities?.length?t.qualities.reduce((e,t)=>Math.abs(t-s){"use strict";Object.defineProperty(s,"__esModule",{value:!0}),Object.defineProperty(s,"default",{enumerable:!0,get:function(){return i}});let r=e.r(670965),a=e.r(543369);function n({config:e,src:t,width:s,quality:i}){let o=(0,a.getDeploymentId)();if(t.startsWith("/")&&!t.startsWith("//")){let e=t.indexOf("?");if(-1!==e){let s=new URLSearchParams(t.slice(e+1)),r=s.get("dpl");if(r){o=r,s.delete("dpl");let a=s.toString();t=t.slice(0,e)+(a?"?"+a:"")}}}if(t.startsWith("/")&&t.includes("?")&&e.localPatterns?.length===1&&"**"===e.localPatterns[0].pathname&&""===e.localPatterns[0].search)throw Object.defineProperty(Error(`Image with src "${t}" is using a query string which is not configured in images.localPatterns. +Read more: https://nextjs.org/docs/messages/next-image-unconfigured-localpatterns`),"__NEXT_ERROR_CODE",{value:"E871",enumerable:!1,configurable:!0});let l=(0,r.findClosestQuality)(i,e);return`${e.path}?url=${encodeURIComponent(t)}&w=${s}&q=${l}${t.startsWith("/")&&o?`&dpl=${o}`:""}`}n.__next_img_default=!0;let i=n},605500,(e,t,s)=>{"use strict";Object.defineProperty(s,"__esModule",{value:!0}),Object.defineProperty(s,"Image",{enumerable:!0,get:function(){return v}});let r=e.r(563141),a=e.r(151836),n=e.r(843476),i=a._(e.r(271645)),o=r._(e.r(174080)),l=r._(e.r(325633)),c=e.r(908927),d=e.r(987690),u=e.r(918556);e.r(233525);let h=e.r(65856),p=r._(e.r(1948)),m=e.r(818581),g={deviceSizes:[640,750,828,1080,1200,1920,2048,3840],imageSizes:[32,48,64,96,128,256,384],qualities:[75],path:"/_next/image",loader:"default",dangerouslyAllowSVG:!1,unoptimized:!0};function f(e,t,s,r,a,n,i){let o=e?.src;e&&e["data-loaded-src"]!==o&&(e["data-loaded-src"]=o,("decode"in e?e.decode():Promise.resolve()).catch(()=>{}).then(()=>{if(e.parentElement&&e.isConnected){if("empty"!==t&&a(!0),s?.current){let t=new Event("load");Object.defineProperty(t,"target",{writable:!1,value:e});let r=!1,a=!1;s.current({...t,nativeEvent:t,currentTarget:e,target:e,isDefaultPrevented:()=>r,isPropagationStopped:()=>a,persist:()=>{},preventDefault:()=>{r=!0,t.preventDefault()},stopPropagation:()=>{a=!0,t.stopPropagation()}})}r?.current&&r.current(e)}}))}function y(e){return i.use?{fetchPriority:e}:{fetchpriority:e}}"u"{let E=(0,i.useCallback)(e=>{e&&(j&&(e.src=e.src),e.complete&&f(e,u,x,b,v,p,_))},[e,u,x,b,v,j,p,_]),T=(0,m.useMergedRef)(N,E);return(0,n.jsx)("img",{...k,...y(d),loading:h,width:a,height:r,decoding:o,"data-nimg":g?"fill":"1",className:l,style:c,sizes:s,srcSet:t,src:e,ref:T,onLoad:e=>{f(e.currentTarget,u,x,b,v,p,_)},onError:e=>{w(!0),"empty"!==u&&v(!0),j&&j(e)}})});function b({isAppRouter:e,imgAttributes:t}){let s={as:"image",imageSrcSet:t.srcSet,imageSizes:t.sizes,crossOrigin:t.crossOrigin,referrerPolicy:t.referrerPolicy,...y(t.fetchPriority)};return e&&o.default.preload?(o.default.preload(t.src,s),null):(0,n.jsx)(l.default,{children:(0,n.jsx)("link",{rel:"preload",href:t.srcSet?void 0:t.src,...s},"__nimg-"+t.src+t.srcSet+t.sizes)})}let v=(0,i.forwardRef)((e,t)=>{let s=(0,i.useContext)(h.RouterContext),r=(0,i.useContext)(u.ImageConfigContext),a=(0,i.useMemo)(()=>{let e=g||r||d.imageConfigDefault,t=[...e.deviceSizes,...e.imageSizes].sort((e,t)=>e-t),s=e.deviceSizes.sort((e,t)=>e-t),a=e.qualities?.sort((e,t)=>e-t);return{...e,allSizes:t,deviceSizes:s,qualities:a,localPatterns:"u"{m.current=o},[o]);let f=(0,i.useRef)(l);(0,i.useEffect)(()=>{f.current=l},[l]);let[y,v]=(0,i.useState)(!1),[w,_]=(0,i.useState)(!1),{props:S,meta:j}=(0,c.getImgProps)(e,{defaultLoader:p.default,imgConf:a,blurComplete:y,showAltText:w});return(0,n.jsxs)(n.Fragment,{children:[(0,n.jsx)(x,{...S,unoptimized:j.unoptimized,placeholder:j.placeholder,fill:j.fill,onLoadRef:m,onLoadingCompleteRef:f,setBlurComplete:v,setShowAltText:_,sizesInput:e.sizes,ref:t}),j.preload?(0,n.jsx)(b,{isAppRouter:!s,imgAttributes:S}):null]})});("function"==typeof s.default||"object"==typeof s.default&&null!==s.default)&&void 0===s.default.__esModule&&(Object.defineProperty(s.default,"__esModule",{value:!0}),Object.assign(s.default,s),t.exports=s.default)},794909,(e,t,s)=>{"use strict";Object.defineProperty(s,"__esModule",{value:!0});var r={default:function(){return d},getImageProps:function(){return c}};for(var a in r)Object.defineProperty(s,a,{enumerable:!0,get:r[a]});let n=e.r(563141),i=e.r(908927),o=e.r(605500),l=n._(e.r(1948));function c(e){let{props:t}=(0,i.getImgProps)(e,{defaultLoader:l.default,imgConf:{deviceSizes:[640,750,828,1080,1200,1920,2048,3840],imageSizes:[32,48,64,96,128,256,384],qualities:[75],path:"/_next/image",loader:"default",dangerouslyAllowSVG:!1,unoptimized:!0}});for(let[e,s]of Object.entries(t))void 0===s&&delete t[e];return{props:t}}let d=o.Image},657688,(e,t,s)=>{t.exports=e.r(794909)},220486,761793,964421,91500,843153,152401,e=>{"use strict";var t=e.i(843476),s=e.i(218129),r=e.i(132104),a=e.i(447593),n=e.i(245094),i=e.i(210612),o=e.i(955135),l=e.i(827252),c=e.i(438957),d=e.i(596239),u=e.i(56456),h=e.i(124608),p=e.i(983561),m=e.i(602073),g=e.i(313603),f=e.i(782273),y=e.i(232164),x=e.i(366308),b=e.i(304967),v=e.i(599724),w=e.i(779241),_=e.i(629569),S=e.i(994388),j=e.i(464571),k=e.i(311451),N=e.i(212931),E=e.i(282786),T=e.i(199133),C=e.i(482725),A=e.i(592968),P=e.i(898586),O=e.i(515831),I=e.i(271645),R=e.i(650056),M=e.i(219470),$=e.i(614677),L=e.i(891547),U=e.i(921511),D=e.i(235267),q=e.i(611052),B=e.i(727749),W=e.i(764205),H=e.i(318059),z=e.i(916940),F=e.i(953860),J=e.i(434788),G=e.i(512882),V=e.i(584976),K=e.i(254530),X=e.i(720762),Y=e.i(921687),Q=e.i(689020);e.i(247167);var Z=e.i(356449);async function ee(e,t,s,r,a,n,i,o){console.log=function(){},console.log("isLocal:",!1);let l=o||(0,W.getProxyBaseUrl)(),c=new Z.default.OpenAI({apiKey:a,baseURL:l,dangerouslyAllowBrowser:!0,defaultHeaders:n&&n.length>0?{"x-litellm-tags":n.join(",")}:void 0});try{let a=Array.isArray(e)?e:[e],n=[];for(let e=0;e1&&B.default.success(`Successfully processed ${n.length} images`)}catch(e){if(console.error("Error making image edit request:",e),i?.aborted)console.log("Image edits request was cancelled");else{let t="Failed to edit image(s)";e?.error?.message?t=e.error.message:e?.message&&(t=e.message),B.default.fromBackend(`Image edit failed: ${t}`)}throw e}}async function et(e,t,s,r,a,n,i){console.log=function(){},console.log("isLocal:",!1);let o=i||(0,W.getProxyBaseUrl)(),l=new Z.default.OpenAI({apiKey:r,baseURL:o,dangerouslyAllowBrowser:!0,defaultHeaders:a&&a.length>0?{"x-litellm-tags":a.join(",")}:void 0});try{let r=await l.images.generate({model:s,prompt:e},{signal:n});if(console.log(r.data),r.data&&r.data[0])if(r.data[0].url)t(r.data[0].url,s);else if(r.data[0].b64_json){let e=r.data[0].b64_json;t(`data:image/png;base64,${e}`,s)}else throw Error("No image data found in response");else throw Error("Invalid response format")}catch(e){throw n?.aborted?console.log("Image generation request was cancelled"):B.default.fromBackend(`Error occurred while generating image. Please try again. Error: ${e}`),e}}var es=e.i(452598),er=e.i(536916),ea=e.i(28651),en=e.i(850627);let ei=({temperature:e=1,maxTokens:s=2048,useAdvancedParams:r,onTemperatureChange:a,onMaxTokensChange:n,onUseAdvancedParamsChange:i,mockTestFallbacks:o,onMockTestFallbacksChange:c})=>{let[d,u]=(0,I.useState)(!1),h=void 0!==r?r:d,[p,m]=(0,I.useState)(e),[g,f]=(0,I.useState)(s);(0,I.useEffect)(()=>{m(e)},[e]),(0,I.useEffect)(()=>{f(s)},[s]);let y=e=>{let t=e??1;m(t),a?.(t)},x=e=>{let t=e??1e3;f(t),n?.(t)},b=h?"text-gray-700":"text-gray-400";return(0,t.jsxs)("div",{className:"space-y-4 p-4 w-80",children:[(0,t.jsx)(er.Checkbox,{checked:h,onChange:e=>{var t;return t=e.target.checked,void(i?i(t):u(t))},children:(0,t.jsx)("span",{className:"font-medium",children:"Use Advanced Parameters"})}),c&&(0,t.jsxs)("div",{className:"flex items-center gap-1",children:[(0,t.jsx)(er.Checkbox,{checked:o??!1,onChange:e=>c(e.target.checked),children:(0,t.jsx)("span",{className:"font-medium",children:"Simulate failure to test fallbacks"})}),(0,t.jsx)(E.Popover,{trigger:"hover",placement:"right",content:(0,t.jsxs)("div",{style:{maxWidth:340},children:[(0,t.jsx)(P.Typography.Paragraph,{className:"text-sm",style:{marginBottom:8},children:"Causes the first request to fail so the router tries fallbacks (if configured). Use this to verify your fallback setup."}),(0,t.jsxs)(P.Typography.Paragraph,{className:"text-sm",style:{marginBottom:0},children:["Behavior can differ when keys, teams, or router settings are configured."," ",(0,t.jsx)("a",{href:"https://docs.litellm.ai/docs/proxy/keys_teams_router_settings",target:"_blank",rel:"noopener noreferrer",className:"text-blue-600 hover:text-blue-800",children:"Learn more"})]})]}),children:(0,t.jsx)(l.InfoCircleOutlined,{className:"text-xs text-gray-400 cursor-pointer shrink-0 hover:text-gray-600","aria-label":"Help: Simulate failure to test fallbacks"})})]}),(0,t.jsxs)("div",{className:"space-y-4 transition-opacity duration-200",style:{opacity:h?1:.4},children:[(0,t.jsxs)("div",{children:[(0,t.jsxs)("div",{className:"flex items-center justify-between mb-2",children:[(0,t.jsxs)("div",{className:"flex items-center gap-1",children:[(0,t.jsx)(v.Text,{className:`text-sm ${b}`,children:"Temperature"}),(0,t.jsx)(A.Tooltip,{title:"Controls randomness. Lower values make output more deterministic, higher values more creative.",children:(0,t.jsx)(l.InfoCircleOutlined,{className:`text-xs ${b} cursor-help`})})]}),(0,t.jsx)(ea.InputNumber,{min:0,max:2,step:.1,value:p,onChange:y,disabled:!h,precision:1,className:"w-20"})]}),(0,t.jsx)(en.Slider,{min:0,max:2,step:.1,value:p,onChange:y,disabled:!h,marks:{0:"0",1:"1.0",2:"2.0"}})]}),(0,t.jsxs)("div",{children:[(0,t.jsxs)("div",{className:"flex items-center justify-between mb-2",children:[(0,t.jsxs)("div",{className:"flex items-center gap-1",children:[(0,t.jsx)(v.Text,{className:`text-sm ${b}`,children:"Max Tokens"}),(0,t.jsx)(A.Tooltip,{title:"Maximum number of tokens to generate in the response.",children:(0,t.jsx)(l.InfoCircleOutlined,{className:`text-xs ${b} cursor-help`})})]}),(0,t.jsx)(ea.InputNumber,{min:1,max:32768,step:1,value:g,onChange:x,disabled:!h})]}),(0,t.jsx)(en.Slider,{min:1,max:32768,step:1,value:g,onChange:x,disabled:!h,marks:{1:"1",32768:"32768"}})]})]})]})};var eo=e.i(785913);let el={ALLOY:"Alloy - Professional and confident",ASH:"Ash - Casual and relaxed",BALAD:"Ballad - Smooth and melodic",CORAL:"Coral - Warm and engaging",ECHO:"Echo - Friendly and conversational",FABLE:"Fable - Wise and measured",NOVA:"Nova - Friendly and conversational",ONYX:"Onyx - Deep and authoritative",SAGE:"Sage - Wise and measured",SHIMMER:"Shimmer - Bright and cheerful"},ec=Object.entries({ALLOY:"alloy",ASH:"ash",BALAD:"ballad",CORAL:"coral",ECHO:"echo",FABLE:"fable",NOVA:"nova",ONYX:"onyx",SAGE:"sage",SHIMMER:"shimmer"}).map(([e,t])=>({value:t,label:el[e]})),ed=[{value:eo.EndpointType.CHAT,label:"/v1/chat/completions"},{value:eo.EndpointType.RESPONSES,label:"/v1/responses"},{value:eo.EndpointType.ANTHROPIC_MESSAGES,label:"/v1/messages"},{value:eo.EndpointType.IMAGE,label:"/v1/images/generations"},{value:eo.EndpointType.IMAGE_EDITS,label:"/v1/images/edits"},{value:eo.EndpointType.EMBEDDINGS,label:"/v1/embeddings"},{value:eo.EndpointType.SPEECH,label:"/v1/audio/speech"},{value:eo.EndpointType.TRANSCRIPTION,label:"/v1/audio/transcriptions"},{value:eo.EndpointType.A2A_AGENTS,label:"/v1/a2a/message/send"},{value:eo.EndpointType.MCP,label:"/mcp-rest/tools/call"},{value:eo.EndpointType.REALTIME,label:"/v1/realtime"}];var eu=e.i(955719),eu=eu;let{Dragger:eh}=O.Upload,ep=({chatUploadedImage:e,chatImagePreviewUrl:s,onImageUpload:r,onRemoveImage:a})=>(0,t.jsx)(t.Fragment,{children:!e&&(0,t.jsx)(eh,{beforeUpload:r,accept:"image/*,.pdf",showUploadList:!1,className:"inline-block",style:{padding:0,border:"none",background:"none"},children:(0,t.jsx)(A.Tooltip,{title:"Attach image or PDF",children:(0,t.jsx)("button",{type:"button",className:"flex items-center justify-center w-8 h-8 text-gray-400 hover:text-gray-600 hover:bg-gray-100 rounded-md transition-colors",children:(0,t.jsx)(eu.default,{style:{fontSize:"16px"}})})})})});e.s(["default",0,ep],761793);let em=async(e,t)=>({role:"user",content:[{type:"text",text:e},{type:"image_url",image_url:{url:await new Promise((e,s)=>{let r=new FileReader;r.onload=()=>{e(r.result)},r.onerror=s,r.readAsDataURL(t)})}}]}),eg=(e,t,s,r)=>{let a="";t&&r&&(a=r.toLowerCase().endsWith(".pdf")?"[PDF attached]":"[Image attached]");let n={role:"user",content:t?`${e} ${a}`:e};return t&&s&&(n.imagePreviewUrl=s),n},ef=e=>"user"===e.role&&"string"==typeof e.content&&(e.content.includes("[Image attached]")||e.content.includes("[PDF attached]"))&&!!e.imagePreviewUrl;e.s(["createChatDisplayMessage",0,eg,"createChatMultimodalMessage",0,em,"shouldShowChatAttachedImage",0,ef],964421);var ey=e.i(790848),ex=e.i(888259),eb=e.i(270377);let ev=({enabled:e,onEnabledChange:s,selectedModel:r,disabled:a=!1})=>{let i=(e=>{if(!e)return!1;let t=e.toLowerCase();return t.startsWith("openai/")||t.startsWith("gpt-")||t.startsWith("o1")||t.startsWith("o3")||t.includes("openai")})(r);return(0,t.jsxs)("div",{className:"border border-gray-200 rounded-lg p-3 bg-gradient-to-r from-blue-50 to-purple-50",children:[(0,t.jsxs)("div",{className:"flex items-center justify-between",children:[(0,t.jsxs)("div",{className:"flex items-center gap-2",children:[(0,t.jsx)(n.CodeOutlined,{className:"text-blue-500"}),(0,t.jsx)(v.Text,{className:"font-medium text-gray-700",children:"Code Interpreter"}),(0,t.jsx)(A.Tooltip,{title:"Run Python code to generate files, charts, and analyze data. Container is created automatically.",children:(0,t.jsx)(l.InfoCircleOutlined,{className:"text-gray-400 text-xs"})})]}),(0,t.jsx)(ey.Switch,{checked:e&&i,onChange:e=>{e&&!i?ex.default.warning("Code Interpreter is only available for OpenAI models"):s(e)},disabled:a||!i,size:"small",className:e&&i?"bg-blue-500":""})]}),!i&&(0,t.jsx)("div",{className:"mt-2 pt-2 border-t border-gray-200",children:(0,t.jsxs)("div",{className:"flex items-start gap-2",children:[(0,t.jsx)(eb.ExclamationCircleOutlined,{className:"text-amber-500 mt-0.5"}),(0,t.jsxs)("div",{className:"text-xs text-gray-600",children:[(0,t.jsx)("span",{children:"Code Interpreter is currently only supported for OpenAI models. "}),(0,t.jsx)("a",{href:"https://github.com/BerriAI/litellm/issues/new?template=feature_request.yml",target:"_blank",rel:"noopener noreferrer",className:"text-blue-600 hover:text-blue-800 underline",children:"Request support for other providers"})]})]})})]})};var ew=e.i(190272);let e_=({endpointType:e,onEndpointChange:s,className:r})=>(0,t.jsx)("div",{className:r,children:(0,t.jsx)(T.Select,{showSearch:!0,value:e,style:{width:"100%"},onChange:s,options:ed,className:"rounded-md",filterOption:(e,t)=>(t?.label??"").toLowerCase().includes(e.toLowerCase())||(t?.value??"").toLowerCase().includes(e.toLowerCase())})});var eS=e.i(931067);let ej={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M531.3 574.4l.3-1.4c5.8-23.9 13.1-53.7 7.4-80.7-3.8-21.3-19.5-29.6-32.9-30.2-15.8-.7-29.9 8.3-33.4 21.4-6.6 24-.7 56.8 10.1 98.6-13.6 32.4-35.3 79.5-51.2 107.5-29.6 15.3-69.3 38.9-75.2 68.7-1.2 5.5.2 12.5 3.5 18.8 3.7 7 9.6 12.4 16.5 15 3 1.1 6.6 2 10.8 2 17.6 0 46.1-14.2 84.1-79.4 5.8-1.9 11.8-3.9 17.6-5.9 27.2-9.2 55.4-18.8 80.9-23.1 28.2 15.1 60.3 24.8 82.1 24.8 21.6 0 30.1-12.8 33.3-20.5 5.6-13.5 2.9-30.5-6.2-39.6-13.2-13-45.3-16.4-95.3-10.2-24.6-15-40.7-35.4-52.4-65.8zM421.6 726.3c-13.9 20.2-24.4 30.3-30.1 34.7 6.7-12.3 19.8-25.3 30.1-34.7zm87.6-235.5c5.2 8.9 4.5 35.8.5 49.4-4.9-19.9-5.6-48.1-2.7-51.4.8.1 1.5.7 2.2 2zm-1.6 120.5c10.7 18.5 24.2 34.4 39.1 46.2-21.6 4.9-41.3 13-58.9 20.2-4.2 1.7-8.3 3.4-12.3 5 13.3-24.1 24.4-51.4 32.1-71.4zm155.6 65.5c.1.2.2.5-.4.9h-.2l-.2.3c-.8.5-9 5.3-44.3-8.6 40.6-1.9 45 7.3 45.1 7.4zm191.4-388.2L639.4 73.4c-6-6-14.1-9.4-22.6-9.4H192c-17.7 0-32 14.3-32 32v832c0 17.7 14.3 32 32 32h640c17.7 0 32-14.3 32-32V311.3c0-8.5-3.4-16.7-9.4-22.7zM790.2 326H602V137.8L790.2 326zm1.8 562H232V136h302v216a42 42 0 0042 42h216v494z"}}]},name:"file-pdf",theme:"outlined"};var ek=e.i(9583),eN=I.forwardRef(function(e,t){return I.createElement(ek.default,(0,eS.default)({},e,{ref:t,icon:ej}))});e.s(["FilePdfOutlined",0,eN],91500);let eE=function({file:e,previewUrl:s,onRemove:r}){let a=e.name.toLowerCase().endsWith(".pdf");return(0,t.jsx)("div",{className:"mb-2",children:(0,t.jsxs)("div",{className:"flex items-center gap-3 p-3 bg-gray-50 rounded-lg border border-gray-200",children:[(0,t.jsx)("div",{className:"relative inline-block",children:a?(0,t.jsx)("div",{className:"w-10 h-10 rounded-md bg-red-500 flex items-center justify-center",children:(0,t.jsx)(eN,{style:{fontSize:"16px",color:"white"}})}):(0,t.jsx)("img",{src:s||"",alt:"Upload preview",className:"w-10 h-10 rounded-md border border-gray-200 object-cover"})}),(0,t.jsxs)("div",{className:"flex-1 min-w-0",children:[(0,t.jsx)("div",{className:"text-sm font-medium text-gray-900 truncate",children:e.name}),(0,t.jsx)("div",{className:"text-xs text-gray-500",children:a?"PDF":"Image"})]}),(0,t.jsx)("button",{className:"flex items-center justify-center w-6 h-6 text-gray-400 hover:text-gray-600 hover:bg-gray-200 rounded-full transition-colors",onClick:r,children:(0,t.jsx)(o.DeleteOutlined,{style:{fontSize:"12px"}})})]})})};var eT=e.i(771674),eC=e.i(918789),eA=e.i(245704),eP=e.i(637235),eO=e.i(166406),eI=e.i(755151),eR=e.i(240647),eM=e.i(993914);let e$=(e,t=8)=>e?e.length>t?`${e.substring(0,t)}…`:e:null,eL=e=>{navigator.clipboard.writeText(e)},eU=({a2aMetadata:e,timeToFirstToken:s,totalLatency:r})=>{let[a,n]=(0,I.useState)(!1);if(!e&&!s&&!r)return null;let{taskId:i,contextId:o,status:l,metadata:c}=e||{},h=(e=>{if(!e)return null;try{return new Date(e).toLocaleTimeString([],{hour:"2-digit",minute:"2-digit"})}catch{return e}})(l?.timestamp);return(0,t.jsxs)("div",{className:"a2a-metrics mt-3 pt-2 border-t border-gray-200 text-xs",children:[(0,t.jsxs)("div",{className:"flex items-center mb-2 text-gray-600",children:[(0,t.jsx)(p.RobotOutlined,{className:"mr-1.5 text-blue-500"}),(0,t.jsx)("span",{className:"font-medium text-gray-700",children:"A2A Metadata"})]}),(0,t.jsxs)("div",{className:"flex flex-wrap items-center gap-2 text-gray-500 ml-4",children:[l?.state&&(0,t.jsxs)("span",{className:`inline-flex items-center px-2 py-0.5 rounded-full text-xs font-medium ${(e=>{switch(e){case"completed":return"bg-green-100 text-green-700";case"working":case"submitted":return"bg-blue-100 text-blue-700";case"failed":case"canceled":return"bg-red-100 text-red-700";default:return"bg-gray-100 text-gray-700"}})(l.state)}`,children:[(e=>{switch(e){case"completed":return(0,t.jsx)(eA.CheckCircleOutlined,{className:"text-green-500"});case"working":case"submitted":return(0,t.jsx)(u.LoadingOutlined,{className:"text-blue-500"});case"failed":case"canceled":return(0,t.jsx)(eb.ExclamationCircleOutlined,{className:"text-red-500"});default:return(0,t.jsx)(eP.ClockCircleOutlined,{className:"text-gray-500"})}})(l.state),(0,t.jsx)("span",{className:"ml-1 capitalize",children:l.state})]}),h&&(0,t.jsx)(A.Tooltip,{title:l?.timestamp,children:(0,t.jsxs)("span",{className:"flex items-center",children:[(0,t.jsx)(eP.ClockCircleOutlined,{className:"mr-1"}),h]})}),void 0!==r&&(0,t.jsx)(A.Tooltip,{title:"Total latency",children:(0,t.jsxs)("span",{className:"flex items-center text-blue-600",children:[(0,t.jsx)(eP.ClockCircleOutlined,{className:"mr-1"}),(r/1e3).toFixed(2),"s"]})}),void 0!==s&&(0,t.jsx)(A.Tooltip,{title:"Time to first token",children:(0,t.jsxs)("span",{className:"flex items-center text-green-600",children:["TTFT: ",(s/1e3).toFixed(2),"s"]})})]}),(0,t.jsxs)("div",{className:"flex flex-wrap items-center gap-3 text-gray-500 ml-4 mt-1.5",children:[i&&(0,t.jsx)(A.Tooltip,{title:`Click to copy: ${i}`,children:(0,t.jsxs)("span",{className:"flex items-center cursor-pointer hover:text-gray-700",onClick:()=>eL(i),children:[(0,t.jsx)(eM.FileTextOutlined,{className:"mr-1"}),"Task: ",e$(i),(0,t.jsx)(eO.CopyOutlined,{className:"ml-1 text-gray-400 hover:text-gray-600"})]})}),o&&(0,t.jsx)(A.Tooltip,{title:`Click to copy: ${o}`,children:(0,t.jsxs)("span",{className:"flex items-center cursor-pointer hover:text-gray-700",onClick:()=>eL(o),children:[(0,t.jsx)(d.LinkOutlined,{className:"mr-1"}),"Session: ",e$(o),(0,t.jsx)(eO.CopyOutlined,{className:"ml-1 text-gray-400 hover:text-gray-600"})]})}),(c||l?.message)&&(0,t.jsxs)(j.Button,{type:"text",size:"small",className:"text-xs text-blue-500 hover:text-blue-700 p-0 h-auto",onClick:()=>n(!a),children:[a?(0,t.jsx)(eI.DownOutlined,{}):(0,t.jsx)(eR.RightOutlined,{}),(0,t.jsx)("span",{className:"ml-1",children:"Details"})]})]}),a&&(0,t.jsxs)("div",{className:"mt-2 ml-4 p-3 bg-gray-50 rounded-md text-gray-600 border border-gray-200",children:[l?.message&&(0,t.jsxs)("div",{className:"mb-2",children:[(0,t.jsx)("span",{className:"font-medium text-gray-700",children:"Status Message:"}),(0,t.jsx)("span",{className:"ml-2",children:l.message})]}),i&&(0,t.jsxs)("div",{className:"mb-1.5 flex items-center",children:[(0,t.jsx)("span",{className:"font-medium text-gray-700 w-24",children:"Task ID:"}),(0,t.jsx)("code",{className:"ml-2 px-2 py-1 bg-white border border-gray-200 rounded text-xs font-mono",children:i}),(0,t.jsx)(eO.CopyOutlined,{className:"ml-2 cursor-pointer text-gray-400 hover:text-blue-500",onClick:()=>eL(i)})]}),o&&(0,t.jsxs)("div",{className:"mb-1.5 flex items-center",children:[(0,t.jsx)("span",{className:"font-medium text-gray-700 w-24",children:"Session ID:"}),(0,t.jsx)("code",{className:"ml-2 px-2 py-1 bg-white border border-gray-200 rounded text-xs font-mono",children:o}),(0,t.jsx)(eO.CopyOutlined,{className:"ml-2 cursor-pointer text-gray-400 hover:text-blue-500",onClick:()=>eL(o)})]}),c&&Object.keys(c).length>0&&(0,t.jsxs)("div",{className:"mt-3",children:[(0,t.jsx)("span",{className:"font-medium text-gray-700",children:"Custom Metadata:"}),(0,t.jsx)("pre",{className:"mt-1.5 p-2 bg-white border border-gray-200 rounded text-xs font-mono overflow-x-auto whitespace-pre-wrap",children:JSON.stringify(c,null,2)})]})]})]})},eD=({message:e})=>e.isAudio&&"string"==typeof e.content?(0,t.jsx)("div",{className:"mb-2",children:(0,t.jsx)("audio",{controls:!0,src:e.content,className:"max-w-full",style:{maxWidth:"500px"},children:"Your browser does not support the audio element."})}):null;var eq=e.i(657688);let eB=({message:e})=>{if(!ef(e))return null;let s="string"==typeof e.content&&e.content.includes("[PDF attached]");return(0,t.jsx)("div",{className:"mb-2",children:s?(0,t.jsx)("div",{className:"w-64 h-32 rounded-md border border-gray-200 bg-red-50 flex items-center justify-center",children:(0,t.jsx)(eN,{style:{fontSize:"48px",color:"#dc2626"}})}):(0,t.jsx)(eq.default,{src:e.imagePreviewUrl||"",alt:"User uploaded image",width:256,height:200,className:"max-w-64 rounded-md border border-gray-200 shadow-sm",style:{maxHeight:"200px",width:"auto",height:"auto"}})})};e.s(["default",0,eB],843153);var eW=e.i(362024),eH=e.i(737434);let ez={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M553.1 509.1l-77.8 99.2-41.1-52.4a8 8 0 00-12.6 0l-99.8 127.2a7.98 7.98 0 006.3 12.9H696c6.7 0 10.4-7.7 6.3-12.9l-136.5-174a8.1 8.1 0 00-12.7 0zM360 442a40 40 0 1080 0 40 40 0 10-80 0zm494.6-153.4L639.4 73.4c-6-6-14.1-9.4-22.6-9.4H192c-17.7 0-32 14.3-32 32v832c0 17.7 14.3 32 32 32h640c17.7 0 32-14.3 32-32V311.3c0-8.5-3.4-16.7-9.4-22.7zM790.2 326H602V137.8L790.2 326zm1.8 562H232V136h302v216a42 42 0 0042 42h216v494z"}}]},name:"file-image",theme:"outlined"};var eF=I.forwardRef(function(e,t){return I.createElement(ek.default,(0,eS.default)({},e,{ref:t,icon:ez}))});let eJ=({code:e,containerId:s,annotations:r=[],accessToken:a})=>{let[i,o]=(0,I.useState)({}),[l,c]=(0,I.useState)({}),d=(0,W.getProxyBaseUrl)();(0,I.useEffect)(()=>{let e=async()=>{for(let e of r)if((e.filename?.toLowerCase().endsWith(".png")||e.filename?.toLowerCase().endsWith(".jpg")||e.filename?.toLowerCase().endsWith(".jpeg")||e.filename?.toLowerCase().endsWith(".gif"))&&e.container_id&&e.file_id){c(t=>({...t,[e.file_id]:!0}));try{let t=await fetch(`${d}/v1/containers/${e.container_id}/files/${e.file_id}/content`,{headers:{[(0,W.getGlobalLitellmHeaderName)()]:`Bearer ${a}`}});if(t.ok){let s=await t.blob(),r=URL.createObjectURL(s);o(t=>({...t,[e.file_id]:r}))}}catch(e){console.error("Error fetching image:",e)}finally{c(t=>({...t,[e.file_id]:!1}))}}};return r.length>0&&a&&e(),()=>{Object.values(i).forEach(e=>URL.revokeObjectURL(e))}},[r,a,d]);let h=async e=>{try{let t=await fetch(`${d}/v1/containers/${e.container_id}/files/${e.file_id}/content`,{headers:{[(0,W.getGlobalLitellmHeaderName)()]:`Bearer ${a}`}});if(t.ok){let s=await t.blob(),r=URL.createObjectURL(s),a=document.createElement("a");a.href=r,a.download=e.filename||`file_${e.file_id}`,document.body.appendChild(a),a.click(),document.body.removeChild(a),URL.revokeObjectURL(r)}}catch(e){console.error("Error downloading file:",e)}},p=r.filter(e=>e.filename?.toLowerCase().endsWith(".png")||e.filename?.toLowerCase().endsWith(".jpg")||e.filename?.toLowerCase().endsWith(".jpeg")||e.filename?.toLowerCase().endsWith(".gif")),m=r.filter(e=>!e.filename?.toLowerCase().endsWith(".png")&&!e.filename?.toLowerCase().endsWith(".jpg")&&!e.filename?.toLowerCase().endsWith(".jpeg")&&!e.filename?.toLowerCase().endsWith(".gif"));return e||0!==r.length?(0,t.jsxs)("div",{className:"mt-3 space-y-3",children:[e&&(0,t.jsx)(eW.Collapse,{size:"small",items:[{key:"code",label:(0,t.jsxs)("span",{className:"flex items-center gap-2 text-sm text-gray-600",children:[(0,t.jsx)(n.CodeOutlined,{})," Python Code Executed"]}),children:(0,t.jsx)(R.Prism,{language:"python",style:M.coy,customStyle:{margin:0,borderRadius:"6px",fontSize:"12px",maxHeight:"300px",overflow:"auto"},children:e})}]}),p.map(e=>(0,t.jsx)("div",{className:"rounded-lg border border-gray-200 overflow-hidden",children:l[e.file_id]?(0,t.jsxs)("div",{className:"flex items-center justify-center p-8 bg-gray-50",children:[(0,t.jsx)(C.Spin,{indicator:(0,t.jsx)(u.LoadingOutlined,{spin:!0})}),(0,t.jsx)("span",{className:"ml-2 text-sm text-gray-500",children:"Loading image..."})]}):i[e.file_id]?(0,t.jsxs)("div",{children:[(0,t.jsx)("img",{src:i[e.file_id],alt:e.filename||"Generated chart",className:"max-w-full",style:{maxHeight:"400px"}}),(0,t.jsxs)("div",{className:"flex items-center justify-between px-3 py-2 bg-gray-50 border-t border-gray-200",children:[(0,t.jsxs)("span",{className:"text-xs text-gray-500 flex items-center gap-1",children:[(0,t.jsx)(eF,{})," ",e.filename]}),(0,t.jsxs)("button",{onClick:()=>h(e),className:"text-xs text-blue-500 hover:text-blue-700 flex items-center gap-1",children:[(0,t.jsx)(eH.DownloadOutlined,{})," Download"]})]})]}):(0,t.jsx)("div",{className:"flex items-center justify-center p-4 bg-gray-50",children:(0,t.jsx)("span",{className:"text-sm text-gray-400",children:"Image not available"})})},e.file_id)),m.length>0&&(0,t.jsx)("div",{className:"flex flex-wrap gap-2",children:m.map(e=>(0,t.jsxs)("button",{onClick:()=>h(e),className:"flex items-center gap-2 px-3 py-2 bg-gray-50 border border-gray-200 rounded-lg hover:bg-gray-100 transition-colors",children:[(0,t.jsx)(eM.FileTextOutlined,{className:"text-blue-500"}),(0,t.jsx)("span",{className:"text-sm",children:e.filename}),(0,t.jsx)(eH.DownloadOutlined,{className:"text-gray-400"})]},e.file_id))})]}):null};var eG=e.i(355343),eV=e.i(966988),eK=e.i(989022);let eX=async(e,t)=>{let s=await new Promise((e,s)=>{let r=new FileReader;r.onload=()=>{e(r.result.split(",")[1])},r.onerror=s,r.readAsDataURL(t)}),r=t.type||(t.name.toLowerCase().endsWith(".pdf")?"application/pdf":"image/jpeg");return{role:"user",content:[{type:"input_text",text:e},{type:"input_image",image_url:`data:${r};base64,${s}`}]}},eY=(e,t,s,r)=>{let a="";t&&r&&(a=r.toLowerCase().endsWith(".pdf")?"[PDF attached]":"[Image attached]");let n={role:"user",content:t?`${e} ${a}`:e};return t&&s&&(n.imagePreviewUrl=s),n},eQ=({message:e})=>{if(!("user"===e.role&&"string"==typeof e.content&&(e.content.includes("[Image attached]")||e.content.includes("[PDF attached]"))&&e.imagePreviewUrl))return null;let s="string"==typeof e.content&&e.content.includes("[PDF attached]");return(0,t.jsx)("div",{className:"mb-2",children:s?(0,t.jsx)("div",{className:"w-64 h-32 rounded-md border border-gray-200 bg-red-50 flex items-center justify-center",children:(0,t.jsx)(eN,{style:{fontSize:"48px",color:"#dc2626"}})}):(0,t.jsx)("img",{src:e.imagePreviewUrl,alt:"User uploaded image",className:"max-w-64 rounded-md border border-gray-200 shadow-sm",style:{maxHeight:"200px"}})})};function eZ({searchResults:e}){let[s,r]=(0,I.useState)(!0),[a,n]=(0,I.useState)({});if(!e||0===e.length)return null;let o=e.reduce((e,t)=>e+t.data.length,0);return(0,t.jsxs)("div",{className:"search-results-content mt-1 mb-2",children:[(0,t.jsxs)(j.Button,{type:"text",className:"flex items-center text-xs text-gray-500 hover:text-gray-700",onClick:()=>r(!s),icon:(0,t.jsx)(i.DatabaseOutlined,{}),children:[s?"Hide sources":`Show sources (${o})`,s?(0,t.jsx)(eI.DownOutlined,{className:"ml-1"}):(0,t.jsx)(eR.RightOutlined,{className:"ml-1"})]}),s&&(0,t.jsx)("div",{className:"mt-2 p-3 bg-gray-50 border border-gray-200 rounded-md text-sm",children:(0,t.jsx)("div",{className:"space-y-3",children:e.map((e,s)=>(0,t.jsxs)("div",{children:[(0,t.jsxs)("div",{className:"text-xs text-gray-600 mb-2 flex items-center gap-2",children:[(0,t.jsx)("span",{className:"font-medium",children:"Query:"}),(0,t.jsxs)("span",{className:"italic",children:['"',e.search_query,'"']}),(0,t.jsx)("span",{className:"text-gray-400",children:"•"}),(0,t.jsxs)("span",{className:"text-gray-500",children:[e.data.length," result",1!==e.data.length?"s":""]})]}),(0,t.jsx)("div",{className:"space-y-2",children:e.data.map((e,r)=>{let i=a[`${s}-${r}`]||!1;return(0,t.jsxs)("div",{className:"border border-gray-200 rounded-md overflow-hidden bg-white",children:[(0,t.jsx)("div",{className:"flex items-center justify-between p-2 cursor-pointer hover:bg-gray-50 transition-colors",onClick:()=>{let e;return e=`${s}-${r}`,void n(t=>({...t,[e]:!t[e]}))},children:(0,t.jsxs)("div",{className:"flex items-center gap-2 flex-1 min-w-0",children:[(0,t.jsx)("svg",{className:`w-4 h-4 text-gray-400 transition-transform flex-shrink-0 ${i?"transform rotate-90":""}`,fill:"none",stroke:"currentColor",viewBox:"0 0 24 24",children:(0,t.jsx)("path",{strokeLinecap:"round",strokeLinejoin:"round",strokeWidth:2,d:"M9 5l7 7-7 7"})}),(0,t.jsx)(eM.FileTextOutlined,{className:"text-gray-400 flex-shrink-0",style:{fontSize:"12px"}}),(0,t.jsx)("span",{className:"text-xs font-medium text-gray-700 truncate",children:e.filename||e.file_id||`Result ${r+1}`}),(0,t.jsx)("span",{className:"text-xs px-2 py-0.5 rounded bg-blue-100 text-blue-700 font-mono flex-shrink-0",children:e.score.toFixed(3)})]})}),i&&(0,t.jsx)("div",{className:"border-t border-gray-200 bg-white",children:(0,t.jsxs)("div",{className:"p-3 space-y-2",children:[e.content.map((e,s)=>(0,t.jsx)("div",{children:(0,t.jsx)("div",{className:"text-xs font-mono bg-gray-50 p-2 rounded text-gray-800 whitespace-pre-wrap break-words",children:e.text})},s)),e.attributes&&Object.keys(e.attributes).length>0&&(0,t.jsxs)("div",{className:"mt-2 pt-2 border-t border-gray-100",children:[(0,t.jsx)("div",{className:"text-xs text-gray-500 mb-1 font-medium",children:"Metadata:"}),(0,t.jsx)("div",{className:"space-y-1",children:Object.entries(e.attributes).map(([e,s])=>(0,t.jsxs)("div",{className:"text-xs flex gap-2",children:[(0,t.jsxs)("span",{className:"text-gray-500 font-medium",children:[e,":"]}),(0,t.jsx)("span",{className:"text-gray-700 font-mono break-all",children:String(s)})]},e))})]})]})})]},r)})})]},s))})})]})}e.s(["SearchResultsDisplay",0,eZ],152401);let e0=function({message:e,isLastMessage:s,endpointType:r,mcpEvents:a,codeInterpreterResult:n,accessToken:i}){let o="user"===e.role;return(0,t.jsx)("div",{className:`mb-4 ${o?"text-right":"text-left"}`,children:(0,t.jsxs)("div",{className:"inline-block max-w-[80%] rounded-lg shadow-sm p-3.5 px-4",style:{backgroundColor:o?"#f0f8ff":"#ffffff",border:o?"1px solid #e6f0fa":"1px solid #f0f0f0",textAlign:"left"},children:[(0,t.jsxs)("div",{className:"flex items-center gap-2 mb-1.5",children:[(0,t.jsx)("div",{className:"flex items-center justify-center w-6 h-6 rounded-full mr-1",style:{backgroundColor:o?"#e6f0fa":"#f5f5f5"},children:o?(0,t.jsx)(eT.UserOutlined,{style:{fontSize:"12px",color:"#2563eb"}}):(0,t.jsx)(p.RobotOutlined,{style:{fontSize:"12px",color:"#4b5563"}})}),(0,t.jsx)("strong",{className:"text-sm capitalize",children:e.role}),"assistant"===e.role&&e.model&&(0,t.jsx)("span",{className:"text-xs px-2 py-0.5 rounded bg-gray-100 text-gray-600 font-normal",children:e.model})]}),e.reasoningContent&&(0,t.jsx)(eV.default,{reasoningContent:e.reasoningContent}),"assistant"===e.role&&s&&a.length>0&&(r===eo.EndpointType.RESPONSES||r===eo.EndpointType.CHAT)&&(0,t.jsx)("div",{className:"mb-3",children:(0,t.jsx)(eG.default,{events:a})}),"assistant"===e.role&&e.searchResults&&(0,t.jsx)(eZ,{searchResults:e.searchResults}),"assistant"===e.role&&s&&n&&r===eo.EndpointType.RESPONSES&&(0,t.jsx)(eJ,{code:n.code,containerId:n.containerId,annotations:n.annotations,accessToken:i}),(0,t.jsxs)("div",{className:"whitespace-pre-wrap break-words max-w-full message-content",style:{wordWrap:"break-word",overflowWrap:"break-word",wordBreak:"break-word",hyphens:"auto"},children:[e.isImage?(0,t.jsx)("img",{src:"string"==typeof e.content?e.content:"",alt:"Generated image",className:"max-w-full rounded-md border border-gray-200 shadow-sm",style:{maxHeight:"500px"}}):e.isAudio?(0,t.jsx)(eD,{message:e}):(0,t.jsxs)(t.Fragment,{children:[r===eo.EndpointType.RESPONSES&&(0,t.jsx)(eQ,{message:e}),r===eo.EndpointType.CHAT&&(0,t.jsx)(eB,{message:e}),(0,t.jsx)(eC.default,{components:{code({node:e,inline:s,className:r,children:a,...n}){let i=/language-(\w+)/.exec(r||"");return!s&&i?(0,t.jsx)(R.Prism,{style:M.coy,language:i[1],PreTag:"div",className:"rounded-md my-2",wrapLines:!0,wrapLongLines:!0,...n,children:String(a).replace(/\n$/,"")}):(0,t.jsx)("code",{className:`${r} px-1.5 py-0.5 rounded bg-gray-100 text-sm font-mono`,style:{wordBreak:"break-word"},...n,children:a})},pre:({node:e,...s})=>(0,t.jsx)("pre",{style:{overflowX:"auto",maxWidth:"100%"},...s})},children:"string"==typeof e.content?e.content:""}),e.image&&(0,t.jsx)("div",{className:"mt-3",children:(0,t.jsx)("img",{src:e.image.url,alt:"Generated image",className:"max-w-full rounded-md border border-gray-200 shadow-sm",style:{maxHeight:"500px"}})})]}),"assistant"===e.role&&(e.timeToFirstToken||e.totalLatency||e.usage)&&!e.a2aMetadata&&(0,t.jsx)(eK.default,{timeToFirstToken:e.timeToFirstToken,totalLatency:e.totalLatency,usage:e.usage,toolName:e.toolName}),"assistant"===e.role&&e.a2aMetadata&&(0,t.jsx)(eU,{a2aMetadata:e.a2aMetadata,timeToFirstToken:e.timeToFirstToken,totalLatency:e.totalLatency})]})]})})};var eu=eu;let{Dragger:e1}=O.Upload,e2=({responsesUploadedImage:e,responsesImagePreviewUrl:s,onImageUpload:r,onRemoveImage:a})=>(0,t.jsx)(t.Fragment,{children:!e&&(0,t.jsx)(e1,{beforeUpload:r,accept:"image/*,.pdf",showUploadList:!1,className:"inline-block",style:{padding:0,border:"none",background:"none"},children:(0,t.jsx)(A.Tooltip,{title:"Attach image or PDF",children:(0,t.jsx)("button",{type:"button",className:"flex items-center justify-center w-8 h-8 text-gray-400 hover:text-gray-600 hover:bg-gray-100 rounded-md transition-colors",children:(0,t.jsx)(eu.default,{style:{fontSize:"16px"}})})})})}),e4=({endpointType:e,responsesSessionId:s,useApiSessionManagement:r,onToggleSessionManagement:a})=>e!==eo.EndpointType.RESPONSES?null:(0,t.jsxs)("div",{className:"mb-4",children:[(0,t.jsxs)("div",{className:"flex items-center justify-between mb-2",children:[(0,t.jsxs)("div",{className:"flex items-center gap-2",children:[(0,t.jsx)("span",{className:"text-sm font-medium text-gray-700",children:"Session Management"}),(0,t.jsx)(A.Tooltip,{title:"Choose between LiteLLM API session management (using previous_response_id) or UI-based session management (using chat history)",children:(0,t.jsx)(l.InfoCircleOutlined,{className:"text-gray-400",style:{fontSize:"12px"}})})]}),(0,t.jsx)(ey.Switch,{checked:r,onChange:a,checkedChildren:"API",unCheckedChildren:"UI",size:"small"})]}),(0,t.jsxs)("div",{className:`text-xs p-2 rounded-md ${s?"bg-green-50 text-green-700 border border-green-200":"bg-blue-50 text-blue-700 border border-blue-200"}`,children:[(0,t.jsxs)("div",{className:"flex items-center justify-between",children:[(0,t.jsxs)("div",{className:"flex items-center gap-1",children:[(0,t.jsx)(l.InfoCircleOutlined,{style:{fontSize:"12px"}}),(()=>{if(!s)return r?"API Session: Ready":"UI Session: Ready";let e=r?"Response ID":"UI Session",t=s.slice(0,10);return`${e}: ${t}...`})()]}),s&&(0,t.jsx)(A.Tooltip,{title:(0,t.jsxs)("div",{className:"text-xs",children:[(0,t.jsx)("div",{className:"mb-1",children:"Copy response ID to continue session:"}),(0,t.jsx)("div",{className:"bg-gray-800 text-gray-100 p-2 rounded font-mono text-xs whitespace-pre-wrap",children:`curl -X POST "your-proxy-url/v1/responses" \\ + -H "Authorization: Bearer your-api-key" \\ + -H "Content-Type: application/json" \\ + -d '{ + "model": "your-model", + "input": [{"role": "user", "content": "your message", "type": "message"}], + "previous_response_id": "${s}", + "stream": true + }'`})]}),overlayStyle:{maxWidth:"500px"},children:(0,t.jsx)("button",{onClick:()=>{s&&(navigator.clipboard.writeText(s),B.default.success("Response ID copied to clipboard!"))},className:"ml-2 p-1 hover:bg-green-100 rounded transition-colors",children:(0,t.jsx)(eO.CopyOutlined,{style:{fontSize:"12px"}})})})]}),(0,t.jsx)("div",{className:"text-xs opacity-75 mt-1",children:s?r?"LiteLLM API session active - context maintained server-side":"UI session active - context maintained client-side":r?"LiteLLM will manage session using previous_response_id":"UI will manage session using chat history"})]})]});var e6={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"defs",attrs:{},children:[{tag:"style",attrs:{}}]},{tag:"path",attrs:{d:"M682 455V311l-76 76v68c-.1 50.7-42 92.1-94 92a95.8 95.8 0 01-52-15l-54 55c29.1 22.4 65.9 36 106 36 93.8 0 170-75.1 170-168z"}},{tag:"path",attrs:{d:"M833 446h-60c-4.4 0-8 3.6-8 8 0 140.3-113.7 254-254 254-63 0-120.7-23-165-61l-54 54a334.01 334.01 0 00179 81v102H326c-13.9 0-24.9 14.3-25 32v36c.1 4.4 2.9 8 6 8h408c3.2 0 6-3.6 6-8v-36c0-17.7-11-32-25-32H547V782c165.3-17.9 294-157.9 294-328 0-4.4-3.6-8-8-8zm13.1-377.7l-43.5-41.9a8 8 0 00-11.2.1l-129 129C634.3 101.2 577 64 511 64c-93.9 0-170 75.3-170 168v224c0 6.7.4 13.3 1.2 19.8l-68 68A252.33 252.33 0 01258 454c-.2-4.4-3.8-8-8-8h-60c-4.4 0-8 3.6-8 8 0 53 12.5 103 34.6 147.4l-137 137a8.03 8.03 0 000 11.3l42.7 42.7c3.1 3.1 8.2 3.1 11.3 0L846.2 79.8l.1-.1c3.1-3.2 3-8.3-.2-11.4zM417 401V232c0-50.6 41.9-92 94-92 46 0 84.1 32.3 92.3 74.7L417 401z"}}]},name:"audio-muted",theme:"outlined"},e3=I.forwardRef(function(e,t){return I.createElement(ek.default,(0,eS.default)({},e,{ref:t,icon:e6}))}),e5=e.i(793916),e8=e.i(518617),e7=e.i(84899);let{Text:e9}=P.Typography,te=({accessToken:e,selectedModel:s,customProxyBaseUrl:r,selectedGuardrails:a})=>{let[n,i]=(0,I.useState)([]),[o,l]=(0,I.useState)(""),[c,d]=(0,I.useState)(!1),[u,h]=(0,I.useState)(!1),[p,m]=(0,I.useState)(!1),[g,y]=(0,I.useState)("alloy"),x=(0,I.useRef)(null),b=(0,I.useRef)(null),v=(0,I.useRef)(null),w=(0,I.useRef)(null);(0,I.useRef)([]),(0,I.useRef)(!1);let _=(0,I.useRef)(null),S=(0,I.useRef)(0),N=(0,I.useCallback)(()=>{_.current?.scrollIntoView({behavior:"smooth"})},[]);(0,I.useEffect)(()=>{N()},[n,N]);let E=(0,I.useCallback)((e,t)=>{i(s=>[...s,{role:e,content:t,timestamp:new Date}])},[]),C=(0,I.useCallback)(e=>{i(t=>{let s=t[t.length-1];return s&&"assistant"===s.role?[...t.slice(0,-1),{...s,content:s.content+e}]:[...t,{role:"assistant",content:e,timestamp:new Date}]})},[]),A=(0,I.useCallback)(e=>{let t=atob(e),s=new Uint8Array(t.length);for(let e=0;e{if(!x.current){if(!s)return void E("status","Please select a model first");h(!0);try{b.current=new AudioContext({sampleRate:24e3});let t=(r||(0,W.getProxyBaseUrl)()).replace(/^http/,"ws"),n=`${t}/v1/realtime?model=${encodeURIComponent(s)}`;a&&a.length>0&&(n+=`&guardrails=${encodeURIComponent(a.join(","))}`);let o=new WebSocket(n,["realtime",`openai-insecure-api-key.${e}`]);o.onopen=()=>{d(!0),h(!1),E("status","Connected to realtime API")},o.onmessage=async e=>{try{let t=e.data;t instanceof Blob?t=await t.text():t instanceof ArrayBuffer&&(t=new TextDecoder().decode(t));let s=JSON.parse(t),r=s.type;"session.created"===r?o.send(JSON.stringify({type:"session.update",session:{type:"realtime",modalities:["text","audio"],voice:g,input_audio_format:"pcm16",output_audio_format:"pcm16",input_audio_transcription:{model:"gpt-4o-mini-transcribe"},turn_detection:null}})):"session.updated"===r||("response.output_audio.delta"===r||"response.audio.delta"===r?s.delta&&A(s.delta):"response.output_text.delta"===r||"response.output_audio_transcript.delta"===r||"response.audio_transcript.delta"===r||"response.text.delta"===r?s.delta&&C(s.delta):"conversation.item.input_audio_transcription.completed"===r?s.transcript&&E("user",s.transcript):"response.done"===r?i(e=>{let t=e[e.length-1];if(t&&"assistant"===t.role&&t.content)return e;let r=s.response?.output||[],a=[];for(let e of r)for(let t of e.content||[]){let e=t.text||t.transcript;e&&a.push(e)}return a.length>0?[...e,{role:"assistant",content:a.join(""),timestamp:new Date}]:e}):"error"===r&&E("status",`Error: ${s.error?.message||JSON.stringify(s.error)}`))}catch{}},o.onerror=()=>{E("status","WebSocket error"),d(!1),h(!1)},o.onclose=()=>{E("status","Disconnected"),d(!1),h(!1),x.current=null},x.current=o}catch(e){E("status",`Connection failed: ${e.message}`),h(!1)}}},[e,s,g,r,a,E,C,A]),O=(0,I.useCallback)(()=>{M(),x.current?.close(),x.current=null,b.current?.close(),b.current=null,S.current=0,$.current=!1,d(!1)},[]),R=(0,I.useCallback)(async()=>{if(x.current&&x.current.readyState===WebSocket.OPEN){x.current.send(JSON.stringify({type:"session.update",session:{type:"realtime",modalities:["text","audio"],voice:g,input_audio_format:"pcm16",output_audio_format:"pcm16",input_audio_transcription:{model:"gpt-4o-mini-transcribe"},turn_detection:{type:"server_vad"}}}));try{let e=await navigator.mediaDevices.getUserMedia({audio:!0});v.current=e;let t=b.current||new AudioContext({sampleRate:24e3});b.current=t;let s=t.createMediaStreamSource(e),r=t.createScriptProcessor(4096,1,1);w.current=r,r.onaudioprocess=e=>{let s;if(!x.current||x.current.readyState!==WebSocket.OPEN)return;let r=e.inputBuffer.getChannelData(0),a=t.sampleRate;if(24e3!==a){let e=a/24e3,t=Math.round(r.length/e);s=new Float32Array(t);for(let a=0;a{w.current?.disconnect(),w.current=null,v.current?.getTracks().forEach(e=>e.stop()),v.current=null,m(!1)},[]),$=(0,I.useRef)(!1),L=(0,I.useCallback)(()=>{!x.current||x.current.readyState!==WebSocket.OPEN||$.current||($.current=!0,x.current.send(JSON.stringify({type:"session.update",session:{type:"realtime",modalities:["text","audio"],voice:g,input_audio_format:"pcm16",output_audio_format:"pcm16",input_audio_transcription:{model:"gpt-4o-mini-transcribe"},turn_detection:null}})))},[g]),U=(0,I.useCallback)(()=>{if(!o.trim()||!x.current||x.current.readyState!==WebSocket.OPEN)return;let e=o.trim();E("user",e),l(""),x.current.send(JSON.stringify({type:"conversation.item.create",item:{type:"message",role:"user",content:[{type:"input_text",text:e}]}})),x.current.send(JSON.stringify({type:"response.create"}))},[o,E,L]);return(0,I.useEffect)(()=>()=>{x.current?.close(),b.current?.close(),v.current?.getTracks().forEach(e=>e.stop())},[]),(0,t.jsxs)("div",{className:"flex flex-col h-full",children:[(0,t.jsxs)("div",{className:"flex items-center justify-between px-4 py-3 border-b border-gray-200 bg-gray-50",children:[(0,t.jsxs)("div",{className:"flex items-center gap-3",children:[(0,t.jsx)(f.SoundOutlined,{className:"text-lg text-blue-500"}),(0,t.jsx)(e9,{className:"font-semibold text-gray-800",children:"Realtime Voice Chat"}),(0,t.jsx)("span",{className:`inline-block w-2 h-2 rounded-full ${c?"bg-green-500":"bg-gray-300"}`}),(0,t.jsx)(e9,{className:"text-xs text-gray-500",children:c?"Connected":u?"Connecting...":"Disconnected"})]}),(0,t.jsxs)("div",{className:"flex items-center gap-2",children:[(0,t.jsx)(T.Select,{size:"small",value:g,onChange:y,options:ec,style:{width:220},disabled:c}),c?(0,t.jsx)(j.Button,{danger:!0,onClick:O,size:"small",icon:(0,t.jsx)(e8.CloseCircleOutlined,{}),children:"Disconnect"}):(0,t.jsx)(j.Button,{type:"primary",onClick:P,loading:u,size:"small",children:"Connect"})]})]}),(0,t.jsxs)("div",{className:"flex-1 overflow-y-auto p-4 space-y-3",children:[0===n.length&&!c&&(0,t.jsxs)("div",{className:"flex flex-col items-center justify-center h-full text-gray-400 gap-3",children:[(0,t.jsx)(f.SoundOutlined,{style:{fontSize:48}}),(0,t.jsx)(e9,{className:"text-lg text-gray-500",children:"Realtime Voice Playground"}),(0,t.jsxs)(e9,{className:"text-sm text-gray-400 text-center max-w-md",children:["Click ",(0,t.jsx)("b",{children:"Connect"})," to start a realtime session. You can speak using your microphone or type messages. The AI will respond with voice and text."]})]}),n.map((e,s)=>(0,t.jsx)("div",{className:`flex ${"user"===e.role?"justify-end":"status"===e.role?"justify-center":"justify-start"}`,children:"status"===e.role?(0,t.jsx)("div",{className:"text-xs text-gray-400 italic px-3 py-1",children:e.content}):(0,t.jsxs)("div",{className:`max-w-[75%] rounded-2xl px-4 py-2.5 ${"user"===e.role?"bg-blue-500 text-white rounded-br-md":"bg-gray-100 text-gray-800 rounded-bl-md"}`,children:[(0,t.jsx)("div",{className:"text-xs font-medium mb-0.5 opacity-70",children:"user"===e.role?"You":"AI"}),(0,t.jsx)("div",{className:"text-sm whitespace-pre-wrap",children:e.content})]})},s)),(0,t.jsx)("div",{ref:_})]}),c&&(0,t.jsxs)("div",{className:"border-t border-gray-200 p-3 bg-white",children:[(0,t.jsxs)("div",{className:"flex items-center gap-2",children:[(0,t.jsx)(j.Button,{shape:"circle",size:"large",type:p?"primary":"default",danger:p,icon:p?(0,t.jsx)(e3,{}):(0,t.jsx)(e5.AudioOutlined,{}),onClick:p?M:R,title:p?"Stop recording":"Start recording",className:p?"animate-pulse":""}),(0,t.jsx)(k.Input,{placeholder:"Type a message or use the mic...",value:o,onChange:e=>l(e.target.value),onPressEnter:U,className:"flex-1",size:"large"}),(0,t.jsx)(j.Button,{type:"primary",icon:(0,t.jsx)(e7.SendOutlined,{}),onClick:U,disabled:!o.trim(),size:"large"})]}),p&&(0,t.jsxs)("div",{className:"mt-2 flex items-center gap-2 text-red-500 text-xs",children:[(0,t.jsx)("span",{className:"inline-block w-2 h-2 rounded-full bg-red-500 animate-pulse"}),"Listening — speak into your microphone. Server VAD will detect when you stop."]})]})]})};var tt=e.i(122550),ts=e.i(434166);let{TextArea:tr}=k.Input,{Dragger:ta}=O.Upload,tn=new Set([eo.EndpointType.CHAT,eo.EndpointType.RESPONSES,eo.EndpointType.MCP]);e.s(["default",0,({accessToken:e,token:k,userRole:O,userID:Z,disabledPersonalKeyCreation:er,proxySettings:ea,simplified:en=!1,fixedModel:el})=>{let[ed,eu]=(0,I.useState)([]),[eh,ef]=(0,I.useState)([]),[ey,ex]=(0,I.useState)(!1),[eb,eS]=(0,I.useState)(null),[ej,ek]=(0,I.useState)(()=>{let e=sessionStorage.getItem("selectedMCPServers");try{return e?JSON.parse(e):[]}catch(e){return console.error("Error parsing selectedMCPServers from sessionStorage",e),[]}}),[eN,eT]=(0,I.useState)(!1),[eC,eA]=(0,I.useState)({}),[eP,eO]=(0,I.useState)(void 0),eI=(0,I.useRef)(null),[eR,eM]=(0,I.useState)(()=>{let e=sessionStorage.getItem("mcpServerToolRestrictions");try{return e?JSON.parse(e):{}}catch(e){return console.error("Error parsing mcpServerToolRestrictions from sessionStorage",e),{}}}),{chatHistory:e$,setChatHistory:eL,mcpEvents:eU,setMCPEvents:eD,messageTraceId:eq,setMessageTraceId:eB,responsesSessionId:eW,setResponsesSessionId:eH,useApiSessionManagement:ez,setUseApiSessionManagement:eF,updateTextUI:eJ,updateReasoningContent:eV,updateTimingData:eK,updateUsageData:eQ,updateA2AMetadata:eZ,updateTotalLatency:e1,updateSearchResults:e6,handleResponseId:e3,handleToggleSessionManagement:e5,handleMCPEvent:e8,updateImageUI:e7,updateEmbeddingsUI:e9,updateAudioUI:ti,updateChatImageUI:to,clearChatHistory:tl,clearMCPEvents:tc}=function({simplified:e}){let[t,s]=(0,I.useState)(()=>{if(e)return[];try{let e=sessionStorage.getItem("chatHistory");return e?JSON.parse(e):[]}catch(e){return console.error("Error parsing chatHistory from sessionStorage",e),[]}}),[r,a]=(0,I.useState)([]),[n,i]=(0,I.useState)(()=>e?null:sessionStorage.getItem("messageTraceId")||null),[o,l]=(0,I.useState)(()=>e?null:sessionStorage.getItem("responsesSessionId")||null),[c,d]=(0,I.useState)(()=>{if(e)return!0;let t=sessionStorage.getItem("useApiSessionManagement");return!t||JSON.parse(t)});return(0,I.useEffect)(()=>{if(e||0===t.length)return;let s=setTimeout(()=>{sessionStorage.setItem("chatHistory",JSON.stringify(t))},500);return()=>{clearTimeout(s)}},[t,e]),(0,I.useEffect)(()=>{e||(n?sessionStorage.setItem("messageTraceId",n):sessionStorage.removeItem("messageTraceId"),o?sessionStorage.setItem("responsesSessionId",o):sessionStorage.removeItem("responsesSessionId"),sessionStorage.setItem("useApiSessionManagement",JSON.stringify(c)))},[n,o,c,e]),{chatHistory:t,setChatHistory:s,mcpEvents:r,setMCPEvents:a,messageTraceId:n,setMessageTraceId:i,responsesSessionId:o,setResponsesSessionId:l,useApiSessionManagement:c,setUseApiSessionManagement:d,updateTextUI:(e,t,r)=>{s(s=>{let a=s[s.length-1];if(!a||a.role!==e||a.isImage||a.isAudio)return[...s,{role:e,content:t,model:r}];{let e={...a,content:a.content+t,model:a.model??r};return[...s.slice(0,-1),e]}})},updateReasoningContent:e=>{s(t=>{let s=t[t.length-1];return!s||"assistant"!==s.role||s.isImage||s.isAudio?t.length>0&&"user"===t[t.length-1].role?[...t,{role:"assistant",content:"",reasoningContent:e}]:t:[...t.slice(0,t.length-1),{...s,reasoningContent:(s.reasoningContent||"")+e}]})},updateTimingData:e=>{s(t=>{let s=t[t.length-1];return s&&"assistant"===s.role?[...t.slice(0,t.length-1),{...s,timeToFirstToken:e}]:s&&"user"===s.role?[...t,{role:"assistant",content:"",timeToFirstToken:e}]:t})},updateUsageData:(e,t)=>{s(s=>{let r=s[s.length-1];if(r&&"assistant"===r.role){let a={...r,usage:e,toolName:t};return[...s.slice(0,s.length-1),a]}return s})},updateA2AMetadata:e=>{s(t=>{let s=t[t.length-1];if(s&&"assistant"===s.role){let r={...s,a2aMetadata:e};return[...t.slice(0,t.length-1),r]}return t})},updateTotalLatency:e=>{s(t=>{let s=t[t.length-1];return s&&"assistant"===s.role?[...t.slice(0,t.length-1),{...s,totalLatency:e}]:t})},updateSearchResults:e=>{s(t=>{let s=t[t.length-1];if(s&&"assistant"===s.role){let r={...s,searchResults:e};return[...t.slice(0,t.length-1),r]}return t})},handleResponseId:e=>{c&&l(e)},handleToggleSessionManagement:e=>{d(e),e||l(null)},handleMCPEvent:e=>{a(t=>e.item_id&&t.some(t=>t.item_id===e.item_id&&t.type===e.type&&(t.sequence_number===e.sequence_number||void 0===t.sequence_number&&void 0===e.sequence_number))?t:[...t,e])},updateImageUI:(e,t)=>{s(s=>[...s,{role:"assistant",content:e,model:t,isImage:!0}])},updateEmbeddingsUI:(e,t)=>{s(s=>[...s,{role:"assistant",content:(0,tt.truncateString)(e,100),model:t,isEmbeddings:!0}])},updateAudioUI:(e,t)=>{s(s=>[...s,{role:"assistant",content:e,model:t,isAudio:!0}])},updateChatImageUI:(e,t)=>{s(s=>{let r=s[s.length-1];if(!r||"assistant"!==r.role||r.isImage||r.isAudio)return[...s,{role:"assistant",content:"",model:t,image:{url:e,detail:"auto"}}];{let a={...r,image:{url:e,detail:"auto"},model:r.model??t};return[...s.slice(0,-1),a]}})},clearChatHistory:()=>{s(e=>(e.forEach(e=>{e.isAudio&&"string"==typeof e.content&&URL.revokeObjectURL(e.content)}),[])),i(null),l(null),a([]),e||(sessionStorage.removeItem("chatHistory"),sessionStorage.removeItem("messageTraceId"),sessionStorage.removeItem("responsesSessionId"))},clearMCPEvents:()=>{a([])}}}({simplified:en}),[td,tu]=(0,I.useState)(()=>{let e=(0,ts.getSecureItem)("apiKeySource");if(e)try{return JSON.parse(e)}catch(e){console.error("Error parsing apiKeySource from sessionStorage",e)}return er?"custom":"session"}),[th,tp]=(0,I.useState)(()=>(0,ts.getSecureItem)("apiKey")||""),[tm,tg]=(0,I.useState)(()=>sessionStorage.getItem("customProxyBaseUrl")||""),[tf,ty]=(0,I.useState)(""),[tx,tb]=(0,I.useState)(en?el:void 0),[tv,tw]=(0,I.useState)(!1),[t_,tS]=(0,I.useState)([]),[tj,tk]=(0,I.useState)([]),[tN,tE]=(0,I.useState)(void 0),tT=(0,I.useRef)(null),[tC,tA]=(0,I.useState)(()=>sessionStorage.getItem("endpointType")||eo.EndpointType.CHAT),[tP,tO]=(0,I.useState)(!1),tI=(0,I.useRef)(null),[tR,tM]=(0,I.useState)(()=>{let e=sessionStorage.getItem("selectedTags");try{return e?JSON.parse(e):[]}catch(e){return console.error("Error parsing selectedTags from sessionStorage",e),[]}}),[t$,tL]=(0,I.useState)(()=>{let e=sessionStorage.getItem("selectedVoice");if(!e)return"alloy";try{return JSON.parse(e)}catch{return e}}),[tU,tD]=(0,I.useState)(()=>{let e=sessionStorage.getItem("selectedVectorStores");try{return e?JSON.parse(e):[]}catch(e){return console.error("Error parsing selectedVectorStores from sessionStorage",e),[]}}),[tq,tB]=(0,I.useState)(()=>{let e=sessionStorage.getItem("selectedGuardrails");try{return e?JSON.parse(e):[]}catch(e){return console.error("Error parsing selectedGuardrails from sessionStorage",e),[]}}),[tW,tH]=(0,I.useState)(()=>{let e=sessionStorage.getItem("selectedPolicies");try{return e?JSON.parse(e):[]}catch(e){return console.error("Error parsing selectedPolicies from sessionStorage",e),[]}}),[tz,tF]=(0,I.useState)([]),[tJ,tG]=(0,I.useState)([]),[tV,tK]=(0,I.useState)(null),[tX,tY]=(0,I.useState)(null),[tQ,tZ]=(0,I.useState)(null),[t0,t1]=(0,I.useState)(null),[t2,t4]=(0,I.useState)(null),[t6,t3]=(0,I.useState)(!1),[t5,t8]=(0,I.useState)(""),[t7,t9]=(0,I.useState)("openai"),[se,st]=(0,I.useState)(1),[ss,sr]=(0,I.useState)(2048),[sa,sn]=(0,I.useState)(!1),[si,so]=(0,I.useState)(!1),sl=function(){let[e,t]=(0,I.useState)(()=>{let e=sessionStorage.getItem("codeInterpreterEnabled");return!!e&&JSON.parse(e)}),[s,r]=(0,I.useState)(null),a=(0,I.useCallback)(e=>{t(e),sessionStorage.setItem("codeInterpreterEnabled",JSON.stringify(e))},[]),n=(0,I.useCallback)(()=>{r(null)},[]),i=(0,I.useCallback)(()=>{a(!e)},[e,a]);return{enabled:e,result:s,setEnabled:a,setResult:r,clearResult:n,toggle:i}}(),sc=(0,I.useRef)(null),sd=async()=>{let t="session"===td?e:th;if(t){eT(!0);try{let[e,s]=await Promise.all([(0,W.fetchMCPServers)(t),(0,W.fetchMCPToolsets)(t).catch(()=>[])]);eu(Array.isArray(e)?e:e.data||[]),ef(Array.isArray(s)?s:[])}catch(e){console.error("Error fetching MCP servers:",e)}finally{eT(!1)}}};(0,I.useEffect)(()=>{en&&el&&(tb(el),tA(eo.EndpointType.CHAT))},[en,el]);let su=async t=>{let s="session"===td?e:th;if(s&&!eC[t])try{let e=await (0,W.listMCPTools)(s,t);eA(s=>({...s,[t]:e.tools||[]}))}catch(e){console.error(`Error fetching tools for server ${t}:`,e)}};(0,I.useEffect)(()=>{if(t6){let t=(0,ew.generateCodeSnippet)({apiKeySource:td,accessToken:e,apiKey:th,inputMessage:tf,chatHistory:e$,selectedTags:tR,selectedVectorStores:tU,selectedGuardrails:tq,selectedPolicies:tW,selectedMCPServers:ej,mcpServers:ed,mcpServerToolRestrictions:eR,endpointType:tC,selectedModel:tx,selectedSdk:t7,selectedVoice:t$,proxySettings:ea});t8(t)}},[t6,t7,td,e,th,tf,e$,tR,tU,tq,tW,ej,ed,eR,tC,tx,ea]),(0,I.useEffect)(()=>{try{(0,ts.setSecureItem)("apiKeySource",JSON.stringify(td)),(0,ts.setSecureItem)("apiKey",th)}catch{}sessionStorage.setItem("endpointType",tC),sessionStorage.setItem("selectedTags",JSON.stringify(tR)),sessionStorage.setItem("selectedVectorStores",JSON.stringify(tU)),sessionStorage.setItem("selectedGuardrails",JSON.stringify(tq)),sessionStorage.setItem("selectedPolicies",JSON.stringify(tW)),sessionStorage.setItem("selectedMCPServers",JSON.stringify(ej)),sessionStorage.setItem("mcpServerToolRestrictions",JSON.stringify(eR)),sessionStorage.setItem("selectedVoice",t$),sessionStorage.removeItem("selectedMCPTools"),en||(tx?sessionStorage.setItem("selectedModel",tx):sessionStorage.removeItem("selectedModel"))},[en,td,th,tx,tC,tR,tU,tq,tW,ej,eR,t$]),(0,I.useEffect)(()=>{let t="session"===td?e:th;if(!t||!k||!O||!Z)return void console.log("userApiKey or token or userRole or userID is missing = ",t,k,O,Z);let s=async()=>{try{if(!t)return void console.log("userApiKey is missing");let e=await (0,Q.fetchAvailableModels)(t);console.log("Fetched models:",e),tS(e);let s=e.some(e=>e.model_group===tx);e.length&&s||tb(void 0)}catch(e){console.error("Error fetching model info:",e)}};en||s(),sd()},[e,Z,O,td,th,k,en]),(0,I.useEffect)(()=>{if(tC===eo.EndpointType.MCP&&1===ej.length&&"__all__"!==ej[0]){let e=ej[0];if(e.startsWith("toolset:")){let t=e.slice(8),s=eh.find(e=>e.toolset_id===t);s&&[...new Set(s.tools.map(e=>e.server_id))].forEach(e=>{eC[e]||su(e)})}else eC[e]||su(e)}},[tC,ej,eC,eh]),(0,I.useEffect)(()=>{let t="session"===td?e:th;t&&tC===eo.EndpointType.A2A_AGENTS&&(async()=>{try{let e=await (0,Y.fetchAvailableAgents)(t,tm||void 0);tk(e),tN&&!e.some(e=>e.agent_name===tN)&&tE(void 0)}catch(e){console.error("Error fetching agents:",e)}})()},[e,td,th,tC,tm,tN]),(0,I.useEffect)(()=>{sc.current&&setTimeout(()=>{sc.current?.scrollIntoView({behavior:"smooth",block:"end"})},100)},[e$]);let sh=e=>{tF(t=>[...t,e]);let t=URL.createObjectURL(e),s=t.startsWith("blob:")?t:"";return tG(e=>[...e,s]),!1},sp=()=>{tJ.forEach(e=>{URL.revokeObjectURL(e)}),tF([]),tG([])},sm=()=>{tX&&URL.revokeObjectURL(tX),tK(null),tY(null)},sg=()=>{t0&&URL.revokeObjectURL(t0),tZ(null),t1(null)},sf=()=>{t4(null)},sy=async()=>{let t;if(""===tf.trim()&&tC!==eo.EndpointType.TRANSCRIPTION&&tC!==eo.EndpointType.MCP)return;if(tC===eo.EndpointType.IMAGE_EDITS&&0===tz.length)return void B.default.fromBackend("Please upload at least one image for editing");if(tC===eo.EndpointType.TRANSCRIPTION&&!t2)return void B.default.fromBackend("Please upload an audio file for transcription");if(tC===eo.EndpointType.A2A_AGENTS&&!tN)return void B.default.fromBackend("Please select an agent to send a message");let s={};if(tC===eo.EndpointType.MCP){let e=1===ej.length&&"__all__"!==ej[0]?ej[0]:null;if(!e)return void B.default.fromBackend("Please select an MCP server to test");if(e.startsWith("toolset:"),!eP)return void B.default.fromBackend("Please select an MCP tool to call");let t=e.startsWith("toolset:")?eh.find(t=>t.toolset_id===e.slice(8)):null,r=[];if(t?[...new Set(t.tools.map(e=>e.server_id))].forEach(e=>{r=r.concat(eC[e]||[])}):r=eC[e]||[],!r.find(e=>e.name===eP))return void B.default.fromBackend("Please wait for tool schema to load");try{s=await eI.current?.getSubmitValues()??{}}catch(e){B.default.fromBackend(e instanceof Error?e.message:"Please fill in all required parameters");return}}if([eo.EndpointType.CHAT,eo.EndpointType.IMAGE,eo.EndpointType.SPEECH,eo.EndpointType.IMAGE_EDITS,eo.EndpointType.RESPONSES,eo.EndpointType.ANTHROPIC_MESSAGES,eo.EndpointType.EMBEDDINGS,eo.EndpointType.TRANSCRIPTION].includes(tC)&&!tx)return void B.default.fromBackend("Please select a model before sending a request");if(!k||!O||!Z)return;let r=en||"session"===td?e:th;if(!r)return void B.default.fromBackend("Please provide a Virtual Key or select Current UI Session");tI.current=new AbortController;let a=tI.current.signal;if(tC===eo.EndpointType.RESPONSES&&tV)try{t=await eX(tf,tV)}catch(e){B.default.fromBackend("Failed to process image. Please try again.");return}else if(tC===eo.EndpointType.CHAT&&tQ)try{t=await em(tf,tQ)}catch(e){B.default.fromBackend("Failed to process image. Please try again.");return}else t={role:"user",content:tf};let n=eq||(0,$.v4)();eq||eB(n),eL([...e$,tC===eo.EndpointType.RESPONSES&&tV?eY(tf,!0,tX||void 0,tV.name):tC===eo.EndpointType.CHAT&&tQ?eg(tf,!0,t0||void 0,tQ.name):tC===eo.EndpointType.TRANSCRIPTION&&t2?eY(tf?`🎵 Audio file: ${t2.name} +Prompt: ${tf}`:`🎵 Audio file: ${t2.name}`,!1):tC===eo.EndpointType.MCP&&eP?eY(`🔧 MCP Tool: ${eP} +Arguments: ${JSON.stringify(s,null,2)}`,!1):eY(tf,!1)]),tc(),sl.clearResult(),tO(!0);try{if(tx)if(tC===eo.EndpointType.CHAT){let e=[...e$.filter(e=>!e.isImage&&!e.isAudio).map(({role:e,content:t})=>({role:e,content:"string"==typeof t?t:""})),t],s=en&&ea?ea.LITELLM_UI_API_DOC_BASE_URL??ea.PROXY_BASE_URL??void 0:tm||void 0;await (0,K.makeOpenAIChatCompletionRequest)(e,(e,t)=>eJ("assistant",e,t),tx,r,tR,a,eV,eK,eQ,n,tU.length>0?tU:void 0,tq.length>0?tq:void 0,tW.length>0?tW:void 0,ej,to,e6,sa?se:void 0,sa?ss:void 0,e1,s,ed,eR,e8,si,eh)}else if(tC===eo.EndpointType.IMAGE)await et(tf,(e,t)=>e7(e,t),tx,r,tR,a,tm||void 0);else if(tC===eo.EndpointType.SPEECH)await (0,G.makeOpenAIAudioSpeechRequest)(tf,t$,(e,t)=>ti(e,t),tx||"",r,tR,a,void 0,void 0,tm||void 0);else if(tC===eo.EndpointType.IMAGE_EDITS)tz.length>0&&await ee(1===tz.length?tz[0]:tz,tf,(e,t)=>e7(e,t),tx,r,tR,a,tm||void 0);else if(tC===eo.EndpointType.RESPONSES){let e;e=ez&&eW?[t]:[...e$.filter(e=>!e.isImage&&!e.isAudio).map(({role:e,content:t})=>({role:e,content:t})),t],await (0,es.makeOpenAIResponsesRequest)(e,(e,t,s)=>eJ(e,t,s),tx,r,tR,a,eV,eK,eQ,n,tU.length>0?tU:void 0,tq.length>0?tq:void 0,tW.length>0?tW:void 0,ej,ez?eW:null,e3,e8,sl.enabled,sl.setResult,tm||void 0,ed,eR,eh)}else if(tC===eo.EndpointType.ANTHROPIC_MESSAGES){let e=[...e$.filter(e=>!e.isImage&&!e.isAudio).map(({role:e,content:t})=>({role:e,content:t})),t];await (0,J.makeAnthropicMessagesRequest)(e,(e,t,s)=>eJ(e,t,s),tx,r,tR,a,eV,eK,eQ,n,tU.length>0?tU:void 0,tq.length>0?tq:void 0,tW.length>0?tW:void 0,ej,tm||void 0)}else tC===eo.EndpointType.EMBEDDINGS?await (0,X.makeOpenAIEmbeddingsRequest)(tf,(e,t)=>e9(e,t),tx,r,tR,tm||void 0):tC===eo.EndpointType.TRANSCRIPTION&&t2&&await (0,V.makeOpenAIAudioTranscriptionRequest)(t2,(e,t)=>eJ("assistant",e,t),tx,r,tR,a,void 0,void 0,void 0,void 0,tm||void 0);if(tC===eo.EndpointType.MCP){let e=1===ej.length&&"__all__"!==ej[0]?ej[0]:null,t=e;if(e?.startsWith("toolset:")){let s=e.slice(8),r=eh.find(e=>e.toolset_id===s),a=r?.tools.find(e=>e.tool_name===eP);t=a?.server_id??e}if(t&&!t.startsWith("toolset:")&&eP){let e=await (0,W.callMCPTool)(r,t,eP,s,tq.length>0?{guardrails:tq}:void 0),a=e?.content?.length>0?JSON.stringify(e.content.map(e=>"text"===e.type?e.text:e).filter(Boolean),null,2):JSON.stringify(e,null,2);eJ("assistant",a||"Tool executed successfully.")}}tC===eo.EndpointType.A2A_AGENTS&&tN&&await (0,F.makeA2ASendMessageRequest)(tN,tf,(e,t)=>eJ("assistant",e,t),r,a,eK,e1,eZ,tm||void 0,tq.length>0?tq:void 0)}catch(e){a.aborted?console.log("Request was cancelled"):(console.error("Error fetching response",e),eJ("assistant","Error fetching response:"+e))}finally{tO(!1),tI.current=null,tC===eo.EndpointType.IMAGE_EDITS&&sp(),tC===eo.EndpointType.RESPONSES&&tV&&sm(),tC===eo.EndpointType.CHAT&&tQ&&sg(),tC===eo.EndpointType.TRANSCRIPTION&&t2&&sf()}ty("")};if(O&&"Admin Viewer"===O){let{Title:e,Paragraph:s}=P.Typography;return(0,t.jsxs)("div",{children:[(0,t.jsx)(e,{level:1,children:"Access Denied"}),(0,t.jsx)(s,{children:"Ask your proxy admin for access to test models"})]})}let sx=(0,t.jsx)(u.LoadingOutlined,{style:{fontSize:24},spin:!0});return(0,t.jsxs)("div",{className:`w-full bg-white ${en?"h-full flex flex-col":"p-4 pb-0"}`,children:[(0,t.jsx)(b.Card,{className:`w-full rounded-xl shadow-md overflow-hidden ${en?"h-full flex flex-col":""}`,children:(0,t.jsxs)("div",{className:`flex w-full gap-4 ${en?"h-full":"h-[80vh]"}`,children:[!en&&(0,t.jsxs)("div",{className:"w-1/4 p-4 bg-gray-50 overflow-y-auto",children:[(0,t.jsx)(_.Title,{className:"text-xl font-semibold mb-6 mt-2",children:"Configurations"}),(0,t.jsxs)("div",{className:"space-y-4",children:[(0,t.jsxs)("div",{children:[(0,t.jsxs)(v.Text,{className:"font-medium block mb-2 text-gray-700 flex items-center",children:[(0,t.jsx)(c.KeyOutlined,{className:"mr-2"})," Virtual Key Source"]}),(0,t.jsx)(T.Select,{disabled:er,value:td,style:{width:"100%"},onChange:e=>{tu(e)},options:[{value:"session",label:"Current UI Session"},{value:"custom",label:"Virtual Key"}],className:"rounded-md"}),"custom"===td&&(0,t.jsx)(w.TextInput,{className:"mt-2",placeholder:"Enter custom Virtual Key",type:"password",onValueChange:tp,value:th,icon:c.KeyOutlined})]}),(0,t.jsxs)("div",{children:[(0,t.jsxs)("div",{className:"flex items-center justify-between mb-2",children:[(0,t.jsxs)(v.Text,{className:"font-medium block text-gray-700 flex items-center",children:[(0,t.jsx)(g.SettingOutlined,{className:"mr-2"})," Custom Proxy Base URL"]}),ea?.LITELLM_UI_API_DOC_BASE_URL&&!tm&&(0,t.jsx)(j.Button,{type:"link",size:"small",icon:(0,t.jsx)(d.LinkOutlined,{}),onClick:()=>{tg(ea.LITELLM_UI_API_DOC_BASE_URL||""),sessionStorage.setItem("customProxyBaseUrl",ea.LITELLM_UI_API_DOC_BASE_URL||"")},className:"text-gray-500 hover:text-gray-700",children:"Fill"}),tm&&(0,t.jsx)(j.Button,{type:"link",size:"small",icon:(0,t.jsx)(a.ClearOutlined,{}),onClick:()=>{tg(""),sessionStorage.removeItem("customProxyBaseUrl")},className:"text-gray-500 hover:text-gray-700",children:"Clear"})]}),(0,t.jsx)(w.TextInput,{placeholder:"Optional: Enter custom proxy URL (e.g., http://localhost:5000)",onValueChange:e=>{tg(e),sessionStorage.setItem("customProxyBaseUrl",e)},value:tm,icon:s.ApiOutlined}),tm&&(0,t.jsxs)(v.Text,{className:"text-xs text-gray-500 mt-1",children:["API calls will be sent to: ",tm]})]}),(0,t.jsxs)("div",{children:[(0,t.jsxs)(v.Text,{className:"font-medium block mb-2 text-gray-700 flex items-center",children:[(0,t.jsx)(s.ApiOutlined,{className:"mr-2"})," Endpoint Type"]}),(0,t.jsx)(e_,{endpointType:tC,onEndpointChange:e=>{tA(e),tb(void 0),tE(void 0),tw(!1),eO(void 0),e===eo.EndpointType.MCP&&ek(e=>1===e.length&&"__all__"!==e[0]?e:[]);try{sessionStorage.removeItem("selectedModel"),sessionStorage.removeItem("selectedAgent")}catch{}},className:"mb-4"}),tC===eo.EndpointType.SPEECH&&(0,t.jsxs)("div",{className:"mb-4",children:[(0,t.jsxs)(v.Text,{className:"font-medium block mb-2 text-gray-700 flex items-center",children:[(0,t.jsx)(f.SoundOutlined,{className:"mr-2"}),"Voice"]}),(0,t.jsx)(T.Select,{value:t$,onChange:e=>{tL(e),sessionStorage.setItem("selectedVoice",e)},style:{width:"100%"},className:"rounded-md",options:ec})]}),(0,t.jsx)(e4,{endpointType:tC,responsesSessionId:eW,useApiSessionManagement:ez,onToggleSessionManagement:e5})]}),tC!==eo.EndpointType.A2A_AGENTS&&tC!==eo.EndpointType.MCP&&(0,t.jsxs)("div",{children:[(0,t.jsxs)(v.Text,{className:"font-medium block mb-2 text-gray-700 flex items-center justify-between",children:[(0,t.jsxs)("span",{className:"flex items-center",children:[(0,t.jsx)(p.RobotOutlined,{className:"mr-2"})," Select Model"]}),(()=>{if(!tx||"custom"===tx)return!1;let e=t_.find(e=>e.model_group===tx);return!!e&&(!e.mode||"chat"===e.mode)})()?(0,t.jsx)(E.Popover,{content:(0,t.jsx)(ei,{temperature:se,maxTokens:ss,useAdvancedParams:sa,onTemperatureChange:st,onMaxTokensChange:sr,onUseAdvancedParamsChange:sn,mockTestFallbacks:si,onMockTestFallbacksChange:so}),title:"Model Settings",trigger:"click",placement:"right",children:(0,t.jsx)(j.Button,{type:"text",size:"small",icon:(0,t.jsx)(g.SettingOutlined,{}),className:"text-gray-500 hover:text-gray-700","aria-label":"Model Settings","data-testid":"model-settings-button"})}):(0,t.jsx)(A.Tooltip,{title:"Advanced parameters are only supported for chat models currently",children:(0,t.jsx)(j.Button,{type:"text",size:"small",icon:(0,t.jsx)(g.SettingOutlined,{}),className:"text-gray-300 cursor-not-allowed",disabled:!0})})]}),(0,t.jsx)(T.Select,{value:tx,placeholder:"Select a Model",onChange:e=>{console.log(`selected ${e}`),tb(e),tw("custom"===e)},options:[{value:"custom",label:"Enter custom model",key:"custom"},...Array.from(new Set(t_.filter(e=>{if(!e.mode)return!0;let t=(0,eo.getEndpointType)(e.mode);return tC===eo.EndpointType.RESPONSES||tC===eo.EndpointType.ANTHROPIC_MESSAGES?t===tC||t===eo.EndpointType.CHAT:tC===eo.EndpointType.IMAGE_EDITS?t===tC||t===eo.EndpointType.IMAGE:t===tC}).map(e=>e.model_group))).map((e,t)=>({value:e,label:e,key:t}))],style:{width:"100%"},showSearch:!0,className:"rounded-md"}),tv&&(0,t.jsx)(w.TextInput,{className:"mt-2",placeholder:"Enter custom model name",onValueChange:e=>{tT.current&&clearTimeout(tT.current),tT.current=setTimeout(()=>{tb(e)},500)}})]}),tC===eo.EndpointType.A2A_AGENTS&&(0,t.jsxs)("div",{children:[(0,t.jsxs)(v.Text,{className:"font-medium block mb-2 text-gray-700 flex items-center",children:[(0,t.jsx)(p.RobotOutlined,{className:"mr-2"})," Select Agent"]}),(0,t.jsx)(T.Select,{value:tN,placeholder:"Select an Agent",onChange:e=>tE(e),options:tj.map(e=>({value:e.agent_name,label:e.agent_name||e.agent_id,key:e.agent_id})),style:{width:"100%"},showSearch:!0,className:"rounded-md",optionLabelProp:"label",children:tj.map(e=>(0,t.jsx)(T.Select.Option,{value:e.agent_name,label:e.agent_name||e.agent_id,children:(0,t.jsxs)("div",{className:"flex flex-col py-1",children:[(0,t.jsx)("span",{className:"font-medium",children:e.agent_name||e.agent_id}),e.agent_card_params?.description&&(0,t.jsx)("span",{className:"text-xs text-gray-500 mt-1",children:e.agent_card_params.description})]})},e.agent_id))}),0===tj.length&&(0,t.jsx)(v.Text,{className:"text-xs text-gray-500 mt-2 block",children:"No agents found. Create agents via /v1/agents endpoint."})]}),(0,t.jsxs)("div",{children:[(0,t.jsxs)(v.Text,{className:"font-medium block mb-2 text-gray-700 flex items-center",children:[(0,t.jsx)(y.TagsOutlined,{className:"mr-2"})," Tags"]}),(0,t.jsx)(H.default,{value:tR,onChange:tM,className:"mb-4",accessToken:e||""})]}),(0,t.jsxs)("div",{children:[(0,t.jsxs)(v.Text,{className:"font-medium block mb-2 text-gray-700 flex items-center",children:[(0,t.jsx)(x.ToolOutlined,{className:"mr-2"}),tC===eo.EndpointType.MCP?"MCP Server":"MCP Servers",(0,t.jsx)(A.Tooltip,{className:"ml-1",title:tC===eo.EndpointType.MCP?"Select an MCP server or toolset to test tools directly.":"Select MCP servers or toolsets to use in your conversation.",children:(0,t.jsx)(l.InfoCircleOutlined,{className:"cursor-pointer",onClick:()=>ex(!0)})})]}),(0,t.jsxs)(T.Select,{mode:tC===eo.EndpointType.MCP?void 0:"multiple",style:{width:"100%"},placeholder:tC===eo.EndpointType.MCP?"Select MCP server":"Select MCP servers",value:tC===eo.EndpointType.MCP?"__all__"!==ej[0]&&1===ej.length?ej[0]:void 0:ej,onChange:e=>{tC===eo.EndpointType.MCP?(ek(e?[e]:[]),eO(void 0),e&&!eC[e]&&su(e)):e.includes("__all__")?(ek(["__all__"]),eM({})):(ek(e),eM(t=>{let s={...t};return Object.keys(s).forEach(t=>{e.includes(t)||delete s[t]}),s}),e.forEach(e=>{eC[e]||su(e)}))},loading:eN,className:"mb-2",allowClear:!0,showSearch:!0,optionLabelProp:"label",disabled:!tn.has(tC),maxTagCount:tC===eo.EndpointType.MCP?1:"responsive",filterOption:(e,t)=>{if(t?.value==="__all__")return"all mcp servers".includes(e.toLowerCase());let s=t?.value;if(s?.startsWith("toolset:")){let t=s.slice(8),r=eh.find(e=>e.toolset_id===t);return!!r&&[r.toolset_name,r.description].filter(Boolean).join(" ").toLowerCase().includes(e.toLowerCase())}let r=ed.find(e=>e.server_id===s);return!!r&&[r.server_name,r.alias,r.server_id,r.description].filter(Boolean).join(" ").toLowerCase().includes(e.toLowerCase())},children:[tC!==eo.EndpointType.MCP&&(0,t.jsx)(T.Select.Option,{value:"__all__",label:"All MCP Servers",children:(0,t.jsxs)("div",{className:"flex flex-col py-1",children:[(0,t.jsx)("span",{className:"font-medium",children:"All MCP Servers"}),(0,t.jsx)("span",{className:"text-xs text-gray-500 mt-1",children:"Use all available MCP servers"})]})},"__all__"),eh.length>0&&(0,t.jsx)(T.Select.OptGroup,{label:"Toolsets",children:eh.map(e=>(0,t.jsx)(T.Select.Option,{value:`toolset:${e.toolset_id}`,label:e.toolset_name,disabled:tC!==eo.EndpointType.MCP&&ej.includes("__all__"),children:(0,t.jsxs)("div",{className:"flex flex-col py-1",children:[(0,t.jsxs)("div",{className:"flex items-center gap-1",children:[(0,t.jsx)("span",{className:"font-medium",children:e.toolset_name}),(0,t.jsx)("span",{className:"text-xs px-1 rounded",style:{background:"#ede9fe",color:"#7c3aed"},children:"Toolset"}),(0,t.jsxs)("span",{className:"text-xs text-gray-500",children:["(",e.tools.length," tools)"]})]}),e.description&&(0,t.jsx)("span",{className:"text-xs text-gray-500 mt-1",children:e.description})]})},`toolset:${e.toolset_id}`))}),ed.length>0&&(0,t.jsx)(T.Select.OptGroup,{label:"Servers",children:ed.map(e=>(0,t.jsx)(T.Select.Option,{value:e.server_id,label:e.alias||e.server_name||e.server_id,disabled:tC!==eo.EndpointType.MCP&&ej.includes("__all__"),children:(0,t.jsxs)("div",{className:"flex flex-col py-1",children:[(0,t.jsx)("span",{className:"font-medium",children:e.alias||e.server_name||e.server_id}),e.description&&(0,t.jsx)("span",{className:"text-xs text-gray-500 mt-1",children:e.description})]})},e.server_id))})]}),tC===eo.EndpointType.MCP&&1===ej.length&&"__all__"!==ej[0]&&(()=>{let e=ej[0],s=e.startsWith("toolset:"),r=[];if(s){let t=e.slice(8),s=eh.find(e=>e.toolset_id===t);s&&(r=s.tools.map(e=>({value:e.tool_name,label:e.tool_name})))}else r=(eC[e]||[]).map(e=>({value:e.name,label:e.name}));return(0,t.jsxs)("div",{className:"mt-3",children:[(0,t.jsx)(v.Text,{className:"text-xs text-gray-600 mb-1 block",children:"Select Tool"}),(0,t.jsx)(T.Select,{style:{width:"100%"},placeholder:"Select a tool to call",value:eP,onChange:e=>eO(e),options:r,allowClear:!0,className:"rounded-md"})]})})(),ej.length>0&&!ej.includes("__all__")&&tC!==eo.EndpointType.MCP&&tn.has(tC)&&(0,t.jsx)("div",{className:"mt-3 space-y-2",children:ej.map(e=>{let s=ed.find(t=>t.server_id===e),r=eC[e]||[];return 0===r.length?null:(0,t.jsxs)("div",{className:"border rounded p-2",children:[(0,t.jsxs)(v.Text,{className:"text-xs text-gray-600 mb-1",children:["Limit tools for ",s?.alias||s?.server_name||e,":"]}),(0,t.jsx)(T.Select,{mode:"multiple",size:"small",style:{width:"100%"},placeholder:"All tools (default)",value:eR[e]||[],onChange:t=>{eM(s=>({...s,[e]:t}))},options:r.map(e=>({value:e.name,label:e.name})),maxTagCount:2})]},e)})}),ej.length>0&&!ej.includes("__all__")&&ej.some(e=>{let t=ed.find(t=>t.server_id===e);return t?.is_byok})&&(0,t.jsx)("div",{className:"mt-3 space-y-2",children:ej.map(e=>{let s=ed.find(t=>t.server_id===e);if(!s?.is_byok)return null;let r=s.alias||s.server_name||e;return(0,t.jsxs)("div",{className:"border border-blue-100 rounded p-2 bg-blue-50 flex items-center justify-between",children:[(0,t.jsxs)(v.Text,{className:"text-xs text-blue-700",children:[r," requires your API key"]}),s.has_user_credential?(0,t.jsxs)("div",{className:"flex items-center gap-2",children:[(0,t.jsxs)("span",{className:"text-green-600 text-xs font-medium flex items-center gap-1",children:[(0,t.jsx)(c.KeyOutlined,{})," Connected"]}),(0,t.jsx)("button",{className:"text-xs text-gray-400 hover:text-blue-500 underline",onClick:()=>eS(s),children:"Reconnect"})]}):(0,t.jsx)("button",{className:"text-xs bg-blue-500 hover:bg-blue-600 text-white px-3 py-1 rounded-lg font-medium",onClick:()=>eS(s),children:"Connect"})]},e)})})]}),(0,t.jsxs)("div",{children:[(0,t.jsxs)(v.Text,{className:"font-medium block mb-2 text-gray-700 flex items-center",children:[(0,t.jsx)(i.DatabaseOutlined,{className:"mr-2"})," Vector Store",(0,t.jsx)(A.Tooltip,{className:"ml-1",title:(0,t.jsxs)("span",{children:["Select vector store(s) to use for this LLM API call. You can set up your vector store"," ",(0,t.jsx)("a",{href:"?page=vector-stores",style:{color:"#1890ff"},children:"here"}),"."]}),children:(0,t.jsx)(l.InfoCircleOutlined,{})})]}),(0,t.jsx)(z.default,{value:tU,onChange:tD,className:"mb-4",accessToken:e||""})]}),(0,t.jsxs)("div",{children:[(0,t.jsxs)(v.Text,{className:"font-medium block mb-2 text-gray-700 flex items-center",children:[(0,t.jsx)(m.SafetyOutlined,{className:"mr-2"})," Guardrails",(0,t.jsx)(A.Tooltip,{className:"ml-1",title:(0,t.jsxs)("span",{children:["Select guardrail(s) to use for this LLM API call. You can set up your guardrails"," ",(0,t.jsx)("a",{href:"?page=guardrails",style:{color:"#1890ff"},children:"here"}),"."]}),children:(0,t.jsx)(l.InfoCircleOutlined,{})})]}),(0,t.jsx)(L.default,{value:tq,onChange:tB,className:"mb-4",accessToken:e||""})]}),(0,t.jsxs)("div",{children:[(0,t.jsxs)(v.Text,{className:"font-medium block mb-2 text-gray-700 flex items-center",children:[(0,t.jsx)(m.SafetyOutlined,{className:"mr-2"})," Policies",(0,t.jsx)(A.Tooltip,{className:"ml-1",title:(0,t.jsxs)("span",{children:["Select policy/policies to apply to this LLM API call. Policies define which guardrails are applied based on conditions. You can set up your policies"," ",(0,t.jsx)("a",{href:"?page=policies",style:{color:"#1890ff"},children:"here"}),"."]}),children:(0,t.jsx)(l.InfoCircleOutlined,{})})]}),(0,t.jsx)(U.default,{value:tW,onChange:tH,className:"mb-4",accessToken:e||""})]}),tC===eo.EndpointType.RESPONSES&&(0,t.jsx)("div",{children:(0,t.jsx)(ev,{accessToken:"session"===td?e||"":th,enabled:sl.enabled,onEnabledChange:sl.setEnabled,selectedContainerId:null,onContainerChange:()=>{},selectedModel:tx||""})})]})]}),(0,t.jsx)("div",{className:`flex flex-col bg-white ${en?"flex-1 w-full":"w-3/4"}`,children:tC===eo.EndpointType.REALTIME?(0,t.jsx)(te,{accessToken:"session"===td?e||"":th,selectedModel:tx||"",customProxyBaseUrl:tm||void 0,selectedGuardrails:tq.length>0?tq:void 0}):(0,t.jsxs)(t.Fragment,{children:[(0,t.jsxs)("div",{className:"p-4 border-b border-gray-200 flex justify-between items-center",children:[(0,t.jsx)(_.Title,{className:"text-xl font-semibold mb-0",children:en?"Chat":"Test Key"}),(0,t.jsxs)("div",{className:"flex gap-2",children:[(0,t.jsx)(S.Button,{onClick:()=>{tl(),sp(),sm(),sg(),sf(),B.default.success("Chat history cleared.")},className:"bg-gray-100 hover:bg-gray-200 text-gray-700 border-gray-300",icon:a.ClearOutlined,children:"Clear Chat"}),!en&&(0,t.jsx)(S.Button,{onClick:()=>t3(!0),className:"bg-gray-100 hover:bg-gray-200 text-gray-700 border-gray-300",icon:n.CodeOutlined,children:"Get Code"})]})]}),(0,t.jsxs)("div",{className:"flex-1 overflow-auto p-4 pb-0",children:[0===e$.length&&(0,t.jsxs)("div",{className:"h-full flex flex-col items-center justify-center text-gray-400",children:[(0,t.jsx)(p.RobotOutlined,{style:{fontSize:"48px",marginBottom:"16px"}}),(0,t.jsx)(v.Text,{children:"Start a conversation, generate an image, or handle audio"})]}),e$.map((s,r)=>(0,t.jsx)("div",{children:(0,t.jsx)(e0,{message:s,isLastMessage:r===e$.length-1,endpointType:tC,mcpEvents:eU,codeInterpreterResult:sl.result,accessToken:"session"===td?e||"":th})},r)),tP&&eU.length>0&&(tC===eo.EndpointType.RESPONSES||tC===eo.EndpointType.CHAT)&&e$.length>0&&"user"===e$[e$.length-1].role&&(0,t.jsx)("div",{className:"text-left mb-4",children:(0,t.jsxs)("div",{className:"inline-block max-w-[80%] rounded-lg shadow-sm p-3.5 px-4",style:{backgroundColor:"#ffffff",border:"1px solid #f0f0f0",textAlign:"left"},children:[(0,t.jsxs)("div",{className:"flex items-center gap-2 mb-1.5",children:[(0,t.jsx)("div",{className:"flex items-center justify-center w-6 h-6 rounded-full mr-1",style:{backgroundColor:"#f5f5f5"},children:(0,t.jsx)(p.RobotOutlined,{style:{fontSize:"12px",color:"#4b5563"}})}),(0,t.jsx)("strong",{className:"text-sm capitalize",children:"Assistant"})]}),(0,t.jsx)(eG.default,{events:eU})]})}),tP&&(0,t.jsx)("div",{className:"flex justify-center items-center my-4",children:(0,t.jsx)(C.Spin,{indicator:sx})}),(0,t.jsx)("div",{ref:sc,style:{height:"1px"}})]}),(0,t.jsxs)("div",{className:"p-4 border-t border-gray-200 bg-white",children:[tC===eo.EndpointType.IMAGE_EDITS&&(0,t.jsx)("div",{className:"mb-4",children:0===tz.length?(0,t.jsxs)(ta,{beforeUpload:sh,accept:"image/*",showUploadList:!1,children:[(0,t.jsx)("p",{className:"ant-upload-drag-icon",children:(0,t.jsx)(h.PictureOutlined,{style:{fontSize:"24px",color:"#666"}})}),(0,t.jsx)("p",{className:"ant-upload-text text-sm",children:"Click or drag images to upload"}),(0,t.jsx)("p",{className:"ant-upload-hint text-xs text-gray-500",children:"Support for PNG, JPG, JPEG formats. Multiple images supported."})]}):(0,t.jsxs)("div",{className:"flex flex-wrap gap-2",children:[tz.map((e,s)=>(0,t.jsxs)("div",{className:"relative inline-block",children:[(0,t.jsx)("img",{src:(()=>{let e=tJ[s];if(!e)return"";try{let t=new URL(e);return"blob:"===t.protocol?t.href:""}catch{return""}})(),alt:`Upload preview ${s+1}`,className:"max-w-32 max-h-32 rounded-md border border-gray-200 object-cover"}),(0,t.jsx)("button",{className:"absolute top-1 right-1 bg-white shadow-sm border border-gray-200 rounded px-1 py-1 text-red-500 hover:bg-red-50 text-xs",onClick:()=>{tJ[s]&&URL.revokeObjectURL(tJ[s]),tF(e=>e.filter((e,t)=>t!==s)),tG(e=>e.filter((e,t)=>t!==s))},children:(0,t.jsx)(o.DeleteOutlined,{})})]},s)),(0,t.jsxs)("div",{className:"flex items-center justify-center w-32 h-32 border-2 border-dashed border-gray-300 rounded-md hover:border-gray-400 cursor-pointer",onClick:()=>document.getElementById("additional-image-upload")?.click(),children:[(0,t.jsxs)("div",{className:"text-center",children:[(0,t.jsx)(h.PictureOutlined,{style:{fontSize:"24px",color:"#666"}}),(0,t.jsx)("p",{className:"text-xs text-gray-500 mt-1",children:"Add more"})]}),(0,t.jsx)("input",{id:"additional-image-upload",type:"file",accept:"image/*",multiple:!0,style:{display:"none"},onChange:e=>{Array.from(e.target.files||[]).forEach(e=>sh(e))}})]})]})}),tC===eo.EndpointType.TRANSCRIPTION&&(0,t.jsx)("div",{className:"mb-4",children:t2?(0,t.jsxs)("div",{className:"flex items-center gap-3 p-3 bg-gray-50 rounded-lg border border-gray-200",children:[(0,t.jsxs)("div",{className:"flex items-center gap-2 flex-1",children:[(0,t.jsx)(f.SoundOutlined,{style:{fontSize:"20px",color:"#666"}}),(0,t.jsx)("span",{className:"text-sm font-medium",children:t2.name}),(0,t.jsxs)("span",{className:"text-xs text-gray-500",children:["(",(t2.size/1024/1024).toFixed(2)," MB)"]})]}),(0,t.jsxs)("button",{className:"bg-white shadow-sm border border-gray-200 rounded px-2 py-1 text-red-500 hover:bg-red-50 text-xs",onClick:sf,children:[(0,t.jsx)(o.DeleteOutlined,{})," Remove"]})]}):(0,t.jsxs)(ta,{beforeUpload:e=>(t4(e),!1),accept:"audio/*,.mp3,.mp4,.mpeg,.mpga,.m4a,.wav,.webm",showUploadList:!1,children:[(0,t.jsx)("p",{className:"ant-upload-drag-icon",children:(0,t.jsx)(f.SoundOutlined,{style:{fontSize:"24px",color:"#666"}})}),(0,t.jsx)("p",{className:"ant-upload-text text-sm",children:"Click or drag audio file to upload"}),(0,t.jsx)("p",{className:"ant-upload-hint text-xs text-gray-500",children:"Support for MP3, MP4, MPEG, MPGA, M4A, WAV, WEBM formats. Max file size: 25 MB."})]})}),tC===eo.EndpointType.RESPONSES&&tV&&(0,t.jsx)(eE,{file:tV,previewUrl:tX,onRemove:sm}),tC===eo.EndpointType.CHAT&&tQ&&(0,t.jsx)(eE,{file:tQ,previewUrl:t0,onRemove:sg}),tC===eo.EndpointType.RESPONSES&&sl.enabled&&(0,t.jsxs)("div",{className:"mb-2 space-y-2",children:[(0,t.jsxs)("div",{className:"px-3 py-2 bg-gradient-to-r from-blue-50 to-purple-50 rounded-lg border border-blue-200 flex items-center justify-between",children:[(0,t.jsx)("div",{className:"flex items-center gap-2",children:tP?(0,t.jsxs)(t.Fragment,{children:[(0,t.jsx)(u.LoadingOutlined,{className:"text-blue-500",spin:!0}),(0,t.jsx)("span",{className:"text-sm text-blue-700 font-medium",children:"Running Python code..."})]}):(0,t.jsxs)(t.Fragment,{children:[(0,t.jsx)(n.CodeOutlined,{className:"text-blue-500"}),(0,t.jsx)("span",{className:"text-sm text-blue-700 font-medium",children:"Code Interpreter Active"})]})}),(0,t.jsx)("button",{className:"text-xs text-blue-500 hover:text-blue-700",onClick:()=>sl.setEnabled(!1),children:"Disable"})]}),!tP&&(0,t.jsx)("div",{className:"flex flex-wrap gap-2",children:["Generate sample sales data CSV and create a chart","Create a PNG bar chart comparing AI gateway providers including LiteLLM","Generate a CSV of LLM pricing data and visualize it as a line chart"].map((e,s)=>(0,t.jsx)("button",{className:"text-xs px-3 py-1.5 bg-white border border-gray-200 rounded-full hover:bg-blue-50 hover:border-blue-300 hover:text-blue-600 transition-colors",onClick:()=>ty(e),children:e},s))})]}),0===e$.length&&!tP&&tC!==eo.EndpointType.MCP&&(0,t.jsx)("div",{className:"flex items-center gap-2 mb-3 overflow-x-auto",children:(tC===eo.EndpointType.A2A_AGENTS?["What can you help me with?","Tell me about yourself","What tasks can you perform?"]:["Write me a poem","Explain quantum computing","Draft a polite email requesting a meeting"]).map(e=>(0,t.jsx)("button",{type:"button",className:"shrink-0 rounded-full border border-gray-200 px-3 py-1 text-xs font-medium text-gray-600 transition-colors hover:bg-blue-50 hover:border-blue-300 hover:text-blue-600 cursor-pointer",onClick:()=>ty(e),children:e},e))}),(0,t.jsxs)("div",{className:"flex items-center gap-2",children:[(0,t.jsxs)("div",{className:"flex items-center flex-1 bg-white border border-gray-300 rounded-xl px-3 py-1 min-h-[44px]",children:[(0,t.jsxs)("div",{className:"flex-shrink-0 mr-2 flex items-center gap-1",children:[tC===eo.EndpointType.RESPONSES&&!tV&&(0,t.jsx)(e2,{responsesUploadedImage:tV,responsesImagePreviewUrl:tX,onImageUpload:e=>(tK(e),tY(URL.createObjectURL(e)),!1),onRemoveImage:sm}),tC===eo.EndpointType.CHAT&&!tQ&&(0,t.jsx)(ep,{chatUploadedImage:tQ,chatImagePreviewUrl:t0,onImageUpload:e=>(tZ(e),t1(URL.createObjectURL(e)),!1),onRemoveImage:sg}),tC===eo.EndpointType.RESPONSES&&(0,t.jsx)(A.Tooltip,{title:sl.enabled?"Code Interpreter enabled (click to disable)":"Enable Code Interpreter",children:(0,t.jsx)("button",{className:`p-1.5 rounded-md transition-colors ${sl.enabled?"bg-blue-100 text-blue-600":"text-gray-400 hover:text-gray-600 hover:bg-gray-100"}`,onClick:()=>{sl.toggle(),sl.enabled||B.default.success("Code Interpreter enabled!")},children:(0,t.jsx)(n.CodeOutlined,{style:{fontSize:"16px"}})})})]}),tC===eo.EndpointType.MCP&&1===ej.length&&"__all__"!==ej[0]&&eP?(0,t.jsx)("div",{className:"flex-1 overflow-y-auto max-h-48 min-h-[44px] p-2 border border-gray-200 rounded-lg bg-gray-50/50",children:(()=>{let e=ej[0],s=[];if(e.startsWith("toolset:")){let t=e.slice(8),r=eh.find(e=>e.toolset_id===t);r&&[...new Set(r.tools.map(e=>e.server_id))].forEach(e=>{s=s.concat(eC[e]||[])})}else s=eC[e]||[];let r=s.find(e=>e.name===eP);return r?(0,t.jsx)(D.default,{ref:eI,tool:r,className:"space-y-2"}):(0,t.jsx)("div",{className:"flex items-center justify-center h-10 text-sm text-gray-500",children:"Loading tool schema..."})})()}):(0,t.jsx)(tr,{value:tf,onChange:e=>ty(e.target.value),onKeyDown:e=>{"Enter"!==e.key||e.shiftKey||(e.preventDefault(),sy())},placeholder:tC===eo.EndpointType.CHAT||tC===eo.EndpointType.EMBEDDINGS||tC===eo.EndpointType.RESPONSES||tC===eo.EndpointType.ANTHROPIC_MESSAGES?"Type your message... (Shift+Enter for new line)":tC===eo.EndpointType.A2A_AGENTS?"Send a message to the A2A agent...":tC===eo.EndpointType.IMAGE_EDITS?"Describe how you want to edit the image...":tC===eo.EndpointType.SPEECH?"Enter text to convert to speech...":tC===eo.EndpointType.TRANSCRIPTION?"Optional: Add context or prompt for transcription...":"Describe the image you want to generate...",disabled:tP,className:"flex-1",autoSize:{minRows:1,maxRows:4},style:{resize:"none",border:"none",boxShadow:"none",background:"transparent",padding:"4px 0",fontSize:"14px",lineHeight:"20px"}}),(0,t.jsx)(S.Button,{onClick:sy,disabled:tP||(tC===eo.EndpointType.MCP?!(1===ej.length&&"__all__"!==ej[0]&&eP):tC===eo.EndpointType.TRANSCRIPTION?!t2:!tf.trim()),className:"flex-shrink-0 ml-2 !w-8 !h-8 !min-w-8 !p-0 !rounded-full !bg-blue-600 hover:!bg-blue-700 disabled:!bg-gray-300 !border-none !text-white disabled:!text-gray-500 !flex !items-center !justify-center",children:(0,t.jsx)(r.ArrowUpOutlined,{style:{fontSize:"14px"}})})]}),tP&&(0,t.jsx)(S.Button,{onClick:()=>{tI.current&&(tI.current.abort(),tI.current=null,tO(!1),B.default.info("Request cancelled"))},className:"bg-red-50 hover:bg-red-100 text-red-600 border-red-200",icon:o.DeleteOutlined,children:"Cancel"})]})]})]})})]})}),(0,t.jsxs)(N.Modal,{title:"Generated Code",open:t6,onCancel:()=>t3(!1),footer:null,width:800,children:[(0,t.jsxs)("div",{className:"flex justify-between items-end my-4",children:[(0,t.jsxs)("div",{children:[(0,t.jsx)(v.Text,{className:"font-medium block mb-1 text-gray-700",children:"SDK Type"}),(0,t.jsx)(T.Select,{value:t7,onChange:e=>t9(e),style:{width:150},options:[{value:"openai",label:"OpenAI SDK"},{value:"azure",label:"Azure SDK"}]})]}),(0,t.jsx)(j.Button,{onClick:()=>{navigator.clipboard.writeText(t5),B.default.success("Copied to clipboard!")},children:"Copy to Clipboard"})]}),(0,t.jsx)(R.Prism,{language:"python",style:M.coy,wrapLines:!0,wrapLongLines:!0,className:"rounded-md",customStyle:{maxHeight:"60vh",overflowY:"auto"},children:t5})]}),eb&&(0,t.jsx)(q.ByokCredentialModal,{server:eb,open:!!eb,onClose:()=>eS(null),onSuccess:e=>{sd(),eS(null)},accessToken:e||""}),(0,t.jsx)(N.Modal,{title:"How Toolsets Work",open:ey,onCancel:()=>ex(!1),footer:[(0,t.jsx)(j.Button,{onClick:()=>ex(!1),children:"Close"},"close")],width:600,children:(0,t.jsxs)("div",{className:"space-y-4 py-2",children:[(0,t.jsxs)("p",{className:"text-gray-700",children:[(0,t.jsx)("strong",{children:"Toolsets"})," are named collections of specific tools from one or more MCP servers. Instead of exposing all tools from a server, a toolset gives an agent exactly the tools it needs."]}),(0,t.jsxs)("div",{children:[(0,t.jsx)("h4",{className:"font-semibold text-gray-800 mb-2",children:"How to use a toolset:"}),(0,t.jsxs)("ol",{className:"list-decimal list-inside space-y-2 text-gray-700",children:[(0,t.jsxs)("li",{children:["Select a ",(0,t.jsx)("span",{style:{color:"#7c3aed",fontWeight:600},children:"Toolset"})," (purple badge) from the MCP Servers dropdown."]}),(0,t.jsx)("li",{children:"The tool picker will show only the tools included in that toolset."}),(0,t.jsx)("li",{children:"Select a tool and fill in its parameters, then send."}),(0,t.jsx)("li",{children:"The tool call is routed to the correct underlying MCP server automatically."})]})]}),(0,t.jsx)("div",{className:"bg-purple-50 border border-purple-200 rounded p-3",children:(0,t.jsxs)("p",{className:"text-sm text-purple-800",children:[(0,t.jsx)("strong",{children:"Example:"}),' A "GitHub Read-only" toolset might include only ',(0,t.jsx)("code",{children:"list_repos"})," and ",(0,t.jsx)("code",{children:"get_file"})," from a GitHub MCP server — preventing agents from making writes."]})}),(0,t.jsxs)("div",{children:[(0,t.jsx)("h4",{className:"font-semibold text-gray-800 mb-1",children:"Creating toolsets:"}),(0,t.jsxs)("p",{className:"text-sm text-gray-600",children:["Admins can create and manage toolsets from the ",(0,t.jsx)("strong",{children:"MCP"})," page → ",(0,t.jsx)("strong",{children:"Toolsets"})," tab. Toolsets can then be assigned to keys and teams to scope their tool access."]})]})]})})]})}],220486)}]); \ No newline at end of file diff --git a/litellm/proxy/_experimental/out/_next/static/chunks/0-wj413-8yde5.js b/litellm/proxy/_experimental/out/_next/static/chunks/0-wj413-8yde5.js new file mode 100644 index 00000000000..050f40e3e2e --- /dev/null +++ b/litellm/proxy/_experimental/out/_next/static/chunks/0-wj413-8yde5.js @@ -0,0 +1,84 @@ +(globalThis.TURBOPACK||(globalThis.TURBOPACK=[])).push(["object"==typeof document?document.currentScript:void 0,185357,180766,782719,969641,476993,824296,64352,230312,e=>{"use strict";var t,a,l=e.i(843476),r=e.i(808613),i=e.i(311451),s=e.i(212931),n=e.i(199133),o=e.i(262218),d=e.i(898586),c=e.i(464571),m=e.i(271645),u=e.i(727749),p=e.i(764205),g=e.i(770914),x=e.i(515831),h=e.i(175712),f=e.i(646563),y=e.i(519756);let{Text:j}=d.Typography,{Option:_}=n.Select,b=({visible:e,prebuiltPatterns:t,categories:a,selectedPatternName:r,patternAction:i,onPatternNameChange:o,onActionChange:d,onAdd:m,onCancel:u})=>(0,l.jsxs)(s.Modal,{title:"Add prebuilt pattern",open:e,onCancel:u,footer:null,width:800,children:[(0,l.jsxs)(g.Space,{direction:"vertical",style:{width:"100%"},size:"large",children:[(0,l.jsxs)("div",{children:[(0,l.jsx)(j,{strong:!0,children:"Pattern type"}),(0,l.jsx)(n.Select,{placeholder:"Choose pattern type",value:r,onChange:o,style:{width:"100%",marginTop:8},showSearch:!0,filterOption:(e,a)=>{let l=t.find(e=>e.name===a?.value);return!!l&&(l.display_name.toLowerCase().includes(e.toLowerCase())||l.name.toLowerCase().includes(e.toLowerCase()))},children:a.map(e=>{let a=t.filter(t=>t.category===e);return 0===a.length?null:(0,l.jsx)(n.Select.OptGroup,{label:e,children:a.map(e=>(0,l.jsx)(_,{value:e.name,children:e.display_name},e.name))},e)})})]}),(0,l.jsxs)("div",{children:[(0,l.jsx)(j,{strong:!0,children:"Action"}),(0,l.jsx)(j,{type:"secondary",style:{display:"block",marginTop:4,marginBottom:8},children:"Choose what action the guardrail should take when this pattern is detected"}),(0,l.jsxs)(n.Select,{value:i,onChange:d,style:{width:"100%"},children:[(0,l.jsx)(_,{value:"BLOCK",children:"Block"}),(0,l.jsx)(_,{value:"MASK",children:"Mask"})]})]})]}),(0,l.jsxs)("div",{style:{display:"flex",justifyContent:"flex-end",gap:"8px",marginTop:"24px"},children:[(0,l.jsx)(c.Button,{onClick:u,children:"Cancel"}),(0,l.jsx)(c.Button,{type:"primary",onClick:m,children:"Add"})]})]}),{Text:v}=d.Typography,{Option:w}=n.Select,N=({visible:e,patternName:t,patternRegex:a,patternAction:r,onNameChange:o,onRegexChange:d,onActionChange:m,onAdd:u,onCancel:p})=>(0,l.jsxs)(s.Modal,{title:"Add custom regex pattern",open:e,onCancel:p,footer:null,width:800,children:[(0,l.jsxs)(g.Space,{direction:"vertical",style:{width:"100%"},size:"large",children:[(0,l.jsxs)("div",{children:[(0,l.jsx)(v,{strong:!0,children:"Pattern name"}),(0,l.jsx)(i.Input,{placeholder:"e.g., internal_id, employee_code",value:t,onChange:e=>o(e.target.value),style:{marginTop:8}})]}),(0,l.jsxs)("div",{children:[(0,l.jsx)(v,{strong:!0,children:"Regex pattern"}),(0,l.jsx)(i.Input,{placeholder:"e.g., ID-[0-9]{6}",value:a,onChange:e=>d(e.target.value),style:{marginTop:8}}),(0,l.jsx)(v,{type:"secondary",style:{fontSize:12},children:"Enter a valid regular expression to match sensitive data"})]}),(0,l.jsxs)("div",{children:[(0,l.jsx)(v,{strong:!0,children:"Action"}),(0,l.jsx)(v,{type:"secondary",style:{display:"block",marginTop:4,marginBottom:8},children:"Choose what action the guardrail should take when this pattern is detected"}),(0,l.jsxs)(n.Select,{value:r,onChange:m,style:{width:"100%"},children:[(0,l.jsx)(w,{value:"BLOCK",children:"Block"}),(0,l.jsx)(w,{value:"MASK",children:"Mask"})]})]})]}),(0,l.jsxs)("div",{style:{display:"flex",justifyContent:"flex-end",gap:"8px",marginTop:"24px"},children:[(0,l.jsx)(c.Button,{onClick:p,children:"Cancel"}),(0,l.jsx)(c.Button,{type:"primary",onClick:u,children:"Add"})]})]}),{Text:C}=d.Typography,{Option:S}=n.Select,k=({visible:e,keyword:t,action:a,description:r,onKeywordChange:o,onActionChange:d,onDescriptionChange:m,onAdd:u,onCancel:p})=>(0,l.jsxs)(s.Modal,{title:"Add blocked keyword",open:e,onCancel:p,footer:null,width:800,children:[(0,l.jsxs)(g.Space,{direction:"vertical",style:{width:"100%"},size:"large",children:[(0,l.jsxs)("div",{children:[(0,l.jsx)(C,{strong:!0,children:"Keyword"}),(0,l.jsx)(i.Input,{placeholder:"Enter sensitive keyword or phrase",value:t,onChange:e=>o(e.target.value),style:{marginTop:8}})]}),(0,l.jsxs)("div",{children:[(0,l.jsx)(C,{strong:!0,children:"Action"}),(0,l.jsx)(C,{type:"secondary",style:{display:"block",marginTop:4,marginBottom:8},children:"Choose what action the guardrail should take when this keyword is detected"}),(0,l.jsxs)(n.Select,{value:a,onChange:d,style:{width:"100%"},children:[(0,l.jsx)(S,{value:"BLOCK",children:"Block"}),(0,l.jsx)(S,{value:"MASK",children:"Mask"})]})]}),(0,l.jsxs)("div",{children:[(0,l.jsx)(C,{strong:!0,children:"Description (optional)"}),(0,l.jsx)(i.Input.TextArea,{placeholder:"Explain why this keyword is sensitive",value:r,onChange:e=>m(e.target.value),rows:3,style:{marginTop:8}})]})]}),(0,l.jsxs)("div",{style:{display:"flex",justifyContent:"flex-end",gap:"8px",marginTop:"24px"},children:[(0,l.jsx)(c.Button,{onClick:p,children:"Cancel"}),(0,l.jsx)(c.Button,{type:"primary",onClick:u,children:"Add"})]})]});var I=e.i(291542),A=e.i(955135);let{Text:O}=d.Typography,{Option:T}=n.Select,P=({patterns:e,onActionChange:t,onRemove:a})=>{let r=[{title:"Type",dataIndex:"type",key:"type",width:100,render:e=>(0,l.jsx)(o.Tag,{color:"prebuilt"===e?"blue":"green",children:"prebuilt"===e?"Prebuilt":"Custom"})},{title:"Pattern name",dataIndex:"name",key:"name",render:(e,t)=>t.display_name||t.name},{title:"Regex pattern",dataIndex:"pattern",key:"pattern",render:e=>e?(0,l.jsxs)(O,{code:!0,style:{fontSize:12},children:[e.substring(0,40),"..."]}):"-"},{title:"Action",dataIndex:"action",key:"action",width:150,render:(e,a)=>(0,l.jsxs)(n.Select,{value:e,onChange:e=>t(a.id,e),style:{width:120},size:"small",children:[(0,l.jsx)(T,{value:"BLOCK",children:"Block"}),(0,l.jsx)(T,{value:"MASK",children:"Mask"})]})},{title:"",key:"actions",width:100,render:(e,t)=>(0,l.jsx)(c.Button,{type:"text",danger:!0,size:"small",icon:(0,l.jsx)(A.DeleteOutlined,{}),onClick:()=>a(t.id),children:"Delete"})}];return 0===e.length?(0,l.jsx)("div",{style:{textAlign:"center",padding:"40px 0",color:"#999"},children:"No patterns added."}):(0,l.jsx)(I.Table,{dataSource:e,columns:r,rowKey:"id",pagination:!1,size:"small"})},{Text:L}=d.Typography,{Option:B}=n.Select,F=({keywords:e,onActionChange:t,onRemove:a})=>{let r=[{title:"Keyword",dataIndex:"keyword",key:"keyword"},{title:"Action",dataIndex:"action",key:"action",width:150,render:(e,a)=>(0,l.jsxs)(n.Select,{value:e,onChange:e=>t(a.id,"action",e),style:{width:120},size:"small",children:[(0,l.jsx)(B,{value:"BLOCK",children:"Block"}),(0,l.jsx)(B,{value:"MASK",children:"Mask"})]})},{title:"Description",dataIndex:"description",key:"description",render:e=>e||"-"},{title:"",key:"actions",width:100,render:(e,t)=>(0,l.jsx)(c.Button,{type:"text",danger:!0,size:"small",icon:(0,l.jsx)(A.DeleteOutlined,{}),onClick:()=>a(t.id),children:"Delete"})}];return 0===e.length?(0,l.jsx)("div",{style:{textAlign:"center",padding:"40px 0",color:"#999"},children:"No keywords added."}):(0,l.jsx)(I.Table,{dataSource:e,columns:r,rowKey:"id",pagination:!1,size:"small"})};var $=e.i(362024),E=e.i(993914);let{Title:M,Text:R}=d.Typography,{Option:G}=n.Select,z=({availableCategories:e,selectedCategories:t,onCategoryAdd:a,onCategoryRemove:r,onCategoryUpdate:i,accessToken:s,pendingSelection:d,onPendingSelectionChange:u})=>{let[g,x]=m.default.useState(""),y=void 0!==d?d:g,j=u||x,[_,b]=m.default.useState({}),[v,w]=m.default.useState({}),[N,C]=m.default.useState({}),[S,k]=m.default.useState([]),[O,T]=m.default.useState(""),[P,L]=m.default.useState(!1),B=async e=>{if(s&&!_[e]){C(t=>({...t,[e]:!0}));try{let t=await (0,p.getCategoryYaml)(s,e),a=t.yaml_content;if("json"===t.file_type)try{let e=JSON.parse(a);a=JSON.stringify(e,null,2)}catch(t){console.warn(`Failed to format JSON for ${e}:`,t)}b(t=>({...t,[e]:a})),w(a=>({...a,[e]:t.file_type||"yaml"}))}catch(t){console.error(`Failed to fetch content for category ${e}:`,t)}finally{C(t=>({...t,[e]:!1}))}}};m.default.useEffect(()=>{if(y&&s){let e=_[y];if(e)return void T(e);L(!0),console.log(`Fetching content for category: ${y}`,{accessToken:s?"present":"missing"}),(0,p.getCategoryYaml)(s,y).then(e=>{console.log(`Successfully fetched content for ${y}:`,e);let t=e.yaml_content;if("json"===e.file_type)try{let e=JSON.parse(t);t=JSON.stringify(e,null,2)}catch(e){console.warn(`Failed to format JSON for ${y}:`,e)}T(t),b(e=>({...e,[y]:t})),w(t=>({...t,[y]:e.file_type||"yaml"}))}).catch(e=>{console.error(`Failed to fetch preview content for category ${y}:`,e),T("")}).finally(()=>{L(!1)})}else T(""),L(!1)},[y,s]);let F=[{title:"Category",dataIndex:"display_name",key:"display_name",render:(t,a)=>{let r=e.find(e=>e.name===a.category);return(0,l.jsxs)("div",{children:[(0,l.jsx)("div",{style:{fontWeight:500},children:t}),r?.description&&(0,l.jsx)("div",{style:{fontSize:"12px",color:"#888",marginTop:"4px"},children:r.description})]})}},{title:"Action",dataIndex:"action",key:"action",width:150,render:(e,t)=>(0,l.jsxs)(n.Select,{value:e,onChange:e=>i(t.id,"action",e),style:{width:"100%"},children:[(0,l.jsx)(G,{value:"BLOCK",children:(0,l.jsx)(o.Tag,{color:"red",children:"BLOCK"})}),(0,l.jsx)(G,{value:"MASK",children:(0,l.jsx)(o.Tag,{color:"orange",children:"MASK"})})]})},{title:"Severity Threshold",dataIndex:"severity_threshold",key:"severity_threshold",width:180,render:(e,t)=>(0,l.jsxs)(n.Select,{value:e,onChange:e=>i(t.id,"severity_threshold",e),style:{width:"100%"},children:[(0,l.jsx)(G,{value:"low",children:"Low"}),(0,l.jsx)(G,{value:"medium",children:"Medium"}),(0,l.jsx)(G,{value:"high",children:"High"})]})},{title:"",key:"actions",width:80,render:(e,t)=>(0,l.jsx)(c.Button,{icon:(0,l.jsx)(A.DeleteOutlined,{}),onClick:()=>r(t.id),size:"small",children:"Remove"})}],z=e.filter(e=>!t.some(t=>t.category===e.name));return(0,l.jsxs)(h.Card,{title:(0,l.jsxs)("div",{style:{display:"flex",justifyContent:"space-between",alignItems:"center",flexWrap:"wrap",gap:8},children:[(0,l.jsx)(M,{level:5,style:{margin:0},children:"Blocked topics"}),(0,l.jsx)(R,{type:"secondary",style:{fontSize:12,fontWeight:400},children:"Select topics to block using keyword and semantic analysis"})]}),size:"small",children:[(0,l.jsxs)("div",{style:{marginBottom:16,display:"flex",gap:8},children:[(0,l.jsx)(n.Select,{placeholder:"Select a content category",value:y||void 0,onChange:j,style:{flex:1},showSearch:!0,optionLabelProp:"label",filterOption:(e,t)=>(t?.label?.toString().toLowerCase()??"").includes(e.toLowerCase()),children:z.map(e=>(0,l.jsx)(G,{value:e.name,label:e.display_name,children:(0,l.jsxs)("div",{children:[(0,l.jsx)("div",{style:{fontWeight:500},children:e.display_name}),(0,l.jsx)("div",{style:{fontSize:"12px",color:"#666",marginTop:"2px"},children:e.description})]})},e.name))}),(0,l.jsx)(c.Button,{type:"primary",onClick:()=>{if(!y)return;let l=e.find(e=>e.name===y);!l||t.some(e=>e.category===y)||(a({id:`category-${Date.now()}`,category:l.name,display_name:l.display_name,action:l.default_action,severity_threshold:"medium"}),j(""),T(""))},disabled:!y,icon:(0,l.jsx)(f.PlusOutlined,{}),children:"Add"})]}),y&&(0,l.jsxs)("div",{style:{marginBottom:16,padding:"12px",background:"#f9f9f9",border:"1px solid #e0e0e0",borderRadius:"4px"},children:[(0,l.jsxs)("div",{style:{marginBottom:8,fontWeight:500,fontSize:"14px"},children:["Preview: ",e.find(e=>e.name===y)?.display_name,v[y]&&(0,l.jsxs)("span",{style:{marginLeft:8,fontSize:"12px",color:"#888",fontWeight:400},children:["(",v[y]?.toUpperCase(),")"]})]}),P?(0,l.jsx)("div",{style:{padding:"16px",textAlign:"center",color:"#888"},children:"Loading content..."}):O?(0,l.jsx)("pre",{style:{background:"#fff",padding:"12px",borderRadius:"4px",overflow:"auto",maxHeight:"300px",maxWidth:"100%",fontSize:"12px",lineHeight:"1.5",margin:0,border:"1px solid #e0e0e0",whiteSpace:"pre-wrap",wordBreak:"break-word"},children:(0,l.jsx)("code",{children:O})}):(0,l.jsx)("div",{style:{padding:"8px",textAlign:"center",color:"#888",fontSize:"12px"},children:"Unable to load category content"})]}),t.length>0?(0,l.jsxs)(l.Fragment,{children:[(0,l.jsx)(I.Table,{dataSource:t,columns:F,pagination:!1,size:"small",rowKey:"id"}),(0,l.jsx)("div",{style:{marginTop:16},children:(0,l.jsx)($.Collapse,{activeKey:S,onChange:e=>{let t=Array.isArray(e)?e:e?[e]:[],a=new Set(S);t.forEach(e=>{a.has(e)||_[e]||B(e)}),k(t)},ghost:!0,items:t.map(e=>{let t=(v[e.category]||"yaml").toUpperCase();return{key:e.category,label:(0,l.jsxs)("div",{style:{display:"flex",alignItems:"center",gap:8},children:[(0,l.jsx)(E.FileTextOutlined,{}),(0,l.jsxs)("span",{children:["View ",t," for ",e.display_name]})]}),children:N[e.category]?(0,l.jsx)("div",{style:{padding:"16px",textAlign:"center",color:"#888"},children:"Loading content..."}):_[e.category]?(0,l.jsx)("pre",{style:{background:"#f5f5f5",padding:"16px",borderRadius:"4px",overflow:"auto",maxHeight:"400px",fontSize:"12px",lineHeight:"1.5",margin:0},children:(0,l.jsx)("code",{children:_[e.category]})}):(0,l.jsx)("div",{style:{padding:"16px",textAlign:"center",color:"#888"},children:"Content will load when expanded"})}})})})]}):(0,l.jsx)("div",{style:{textAlign:"center",padding:"24px",color:"#888",border:"1px dashed #d9d9d9",borderRadius:"4px"},children:"No blocked topics selected. Add topics to detect and block harmful content."})]})};var D=e.i(790848),K=e.i(28651);let{Title:q,Text:H}=d.Typography,{Option:J}=n.Select,U={competitor_intent_type:"airline",brand_self:[],locations:[],policy:{competitor_comparison:"refuse",possible_competitor_comparison:"reframe"},threshold_high:.7,threshold_medium:.45,threshold_low:.3},W=({enabled:e,config:t,onChange:a,accessToken:i})=>{let s=t??U,[o,d]=(0,m.useState)([]),[c,u]=(0,m.useState)(!1);(0,m.useEffect)(()=>{"airline"===s.competitor_intent_type&&i&&0===o.length&&(u(!0),(0,p.getMajorAirlines)(i).then(e=>d(e.airlines??[])).catch(()=>d([])).finally(()=>u(!1)))},[s.competitor_intent_type,i,o.length]);let x=e=>{a(e,e?{...U}:null)},f=(t,l)=>{a(e,{...s,[t]:l})},y=(t,l)=>{a(e,{...s,policy:{...s.policy,[t]:l}})},j=(t,l)=>{a(e,{...s,[t]:l.filter(Boolean)})};return e?(0,l.jsxs)(h.Card,{title:(0,l.jsxs)("div",{style:{display:"flex",justifyContent:"space-between",alignItems:"center"},children:[(0,l.jsx)(q,{level:5,style:{margin:0},children:"Competitor Intent Filter"}),(0,l.jsx)(D.Switch,{checked:e,onChange:x})]}),size:"small",children:[(0,l.jsx)(H,{type:"secondary",style:{display:"block",marginBottom:16},children:"Block or reframe competitor comparison questions. Airline type uses major airlines (excluding your brand); generic requires manual competitor list."}),(0,l.jsxs)(r.Form,{layout:"vertical",size:"small",children:[(0,l.jsx)(r.Form.Item,{label:"Type",children:(0,l.jsxs)(n.Select,{value:s.competitor_intent_type,onChange:e=>f("competitor_intent_type",e),style:{width:"100%"},children:[(0,l.jsx)(J,{value:"airline",children:"Airline (auto-load competitors from IATA)"}),(0,l.jsx)(J,{value:"generic",children:"Generic (specify competitors manually)"})]})}),(0,l.jsx)(r.Form.Item,{label:"Your Brand (brand_self)",required:!0,help:"airline"===s.competitor_intent_type?"Select your airline from the list (excluded from competitors) or type to add a custom term":"Names/codes users use for your brand",children:(0,l.jsx)(n.Select,{mode:"tags",style:{width:"100%"},placeholder:c?"Loading airlines...":"airline"===s.competitor_intent_type?"Search or select airline, or type to add custom":"Type and press Enter to add",value:s.brand_self,onChange:t=>"airline"===s.competitor_intent_type&&o.length>0?(t=>{let l=t.filter(Boolean),r=[],i=new Set;for(let e of l){let t=o.find(t=>t.match.split("|")[0]?.trim().toLowerCase()===e.toLowerCase());if(t)for(let e of t.match.split("|").map(e=>e.trim().toLowerCase()).filter(Boolean))i.has(e)||(i.add(e),r.push(e));else i.has(e.toLowerCase())||(i.add(e.toLowerCase()),r.push(e))}a(e,{...s,brand_self:r})})(t??[]):j("brand_self",t??[]),tokenSeparators:[","],loading:c,showSearch:!0,filterOption:(e,t)=>(t?.label?.toString().toLowerCase()??"").includes(e.toLowerCase()),optionFilterProp:"label",options:"airline"===s.competitor_intent_type&&o.length>0?o.map(e=>{let t=e.match.split("|")[0]?.trim()??e.id,a=e.match.split("|").map(e=>e.trim().toLowerCase()).filter(Boolean);return{value:t.toLowerCase(),label:`${t}${a.length>1?` (${a.slice(1).join(", ")})`:""}`}}):void 0})}),"airline"===s.competitor_intent_type&&(0,l.jsx)(r.Form.Item,{label:"Locations (optional)",help:"Countries, cities, airports for disambiguation (e.g. qatar, doha)",children:(0,l.jsx)(n.Select,{mode:"tags",style:{width:"100%"},placeholder:"Type and press Enter to add",value:s.locations??[],onChange:e=>j("locations",e??[]),tokenSeparators:[","]})}),"generic"===s.competitor_intent_type&&(0,l.jsx)(r.Form.Item,{label:"Competitors",required:!0,help:"Competitor names to detect (required for generic type)",children:(0,l.jsx)(n.Select,{mode:"tags",style:{width:"100%"},placeholder:"Type and press Enter to add",value:s.competitors??[],onChange:e=>j("competitors",e??[]),tokenSeparators:[","]})}),(0,l.jsx)(r.Form.Item,{label:"Policy: Competitor comparison",children:(0,l.jsxs)(n.Select,{value:s.policy?.competitor_comparison??"refuse",onChange:e=>y("competitor_comparison",e),style:{width:"100%"},children:[(0,l.jsx)(J,{value:"refuse",children:"Refuse (block request)"}),(0,l.jsx)(J,{value:"reframe",children:"Reframe (suggest alternative)"})]})}),(0,l.jsx)(r.Form.Item,{label:"Policy: Possible competitor comparison",children:(0,l.jsxs)(n.Select,{value:s.policy?.possible_competitor_comparison??"reframe",onChange:e=>y("possible_competitor_comparison",e),style:{width:"100%"},children:[(0,l.jsx)(J,{value:"refuse",children:"Refuse (block request)"}),(0,l.jsx)(J,{value:"reframe",children:"Reframe (suggest alternative to backend LLM)"})]})}),(0,l.jsx)(r.Form.Item,{label:"Confidence thresholds",help:(0,l.jsxs)(l.Fragment,{children:["Classify competitor intent by confidence (0–1). Higher confidence → stronger intent.",(0,l.jsxs)("ul",{style:{marginBottom:0,marginTop:4,paddingLeft:20},children:[(0,l.jsxs)("li",{children:[(0,l.jsx)("strong",{children:"High (≥)"}),': Treat as full competitor comparison → uses "Competitor comparison" policy']}),(0,l.jsxs)("li",{children:[(0,l.jsx)("strong",{children:"Medium (≥)"}),': Treat as possible comparison → uses "Possible competitor comparison" policy']}),(0,l.jsxs)("li",{children:[(0,l.jsx)("strong",{children:"Low (≥)"}),": Log only; allow request. Below Low → allow with no action"]})]}),"Raise thresholds to be more permissive; lower them to be stricter."]}),children:(0,l.jsxs)(g.Space,{wrap:!0,children:[(0,l.jsx)(r.Form.Item,{label:"High",style:{marginBottom:0},help:"e.g. 0.7",children:(0,l.jsx)(K.InputNumber,{min:0,max:1,step:.05,value:s.threshold_high??.7,onChange:e=>f("threshold_high",e??.7),style:{width:80}})}),(0,l.jsx)(r.Form.Item,{label:"Medium",style:{marginBottom:0},help:"e.g. 0.45",children:(0,l.jsx)(K.InputNumber,{min:0,max:1,step:.05,value:s.threshold_medium??.45,onChange:e=>f("threshold_medium",e??.45),style:{width:80}})}),(0,l.jsx)(r.Form.Item,{label:"Low",style:{marginBottom:0},help:"e.g. 0.3",children:(0,l.jsx)(K.InputNumber,{min:0,max:1,step:.05,value:s.threshold_low??.3,onChange:e=>f("threshold_low",e??.3),style:{width:80}})})]})})]})]}):(0,l.jsx)(h.Card,{title:(0,l.jsxs)("div",{style:{display:"flex",justifyContent:"space-between",alignItems:"center"},children:[(0,l.jsx)(q,{level:5,style:{margin:0},children:"Competitor Intent Filter"}),(0,l.jsx)(D.Switch,{checked:!1,onChange:x})]}),size:"small",children:(0,l.jsx)(H,{type:"secondary",children:"Block or reframe competitor comparison questions. When enabled, airline type auto-loads competitors from IATA; generic type requires manual competitor list."})})},{Title:V,Text:Y}=d.Typography,Q=({prebuiltPatterns:e,categories:t,selectedPatterns:a,blockedWords:r,onPatternAdd:i,onPatternRemove:s,onPatternActionChange:n,onBlockedWordAdd:o,onBlockedWordRemove:d,onBlockedWordUpdate:j,onFileUpload:_,accessToken:v,showStep:w,contentCategories:C=[],selectedContentCategories:S=[],onContentCategoryAdd:I,onContentCategoryRemove:A,onContentCategoryUpdate:O,pendingCategorySelection:T,onPendingCategorySelectionChange:L,competitorIntentEnabled:B=!1,competitorIntentConfig:$=null,onCompetitorIntentChange:E})=>{let[M,R]=(0,m.useState)(!1),[G,D]=(0,m.useState)(!1),[K,q]=(0,m.useState)(!1),[H,J]=(0,m.useState)(""),[U,Q]=(0,m.useState)("BLOCK"),[X,Z]=(0,m.useState)(""),[ee,et]=(0,m.useState)(""),[ea,el]=(0,m.useState)("BLOCK"),[er,ei]=(0,m.useState)(""),[es,en]=(0,m.useState)("BLOCK"),[eo,ed]=(0,m.useState)(""),[ec,em]=(0,m.useState)(!1),eu=async e=>{em(!0);try{let t=await e.text();if(v){let e=await (0,p.validateBlockedWordsFile)(v,t);if(e.valid)_&&_(t),u.default.success(e.message||"File uploaded successfully");else{let t=e.error||e.errors&&e.errors.join(", ")||"Invalid file";u.default.error(`Validation failed: ${t}`)}}}catch(e){u.default.error(`Failed to upload file: ${e}`)}finally{em(!1)}return!1};return(0,l.jsxs)("div",{className:"space-y-6",children:[!w&&(0,l.jsx)("div",{children:(0,l.jsx)(Y,{type:"secondary",children:"Configure patterns, keywords, and content categories to detect and filter sensitive information in requests and responses."})}),(!w||"patterns"===w)&&(0,l.jsxs)(h.Card,{title:(0,l.jsxs)("div",{style:{display:"flex",justifyContent:"space-between",alignItems:"center"},children:[(0,l.jsx)(V,{level:5,style:{margin:0},children:"Pattern Detection"}),(0,l.jsx)(Y,{type:"secondary",style:{fontSize:14,fontWeight:400},children:"Detect sensitive information using regex patterns (SSN, credit cards, API keys, etc.)"})]}),size:"small",children:[(0,l.jsx)("div",{style:{marginBottom:16},children:(0,l.jsxs)(g.Space,{children:[(0,l.jsx)(c.Button,{type:"primary",onClick:()=>R(!0),icon:(0,l.jsx)(f.PlusOutlined,{}),children:"Add prebuilt pattern"}),(0,l.jsx)(c.Button,{onClick:()=>q(!0),icon:(0,l.jsx)(f.PlusOutlined,{}),children:"Add custom regex"})]})}),(0,l.jsx)(P,{patterns:a,onActionChange:n,onRemove:s})]}),(!w||"keywords"===w)&&(0,l.jsxs)(h.Card,{title:(0,l.jsxs)("div",{style:{display:"flex",justifyContent:"space-between",alignItems:"center"},children:[(0,l.jsx)(V,{level:5,style:{margin:0},children:"Blocked Keywords"}),(0,l.jsx)(Y,{type:"secondary",style:{fontSize:14,fontWeight:400},children:"Block or mask specific sensitive terms and phrases"})]}),size:"small",children:[(0,l.jsx)("div",{style:{marginBottom:16},children:(0,l.jsxs)(g.Space,{children:[(0,l.jsx)(c.Button,{type:"primary",onClick:()=>D(!0),icon:(0,l.jsx)(f.PlusOutlined,{}),children:"Add keyword"}),(0,l.jsx)(x.Upload,{beforeUpload:eu,accept:".yaml,.yml",showUploadList:!1,children:(0,l.jsx)(c.Button,{icon:(0,l.jsx)(y.UploadOutlined,{}),loading:ec,children:"Upload YAML file"})})]})}),(0,l.jsx)(F,{keywords:r,onActionChange:j,onRemove:d})]}),(!w||"competitor_intent"===w||"categories"===w)&&E&&(0,l.jsx)(W,{enabled:B,config:$,onChange:E,accessToken:v}),(!w||"categories"===w)&&C.length>0&&I&&A&&O&&(0,l.jsx)(z,{availableCategories:C,selectedCategories:S,onCategoryAdd:I,onCategoryRemove:A,onCategoryUpdate:O,accessToken:v,pendingSelection:T,onPendingSelectionChange:L}),(0,l.jsx)(b,{visible:M,prebuiltPatterns:e,categories:t,selectedPatternName:H,patternAction:U,onPatternNameChange:J,onActionChange:e=>Q(e),onAdd:()=>{if(!H)return void u.default.error("Please select a pattern");let t=e.find(e=>e.name===H);i({id:`pattern-${Date.now()}`,type:"prebuilt",name:H,display_name:t?.display_name,action:U}),R(!1),J(""),Q("BLOCK")},onCancel:()=>{R(!1),J(""),Q("BLOCK")}}),(0,l.jsx)(N,{visible:K,patternName:X,patternRegex:ee,patternAction:ea,onNameChange:Z,onRegexChange:et,onActionChange:e=>el(e),onAdd:()=>{X&&ee?(i({id:`custom-${Date.now()}`,type:"custom",name:X,pattern:ee,action:ea}),q(!1),Z(""),et(""),el("BLOCK")):u.default.error("Please provide pattern name and regex")},onCancel:()=>{q(!1),Z(""),et(""),el("BLOCK")}}),(0,l.jsx)(k,{visible:G,keyword:er,action:es,description:eo,onKeywordChange:ei,onActionChange:e=>en(e),onDescriptionChange:ed,onAdd:()=>{er?(o({id:`word-${Date.now()}`,keyword:er,action:es,description:eo||void 0}),D(!1),ei(""),ed(""),en("BLOCK")):u.default.error("Please enter a keyword")},onCancel:()=>{D(!1),ei(""),ed(""),en("BLOCK")}})]})};var X=((t={}).PresidioPII="Presidio PII",t.Bedrock="Bedrock Guardrail",t.Lakera="Lakera",t);let Z={},ee=e=>{let t={};return t.PresidioPII="Presidio PII",t.Bedrock="Bedrock Guardrail",t.Lakera="Lakera",t.LlmAsAJudge="LiteLLM LLM as a Judge",Object.entries(e).forEach(([e,a])=>{a&&"object"==typeof a&&"ui_friendly_name"in a&&(t[e.split("_").map((e,t)=>e.charAt(0).toUpperCase()+e.slice(1)).join("")]=a.ui_friendly_name)}),Z=t,t},et=()=>Object.keys(Z).length>0?Z:X,ea={PresidioPII:"presidio",Bedrock:"bedrock",Lakera:"lakera_v2",LitellmContentFilter:"litellm_content_filter",ToolPermission:"tool_permission",BlockCodeExecution:"block_code_execution",Promptguard:"promptguard",LlmAsAJudge:"llm_as_a_judge",Xecguard:"xecguard",Deepkeep:"deepkeep",QostodianNexus:"qostodian_nexus"},el=e=>{Object.entries(e).forEach(([e,t])=>{t&&"object"==typeof t&&"ui_friendly_name"in t&&(ea[e.split("_").map((e,t)=>e.charAt(0).toUpperCase()+e.slice(1)).join("")]=e)})},er=e=>!!e&&"Presidio PII"===et()[e],ei=e=>!!e&&"LiteLLM Content Filter"===et()[e],es=e=>!!e&&"llm_as_a_judge"===ea[e],en="../ui/assets/logos/",eo={"Zscaler AI Guard":`${en}zscaler.svg`,"Presidio PII":`${en}microsoft_azure.svg`,"Bedrock Guardrail":`${en}bedrock.svg`,Lakera:`${en}lakeraai.jpeg`,"Azure Content Safety Prompt Shield":`${en}microsoft_azure.svg`,"Azure Content Safety Text Moderation":`${en}microsoft_azure.svg`,"Aporia AI":`${en}aporia.png`,"PANW Prisma AIRS":`${en}palo_alto_networks.jpeg`,"Noma Security":`${en}noma_security.png`,"Javelin Guardrails":`${en}javelin.png`,"Pillar Guardrail":`${en}pillar.jpeg`,"Google Cloud Model Armor":`${en}google.svg`,"Guardrails AI":`${en}guardrails_ai.jpeg`,"Lasso Guardrail":`${en}lasso.png`,"Pangea Guardrail":`${en}pangea.png`,"AIM Guardrail":`${en}aim_security.jpeg`,"OpenAI Moderation":`${en}openai_small.svg`,EnkryptAI:`${en}enkrypt_ai.avif`,"Prompt Security":`${en}prompt_security.png`,PromptGuard:`${en}promptguard.svg`,XecGuard:`${en}xecguard.svg`,"LiteLLM Content Filter":`${en}litellm_logo.jpg`,"LiteLLM LLM as a Judge":`${en}litellm_logo.jpg`,Akto:`${en}akto.svg`,"DeepKeep AI Firewall":`${en}deepkeep.svg`,"Qostodian Nexus":`${en}qohash.jpg`},ed=e=>{if(!e)return{logo:"",displayName:"-"};let t=Object.keys(ea).find(t=>ea[t].toLowerCase()===e.toLowerCase());if(!t)return{logo:"",displayName:e};let a=et()[t];return{logo:eo[a]||"",displayName:a||e}};function ec(e){return!0===e?"yes":!1===e?"no":"inherit"}function em(e){return"yes"===e||"no"!==e&&void 0}function eu(e){return!0===e?"yes":!1===e?"no":"inherit"}function ep(e){return"yes"===e||"no"!==e&&void 0}e.s(["choiceToSkipSystemForCreate",0,em,"choiceToSkipToolForCreate",0,ep,"getGuardrailLogoAndName",0,ed,"getGuardrailProviders",0,et,"guardrailLogoMap",0,eo,"guardrail_provider_map",0,ea,"populateGuardrailProviderMap",0,el,"populateGuardrailProviders",0,ee,"shouldRenderContentFilterConfigSettings",0,ei,"shouldRenderLLMJudgeFields",0,es,"shouldRenderPIIConfigSettings",0,er,"skipSystemMessageToChoice",0,ec,"skipToolMessageToChoice",0,eu],180766);var eg=e.i(435451);let{Title:ex}=d.Typography,eh=({field:e,fieldKey:t,fullFieldKey:a,value:s})=>{let[o,d]=m.default.useState([]),[u,p]=m.default.useState(e.dict_key_options||[]);return m.default.useEffect(()=>{if(s&&"object"==typeof s){let t=Object.keys(s);d(t.map(e=>({key:e,id:`${e}_${Date.now()}_${Math.random()}`}))),p((e.dict_key_options||[]).filter(e=>!t.includes(e)))}},[s,e.dict_key_options]),(0,l.jsxs)("div",{className:"space-y-3",children:[o.map(t=>(0,l.jsxs)("div",{className:"flex items-center space-x-3 p-3 border rounded-lg",children:[(0,l.jsx)("div",{className:"w-24 font-medium text-sm",children:t.key}),(0,l.jsx)("div",{className:"flex-1",children:(0,l.jsx)(r.Form.Item,{name:Array.isArray(a)?[...a,t.key]:[a,t.key],style:{marginBottom:0},initialValue:s&&"object"==typeof s?s[t.key]:void 0,normalize:"number"===e.dict_value_type?e=>{if(null==e||""===e)return;let t=Number(e);return isNaN(t)?e:t}:void 0,children:"number"===e.dict_value_type?(0,l.jsx)(eg.default,{step:1,width:200,placeholder:`Enter ${t.key} value`}):"boolean"===e.dict_value_type?(0,l.jsxs)(n.Select,{placeholder:`Select ${t.key} value`,children:[(0,l.jsx)(n.Select.Option,{value:!0,children:"True"}),(0,l.jsx)(n.Select.Option,{value:!1,children:"False"})]}):(0,l.jsx)(i.Input,{placeholder:`Enter ${t.key} value`})})}),(0,l.jsx)(c.Button,{type:"text",danger:!0,size:"small",onClick:()=>{var e,a;return e=t.id,a=t.key,void(d(o.filter(t=>t.id!==e)),p([...u,a].sort()))},children:"Remove"})]},t.id)),u.length>0&&(0,l.jsxs)("div",{className:"flex items-center space-x-3 mt-2",children:[(0,l.jsx)(n.Select,{placeholder:"Select category to configure",style:{width:200},onSelect:e=>e&&void(!e||(d([...o,{key:e,id:`${e}_${Date.now()}`}]),p(u.filter(t=>t!==e)))),value:void 0,children:u.map(e=>(0,l.jsx)(n.Select.Option,{value:e,children:e},e))}),(0,l.jsx)("span",{className:"text-sm text-gray-500",children:"Select a category to add threshold configuration"})]})]})},ef=({optionalParams:e,parentFieldKey:t,values:a})=>e.fields&&0!==Object.keys(e.fields).length?(0,l.jsxs)("div",{className:"guardrail-optional-params",children:[(0,l.jsxs)("div",{className:"mb-8 pb-4 border-b border-gray-100",children:[(0,l.jsx)(ex,{level:3,className:"mb-2 font-semibold text-gray-900",children:"Optional Parameters"}),(0,l.jsx)("p",{className:"text-gray-600 text-sm",children:e.description||"Configure additional settings for this guardrail provider"})]}),(0,l.jsx)("div",{className:"space-y-8",children:Object.entries(e.fields).map(([e,s])=>{let o,d;return o=`${t}.${e}`,(console.log("value",d=a?.[e]),"dict"===s.type&&s.dict_key_options)?(0,l.jsxs)("div",{className:"mb-8 p-6 bg-gray-50 rounded-lg border border-gray-200",children:[(0,l.jsx)("div",{className:"mb-4 font-medium text-gray-900 text-base",children:e}),(0,l.jsx)("p",{className:"text-sm text-gray-600 mb-4",children:s.description}),(0,l.jsx)(eh,{field:s,fieldKey:e,fullFieldKey:[t,e],value:d})]},o):(0,l.jsx)("div",{className:"mb-8 p-6 bg-white rounded-lg border border-gray-200 shadow-sm",children:(0,l.jsx)(r.Form.Item,{name:[t,e],label:(0,l.jsxs)("div",{className:"mb-2",children:[(0,l.jsx)("div",{className:"font-medium text-gray-900 text-base",children:e}),(0,l.jsx)("p",{className:"text-sm text-gray-600 mt-1",children:s.description})]}),rules:s.required?[{required:!0,message:`${e} is required`}]:void 0,className:"mb-0",initialValue:void 0!==d?d:s.default_value,normalize:"number"===s.type?e=>{if(null==e||""===e)return;let t=Number(e);return isNaN(t)?e:t}:void 0,children:"select"===s.type&&s.options?(0,l.jsx)(n.Select,{placeholder:s.description,children:s.options.map(e=>(0,l.jsx)(n.Select.Option,{value:e,children:e},e))}):"multiselect"===s.type&&s.options?(0,l.jsx)(n.Select,{mode:"multiple",placeholder:s.description,children:s.options.map(e=>(0,l.jsx)(n.Select.Option,{value:e,children:e},e))}):"bool"===s.type||"boolean"===s.type?(0,l.jsxs)(n.Select,{placeholder:s.description,children:[(0,l.jsx)(n.Select.Option,{value:!0,children:"True"}),(0,l.jsx)(n.Select.Option,{value:!1,children:"False"})]}):"number"===s.type?(0,l.jsx)(eg.default,{step:1,width:400,placeholder:s.description}):e.includes("password")||e.includes("secret")||e.includes("key")?(0,l.jsx)(i.Input.Password,{placeholder:s.description}):(0,l.jsx)(i.Input,{placeholder:s.description})})},o)})})]}):null;var ey=e.i(482725),ej=e.i(850627);let e_=({selectedProvider:e,accessToken:t,providerParams:a=null,value:s=null})=>{let[o,d]=(0,m.useState)(!1),[c,u]=(0,m.useState)(a),[g,x]=(0,m.useState)(null);if((0,m.useEffect)(()=>{if(a)return void u(a);let e=async()=>{if(t){d(!0),x(null);try{let e=await (0,p.getGuardrailProviderSpecificParams)(t);console.log("Provider params API response:",e),u(e),ee(e),el(e)}catch(e){console.error("Error fetching provider params:",e),x("Failed to load provider parameters")}finally{d(!1)}}};a||e()},[t,a]),!e)return null;if(o)return(0,l.jsx)(ey.Spin,{tip:"Loading provider parameters..."});if(g)return(0,l.jsx)("div",{className:"text-red-500",children:g});let h=ea[e]?.toLowerCase(),f=c&&c[h];if(console.log("Provider key:",h),console.log("Provider fields:",f),!f||0===Object.keys(f).length)return(0,l.jsx)("div",{children:"No configuration fields available for this provider."});console.log("Value:",s);let y=new Set(["patterns","blocked_words","blocked_words_file","categories","severity_threshold","pattern_redaction_format","keyword_redaction_tag"]),j=ei(e),_=(e,t="",a)=>Object.entries(e).map(([e,o])=>{let d=t?`${t}.${e}`:e,c=a?a[e]:s?.[e];if(console.log("Field value:",c),"ui_friendly_name"===e||"optional_params"===e&&"nested"===o.type&&o.fields||j&&y.has(e))return null;if("nested"===o.type&&o.fields)return(0,l.jsxs)("div",{children:[(0,l.jsx)("div",{className:"mb-2 font-medium",children:e}),(0,l.jsx)("div",{className:"ml-4 border-l-2 border-gray-200 pl-4",children:_(o.fields,d,c)})]},d);let m=void 0!==c?c:o.default_value??("percentage"===o.type?.5:void 0);return(0,l.jsx)(r.Form.Item,{name:d,label:e,tooltip:o.description,rules:o.required?[{required:!0,message:`${e} is required`}]:void 0,initialValue:m,children:"select"===o.type&&o.options?(0,l.jsx)(n.Select,{placeholder:o.description,defaultValue:c||o.default_value,children:o.options.map(e=>(0,l.jsx)(n.Select.Option,{value:e,children:e},e))}):"multiselect"===o.type&&o.options?(0,l.jsx)(n.Select,{mode:"multiple",placeholder:o.description,defaultValue:c||o.default_value,children:o.options.map(e=>(0,l.jsx)(n.Select.Option,{value:e,children:e},e))}):"bool"===o.type||"boolean"===o.type?(0,l.jsxs)(n.Select,{placeholder:o.description,children:[(0,l.jsx)(n.Select.Option,{value:!0,children:"True"}),(0,l.jsx)(n.Select.Option,{value:!1,children:"False"})]}):"percentage"===o.type&&null!=o.min&&null!=o.max?(0,l.jsx)(ej.Slider,{min:o.min,max:o.max,step:o.step??.1,marks:{[o.min]:"0%",[(o.min+o.max)/2]:"50%",[o.max]:"100%"}}):"number"===o.type?(0,l.jsx)(eg.default,{step:1,width:400,placeholder:o.description,defaultValue:void 0!==c?Number(c):void 0}):e.includes("password")||e.includes("secret")||e.includes("key")?(0,l.jsx)(i.Input.Password,{placeholder:o.description,defaultValue:c||""}):(0,l.jsx)(i.Input,{placeholder:o.description,defaultValue:c||""})},d)});return(0,l.jsx)(l.Fragment,{children:_(f)})};var eb=e.i(592968),ev=e.i(750113);let ew=({availableModels:e,form:t})=>(0,l.jsxs)(l.Fragment,{children:[(0,l.jsxs)("div",{style:{background:"#f6ffed",border:"1px solid #b7eb8f",borderRadius:6,padding:"10px 14px",marginBottom:16,fontSize:13,color:"#389e0d"},children:["After each LLM response, the ",(0,l.jsx)("strong",{children:"Judge Model"})," scores it 0–100 against your criteria. If the weighted average falls below the threshold, the response is blocked (or logged)."]}),(0,l.jsx)(r.Form.Item,{name:"judge_model",label:(0,l.jsxs)("span",{children:["Judge Model ",(0,l.jsx)(eb.Tooltip,{title:"The LLM that reads each response and grades it. Pick a capable model — it never sees end-user data beyond what the LLM returned.",children:(0,l.jsx)(ev.QuestionCircleOutlined,{style:{color:"#8c8c8c"}})})]}),rules:[{required:!0,message:"Select a judge model"}],children:(0,l.jsx)(n.Select,{showSearch:!0,placeholder:"Select a model",options:e.map(e=>({label:e,value:e}))})}),(0,l.jsx)(r.Form.Item,{name:"overall_threshold",label:(0,l.jsxs)("span",{children:["Minimum Score to Pass ",(0,l.jsx)(eb.Tooltip,{title:"0–100. If the weighted average of criterion scores falls below this, the guardrail triggers. 80 is a good default.",children:(0,l.jsx)(ev.QuestionCircleOutlined,{style:{color:"#8c8c8c"}})})]}),initialValue:80,children:(0,l.jsx)(K.InputNumber,{min:0,max:100,addonAfter:"/ 100",style:{width:"100%"}})}),(0,l.jsx)(r.Form.Item,{name:"on_failure",label:(0,l.jsxs)("span",{children:["On Failure ",(0,l.jsx)(eb.Tooltip,{title:"Block: return HTTP 422 when the score is too low. Log: record the result but let the response through.",children:(0,l.jsx)(ev.QuestionCircleOutlined,{style:{color:"#8c8c8c"}})})]}),initialValue:"block",children:(0,l.jsxs)(n.Select,{children:[(0,l.jsx)(n.Select.Option,{value:"block",children:"Block (return 422)"}),(0,l.jsx)(n.Select.Option,{value:"log",children:"Log only"})]})}),(0,l.jsx)(r.Form.Item,{label:(0,l.jsxs)("span",{children:["Evaluation Criteria ",(0,l.jsx)(eb.Tooltip,{title:"Each criterion is something the judge checks. Weights must add up to 100%.",children:(0,l.jsx)(ev.QuestionCircleOutlined,{style:{color:"#8c8c8c"}})})]}),children:(0,l.jsx)(r.Form.List,{name:"criteria",initialValue:[{name:"",weight:100,description:""}],children:(e,{add:a,remove:s})=>(0,l.jsxs)(l.Fragment,{children:[e.map(({key:e,name:t,...a})=>(0,l.jsxs)("div",{style:{border:"1px solid #f0f0f0",borderRadius:6,padding:"12px 12px 0",marginBottom:8},children:[(0,l.jsxs)("div",{style:{display:"flex",gap:8,alignItems:"flex-end"},children:[(0,l.jsx)(r.Form.Item,{...a,name:[t,"name"],rules:[{required:!0,message:"Enter criterion name"}],style:{flex:2,marginBottom:8},children:(0,l.jsx)(i.Input,{placeholder:"Criterion name (e.g. Policy accuracy)"})}),(0,l.jsx)(r.Form.Item,{...a,name:[t,"weight"],label:(0,l.jsx)(eb.Tooltip,{title:"How much this criterion counts toward the final score. All weights must add up to 100%.",children:(0,l.jsxs)("span",{style:{fontSize:12,color:"#595959"},children:["Weight ",(0,l.jsx)(ev.QuestionCircleOutlined,{style:{color:"#bfbfbf"}})]})}),rules:[{required:!0,message:"Enter weight"}],style:{flex:1,marginBottom:8},children:(0,l.jsx)(K.InputNumber,{min:0,max:100,addonAfter:"%",style:{width:"100%"},placeholder:"e.g. 50"})}),(0,l.jsx)("div",{style:{marginBottom:8},children:(0,l.jsx)(c.Button,{type:"text",danger:!0,size:"small",onClick:()=>s(t),children:"×"})})]}),(0,l.jsx)(r.Form.Item,{...a,name:[t,"description"],rules:[{required:!0,message:"Describe what to check"}],style:{marginBottom:8},children:(0,l.jsx)(i.Input,{placeholder:"What should the judge check for this criterion?"})})]},e)),(0,l.jsx)(c.Button,{type:"dashed",block:!0,style:{marginTop:4},onClick:()=>a({name:"",weight:0,description:""}),icon:(0,l.jsx)(f.PlusOutlined,{}),children:"Add Criterion"}),e.length>0&&(0,l.jsx)(r.Form.Item,{shouldUpdate:!0,noStyle:!0,children:()=>{let e=(t.getFieldValue("criteria")||[]).reduce((e,t)=>e+(Number(t?.weight)||0),0),a=100===e;return(0,l.jsxs)("div",{style:{marginTop:6,fontSize:12,color:a?"#52c41a":"#faad14"},children:["Weights total: ",e,"%",a?" ✓":" — must add up to 100%"]})}})]})})})]});var eN=e.i(536916),eC=e.i(149192),eS=e.i(741585),eS=eS,ek=e.i(724154);e.i(247167);var eI=e.i(931067);let eA={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M880.1 154H143.9c-24.5 0-39.8 26.7-27.5 48L349 597.4V838c0 17.7 14.2 32 31.8 32h262.4c17.6 0 31.8-14.3 31.8-32V597.4L907.7 202c12.2-21.3-3.1-48-27.6-48zM603.4 798H420.6V642h182.9v156zm9.6-236.6l-9.5 16.6h-183l-9.5-16.6L212.7 226h598.6L613 561.4z"}}]},name:"filter",theme:"outlined"};var eO=e.i(9583),eT=m.forwardRef(function(e,t){return m.createElement(eO.default,(0,eI.default)({},e,{ref:t,icon:eA}))});let{Text:eP}=d.Typography,{Option:eL}=n.Select,eB=({categories:e,selectedCategories:t,onChange:a})=>(0,l.jsxs)("div",{children:[(0,l.jsxs)("div",{className:"flex items-center mb-2",children:[(0,l.jsx)(eT,{className:"text-gray-500 mr-1"}),(0,l.jsx)(eP,{className:"text-gray-500 font-medium",children:"Filter by category"})]}),(0,l.jsx)(n.Select,{mode:"multiple",placeholder:"Select categories to filter by",style:{width:"100%"},onChange:a,value:t,allowClear:!0,showSearch:!0,optionFilterProp:"children",className:"mb-4",tagRender:e=>(0,l.jsx)(o.Tag,{color:"blue",closable:e.closable,onClose:e.onClose,className:"mr-2 mb-2",children:e.label}),children:e.map(e=>(0,l.jsx)(eL,{value:e.category,children:e.category},e.category))})]}),eF=({onSelectAll:e,onUnselectAll:t,hasSelectedEntities:a})=>(0,l.jsxs)("div",{className:"bg-gray-50 p-5 rounded-lg mb-6 border border-gray-200 shadow-sm",children:[(0,l.jsxs)("div",{className:"flex items-center justify-between mb-4",children:[(0,l.jsxs)("div",{className:"flex items-center",children:[(0,l.jsx)(eP,{strong:!0,className:"text-gray-700 text-base",children:"Quick Actions"}),(0,l.jsx)(eb.Tooltip,{title:"Apply action to all PII types at once",children:(0,l.jsx)("div",{className:"ml-2 text-gray-400 cursor-help text-xs",children:"ⓘ"})})]}),(0,l.jsx)(c.Button,{color:"danger",variant:"outlined",onClick:t,disabled:!a,icon:(0,l.jsx)(eC.CloseOutlined,{}),children:"Unselect All"})]}),(0,l.jsxs)("div",{className:"grid grid-cols-2 gap-4",children:[(0,l.jsx)(c.Button,{color:"primary",variant:"outlined",onClick:()=>e("MASK"),className:"h-10",block:!0,icon:(0,l.jsx)(eS.default,{}),children:"Select All & Mask"}),(0,l.jsx)(c.Button,{color:"danger",variant:"outlined",onClick:()=>e("BLOCK"),className:"h-10 hover:bg-red-100",block:!0,icon:(0,l.jsx)(ek.StopOutlined,{}),children:"Select All & Block"})]})]}),e$=({entities:e,selectedEntities:t,selectedActions:a,actions:r,onEntitySelect:i,onActionSelect:s,entityToCategoryMap:d})=>(0,l.jsxs)("div",{className:"border rounded-lg overflow-hidden shadow-sm",children:[(0,l.jsxs)("div",{className:"bg-gray-50 px-5 py-3 border-b flex",children:[(0,l.jsx)(eP,{strong:!0,className:"flex-1 text-gray-700",children:"PII Type"}),(0,l.jsx)(eP,{strong:!0,className:"w-32 text-right text-gray-700",children:"Action"})]}),(0,l.jsx)("div",{className:"max-h-[400px] overflow-y-auto",children:0===e.length?(0,l.jsx)("div",{className:"py-10 text-center text-gray-500",children:"No PII types match your filter criteria"}):e.map(e=>(0,l.jsxs)("div",{className:`px-5 py-3 flex items-center justify-between hover:bg-gray-50 border-b ${t.includes(e)?"bg-blue-50":""}`,children:[(0,l.jsxs)("div",{className:"flex items-center flex-1",children:[(0,l.jsx)(eN.Checkbox,{checked:t.includes(e),onChange:()=>i(e),className:"mr-3"}),(0,l.jsx)(eP,{className:t.includes(e)?"font-medium text-gray-900":"text-gray-700",children:e.replace(/_/g," ")}),d.get(e)&&(0,l.jsx)(o.Tag,{className:"ml-2 text-xs",color:"blue",children:d.get(e)})]}),(0,l.jsx)("div",{className:"w-32",children:(0,l.jsx)(n.Select,{value:t.includes(e)&&a[e]||"MASK",onChange:t=>s(e,t),style:{width:120},disabled:!t.includes(e),className:`${!t.includes(e)?"opacity-50":""}`,dropdownMatchSelectWidth:!1,children:r.map(e=>(0,l.jsx)(eL,{value:e,children:(0,l.jsxs)("div",{className:"flex items-center",children:[(e=>{switch(e){case"MASK":return(0,l.jsx)(eS.default,{style:{marginRight:4}});case"BLOCK":return(0,l.jsx)(ek.StopOutlined,{style:{marginRight:4}});default:return null}})(e),e]})},e))})})]},e))})]}),{Title:eE,Text:eM}=d.Typography,eR=({entities:e,actions:t,selectedEntities:a,selectedActions:r,onEntitySelect:i,onActionSelect:s,entityCategories:n=[]})=>{let[o,d]=(0,m.useState)([]),c=new Map;n.forEach(e=>{e.entities.forEach(t=>{c.set(t,e.category)})});let u=e.filter(e=>0===o.length||o.includes(c.get(e)||""));return(0,l.jsxs)("div",{className:"pii-configuration",children:[(0,l.jsxs)("div",{className:"flex justify-between items-center mb-5",children:[(0,l.jsx)("div",{className:"flex items-center",children:(0,l.jsx)(eE,{level:4,className:"!m-0 font-semibold text-gray-800",children:"Configure PII Protection"})}),(0,l.jsxs)(eM,{className:"text-gray-500",children:[a.length," items selected"]})]}),(0,l.jsxs)("div",{className:"mb-6",children:[(0,l.jsx)(eB,{categories:n,selectedCategories:o,onChange:d}),(0,l.jsx)(eF,{onSelectAll:t=>{e.forEach(e=>{a.includes(e)||i(e),s(e,t)})},onUnselectAll:()=>{a.forEach(e=>{i(e)})},hasSelectedEntities:a.length>0})]}),(0,l.jsx)(e$,{entities:u,selectedEntities:a,selectedActions:r,actions:t,onEntitySelect:i,onActionSelect:s,entityToCategoryMap:c})]})};var eG=e.i(304967),ez=e.i(599724),eD=e.i(312361),eK=e.i(21548),eq=e.i(827252);let eH={rules:[],default_action:"deny",on_disallowed_action:"block",violation_message_template:""},eJ=({value:e,onChange:t,disabled:a=!1})=>{let r={...eH,...e||{},rules:e?.rules?[...e.rules]:[]},s=e=>{let a={...r,...e};t?.(a)},o=(e,t)=>{s({rules:r.rules.map((a,l)=>l===e?{...a,...t}:a)})},d=(e,t)=>{let a=r.rules[e];if(!a)return;let l=Object.entries(a.allowed_param_patterns||{});t(l);let i={};l.forEach(([e,t])=>{i[e]=t}),o(e,{allowed_param_patterns:Object.keys(i).length>0?i:void 0})};return(0,l.jsxs)(eG.Card,{children:[(0,l.jsxs)("div",{className:"flex items-center justify-between",children:[(0,l.jsxs)("div",{children:[(0,l.jsx)(ez.Text,{className:"text-lg font-semibold",children:"LiteLLM Tool Permission Guardrail"}),(0,l.jsx)(ez.Text,{className:"text-sm text-gray-500",children:"Provide regex patterns (e.g., ^mcp__github_.*$) for tool names or types and optionally constrain payload fields."})]}),!a&&(0,l.jsx)(c.Button,{icon:(0,l.jsx)(f.PlusOutlined,{}),type:"primary",onClick:()=>{s({rules:[...r.rules,{id:`rule_${Math.random().toString(36).slice(2,8)}`,decision:"allow",allowed_param_patterns:void 0}]})},className:"!bg-blue-600 !text-white hover:!bg-blue-500",children:"Add Rule"})]}),(0,l.jsx)(eD.Divider,{}),0===r.rules.length?(0,l.jsx)(eK.Empty,{description:"No tool rules added yet"}):(0,l.jsx)("div",{className:"space-y-4",children:r.rules.map((e,t)=>{let m;return(0,l.jsxs)(eG.Card,{className:"bg-gray-50",children:[(0,l.jsxs)("div",{className:"flex items-center justify-between mb-3",children:[(0,l.jsxs)(ez.Text,{className:"font-semibold",children:["Rule ",t+1]}),(0,l.jsx)(c.Button,{icon:(0,l.jsx)(A.DeleteOutlined,{}),danger:!0,type:"text",disabled:a,onClick:()=>{s({rules:r.rules.filter((e,a)=>a!==t)})},children:"Remove"})]}),(0,l.jsxs)("div",{className:"grid grid-cols-1 gap-4 md:grid-cols-2",children:[(0,l.jsxs)("div",{children:[(0,l.jsx)(ez.Text,{className:"text-sm font-medium",children:"Rule ID"}),(0,l.jsx)(i.Input,{disabled:a,placeholder:"unique_rule_id",value:e.id,onChange:e=>o(t,{id:e.target.value})})]}),(0,l.jsxs)("div",{children:[(0,l.jsx)(ez.Text,{className:"text-sm font-medium",children:"Tool Name (optional)"}),(0,l.jsx)(i.Input,{disabled:a,placeholder:"^mcp__github_.*$",value:e.tool_name??"",onChange:e=>o(t,{tool_name:""===e.target.value.trim()?void 0:e.target.value})})]})]}),(0,l.jsx)("div",{className:"grid grid-cols-1 gap-4 md:grid-cols-2 mt-4",children:(0,l.jsxs)("div",{children:[(0,l.jsx)(ez.Text,{className:"text-sm font-medium",children:"Tool Type (optional)"}),(0,l.jsx)(i.Input,{disabled:a,placeholder:"^function$",value:e.tool_type??"",onChange:e=>o(t,{tool_type:""===e.target.value.trim()?void 0:e.target.value})})]})}),(0,l.jsxs)("div",{className:"mt-4 flex flex-col gap-2",children:[(0,l.jsx)(ez.Text,{className:"text-sm font-medium",children:"Decision"}),(0,l.jsxs)(n.Select,{disabled:a,value:e.decision,style:{width:200},onChange:e=>o(t,{decision:e}),children:[(0,l.jsx)(n.Select.Option,{value:"allow",children:"Allow"}),(0,l.jsx)(n.Select.Option,{value:"deny",children:"Deny"})]})]}),(0,l.jsx)("div",{className:"mt-4",children:0===(m=Object.entries(e.allowed_param_patterns||{})).length?(0,l.jsx)(c.Button,{disabled:a,size:"small",onClick:()=>o(t,{allowed_param_patterns:{"":""}}),children:"+ Restrict tool arguments (optional)"}):(0,l.jsxs)("div",{className:"space-y-2",children:[(0,l.jsx)(ez.Text,{className:"text-sm text-gray-600",children:"Argument constraints (dot or array paths)"}),m.map(([r,s],n)=>(0,l.jsxs)(g.Space,{align:"start",children:[(0,l.jsx)(i.Input,{disabled:a,placeholder:"messages[0].content",value:r,onChange:e=>{var a;return a=e.target.value,void d(t,e=>{if(!e[n])return;let[,t]=e[n];e[n]=[a,t]})}}),(0,l.jsx)(i.Input,{disabled:a,placeholder:"^email@.*$",value:s,onChange:e=>{var a;return a=e.target.value,void d(t,e=>{if(!e[n])return;let[t]=e[n];e[n]=[t,a]})}}),(0,l.jsx)(c.Button,{disabled:a,icon:(0,l.jsx)(A.DeleteOutlined,{}),danger:!0,onClick:()=>d(t,e=>{e.splice(n,1)})})]},`${e.id||t}-${n}`)),(0,l.jsx)(c.Button,{disabled:a,size:"small",onClick:()=>o(t,{allowed_param_patterns:{...e.allowed_param_patterns||{},"":""}}),children:"+ Add another constraint"})]})})]},e.id||t)})}),(0,l.jsx)(eD.Divider,{}),(0,l.jsxs)("div",{className:"grid gap-4 md:grid-cols-2",children:[(0,l.jsxs)("div",{children:[(0,l.jsx)(ez.Text,{className:"text-sm font-medium",children:"Default action"}),(0,l.jsxs)(n.Select,{disabled:a,value:r.default_action,onChange:e=>s({default_action:e}),children:[(0,l.jsx)(n.Select.Option,{value:"allow",children:"Allow"}),(0,l.jsx)(n.Select.Option,{value:"deny",children:"Deny"})]})]}),(0,l.jsxs)("div",{children:[(0,l.jsxs)(ez.Text,{className:"text-sm font-medium flex items-center gap-1",children:["On disallowed action",(0,l.jsx)(eb.Tooltip,{title:"Block returns an error when a forbidden tool is invoked. Rewrite strips the tool call but lets the rest of the response continue.",children:(0,l.jsx)(eq.InfoCircleOutlined,{})})]}),(0,l.jsxs)(n.Select,{disabled:a,value:r.on_disallowed_action,onChange:e=>s({on_disallowed_action:e}),children:[(0,l.jsx)(n.Select.Option,{value:"block",children:"Block"}),(0,l.jsx)(n.Select.Option,{value:"rewrite",children:"Rewrite"})]})]})]}),(0,l.jsxs)("div",{className:"mt-4",children:[(0,l.jsx)(ez.Text,{className:"text-sm font-medium",children:"Violation message (optional)"}),(0,l.jsx)(i.Input.TextArea,{disabled:a,rows:3,placeholder:"This violates our org policy...",value:r.violation_message_template,onChange:e=>s({violation_message_template:e.target.value})})]})]})},{Title:eU,Text:eW,Link:eV}=d.Typography,{Option:eY}=n.Select,eQ={pre_call:"Before LLM Call - Runs before the LLM call and checks the input (Recommended)",during_call:"During LLM Call - Runs in parallel with the LLM call, with response held until check completes",post_call:"After LLM Call - Runs after the LLM call and checks only the output",logging_only:"Logging Only - Only runs on logging callbacks without affecting the LLM call",pre_mcp_call:"Before MCP Tool Call - Runs before MCP tool execution and validates tool calls",during_mcp_call:"During MCP Tool Call - Runs in parallel with MCP tool execution for monitoring"};e.s(["default",0,({visible:e,onClose:t,accessToken:a,onSuccess:d,preset:g})=>{let[x]=r.Form.useForm(),[h,f]=(0,m.useState)(!1),[y,j]=(0,m.useState)(null),[_,b]=(0,m.useState)(null),[v,w]=(0,m.useState)([]),[N,C]=(0,m.useState)({}),[S,k]=(0,m.useState)(0),[I,A]=(0,m.useState)(null),[O,T]=(0,m.useState)([]),[P,L]=(0,m.useState)(2),[B,F]=(0,m.useState)({}),[$,E]=(0,m.useState)([]),[M,R]=(0,m.useState)([]),[G,z]=(0,m.useState)([]),[D,K]=(0,m.useState)(""),[q,H]=(0,m.useState)(!1),[J,U]=(0,m.useState)(null),[W,V]=(0,m.useState)(""),[Y,X]=(0,m.useState)(void 0),[Z,en]=(0,m.useState)("warn"),[ed,ec]=(0,m.useState)(""),[eu,eg]=(0,m.useState)(!1),[ex,eh]=(0,m.useState)([]),[ey,ej]=(0,m.useState)({rules:[],default_action:"deny",on_disallowed_action:"block",violation_message_template:""}),eb=(0,m.useMemo)(()=>!!y&&"tool_permission"===(ea[y]||"").toLowerCase(),[y]);(0,m.useEffect)(()=>{a&&(async()=>{try{let[e,t,l]=await Promise.all([(0,p.getGuardrailUISettings)(a),(0,p.getGuardrailProviderSpecificParams)(a),(0,p.modelAvailableCall)(a,"","").catch(()=>null)]);b(e),A(t),l?.data&&eh(l.data.map(e=>e.id)),ee(t),el(t)}catch(e){console.error("Error fetching guardrail data:",e),u.default.fromBackend("Failed to load guardrail configuration")}})()},[a]),(0,m.useEffect)(()=>{if(!g||!e||!_)return;j(g.provider);let t={provider:g.provider,guardrail_name:g.guardrailNameSuggestion,mode:g.mode,default_on:g.defaultOn,skip_system_message_choice:"inherit",skip_tool_message_choice:"inherit"};if("BlockCodeExecution"===g.provider&&(t.confidence_threshold=.5),x.setFieldsValue(t),g.categoryName&&_.content_filter_settings?.content_categories){let e=_.content_filter_settings.content_categories.find(e=>e.name===g.categoryName);e&&z([{id:`category-${Date.now()}`,category:e.name,display_name:e.display_name,action:e.default_action,severity_threshold:"medium"}])}},[g,e,_]);let ev=e=>{j(e);let t={config:void 0,presidio_analyzer_api_base:void 0,presidio_anonymizer_api_base:void 0};"BlockCodeExecution"===e&&(t.confidence_threshold=.5),x.setFieldsValue(t),w([]),C({}),T([]),L(2),F({}),E([]),R([]),z([]),K(""),H(!1),U(null),ej({rules:[],default_action:"deny",on_disallowed_action:"block",violation_message_template:""}),"LlmAsAJudge"===e&&x.setFieldsValue({mode:"post_call"})},eN=e=>{w(t=>t.includes(e)?t.filter(t=>t!==e):[...t,e])},eC=(e,t)=>{C(a=>({...a,[e]:t}))},eS=async()=>{try{if(0===S&&(await x.validateFields(["guardrail_name","provider","mode","default_on"]),y)){let e=["guardrail_name","provider","mode","default_on"];"PresidioPII"===y&&e.push("presidio_analyzer_api_base","presidio_anonymizer_api_base"),await x.validateFields(e)}if(1===S&&er(y)&&0===v.length)return void u.default.fromBackend("Please select at least one PII entity to continue");k(S+1)}catch(e){console.error("Form validation failed:",e)}},ek=()=>{x.resetFields(),j(null),w([]),C({}),T([]),L(2),F({}),E([]),R([]),z([]),K(""),ej({rules:[],default_action:"deny",on_disallowed_action:"block",violation_message_template:""}),V(""),X(void 0),en("warn"),ec(""),eg(!1),k(0)},eI=()=>{ek(),t()},eA=async()=>{try{f(!0),await x.validateFields();let e=x.getFieldsValue(!0),l=ea[e.provider],r={guardrail_name:e.guardrail_name,litellm_params:{guardrail:l,mode:e.mode,default_on:e.default_on},guardrail_info:{}},i=em(e.skip_system_message_choice);void 0!==i&&(r.litellm_params.skip_system_message_in_guardrail=i);let s=ep(e.skip_tool_message_choice);if(void 0!==s&&(r.litellm_params.skip_tool_message_in_guardrail=s),"PresidioPII"===e.provider&&v.length>0){let t={};v.forEach(e=>{t[e]=N[e]||"MASK"}),r.litellm_params.pii_entities_config=t,e.presidio_analyzer_api_base&&(r.litellm_params.presidio_analyzer_api_base=e.presidio_analyzer_api_base),e.presidio_anonymizer_api_base&&(r.litellm_params.presidio_anonymizer_api_base=e.presidio_anonymizer_api_base)}if(ei(e.provider)){let e=q&&J?.brand_self?.length>0;if(0===$.length&&0===M.length&&0===G.length&&!e){u.default.fromBackend("Please configure at least one content filter setting (category, pattern, keyword, or competitor intent)"),f(!1);return}$.length>0&&(r.litellm_params.patterns=$.map(e=>({pattern_type:"prebuilt"===e.type?"prebuilt":"regex",pattern_name:"prebuilt"===e.type?e.name:void 0,pattern:"custom"===e.type?e.pattern:void 0,name:e.name,action:e.action}))),M.length>0&&(r.litellm_params.blocked_words=M.map(e=>({keyword:e.keyword,action:e.action,description:e.description}))),G.length>0&&(r.litellm_params.categories=G.map(e=>({category:e.category,enabled:!0,action:e.action,severity_threshold:e.severity_threshold||"medium"}))),q&&J?.brand_self?.length>0&&(r.litellm_params.competitor_intent_config={competitor_intent_type:J.competitor_intent_type??"airline",brand_self:J.brand_self,locations:J.locations?.length>0?J.locations:void 0,competitors:"generic"===J.competitor_intent_type&&J.competitors?.length>0?J.competitors:void 0,policy:J.policy,threshold_high:J.threshold_high,threshold_medium:J.threshold_medium,threshold_low:J.threshold_low})}else if(e.config)try{r.guardrail_info=JSON.parse(e.config)}catch(e){u.default.fromBackend("Invalid JSON in configuration"),f(!1);return}if("llm_as_a_judge"===l){let t=e.criteria||[];if(0===t.length){u.default.fromBackend("Add at least one evaluation criterion"),f(!1);return}let a=t.reduce((e,t)=>e+(Number(t?.weight)||0),0);if(100!==a){u.default.fromBackend(`Criterion weights must sum to 100% (currently ${a}%)`),f(!1);return}r.litellm_params.judge_model=e.judge_model,r.litellm_params.overall_threshold=e.overall_threshold??80,r.litellm_params.on_failure=e.on_failure??"block",r.litellm_params.criteria=t.map(e=>({name:e.name,weight:Number(e.weight),description:e.description||""}))}if("tool_permission"===l){if(0===ey.rules.length){u.default.fromBackend("Add at least one tool permission rule"),f(!1);return}r.litellm_params.rules=ey.rules,r.litellm_params.default_action=ey.default_action,r.litellm_params.on_disallowed_action=ey.on_disallowed_action,ey.violation_message_template&&(r.litellm_params.violation_message_template=ey.violation_message_template)}if(ei(e.provider)&&(void 0!==Y&&Y>0&&(r.litellm_params.end_session_after_n_fails=Y),Z&&"realtime"===W&&(r.litellm_params.on_violation=Z),ed.trim()&&(r.litellm_params.realtime_violation_message=ed.trim())),console.log("values: ",JSON.stringify(e)),I&&y&&"llm_as_a_judge"!==l){let t=ea[y]?.toLowerCase();console.log("providerKey: ",t);let a=I[t]||{},l=new Set;console.log("providerSpecificParams: ",JSON.stringify(a)),Object.keys(a).forEach(e=>{"optional_params"!==e&&l.add(e)}),a.optional_params&&a.optional_params.fields&&Object.keys(a.optional_params.fields).forEach(e=>{l.add(e)}),console.log("allowedParams: ",l),l.forEach(t=>{let a=e[t];(null==a||""===a)&&(a=e.optional_params?.[t]),null!=a&&""!==a&&(r.litellm_params[t]=a)})}if(!a)throw Error("No access token available");console.log("Sending guardrail data:",JSON.stringify(r)),await (0,p.createGuardrailCall)(a,r),u.default.success("Guardrail created successfully"),ek(),d(),t()}catch(e){console.error("Failed to create guardrail:",e),u.default.fromBackend("Failed to create guardrail: "+(e instanceof Error?e.message:String(e)))}finally{f(!1)}},eO=e=>{if(!_||!ei(y))return null;let t=_.content_filter_settings;return t?(0,l.jsx)(Q,{prebuiltPatterns:t.prebuilt_patterns||[],categories:t.pattern_categories||[],selectedPatterns:$,blockedWords:M,onPatternAdd:e=>E([...$,e]),onPatternRemove:e=>E($.filter(t=>t.id!==e)),onPatternActionChange:(e,t)=>{E($.map(a=>a.id===e?{...a,action:t}:a))},onBlockedWordAdd:e=>R([...M,e]),onBlockedWordRemove:e=>R(M.filter(t=>t.id!==e)),onBlockedWordUpdate:(e,t,a)=>{R(M.map(l=>l.id===e?{...l,[t]:a}:l))},contentCategories:t.content_categories||[],selectedContentCategories:G,onContentCategoryAdd:e=>z([...G,e]),onContentCategoryRemove:e=>z(G.filter(t=>t.id!==e)),onContentCategoryUpdate:(e,t,a)=>{z(G.map(l=>l.id===e?{...l,[t]:a}:l))},pendingCategorySelection:D,onPendingCategorySelectionChange:K,accessToken:a,showStep:e,competitorIntentEnabled:q,competitorIntentConfig:J,onCompetitorIntentChange:(e,t)=>{H(e),U(t)}}):null},eT=ei(y)?[{title:"Basic Info",optional:!1},{title:"Topics",optional:!1},{title:"Patterns",optional:!1},{title:"Keywords",optional:!1},{title:"Endpoint Settings (Optional)",optional:!0}]:er(y)?[{title:"Basic Info",optional:!1},{title:"PII Configuration",optional:!1}]:[{title:"Basic Info",optional:!1},{title:"Provider Configuration",optional:!1}];return(0,l.jsx)(s.Modal,{title:null,open:e,onCancel:eI,footer:null,width:1e3,closable:!1,className:"top-8",styles:{body:{padding:0}},children:(0,l.jsxs)("div",{className:"flex flex-col",children:[(0,l.jsxs)("div",{className:"flex items-center justify-between px-6 py-4 border-b border-gray-200",children:[(0,l.jsx)("h3",{className:"text-base font-semibold text-gray-900 m-0",children:"Create guardrail"}),(0,l.jsx)("button",{onClick:eI,className:"text-gray-400 hover:text-gray-600 bg-transparent border-none cursor-pointer text-base leading-none p-1",children:"✕"})]}),(0,l.jsx)("div",{className:"overflow-auto px-6 py-4",style:{maxHeight:"calc(80vh - 120px)"},children:(0,l.jsx)(r.Form,{form:x,layout:"vertical",preserve:!0,initialValues:{mode:"pre_call",default_on:!1,skip_system_message_choice:"inherit",skip_tool_message_choice:"inherit"},children:eT.map((e,t)=>{let s=t{s&&k(t)},style:{minHeight:24},children:[(0,l.jsx)("span",{className:"text-sm",style:{fontWeight:d?600:500,color:d?"#1e293b":s?"#4f46e5":"#94a3b8"},children:e.title}),e.optional&&!d&&(0,l.jsx)("span",{className:"text-[11px] text-slate-400",children:"optional"}),s&&(0,l.jsx)("span",{className:"text-[11px] text-indigo-500 hover:underline",children:"Edit"})]}),d&&(0,l.jsx)("div",{className:"mt-3",children:(()=>{switch(S){case 0:return(0,l.jsxs)(l.Fragment,{children:[(0,l.jsx)(r.Form.Item,{name:"guardrail_name",label:"Guardrail Name",rules:[{required:!0,message:"Please enter a guardrail name"}],children:(0,l.jsx)(i.Input,{placeholder:"Enter a name for this guardrail"})}),(0,l.jsx)(r.Form.Item,{name:"provider",label:"Guardrail Provider",rules:[{required:!0,message:"Please select a provider"}],children:(0,l.jsx)(n.Select,{placeholder:"Select a guardrail provider",onChange:ev,labelInValue:!1,optionLabelProp:"label",dropdownRender:e=>e,showSearch:!0,children:Object.entries(et()).map(([e,t])=>(0,l.jsx)(eY,{value:e,label:(0,l.jsxs)("div",{style:{display:"flex",alignItems:"center"},children:[eo[t]&&(0,l.jsx)("img",{src:eo[t],alt:"",style:{height:"20px",width:"20px",marginRight:"8px",objectFit:"contain"},onError:e=>{e.currentTarget.style.display="none"}}),(0,l.jsx)("span",{children:t})]}),children:(0,l.jsxs)("div",{style:{display:"flex",alignItems:"center"},children:[eo[t]&&(0,l.jsx)("img",{src:eo[t],alt:"",style:{height:"20px",width:"20px",marginRight:"8px",objectFit:"contain"},onError:e=>{e.currentTarget.style.display="none"}}),(0,l.jsx)("span",{children:t})]})},e))})}),(0,l.jsx)(r.Form.Item,{name:"mode",label:"Mode",tooltip:"How the guardrail should be applied",rules:[{required:!0,message:"Please select a mode"}],children:(0,l.jsx)(n.Select,{optionLabelProp:"label",mode:"multiple",children:_?.supported_modes?.map(e=>(0,l.jsx)(eY,{value:e,label:e,children:(0,l.jsxs)("div",{children:[(0,l.jsxs)("div",{children:[(0,l.jsx)("strong",{children:e}),"pre_call"===e&&(0,l.jsx)(o.Tag,{color:"green",style:{marginLeft:"8px"},children:"Recommended"})]}),(0,l.jsx)("div",{style:{fontSize:"12px",color:"#888"},children:eQ[e]})]})},e))||(0,l.jsxs)(l.Fragment,{children:[(0,l.jsx)(eY,{value:"pre_call",label:"pre_call",children:(0,l.jsxs)("div",{children:[(0,l.jsxs)("div",{children:[(0,l.jsx)("strong",{children:"pre_call"})," ",(0,l.jsx)(o.Tag,{color:"green",children:"Recommended"})]}),(0,l.jsx)("div",{style:{fontSize:"12px",color:"#888"},children:eQ.pre_call})]})}),(0,l.jsx)(eY,{value:"during_call",label:"during_call",children:(0,l.jsxs)("div",{children:[(0,l.jsx)("div",{children:(0,l.jsx)("strong",{children:"during_call"})}),(0,l.jsx)("div",{style:{fontSize:"12px",color:"#888"},children:eQ.during_call})]})}),(0,l.jsx)(eY,{value:"post_call",label:"post_call",children:(0,l.jsxs)("div",{children:[(0,l.jsx)("div",{children:(0,l.jsx)("strong",{children:"post_call"})}),(0,l.jsx)("div",{style:{fontSize:"12px",color:"#888"},children:eQ.post_call})]})}),(0,l.jsx)(eY,{value:"logging_only",label:"logging_only",children:(0,l.jsxs)("div",{children:[(0,l.jsx)("div",{children:(0,l.jsx)("strong",{children:"logging_only"})}),(0,l.jsx)("div",{style:{fontSize:"12px",color:"#888"},children:eQ.logging_only})]})})]})})}),(0,l.jsx)(r.Form.Item,{name:"default_on",label:"Always On",tooltip:"If enabled, this guardrail will be applied to all requests by default.",children:(0,l.jsxs)(n.Select,{children:[(0,l.jsx)(n.Select.Option,{value:!0,children:"Yes"}),(0,l.jsx)(n.Select.Option,{value:!1,children:"No"})]})}),(0,l.jsx)(r.Form.Item,{name:"skip_system_message_choice",label:"Skip system messages in guardrail",tooltip:"Unified guardrails only: omit role: system from guardrail evaluation input (OpenAI chat + Anthropic messages). The model still receives full messages. Use global default follows litellm_settings.skip_system_message_in_guardrail.",children:(0,l.jsxs)(n.Select,{children:[(0,l.jsx)(n.Select.Option,{value:"inherit",children:"Use global default"}),(0,l.jsx)(n.Select.Option,{value:"yes",children:"Yes — exclude from guardrail scan"}),(0,l.jsx)(n.Select.Option,{value:"no",children:"No — always include in scan"})]})}),(0,l.jsx)(r.Form.Item,{name:"skip_tool_message_choice",label:"Skip tool messages in guardrail",tooltip:"Unified guardrails only: omit role: tool from guardrail evaluation input (OpenAI chat + Anthropic messages). The model still receives full messages. Use global default follows litellm_settings.skip_tool_message_in_guardrail.",children:(0,l.jsxs)(n.Select,{children:[(0,l.jsx)(n.Select.Option,{value:"inherit",children:"Use global default"}),(0,l.jsx)(n.Select.Option,{value:"yes",children:"Yes — exclude from guardrail scan"}),(0,l.jsx)(n.Select.Option,{value:"no",children:"No — always include in scan"})]})}),!eb&&!ei(y)&&!es(y)&&(0,l.jsx)(e_,{selectedProvider:y,accessToken:a,providerParams:I})]});case 1:if(er(y))return _&&"PresidioPII"===y?(0,l.jsx)(eR,{entities:_.supported_entities,actions:_.supported_actions,selectedEntities:v,selectedActions:N,onEntitySelect:eN,onActionSelect:eC,entityCategories:_.pii_entity_categories}):null;if(ei(y))return eO("categories");if(es(y))return(0,l.jsx)(ew,{availableModels:ex,form:x});if(!y)return null;if(eb)return(0,l.jsx)(eJ,{value:ey,onChange:ej});if(!I)return null;console.log("guardrail_provider_map: ",ea),console.log("selectedProvider: ",y);let e=ea[y]?.toLowerCase(),t=I&&I[e];return t&&t.optional_params?(0,l.jsx)(ef,{optionalParams:t.optional_params,parentFieldKey:"optional_params"}):null;case 2:if(ei(y))return eO("patterns");return null;case 3:if(ei(y))return eO("keywords");return null;case 4:return(0,l.jsxs)("div",{className:"space-y-6",children:[(0,l.jsx)("div",{children:(0,l.jsxs)("p",{className:"text-sm text-gray-500",children:["Configure settings for a specific call type. Most guardrails don't need this — skip it unless you're using a specific endpoint like ",(0,l.jsx)("code",{children:"/v1/realtime"}),"."]})}),(0,l.jsxs)("div",{children:[(0,l.jsx)("label",{className:"block text-sm font-medium text-gray-700 mb-1",children:"Call type"}),(0,l.jsx)(n.Select,{placeholder:"Select a call type",value:W||void 0,onChange:e=>{V(e),eg(!1)},style:{width:260},allowClear:!0,options:[{value:"realtime",label:"/v1/realtime"}]}),(0,l.jsx)("p",{className:"text-xs text-gray-400 mt-1",children:"More call types coming soon."})]}),"realtime"===W&&(0,l.jsxs)("div",{className:"border border-gray-200 rounded-lg overflow-hidden",children:[(0,l.jsxs)("button",{type:"button",onClick:()=>eg(e=>!e),className:"w-full flex items-center justify-between px-4 py-3 bg-gray-50 hover:bg-gray-100 text-sm font-medium text-gray-700",children:[(0,l.jsx)("span",{children:"/v1/realtime settings"}),(0,l.jsx)("svg",{className:`w-4 h-4 text-gray-500 transition-transform ${eu?"rotate-180":""}`,fill:"none",viewBox:"0 0 24 24",stroke:"currentColor",strokeWidth:2,children:(0,l.jsx)("path",{strokeLinecap:"round",strokeLinejoin:"round",d:"M19 9l-7 7-7-7"})})]}),eu&&(0,l.jsxs)("div",{className:"space-y-5 px-4 py-4 border-t border-gray-200",children:[(0,l.jsxs)("div",{children:[(0,l.jsx)("label",{className:"block text-sm font-medium text-gray-700 mb-1",children:"End session after X violations"}),(0,l.jsx)("p",{className:"text-xs text-gray-400 mb-2",children:"Automatically close the session after this many guardrail violations. Leave empty to never auto-close."}),(0,l.jsx)("input",{type:"number",min:1,placeholder:"e.g. 3",value:Y??"",onChange:e=>X(e.target.value?parseInt(e.target.value,10):void 0),className:"border border-gray-300 rounded px-3 py-1.5 text-sm w-32"})]}),(0,l.jsxs)("div",{children:[(0,l.jsx)("label",{className:"block text-sm font-medium text-gray-700 mb-2",children:"On violation"}),(0,l.jsx)("div",{className:"space-y-2",children:["warn","end_session"].map(e=>(0,l.jsxs)("label",{className:"flex items-start gap-2 cursor-pointer",children:[(0,l.jsx)("input",{type:"radio",name:"on_violation",value:e,checked:Z===e,onChange:()=>en(e),className:"mt-0.5"}),(0,l.jsxs)("div",{children:[(0,l.jsx)("span",{className:"text-sm font-medium text-gray-800",children:"warn"===e?"Warn":"End session"}),(0,l.jsx)("p",{className:"text-xs text-gray-400 m-0",children:"warn"===e?"Bot speaks the message, session continues":"Bot speaks the message, connection closes immediately"})]})]},e))})]}),(0,l.jsxs)("div",{children:[(0,l.jsx)("label",{className:"block text-sm font-medium text-gray-700 mb-1",children:"Message the user hears"}),(0,l.jsx)("p",{className:"text-xs text-gray-400 mb-2",children:"What the bot says aloud when this guardrail fires. Falls back to the default violation message if empty."}),(0,l.jsx)("textarea",{rows:3,placeholder:"e.g. I'm not able to continue this conversation. Please contact us at 1-800-774-2678.",value:ed,onChange:e=>ec(e.target.value),className:"border border-gray-300 rounded px-3 py-2 text-sm w-full resize-none"})]})]})]})]});default:return null}})()})]})]},t)})})}),(0,l.jsxs)("div",{className:"flex items-center justify-end space-x-3 px-6 py-3 border-t border-gray-200",children:[(0,l.jsx)(c.Button,{onClick:eI,children:"Cancel"}),S>0&&(0,l.jsx)(c.Button,{onClick:()=>{k(S-1)},children:"Previous"}),S{let[x]=r.Form.useForm(),[h,f]=(0,m.useState)(!1),[y,j]=(0,m.useState)(g?.provider||null),[_,b]=(0,m.useState)(null),[v,w]=(0,m.useState)([]),[N,C]=(0,m.useState)({});(0,m.useEffect)(()=>{(async()=>{try{if(!a)return;let e=await (0,p.getGuardrailUISettings)(a);b(e)}catch(e){console.error("Error fetching guardrail settings:",e),u.default.fromBackend("Failed to load guardrail settings")}})()},[a]),(0,m.useEffect)(()=>{g?.pii_entities_config&&Object.keys(g.pii_entities_config).length>0&&(w(Object.keys(g.pii_entities_config)),C(g.pii_entities_config))},[g]);let S=e=>{w(t=>t.includes(e)?t.filter(t=>t!==e):[...t,e])},k=(e,t)=>{C(a=>({...a,[e]:t}))},I=async()=>{try{f(!0);let e=await x.validateFields(),l=ea[e.provider],r=c&&"object"==typeof c?{...c}:{};r.guardrail=l,r.mode=e.mode,r.default_on=e.default_on;let i=e.skip_system_message_choice;"yes"===i?r.skip_system_message_in_guardrail=!0:"no"===i?r.skip_system_message_in_guardrail=!1:delete r.skip_system_message_in_guardrail;let s=e.skip_tool_message_choice;"yes"===s?r.skip_tool_message_in_guardrail=!0:"no"===s?r.skip_tool_message_in_guardrail=!1:delete r.skip_tool_message_in_guardrail;let n={};if("PresidioPII"===e.provider&&v.length>0){let e={};v.forEach(t=>{e[t]=N[t]||"MASK"}),r.pii_entities_config=e}else if(e.config)try{let t=JSON.parse(e.config);"Bedrock"===e.provider&&t?(t.guardrail_id&&(r.guardrailIdentifier=t.guardrail_id),t.guardrail_version&&(r.guardrailVersion=t.guardrail_version)):n=t}catch(e){u.default.fromBackend("Invalid JSON in configuration"),f(!1);return}let m={guardrail_id:d,guardrail:{guardrail_name:e.guardrail_name,litellm_params:r,guardrail_info:n}};if(!a)throw Error("No access token available");console.log("Sending guardrail update data:",JSON.stringify(m));let g=`/guardrails/${d}`,h=await fetch(g,{method:"PUT",headers:{[(0,p.getGlobalLitellmHeaderName)()]:`Bearer ${a}`,"Content-Type":"application/json"},body:JSON.stringify(m)});if(!h.ok){let e=await h.text();throw Error(e||"Failed to update guardrail")}u.default.success("Guardrail updated successfully"),o(),t()}catch(e){console.error("Failed to update guardrail:",e),u.default.fromBackend("Failed to update guardrail: "+(e instanceof Error?e.message:String(e)))}finally{f(!1)}};return(0,l.jsx)(s.Modal,{title:"Edit Guardrail",open:e,onCancel:t,footer:null,width:700,children:(0,l.jsxs)(r.Form,{form:x,layout:"vertical",initialValues:g,children:[(0,l.jsx)(r.Form.Item,{name:"guardrail_name",label:"Guardrail Name",rules:[{required:!0,message:"Please enter a guardrail name"}],children:(0,l.jsx)(tl.TextInput,{placeholder:"Enter a name for this guardrail"})}),(0,l.jsx)(r.Form.Item,{name:"provider",label:"Guardrail Provider",rules:[{required:!0,message:"Please select a provider"}],children:(0,l.jsx)(n.Select,{placeholder:"Select a guardrail provider",onChange:e=>{j(e),x.setFieldsValue({config:void 0}),w([]),C({})},disabled:!0,optionLabelProp:"label",children:Object.entries(et()).map(([e,t])=>(0,l.jsx)(ts,{value:e,label:t,children:(0,l.jsxs)("div",{style:{display:"flex",alignItems:"center"},children:[eo[t]&&(0,l.jsx)("img",{src:eo[t],alt:"",style:{height:"20px",width:"20px",marginRight:"8px",objectFit:"contain"},onError:e=>{e.currentTarget.style.display="none"}}),(0,l.jsx)("span",{children:t})]})},e))})}),(0,l.jsx)(r.Form.Item,{name:"mode",label:"Mode",tooltip:"How the guardrail should be applied",rules:[{required:!0,message:"Please select a mode"}],children:(0,l.jsx)(n.Select,{children:_?.supported_modes?.map(e=>(0,l.jsx)(ts,{value:e,children:e},e))||(0,l.jsxs)(l.Fragment,{children:[(0,l.jsx)(ts,{value:"pre_call",children:"pre_call"}),(0,l.jsx)(ts,{value:"post_call",children:"post_call"})]})})}),(0,l.jsx)(r.Form.Item,{name:"default_on",label:"Always On",tooltip:"If enabled, this guardrail will be applied to all requests by default",valuePropName:"checked",children:(0,l.jsx)(D.Switch,{})}),(0,l.jsx)(r.Form.Item,{name:"skip_system_message_choice",label:"Skip system messages in guardrail",tooltip:"Unified guardrails only: whether role: system content is omitted from guardrail input (LLM still receives full messages). Use global default follows litellm_settings.skip_system_message_in_guardrail.",children:(0,l.jsxs)(n.Select,{children:[(0,l.jsx)(ts,{value:"inherit",children:"Use global default"}),(0,l.jsx)(ts,{value:"yes",children:"Yes — exclude from guardrail scan"}),(0,l.jsx)(ts,{value:"no",children:"No — always include in scan"})]})}),(0,l.jsx)(r.Form.Item,{name:"skip_tool_message_choice",label:"Skip tool messages in guardrail",tooltip:"Unified guardrails only: whether role: tool content is omitted from guardrail input (LLM still receives full messages). Use global default follows litellm_settings.skip_tool_message_in_guardrail.",children:(0,l.jsxs)(n.Select,{children:[(0,l.jsx)(ts,{value:"inherit",children:"Use global default"}),(0,l.jsx)(ts,{value:"yes",children:"Yes — exclude from guardrail scan"}),(0,l.jsx)(ts,{value:"no",children:"No — always include in scan"})]})}),(()=>{if(!y)return null;if("PresidioPII"===y)return _&&y&&"PresidioPII"===y?(0,l.jsx)(eR,{entities:_.supported_entities,actions:_.supported_actions,selectedEntities:v,selectedActions:N,onEntitySelect:S,onActionSelect:k,entityCategories:_.pii_entity_categories}):null;switch(y){case"Aporia":return(0,l.jsx)(r.Form.Item,{label:"Aporia Configuration",name:"config",tooltip:"JSON configuration for Aporia",children:(0,l.jsx)(i.Input.TextArea,{rows:4,placeholder:`{ + "api_key": "your_aporia_api_key", + "project_name": "your_project_name" +}`})});case"AimSecurity":return(0,l.jsx)(r.Form.Item,{label:"Aim Security Configuration",name:"config",tooltip:"JSON configuration for Aim Security",children:(0,l.jsx)(i.Input.TextArea,{rows:4,placeholder:`{ + "api_key": "your_aim_api_key" +}`})});case"Bedrock":return(0,l.jsx)(r.Form.Item,{label:"Amazon Bedrock Configuration",name:"config",tooltip:"JSON configuration for Amazon Bedrock guardrails",children:(0,l.jsx)(i.Input.TextArea,{rows:4,placeholder:`{ + "guardrail_id": "your_guardrail_id", + "guardrail_version": "your_guardrail_version" +}`})});case"GuardrailsAI":return(0,l.jsx)(r.Form.Item,{label:"Guardrails.ai Configuration",name:"config",tooltip:"JSON configuration for Guardrails.ai",children:(0,l.jsx)(i.Input.TextArea,{rows:4,placeholder:`{ + "api_key": "your_guardrails_api_key", + "guardrail_id": "your_guardrail_id" +}`})});case"LakeraAI":return(0,l.jsx)(r.Form.Item,{label:"Lakera AI Configuration",name:"config",tooltip:"JSON configuration for Lakera AI",children:(0,l.jsx)(i.Input.TextArea,{rows:4,placeholder:`{ + "api_key": "your_lakera_api_key" +}`})});case"PromptInjection":return(0,l.jsx)(r.Form.Item,{label:"Prompt Injection Configuration",name:"config",tooltip:"JSON configuration for prompt injection detection",children:(0,l.jsx)(i.Input.TextArea,{rows:4,placeholder:`{ + "threshold": 0.8 +}`})});default:return(0,l.jsx)(r.Form.Item,{label:"Custom Configuration",name:"config",tooltip:"JSON configuration for your custom guardrail",children:(0,l.jsx)(i.Input.TextArea,{rows:4,placeholder:`{ + "key1": "value1", + "key2": "value2" +}`})})}})(),(0,l.jsxs)("div",{className:"flex justify-end space-x-2 mt-4",children:[(0,l.jsx)(e6.Button,{variant:"secondary",onClick:t,children:"Cancel"}),(0,l.jsx)(e6.Button,{onClick:I,loading:h,children:"Update Guardrail"})]})]})})};var to=((a={}).DB="db",a.CONFIG="config",a);e.s(["default",0,({guardrailsList:e,isLoading:t,onDeleteClick:a,accessToken:r,onGuardrailUpdated:i,isAdmin:s=!1,onGuardrailClick:n})=>{let[o,d]=(0,m.useState)([{id:"created_at",desc:!0}]),[c,u]=(0,m.useState)(!1),[p,g]=(0,m.useState)(null),x=e=>e?new Date(e).toLocaleString():"-",h=[{header:"Guardrail ID",accessorKey:"guardrail_id",cell:e=>(0,l.jsx)(eb.Tooltip,{title:String(e.getValue()||""),children:(0,l.jsx)(e6.Button,{size:"xs",variant:"light",className:"font-mono text-blue-500 bg-blue-50 hover:bg-blue-100 text-xs font-normal px-2 py-0.5 text-left overflow-hidden truncate max-w-[200px]",onClick:()=>e.getValue()&&n(e.getValue()),children:e.getValue()?`${String(e.getValue()).slice(0,7)}...`:""})})},{header:"Name",accessorKey:"guardrail_name",cell:({row:e})=>{let t=e.original;return(0,l.jsx)(eb.Tooltip,{title:t.guardrail_name,children:(0,l.jsx)("span",{className:"text-xs font-medium",children:t.guardrail_name||"-"})})}},{header:"Provider",accessorKey:"litellm_params.guardrail",cell:({row:e})=>{let{logo:t,displayName:a}=ed(e.original.litellm_params.guardrail);return(0,l.jsxs)("div",{className:"flex items-center space-x-2",children:[t&&(0,l.jsx)("img",{src:t,alt:`${a} logo`,className:"w-4 h-4",onError:e=>{e.target.style.display="none"}}),(0,l.jsx)("span",{className:"text-xs",children:a})]})}},{header:"Mode",accessorKey:"litellm_params.mode",cell:({row:e})=>{let t=e.original;return(0,l.jsx)("span",{className:"text-xs",children:t.litellm_params.mode})}},{header:"Default On",accessorKey:"litellm_params.default_on",cell:({row:e})=>{let t=e.original;return(0,l.jsx)(te.Badge,{color:t.litellm_params?.default_on?"green":"gray",className:"text-xs font-normal",size:"xs",children:t.litellm_params?.default_on?"Default On":"Default Off"})}},{header:"Created At",accessorKey:"created_at",cell:({row:e})=>{let t=e.original;return(0,l.jsx)(eb.Tooltip,{title:t.created_at,children:(0,l.jsx)("span",{className:"text-xs",children:x(t.created_at)})})}},{header:"Updated At",accessorKey:"updated_at",cell:({row:e})=>{let t=e.original;return(0,l.jsx)(eb.Tooltip,{title:t.updated_at,children:(0,l.jsx)("span",{className:"text-xs",children:x(t.updated_at)})})}},{id:"actions",header:"Actions",cell:({row:e})=>{let t=e.original,r=t.guardrail_definition_location===to.CONFIG;return(0,l.jsx)("div",{className:"flex space-x-2",children:r?(0,l.jsx)(eb.Tooltip,{title:"Config guardrail cannot be deleted on the dashboard. Please delete it from the config file.",children:(0,l.jsx)(e5.Icon,{"data-testid":"config-delete-icon",icon:e8.TrashIcon,size:"sm",className:"cursor-not-allowed text-gray-400",title:"Config guardrail cannot be deleted on the dashboard. Please delete it from the config file.","aria-label":"Delete guardrail (config)"})}):(0,l.jsx)(eb.Tooltip,{title:"Delete guardrail",children:(0,l.jsx)(e5.Icon,{icon:e8.TrashIcon,size:"sm",onClick:()=>t.guardrail_id&&a(t.guardrail_id,t.guardrail_name||"Unnamed Guardrail"),className:"cursor-pointer hover:text-red-500"})})})}}],f=(0,tt.useReactTable)({data:e,columns:h,state:{sorting:o},onSortingChange:d,getCoreRowModel:(0,ta.getCoreRowModel)(),getSortedRowModel:(0,ta.getSortedRowModel)(),enableSorting:!0});return(0,l.jsxs)("div",{className:"rounded-lg custom-border relative",children:[(0,l.jsx)("div",{className:"overflow-x-auto",children:(0,l.jsxs)(eX.Table,{className:"[&_td]:py-0.5 [&_th]:py-1",children:[(0,l.jsx)(e1.TableHead,{children:f.getHeaderGroups().map(e=>(0,l.jsx)(e4.TableRow,{children:e.headers.map(e=>(0,l.jsx)(e2.TableHeaderCell,{className:`py-1 h-8 ${"actions"===e.id?"sticky right-0 bg-white shadow-[-4px_0_8px_-6px_rgba(0,0,0,0.1)]":""}`,onClick:e.column.getToggleSortingHandler(),children:(0,l.jsxs)("div",{className:"flex items-center justify-between gap-2",children:[(0,l.jsx)("div",{className:"flex items-center",children:e.isPlaceholder?null:(0,tt.flexRender)(e.column.columnDef.header,e.getContext())}),"actions"!==e.id&&(0,l.jsx)("div",{className:"w-4",children:e.column.getIsSorted()?({asc:(0,l.jsx)(e7.ChevronUpIcon,{className:"h-4 w-4 text-blue-500"}),desc:(0,l.jsx)(e9.ChevronDownIcon,{className:"h-4 w-4 text-blue-500"})})[e.column.getIsSorted()]:(0,l.jsx)(e3.SwitchVerticalIcon,{className:"h-4 w-4 text-gray-400"})})]})},e.id))},e.id))}),(0,l.jsx)(eZ.TableBody,{children:t?(0,l.jsx)(e4.TableRow,{children:(0,l.jsx)(e0.TableCell,{colSpan:h.length,className:"h-8 text-center",children:(0,l.jsx)("div",{className:"text-center text-gray-500",children:(0,l.jsx)("p",{children:"Loading..."})})})}):e.length>0?f.getRowModel().rows.map(e=>(0,l.jsx)(e4.TableRow,{className:"h-8",children:e.getVisibleCells().map(e=>(0,l.jsx)(e0.TableCell,{className:`py-0.5 max-h-8 overflow-hidden text-ellipsis whitespace-nowrap ${"actions"===e.column.id?"sticky right-0 bg-white shadow-[-4px_0_8px_-6px_rgba(0,0,0,0.1)]":""}`,children:(0,tt.flexRender)(e.column.columnDef.cell,e.getContext())},e.id))},e.id)):(0,l.jsx)(e4.TableRow,{children:(0,l.jsx)(e0.TableCell,{colSpan:h.length,className:"h-8 text-center",children:(0,l.jsx)("div",{className:"text-center text-gray-500",children:(0,l.jsx)("p",{children:"No guardrails found"})})})})})]})}),p&&(0,l.jsx)(tn,{visible:c,onClose:()=>u(!1),accessToken:r,onSuccess:()=>{u(!1),g(null),i()},guardrailId:p.guardrail_id||"",fullLitellmParams:p.litellm_params,initialValues:{guardrail_name:p.guardrail_name||"",provider:Object.keys(ea).find(e=>ea[e]===p?.litellm_params.guardrail)||"",mode:p.litellm_params.mode,default_on:p.litellm_params.default_on,pii_entities_config:p.litellm_params.pii_entities_config,skip_system_message_choice:ec(p.litellm_params?.skip_system_message_in_guardrail),skip_tool_message_choice:eu(p.litellm_params?.skip_tool_message_in_guardrail),...p.guardrail_info}})]})}],782719);var td=e.i(500330),tc=e.i(245094),eS=eS,tm=e.i(530212),tu=e.i(350967),tp=e.i(197647),tg=e.i(653824),tx=e.i(881073),th=e.i(404206),tf=e.i(723731),ty=e.i(629569),tj=e.i(678784),t_=e.i(118366),tb=e.i(560445);let{Text:tv}=d.Typography,{Option:tw}=n.Select,tN=({categories:e,onActionChange:t,onSeverityChange:a,onRemove:r,readOnly:i=!1})=>{let s=[{title:"Category",dataIndex:"display_name",key:"display_name",render:(e,t)=>(0,l.jsxs)("div",{children:[(0,l.jsx)(tv,{strong:!0,children:e}),e!==t.category&&(0,l.jsx)("div",{children:(0,l.jsx)(tv,{type:"secondary",style:{fontSize:12},children:t.category})})]})},{title:"Severity Threshold",dataIndex:"severity_threshold",key:"severity_threshold",width:180,render:(e,t)=>i?(0,l.jsx)(o.Tag,{color:{high:"red",medium:"orange",low:"yellow"}[e],children:e.toUpperCase()}):(0,l.jsxs)(n.Select,{value:e,onChange:e=>a?.(t.id,e),style:{width:150},size:"small",children:[(0,l.jsx)(tw,{value:"high",children:"High"}),(0,l.jsx)(tw,{value:"medium",children:"Medium"}),(0,l.jsx)(tw,{value:"low",children:"Low"})]})},{title:"Action",dataIndex:"action",key:"action",width:150,render:(e,a)=>i?(0,l.jsx)(o.Tag,{color:"BLOCK"===e?"red":"blue",children:e}):(0,l.jsxs)(n.Select,{value:e,onChange:e=>t?.(a.id,e),style:{width:120},size:"small",children:[(0,l.jsx)(tw,{value:"BLOCK",children:"Block"}),(0,l.jsx)(tw,{value:"MASK",children:"Mask"})]})}];return(i||s.push({title:"",key:"actions",width:100,render:(e,t)=>(0,l.jsx)(c.Button,{type:"text",danger:!0,size:"small",icon:(0,l.jsx)(A.DeleteOutlined,{}),onClick:()=>r?.(t.id),children:"Delete"})}),0===e.length)?(0,l.jsx)("div",{style:{textAlign:"center",padding:"40px 0",color:"#999"},children:"No categories configured."}):(0,l.jsx)(I.Table,{dataSource:e,columns:s,rowKey:"id",pagination:!1,size:"small"})},tC=({patterns:e,blockedWords:t,categories:a=[],readOnly:r=!0,onPatternActionChange:i,onPatternRemove:s,onBlockedWordUpdate:n,onBlockedWordRemove:o,onCategoryActionChange:d,onCategorySeverityChange:c,onCategoryRemove:m})=>{if(0===e.length&&0===t.length&&0===a.length)return null;let u=()=>{};return(0,l.jsxs)(l.Fragment,{children:[a.length>0&&(0,l.jsxs)(eG.Card,{className:"mt-6",children:[(0,l.jsxs)("div",{className:"flex justify-between items-center mb-4",children:[(0,l.jsx)(ez.Text,{className:"text-lg font-semibold",children:"Content Categories"}),(0,l.jsxs)(te.Badge,{color:"blue",children:[a.length," categories configured"]})]}),(0,l.jsx)(tN,{categories:a,onActionChange:r?void 0:d,onSeverityChange:r?void 0:c,onRemove:r?void 0:m,readOnly:r})]}),e.length>0&&(0,l.jsxs)(eG.Card,{className:"mt-6",children:[(0,l.jsxs)("div",{className:"flex justify-between items-center mb-4",children:[(0,l.jsx)(ez.Text,{className:"text-lg font-semibold",children:"Pattern Detection"}),(0,l.jsxs)(te.Badge,{color:"blue",children:[e.length," patterns configured"]})]}),(0,l.jsx)(P,{patterns:e,onActionChange:r?u:i||u,onRemove:r?u:s||u})]}),t.length>0&&(0,l.jsxs)(eG.Card,{className:"mt-6",children:[(0,l.jsxs)("div",{className:"flex justify-between items-center mb-4",children:[(0,l.jsx)(ez.Text,{className:"text-lg font-semibold",children:"Blocked Keywords"}),(0,l.jsxs)(te.Badge,{color:"blue",children:[t.length," keywords configured"]})]}),(0,l.jsx)(F,{keywords:t,onActionChange:r?u:n||u,onRemove:r?u:o||u})]})]})},{Text:tS}=d.Typography,tk=({guardrailData:e,guardrailSettings:t,isEditing:a,accessToken:r,onDataChange:i,onUnsavedChanges:s})=>{let[n,o]=(0,m.useState)([]),[d,c]=(0,m.useState)([]),[u,p]=(0,m.useState)([]),[g,x]=(0,m.useState)([]),[h,f]=(0,m.useState)([]),[y,j]=(0,m.useState)([]),[_,b]=(0,m.useState)(!1),[v,w]=(0,m.useState)(null),[N,C]=(0,m.useState)(!1),[S,k]=(0,m.useState)(null);(0,m.useEffect)(()=>{if(e?.litellm_params?.patterns){let t=e.litellm_params.patterns.map((e,t)=>({id:`pattern-${t}`,type:"prebuilt"===e.pattern_type?"prebuilt":"custom",name:e.pattern_name||e.name,display_name:e.display_name,pattern:e.pattern,action:e.action||"BLOCK"}));o(t),x(t)}else o([]),x([]);if(e?.litellm_params?.blocked_words){let t=e.litellm_params.blocked_words.map((e,t)=>({id:`word-${t}`,keyword:e.keyword,action:e.action||"BLOCK",description:e.description}));c(t),f(t)}else c([]),f([]);if(e?.litellm_params?.categories?.length>0){let a=t?.content_filter_settings?.content_categories?Object.fromEntries(t.content_filter_settings.content_categories.map(e=>[e.name,e])):{},l=e.litellm_params.categories.map((e,t)=>{let l=a[e.category];return{id:`category-${t}`,category:e.category,display_name:l?.display_name??e.category,action:e.action||"BLOCK",severity_threshold:e.severity_threshold||"medium"}});p(l),j(l)}else p([]),j([]);let a=e?.litellm_params?.competitor_intent_config;if(a&&"object"==typeof a){let e=!!(a.brand_self&&Array.isArray(a.brand_self)&&a.brand_self.length>0),t={competitor_intent_type:a.competitor_intent_type??"airline",brand_self:Array.isArray(a.brand_self)?a.brand_self:[],locations:Array.isArray(a.locations)?a.locations:[],competitors:Array.isArray(a.competitors)?a.competitors:[],policy:a.policy??{competitor_comparison:"refuse",possible_competitor_comparison:"reframe"},threshold_high:"number"==typeof a.threshold_high?a.threshold_high:.7,threshold_medium:"number"==typeof a.threshold_medium?a.threshold_medium:.45,threshold_low:"number"==typeof a.threshold_low?a.threshold_low:.3};b(e),w(t),C(e),k(t)}else b(!1),w(null),C(!1),k(null)},[e,t?.content_filter_settings?.content_categories]),(0,m.useEffect)(()=>{i&&i(n,d,u,_,v)},[n,d,u,_,v,i]);let I=m.default.useMemo(()=>{let e=JSON.stringify(n)!==JSON.stringify(g),t=JSON.stringify(d)!==JSON.stringify(h),a=JSON.stringify(u)!==JSON.stringify(y),l=_!==N||JSON.stringify(v)!==JSON.stringify(S);return e||t||a||l},[n,d,u,_,v,g,h,y,N,S]);return((0,m.useEffect)(()=>{a&&s&&s(I)},[I,a,s]),e?.litellm_params?.guardrail!=="litellm_content_filter")?null:a?(0,l.jsxs)(l.Fragment,{children:[(0,l.jsx)(eD.Divider,{orientation:"left",children:"Content Filter Configuration"}),I&&(0,l.jsx)(tb.Alert,{type:"warning",showIcon:!0,className:"mb-4",message:(0,l.jsx)(tS,{children:'You have unsaved changes to patterns or keywords. Remember to click "Save Changes" at the bottom.'})}),(0,l.jsx)("div",{className:"mb-6",children:t&&t.content_filter_settings&&(0,l.jsx)(Q,{prebuiltPatterns:t.content_filter_settings.prebuilt_patterns||[],categories:t.content_filter_settings.pattern_categories||[],selectedPatterns:n,blockedWords:d,onPatternAdd:e=>o([...n,e]),onPatternRemove:e=>o(n.filter(t=>t.id!==e)),onPatternActionChange:(e,t)=>o(n.map(a=>a.id===e?{...a,action:t}:a)),onBlockedWordAdd:e=>c([...d,e]),onBlockedWordRemove:e=>c(d.filter(t=>t.id!==e)),onBlockedWordUpdate:(e,t,a)=>c(d.map(l=>l.id===e?{...l,[t]:a}:l)),onFileUpload:e=>{console.log("File uploaded:",e)},accessToken:r,contentCategories:t.content_filter_settings.content_categories||[],selectedContentCategories:u,onContentCategoryAdd:e=>p([...u,e]),onContentCategoryRemove:e=>p(u.filter(t=>t.id!==e)),onContentCategoryUpdate:(e,t,a)=>p(u.map(l=>l.id===e?{...l,[t]:a}:l)),competitorIntentEnabled:_,competitorIntentConfig:v,onCompetitorIntentChange:(e,t)=>{b(e),w(t)}})})]}):(0,l.jsx)(tC,{patterns:n,blockedWords:d,categories:u,readOnly:!0})};var tI=e.i(788191),tA=e.i(245704),tO=e.i(518617);let tT={icon:{tag:"svg",attrs:{viewBox:"0 0 1024 1024",focusable:"false"},children:[{tag:"path",attrs:{d:"M715.8 493.5L335 165.1c-14.2-12.2-35-1.2-35 18.5v656.8c0 19.7 20.8 30.7 35 18.5l380.8-328.4c10.9-9.4 10.9-27.6 0-37z"}}]},name:"caret-right",theme:"outlined"};var tP=m.forwardRef(function(e,t){return m.createElement(eO.default,(0,eI.default)({},e,{ref:t,icon:tT}))}),tL=e.i(987432);let tB={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M892 772h-80v-80c0-4.4-3.6-8-8-8h-48c-4.4 0-8 3.6-8 8v80h-80c-4.4 0-8 3.6-8 8v48c0 4.4 3.6 8 8 8h80v80c0 4.4 3.6 8 8 8h48c4.4 0 8-3.6 8-8v-80h80c4.4 0 8-3.6 8-8v-48c0-4.4-3.6-8-8-8zM373.5 498.4c-.9-8.7-1.4-17.5-1.4-26.4 0-15.9 1.5-31.4 4.3-46.5.7-3.6-1.2-7.3-4.5-8.8-13.6-6.1-26.1-14.5-36.9-25.1a127.54 127.54 0 01-38.7-95.4c.9-32.1 13.8-62.6 36.3-85.6 24.7-25.3 57.9-39.1 93.2-38.7 31.9.3 62.7 12.6 86 34.4 7.9 7.4 14.7 15.6 20.4 24.4 2 3.1 5.9 4.4 9.3 3.2 17.6-6.1 36.2-10.4 55.3-12.4 5.6-.6 8.8-6.6 6.3-11.6-32.5-64.3-98.9-108.7-175.7-109.9-110.8-1.7-203.2 89.2-203.2 200 0 62.8 28.9 118.8 74.2 155.5-31.8 14.7-61.1 35-86.5 60.4-54.8 54.7-85.8 126.9-87.8 204a8 8 0 008 8.2h56.1c4.3 0 7.9-3.4 8-7.7 1.9-58 25.4-112.3 66.7-153.5 29.4-29.4 65.4-49.8 104.7-59.7 3.8-1.1 6.4-4.8 5.9-8.8zM824 472c0-109.4-87.9-198.3-196.9-200C516.3 270.3 424 361.2 424 472c0 62.8 29 118.8 74.2 155.5a300.95 300.95 0 00-86.4 60.4C357 742.6 326 814.8 324 891.8a8 8 0 008 8.2h56c4.3 0 7.9-3.4 8-7.7 1.9-58 25.4-112.3 66.7-153.5C505.8 695.7 563 672 624 672c110.4 0 200-89.5 200-200zm-109.5 90.5C690.3 586.7 658.2 600 624 600s-66.3-13.3-90.5-37.5a127.26 127.26 0 01-37.5-91.8c.3-32.8 13.4-64.5 36.3-88 24-24.6 56.1-38.3 90.4-38.7 33.9-.3 66.8 12.9 91 36.6 24.8 24.3 38.4 56.8 38.4 91.4-.1 34.2-13.4 66.3-37.6 90.5z"}}]},name:"usergroup-add",theme:"outlined"};var tF=m.forwardRef(function(e,t){return m.createElement(eO.default,(0,eI.default)({},e,{ref:t,icon:tB}))}),t$=e.i(872934);let{Panel:tE}=$.Collapse,{TextArea:tM}=i.Input,tR={empty:{name:"Empty Template",code:`async def apply_guardrail(inputs, request_data, input_type): + # inputs: {texts, images, tools, tool_calls, structured_messages, model} + # request_data: {model, user_id, team_id, end_user_id, metadata} + # input_type: "request" or "response" + return allow()`},blockSSN:{name:"Block SSN",code:`def apply_guardrail(inputs, request_data, input_type): + for text in inputs["texts"]: + if regex_match(text, r"\\d{3}-\\d{2}-\\d{4}"): + return block("SSN detected") + return allow()`},redactEmail:{name:"Redact Emails",code:`def apply_guardrail(inputs, request_data, input_type): + pattern = r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}" + modified = [] + for text in inputs["texts"]: + modified.append(regex_replace(text, pattern, "[EMAIL REDACTED]")) + return modify(texts=modified)`},blockSQL:{name:"Block SQL Injection",code:`def apply_guardrail(inputs, request_data, input_type): + if input_type != "request": + return allow() + for text in inputs["texts"]: + if contains_code_language(text, ["sql"]): + return block("SQL code not allowed") + return allow()`},validateJSON:{name:"Validate JSON",code:`def apply_guardrail(inputs, request_data, input_type): + if input_type != "response": + return allow() + + schema = {"type": "object", "required": ["name", "value"]} + + for text in inputs["texts"]: + obj = json_parse(text) + if obj is None: + return block("Invalid JSON response") + if not json_schema_valid(obj, schema): + return block("Response missing required fields") + return allow()`},externalAPI:{name:"External API Check (async)",code:`async def apply_guardrail(inputs, request_data, input_type): + # Call an external moderation API (async for non-blocking) + for text in inputs["texts"]: + response = await http_post( + "https://api.example.com/moderate", + body={"text": text, "user_id": request_data["user_id"]}, + headers={"Authorization": "Bearer YOUR_API_KEY"}, + timeout=10 + ) + + if not response["success"]: + # API call failed, allow by default or block + return allow() + + if response["body"].get("flagged"): + return block(response["body"].get("reason", "Content flagged")) + + return allow()`}},tG={"Return Values":[{name:"allow()",desc:"Let request/response through"},{name:"block(reason)",desc:"Reject with message"},{name:"modify(texts=[], images=[], tool_calls=[])",desc:"Transform content"}],"HTTP Requests (async)":[{name:"await http_request(url, method, headers, body)",desc:"Make async HTTP request"},{name:"await http_get(url, headers)",desc:"Async GET request"},{name:"await http_post(url, body, headers)",desc:"Async POST request"}],"Regex Functions":[{name:"regex_match(text, pattern)",desc:"Returns True if pattern found"},{name:"regex_replace(text, pattern, replacement)",desc:"Replace all matches"},{name:"regex_find_all(text, pattern)",desc:"Return list of matches"}],"JSON Functions":[{name:"json_parse(text)",desc:"Parse JSON string, returns None on error"},{name:"json_stringify(obj)",desc:"Convert to JSON string"},{name:"json_schema_valid(obj, schema)",desc:"Validate against JSON schema"}],"URL Functions":[{name:"extract_urls(text)",desc:"Extract all URLs from text"},{name:"is_valid_url(url)",desc:"Check if URL is valid"},{name:"all_urls_valid(text)",desc:"Check all URLs in text are valid"}],"Code Detection":[{name:"detect_code(text)",desc:"Returns True if code detected"},{name:"detect_code_languages(text)",desc:"Returns list of detected languages"},{name:'contains_code_language(text, ["sql"])',desc:"Check for specific languages"}],"Text Utilities":[{name:"contains(text, substring)",desc:"Check if substring exists"},{name:"contains_any(text, [substr1, substr2])",desc:"Check if any substring exists"},{name:"word_count(text)",desc:"Count words"},{name:"char_count(text)",desc:"Count characters"},{name:"lower(text) / upper(text) / trim(text)",desc:"String transforms"}]},tz=[{value:"pre_call",label:"pre_call (Request)"},{value:"post_call",label:"post_call (Response)"},{value:"during_call",label:"during_call (Parallel)"},{value:"logging_only",label:"logging_only"},{value:"pre_mcp_call",label:"pre_mcp_call (Before MCP Tool Call)"},{value:"post_mcp_call",label:"post_mcp_call (After MCP Tool Call)"},{value:"during_mcp_call",label:"during_mcp_call (During MCP Tool Call)"}],tD=({visible:e,onClose:t,onSuccess:a,accessToken:r,editData:i})=>{let o=!!i,[d,c]=(0,m.useState)(""),[g,x]=(0,m.useState)(["pre_call"]),[h,f]=(0,m.useState)(!1),[y,j]=(0,m.useState)("empty"),[_,b]=(0,m.useState)(tR.empty.code),[v,w]=(0,m.useState)(!1),[N,C]=(0,m.useState)(!1),[S,k]=(0,m.useState)(!1),I={texts:["Hello, my SSN is 123-45-6789"],images:[],tools:[{type:"function",function:{name:"get_weather",description:"Get the current weather in a location",parameters:{type:"object",properties:{location:{type:"string",description:"City name"}},required:["location"]}}}],tool_calls:[],structured_messages:[{role:"system",content:"You are a helpful assistant."},{role:"user",content:"Hello, my SSN is 123-45-6789"}],model:"gpt-4"},A={texts:["The weather in San Francisco is 72°F and sunny."],images:[],tools:[],tool_calls:[{id:"call_abc123",type:"function",function:{name:"get_weather",arguments:'{"location": "San Francisco"}'}}],structured_messages:[],model:"gpt-4"},O={texts:['Tool: read_wiki_structure\nArguments: {"repoName": "BerriAI/litellm"}'],images:[],tools:[{type:"function",function:{name:"read_wiki_structure",description:"Read the structure of a GitHub repository (MCP tool passed as OpenAI tool)",parameters:{type:"object",properties:{repoName:{type:"string",description:"Repository name, e.g. BerriAI/litellm"}},required:["repoName"]}}}],tool_calls:[{id:"call_mcp_001",type:"function",function:{name:"read_wiki_structure",arguments:'{"repoName": "BerriAI/litellm"}'}}],structured_messages:[{role:"user",content:'Tool: read_wiki_structure\nArguments: {"repoName": "BerriAI/litellm"}'}],model:"mcp-tool-call"},[T,P]=(0,m.useState)(JSON.stringify(I,null,2)),[L,B]=(0,m.useState)(null),[F,E]=(0,m.useState)(null),M=(0,m.useRef)(null),R=e=>null==e?["pre_call"]:Array.isArray(e)?e.length?e:["pre_call"]:[e];(0,m.useEffect)(()=>{e&&(i?(c(i.guardrail_name||""),x(R(i.litellm_params?.mode)),f(i.litellm_params?.default_on||!1),b(i.litellm_params?.custom_code||tR.empty.code),j("")):(c(""),x(["pre_call"]),f(!1),j("empty"),b(tR.empty.code)),B(null),k(!1))},[e,i]);let G=async e=>{try{await navigator.clipboard.writeText(e),E(e),setTimeout(()=>E(null),2e3)}catch(e){console.error("Failed to copy:",e)}},z=async()=>{if(!d.trim())return void u.default.fromBackend("Please enter a guardrail name");if(!_.trim())return void u.default.fromBackend("Please enter custom code");if(!r)return void u.default.fromBackend("No access token available");w(!0);try{if(o&&i){let e={litellm_params:{custom_code:_}};d!==i.guardrail_name&&(e.guardrail_name=d);let t=R(i.litellm_params?.mode);(g.length!==t.length||g.some((e,a)=>e!==t[a]))&&(e.litellm_params.mode=g),h!==i.litellm_params?.default_on&&(e.litellm_params.default_on=h),await (0,p.updateGuardrailCall)(r,i.guardrail_id,e),u.default.success("Custom code guardrail updated successfully")}else await (0,p.createGuardrailCall)(r,{guardrail_name:d,litellm_params:{guardrail:"custom_code",mode:g,default_on:h,custom_code:_},guardrail_info:{}}),u.default.success("Custom code guardrail created successfully");a(),t()}catch(e){console.error("Failed to save guardrail:",e),u.default.fromBackend(`Failed to ${o?"update":"create"} guardrail: `+(e instanceof Error?e.message:String(e)))}finally{w(!1)}},K=async()=>{if(!r)return void B({error:"No access token available"});C(!0),B(null);try{let e;try{e=JSON.parse(T)}catch(e){B({error:"Invalid test input JSON"}),C(!1);return}e.texts||(e.texts=[]);let t=["pre_call","pre_mcp_call"],a=["post_call","post_mcp_call"],l=g.some(e=>t.includes(e))?"request":g.some(e=>a.includes(e))?"response":"request",i=await (0,p.testCustomCodeGuardrail)(r,{custom_code:_,test_input:e,input_type:l,request_data:{model:"test-model",metadata:{}}});i.success&&i.result?B(i.result):i.error?B({error:i.error,error_type:i.error_type}):B({error:"Unknown error occurred"})}catch(e){console.error("Failed to test custom code:",e),B({error:e instanceof Error?e.message:"Failed to test custom code"})}finally{C(!1)}},q=_.split("\n").length;return(0,l.jsxs)(s.Modal,{open:e,onCancel:t,footer:null,width:1400,className:"custom-code-modal",closable:!0,destroyOnClose:!0,children:[(0,l.jsxs)("div",{className:"flex flex-col h-[80vh]",children:[(0,l.jsxs)("div",{className:"pb-4 border-b border-gray-200",children:[(0,l.jsx)("h2",{className:"text-xl font-semibold text-gray-900",children:o?"Edit Custom Guardrail":"Create Custom Guardrail"}),(0,l.jsx)("p",{className:"text-sm text-gray-500 mt-1",children:"Define custom logic using Python-like syntax"})]}),(0,l.jsxs)("div",{className:"flex items-center gap-4 py-4 border-b border-gray-100",children:[(0,l.jsxs)("div",{className:"flex-1 max-w-[200px]",children:[(0,l.jsx)("label",{className:"block text-xs font-medium text-gray-600 mb-1",children:"Guardrail Name"}),(0,l.jsx)(tl.TextInput,{value:d,onValueChange:c,placeholder:"e.g., block-pii-custom"})]}),(0,l.jsxs)("div",{className:"w-[280px]",children:[(0,l.jsx)("label",{className:"block text-xs font-medium text-gray-600 mb-1",children:"Mode (can select multiple)"}),(0,l.jsx)(n.Select,{mode:"multiple",value:g,onChange:x,options:tz,className:"w-full",size:"middle",placeholder:"Select modes"})]}),(0,l.jsxs)("div",{className:"w-[180px]",children:[(0,l.jsx)("label",{className:"block text-xs font-medium text-gray-600 mb-1",children:"Template"}),(0,l.jsx)(n.Select,{value:y,onChange:e=>{j(e),b(tR[e].code)},className:"w-full",size:"middle",dropdownRender:e=>(0,l.jsxs)(l.Fragment,{children:[e,(0,l.jsx)(eD.Divider,{style:{margin:"8px 0"}}),(0,l.jsxs)("div",{style:{padding:"8px 12px",cursor:"pointer",color:"#1890ff",fontSize:"12px",display:"flex",alignItems:"center",gap:"4px"},onClick:e=>{e.preventDefault(),window.open("https://models.litellm.ai/guardrails","_blank")},onMouseEnter:e=>{e.currentTarget.style.backgroundColor="#f0f0f0"},onMouseLeave:e=>{e.currentTarget.style.backgroundColor="transparent"},children:[(0,l.jsx)(tF,{}),(0,l.jsx)("span",{children:"Browse Community templates"}),(0,l.jsx)(t$.ExportOutlined,{style:{fontSize:"10px"}})]})]}),children:(0,l.jsx)(n.Select.OptGroup,{label:"STANDARD",children:Object.entries(tR).map(([e,t])=>(0,l.jsx)(n.Select.Option,{value:e,children:t.name},e))})})]}),(0,l.jsxs)("div",{className:"flex items-center gap-2 pt-5",children:[(0,l.jsx)("span",{className:"text-sm text-gray-600",children:"Default On"}),(0,l.jsx)(D.Switch,{checked:h,onChange:f})]})]}),(0,l.jsxs)("div",{className:"flex flex-1 overflow-hidden mt-4 gap-6",children:[(0,l.jsxs)("div",{className:"flex-[2] flex flex-col min-w-0 overflow-y-auto",children:[(0,l.jsxs)("div",{className:"flex items-center justify-between mb-2 flex-shrink-0",children:[(0,l.jsx)("span",{className:"text-xs font-semibold text-gray-500 uppercase tracking-wide",children:"Python Logic"}),(0,l.jsx)("span",{className:"text-xs text-gray-400",children:"Restricted environment (no imports)"})]}),(0,l.jsxs)("div",{className:"relative rounded-lg overflow-hidden border border-gray-700 bg-[#1e1e1e] flex-shrink-0",style:{minHeight:"300px",maxHeight:"400px"},children:[(0,l.jsx)("div",{className:"absolute left-0 top-0 bottom-0 w-12 bg-[#1e1e1e] border-r border-gray-700 text-right pr-3 pt-3 select-none overflow-hidden",style:{fontFamily:"'Fira Code', 'Monaco', 'Consolas', monospace",fontSize:"14px",lineHeight:"1.6"},children:Array.from({length:Math.max(q,20)},(e,t)=>(0,l.jsx)("div",{className:"text-gray-500 h-[22.4px]",children:t+1},t+1))}),(0,l.jsx)("textarea",{ref:M,value:_,onChange:e=>b(e.target.value),onKeyDown:e=>{if("Tab"===e.key){e.preventDefault();let t=e.currentTarget,a=t.selectionStart,l=t.selectionEnd;b(_.substring(0,a)+" "+_.substring(l)),setTimeout(()=>{t.selectionStart=t.selectionEnd=a+4},0)}},spellCheck:!1,className:"w-full h-full pl-14 pr-4 pt-3 pb-3 resize-none focus:outline-none bg-transparent text-gray-200",style:{fontFamily:"'Fira Code', 'Monaco', 'Consolas', monospace",fontSize:"14px",lineHeight:"1.6",tabSize:4}})]}),(0,l.jsx)($.Collapse,{activeKey:S?["test"]:[],onChange:e=>k(e.includes("test")),className:"mt-3 bg-white border border-gray-200 rounded-lg flex-shrink-0",expandIcon:({isActive:e})=>(0,l.jsx)(tP,{rotate:90*!!e}),children:(0,l.jsx)(tE,{header:(0,l.jsxs)("span",{className:"flex items-center gap-2 text-sm font-medium",children:[(0,l.jsx)(tI.PlayCircleOutlined,{className:"text-blue-500"}),"Test Your Guardrail"]}),children:(0,l.jsxs)("div",{className:"space-y-3",children:[(0,l.jsxs)("div",{children:[(0,l.jsxs)("div",{className:"flex items-center justify-between mb-2",children:[(0,l.jsx)("label",{className:"block text-xs font-medium text-gray-600",children:"Test Input (JSON)"}),(0,l.jsxs)("div",{className:"flex items-center gap-2",children:[(0,l.jsx)("span",{className:"text-xs text-gray-500",children:"Load example:"}),(0,l.jsx)("button",{type:"button",onClick:()=>P(JSON.stringify(I,null,2)),className:"px-2 py-1 text-xs rounded border border-orange-200 bg-orange-50 text-orange-700 hover:bg-orange-100 transition-colors",children:"Pre-call"}),(0,l.jsx)("button",{type:"button",onClick:()=>P(JSON.stringify(O,null,2)),className:"px-2 py-1 text-xs rounded border border-purple-200 bg-purple-50 text-purple-700 hover:bg-purple-100 transition-colors",children:"Pre MCP"}),(0,l.jsx)("button",{type:"button",onClick:()=>P(JSON.stringify(A,null,2)),className:"px-2 py-1 text-xs rounded border border-green-200 bg-green-50 text-green-700 hover:bg-green-100 transition-colors",children:"Post-call"})]})]}),(0,l.jsx)("div",{className:"mb-2 p-2 bg-gray-50 rounded text-xs text-gray-600 border border-gray-200",children:(0,l.jsxs)("div",{className:"grid grid-cols-2 gap-x-4 gap-y-1",children:[(0,l.jsxs)("div",{children:[(0,l.jsx)("strong",{children:"texts"}),": Message content (always)"]}),(0,l.jsxs)("div",{children:[(0,l.jsx)("strong",{children:"images"}),": Base64 images (vision)"]}),(0,l.jsxs)("div",{children:[(0,l.jsx)("strong",{children:"tools"}),": Tool definitions ",(0,l.jsx)("span",{className:"text-orange-600",children:"(pre_call)"}),", MCP as OpenAI tool ",(0,l.jsx)("span",{className:"text-purple-600",children:"(pre_mcp_call)"})]}),(0,l.jsxs)("div",{children:[(0,l.jsx)("strong",{children:"tool_calls"}),": LLM tool calls ",(0,l.jsx)("span",{className:"text-green-600",children:"(post_call)"})]}),(0,l.jsxs)("div",{children:[(0,l.jsx)("strong",{children:"structured_messages"}),": Full messages ",(0,l.jsx)("span",{className:"text-orange-600",children:"(pre_call)"})]}),(0,l.jsxs)("div",{children:[(0,l.jsx)("strong",{children:"model"}),": Model name (always)"]})]})}),(0,l.jsx)(tM,{value:T,onChange:e=>P(e.target.value),rows:8,className:"font-mono text-xs",placeholder:'{"texts": ["test message"], ...}'})]}),(0,l.jsxs)("div",{className:"flex items-center gap-3",children:[(0,l.jsx)(e6.Button,{size:"xs",onClick:K,disabled:N,icon:tI.PlayCircleOutlined,children:N?"Running...":"Run Test"}),L&&(0,l.jsx)("div",{className:`flex items-center gap-2 text-sm ${L.error?"text-red-600":"allow"===L.action?"text-green-600":"block"===L.action?"text-orange-600":"text-blue-600"}`,children:L.error?(0,l.jsxs)(l.Fragment,{children:[(0,l.jsx)(tO.CloseCircleOutlined,{}),(0,l.jsxs)("span",{children:[L.error_type&&(0,l.jsxs)("span",{className:"font-medium",children:["[",L.error_type,"] "]}),L.error]})]}):"allow"===L.action?(0,l.jsxs)(l.Fragment,{children:[(0,l.jsx)(tA.CheckCircleOutlined,{})," Allowed"]}):"block"===L.action?(0,l.jsxs)(l.Fragment,{children:[(0,l.jsx)(tO.CloseCircleOutlined,{})," Blocked: ",L.reason]}):"modify"===L.action?(0,l.jsxs)(l.Fragment,{children:[(0,l.jsx)(tA.CheckCircleOutlined,{})," Modified",L.texts&&L.texts.length>0&&(0,l.jsxs)("span",{className:"text-xs text-gray-500 ml-1",children:["→ ",L.texts[0].substring(0,50),L.texts[0].length>50?"...":""]})]}):(0,l.jsxs)(l.Fragment,{children:[(0,l.jsx)(tA.CheckCircleOutlined,{})," ",L.action||"Unknown"]})})]})]})},"test")}),(0,l.jsxs)("div",{className:"mt-3 p-4 bg-gradient-to-r from-blue-50 to-indigo-50 border border-blue-200 rounded-lg flex items-center justify-between flex-shrink-0",children:[(0,l.jsxs)("div",{className:"flex items-center gap-3",children:[(0,l.jsx)("div",{className:"bg-blue-100 rounded-full p-2",children:(0,l.jsx)(tF,{className:"text-blue-600 text-lg"})}),(0,l.jsxs)("div",{children:[(0,l.jsx)("div",{className:"text-sm font-medium text-gray-900",children:"Built a useful guardrail?"}),(0,l.jsx)("div",{className:"text-xs text-gray-600",children:"Share it with the community and help others build faster"})]})]}),(0,l.jsx)(e6.Button,{size:"xs",onClick:()=>window.open("https://github.com/BerriAI/litellm-guardrails","_blank"),icon:t$.ExportOutlined,className:"bg-blue-600 hover:bg-blue-700 text-white border-0",children:"Contribute Template"})]})]}),(0,l.jsxs)("div",{className:"w-[300px] flex-shrink-0 overflow-auto border-l border-gray-200 pl-6",children:[(0,l.jsxs)("div",{className:"flex items-center gap-2 mb-3",children:[(0,l.jsx)(tc.CodeOutlined,{className:"text-blue-500"}),(0,l.jsx)("span",{className:"font-semibold text-gray-700",children:"Available Primitives"})]}),(0,l.jsx)("p",{className:"text-xs text-gray-500 mb-3",children:"Click to copy functions to clipboard"}),(0,l.jsx)($.Collapse,{defaultActiveKey:["Return Values"],className:"primitives-collapse bg-transparent border-0",expandIconPosition:"end",children:Object.entries(tG).map(([e,t])=>(0,l.jsx)(tE,{header:(0,l.jsx)("span",{className:"text-sm font-medium text-gray-700",children:e}),className:"bg-white mb-2 rounded-lg border border-gray-200",children:(0,l.jsx)("div",{className:"space-y-2",children:t.map(e=>(0,l.jsx)("button",{onClick:()=>G(e.name),className:`w-full text-left px-2 py-2 rounded transition-colors ${F===e.name?"bg-green-100":"bg-gray-50 hover:bg-blue-50"}`,children:F===e.name?(0,l.jsxs)("span",{className:"flex items-center gap-1 text-xs font-mono text-green-700",children:[(0,l.jsx)(tA.CheckCircleOutlined,{})," Copied!"]}):(0,l.jsxs)(l.Fragment,{children:[(0,l.jsx)("div",{className:"text-xs font-mono text-gray-800",children:e.name}),(0,l.jsx)("div",{className:"text-[10px] text-gray-500 mt-0.5",children:e.desc})]})},e.name))})},e))})]})]}),(0,l.jsxs)("div",{className:"flex items-center justify-between pt-4 mt-4 border-t border-gray-200",children:[(0,l.jsx)("span",{className:"text-xs text-gray-400",children:"Changes are auto-saved to local draft"}),(0,l.jsxs)("div",{className:"flex items-center gap-3",children:[(0,l.jsx)(e6.Button,{variant:"secondary",onClick:t,children:"Cancel"}),(0,l.jsx)(e6.Button,{onClick:z,loading:v,disabled:v||!d.trim(),icon:tL.SaveOutlined,children:o?"Update Guardrail":"Save Guardrail"})]})]})]}),(0,l.jsx)("style",{children:` + .custom-code-modal .ant-modal-content { + padding: 24px; + } + .custom-code-modal .ant-modal-close { + top: 20px; + right: 20px; + } + .primitives-collapse .ant-collapse-item { + border: none !important; + } + .primitives-collapse .ant-collapse-header { + padding: 8px 12px !important; + } + .primitives-collapse .ant-collapse-content-box { + padding: 8px 12px !important; + } + `})]})};e.s(["default",0,({guardrailId:e,onClose:t,accessToken:a,isAdmin:s})=>{let o,[d,g]=(0,m.useState)(null),[x,h]=(0,m.useState)(null),[f,y]=(0,m.useState)(!0),[j,_]=(0,m.useState)(!1),[b]=r.Form.useForm(),[v,w]=(0,m.useState)([]),[N,C]=(0,m.useState)({}),[S,k]=(0,m.useState)(null),[I,A]=(0,m.useState)({}),[O,T]=(0,m.useState)(!1),P={rules:[],default_action:"deny",on_disallowed_action:"block",violation_message_template:""},[L,B]=(0,m.useState)(P),[F,$]=(0,m.useState)(!1),[E,M]=(0,m.useState)(!1),R=m.default.useRef({patterns:[],blockedWords:[],categories:[]}),G=(0,m.useCallback)((e,t,a,l,r)=>{R.current={patterns:e,blockedWords:t,categories:a||[],competitorIntentEnabled:l,competitorIntentConfig:r}},[]),z=async()=>{try{if(y(!0),!a)return;let t=await (0,p.getGuardrailInfo)(a,e);if(g(t),t.litellm_params?.pii_entities_config){let e=t.litellm_params.pii_entities_config;if(w([]),C({}),Object.keys(e).length>0){let t=[],a={};Object.entries(e).forEach(([e,l])=>{t.push(e),a[e]="string"==typeof l?l:"MASK"}),w(t),C(a)}}else w([]),C({})}catch(e){u.default.fromBackend("Failed to load guardrail information"),console.error("Error fetching guardrail info:",e)}finally{y(!1)}},D=async()=>{try{if(!a)return;let e=await (0,p.getGuardrailProviderSpecificParams)(a);h(e)}catch(e){console.error("Error fetching guardrail provider specific params:",e)}},K=async()=>{try{if(!a)return;let e=await (0,p.getGuardrailUISettings)(a);k(e)}catch(e){console.error("Error fetching guardrail UI settings:",e)}};(0,m.useEffect)(()=>{D()},[a]),(0,m.useEffect)(()=>{z(),K()},[e,a]),(0,m.useEffect)(()=>{if(d&&b){let e={...d.litellm_params||{}};delete e.skip_system_message_in_guardrail,delete e.skip_tool_message_in_guardrail,b.setFieldsValue({guardrail_name:d.guardrail_name,...e,skip_system_message_choice:ec(d.litellm_params?.skip_system_message_in_guardrail),skip_tool_message_choice:eu(d.litellm_params?.skip_tool_message_in_guardrail),guardrail_info:d.guardrail_info?JSON.stringify(d.guardrail_info,null,2):"",...d.litellm_params?.optional_params&&{optional_params:d.litellm_params.optional_params}})}},[d,x,b]);let q=(0,m.useCallback)(()=>{d?.litellm_params?.guardrail==="tool_permission"?B({rules:d.litellm_params?.rules||[],default_action:(d.litellm_params?.default_action||"deny").toLowerCase(),on_disallowed_action:(d.litellm_params?.on_disallowed_action||"block").toLowerCase(),violation_message_template:d.litellm_params?.violation_message_template||""}):B(P),$(!1)},[d]);(0,m.useEffect)(()=>{q()},[q]);let H=async t=>{try{if(!a)return;let o={litellm_params:{}};t.guardrail_name!==d.guardrail_name&&(o.guardrail_name=t.guardrail_name),t.default_on!==d.litellm_params?.default_on&&(o.litellm_params.default_on=t.default_on);let c=ec(d.litellm_params?.skip_system_message_in_guardrail),m=t.skip_system_message_choice;void 0!==m&&m!==c&&("inherit"===m?o.litellm_params.skip_system_message_in_guardrail=null:"yes"===m?o.litellm_params.skip_system_message_in_guardrail=!0:o.litellm_params.skip_system_message_in_guardrail=!1);let g=eu(d.litellm_params?.skip_tool_message_in_guardrail),h=t.skip_tool_message_choice;void 0!==h&&h!==g&&("inherit"===h?o.litellm_params.skip_tool_message_in_guardrail=null:"yes"===h?o.litellm_params.skip_tool_message_in_guardrail=!0:o.litellm_params.skip_tool_message_in_guardrail=!1);let f=d.guardrail_info,y=t.guardrail_info?JSON.parse(t.guardrail_info):void 0;JSON.stringify(f)!==JSON.stringify(y)&&(o.guardrail_info=y);let j=d.litellm_params?.pii_entities_config||{},b={};if(v.forEach(e=>{b[e]=N[e]||"MASK"}),JSON.stringify(j)!==JSON.stringify(b)&&(o.litellm_params.pii_entities_config=b),d.litellm_params?.guardrail==="litellm_content_filter"&&O){var l,r,i,s,n;let e,t=(l=R.current.patterns||[],r=R.current.blockedWords||[],i=R.current.categories||[],s=R.current.competitorIntentEnabled,n=R.current.competitorIntentConfig,e={patterns:l.map(e=>({pattern_type:"prebuilt"===e.type?"prebuilt":"regex",pattern_name:"prebuilt"===e.type?e.name:void 0,pattern:"custom"===e.type?e.pattern:void 0,name:e.name,action:e.action})),blocked_words:r.map(e=>({keyword:e.keyword,action:e.action,description:e.description}))},void 0!==i&&(e.categories=i.map(e=>({category:e.category,enabled:!0,action:e.action,severity_threshold:e.severity_threshold||"medium"}))),s&&n&&n.brand_self.length>0&&(e.competitor_intent_config={competitor_intent_type:n.competitor_intent_type,brand_self:n.brand_self,locations:n.locations?.length?n.locations:void 0,competitors:"generic"===n.competitor_intent_type&&n.competitors?.length?n.competitors:void 0,policy:n.policy,threshold_high:n.threshold_high,threshold_medium:n.threshold_medium,threshold_low:n.threshold_low}),e);o.litellm_params.patterns=t.patterns,o.litellm_params.blocked_words=t.blocked_words,o.litellm_params.categories=t.categories,o.litellm_params.competitor_intent_config=t.competitor_intent_config??null}if(d.litellm_params?.guardrail==="tool_permission"){let e=d.litellm_params?.rules||[],t=L.rules||[],a=JSON.stringify(e)!==JSON.stringify(t),l=(d.litellm_params?.default_action||"deny").toLowerCase(),r=(L.default_action||"deny").toLowerCase(),i=l!==r,s=(d.litellm_params?.on_disallowed_action||"block").toLowerCase(),n=(L.on_disallowed_action||"block").toLowerCase(),c=s!==n,m=d.litellm_params?.violation_message_template||"",u=L.violation_message_template||"",p=m!==u;(F||a||i||c||p)&&(o.litellm_params.rules=t,o.litellm_params.default_action=r,o.litellm_params.on_disallowed_action=n,o.litellm_params.violation_message_template=u||null)}let w=Object.keys(ea).find(e=>ea[e]===d.litellm_params?.guardrail);console.log("values: ",JSON.stringify(t)),console.log("currentProvider: ",w);let C=d.litellm_params?.guardrail==="tool_permission";if(x&&w&&!C){let e=x[ea[w]?.toLowerCase()]||{},a=new Set;console.log("providerSpecificParams: ",JSON.stringify(e)),Object.keys(e).forEach(e=>{"optional_params"!==e&&a.add(e)}),e.optional_params&&e.optional_params.fields&&Object.keys(e.optional_params.fields).forEach(e=>{a.add(e)}),console.log("allowedParams: ",a),a.forEach(e=>{if("patterns"===e||"blocked_words"===e||"categories"===e)return;let a=t[e];(null==a||""===a)&&(a=t.optional_params?.[e]);let l=d.litellm_params?.[e];JSON.stringify(a)!==JSON.stringify(l)&&(null!=a&&""!==a?o.litellm_params[e]=a:null!=l&&""!==l&&(o.litellm_params[e]=null))})}if(0===Object.keys(o.litellm_params).length&&delete o.litellm_params,0===Object.keys(o).length){u.default.info("No changes detected"),_(!1);return}await (0,p.updateGuardrailCall)(a,e,o),u.default.success("Guardrail updated successfully"),T(!1),z(),_(!1)}catch(e){console.error("Error updating guardrail:",e),u.default.fromBackend("Failed to update guardrail")}};if(f)return(0,l.jsx)("div",{className:"p-4",children:"Loading..."});if(!d)return(0,l.jsx)("div",{className:"p-4",children:"Guardrail not found"});let J=e=>e?new Date(e).toLocaleString():"-",{logo:U,displayName:W}=ed(d.litellm_params?.guardrail||""),V=async(e,t)=>{await (0,td.copyToClipboard)(e)&&(A(e=>({...e,[t]:!0})),setTimeout(()=>{A(e=>({...e,[t]:!1}))},2e3))},Y="config"===d.guardrail_definition_location;return(0,l.jsxs)("div",{className:"p-4",children:[(0,l.jsxs)("div",{children:[(0,l.jsx)(c.Button,{type:"text",icon:(0,l.jsx)(tm.ArrowLeftIcon,{className:"w-4 h-4"}),onClick:t,className:"mb-4",children:"Back to Guardrails"}),(0,l.jsx)(ty.Title,{children:d.guardrail_name||"Unnamed Guardrail"}),(0,l.jsxs)("div",{className:"flex items-center cursor-pointer",children:[(0,l.jsx)(ez.Text,{className:"text-gray-500 font-mono",children:d.guardrail_id}),(0,l.jsx)(c.Button,{type:"text",size:"small",icon:I["guardrail-id"]?(0,l.jsx)(tj.CheckIcon,{size:12}):(0,l.jsx)(t_.CopyIcon,{size:12}),onClick:()=>V(d.guardrail_id,"guardrail-id"),className:`left-2 z-10 transition-all duration-200 ${I["guardrail-id"]?"text-green-600 bg-green-50 border-green-200":"text-gray-500 hover:text-gray-700 hover:bg-gray-100"}`})]})]}),(0,l.jsxs)(tg.TabGroup,{children:[(0,l.jsxs)(tx.TabList,{className:"mb-4",children:[(0,l.jsx)(tp.Tab,{children:"Overview"},"overview"),s?(0,l.jsx)(tp.Tab,{children:"Settings"},"settings"):(0,l.jsx)(l.Fragment,{})]}),(0,l.jsxs)(tf.TabPanels,{children:[(0,l.jsxs)(th.TabPanel,{children:[(0,l.jsxs)(tu.Grid,{numItems:1,numItemsSm:2,numItemsLg:3,className:"gap-6",children:[(0,l.jsxs)(eG.Card,{children:[(0,l.jsx)(ez.Text,{children:"Provider"}),(0,l.jsxs)("div",{className:"mt-2 flex items-center space-x-2",children:[U&&(0,l.jsx)("img",{src:U,alt:`${W} logo`,className:"w-6 h-6",onError:e=>{e.target.style.display="none"}}),(0,l.jsx)(ty.Title,{children:W})]})]}),(0,l.jsxs)(eG.Card,{children:[(0,l.jsx)(ez.Text,{children:"Mode"}),(0,l.jsxs)("div",{className:"mt-2",children:[(0,l.jsx)(ty.Title,{children:d.litellm_params?.mode||"-"}),(0,l.jsx)(te.Badge,{color:d.litellm_params?.default_on?"green":"gray",children:d.litellm_params?.default_on?"Default On":"Default Off"})]})]}),(0,l.jsxs)(eG.Card,{children:[(0,l.jsx)(ez.Text,{children:"Created At"}),(0,l.jsxs)("div",{className:"mt-2",children:[(0,l.jsx)(ty.Title,{children:J(d.created_at)}),(0,l.jsxs)(ez.Text,{children:["Last Updated: ",J(d.updated_at)]})]})]})]}),d.litellm_params?.pii_entities_config&&Object.keys(d.litellm_params.pii_entities_config).length>0&&(0,l.jsx)(eG.Card,{className:"mt-6",children:(0,l.jsxs)("div",{className:"flex justify-between items-center",children:[(0,l.jsx)(ez.Text,{className:"font-medium",children:"PII Protection"}),(0,l.jsxs)(te.Badge,{color:"blue",children:[Object.keys(d.litellm_params.pii_entities_config).length," PII entities configured"]})]})}),d.litellm_params?.pii_entities_config&&Object.keys(d.litellm_params.pii_entities_config).length>0&&(0,l.jsxs)(eG.Card,{className:"mt-6",children:[(0,l.jsx)(ez.Text,{className:"mb-4 text-lg font-semibold",children:"PII Entity Configuration"}),(0,l.jsxs)("div",{className:"border rounded-lg overflow-hidden shadow-sm",children:[(0,l.jsxs)("div",{className:"bg-gray-50 px-5 py-3 border-b flex",children:[(0,l.jsx)(ez.Text,{className:"flex-1 font-semibold text-gray-700",children:"Entity Type"}),(0,l.jsx)(ez.Text,{className:"flex-1 font-semibold text-gray-700",children:"Configuration"})]}),(0,l.jsx)("div",{className:"max-h-[400px] overflow-y-auto",children:Object.entries(d.litellm_params?.pii_entities_config).map(([e,t])=>(0,l.jsxs)("div",{className:"px-5 py-3 flex border-b hover:bg-gray-50 transition-colors",children:[(0,l.jsx)(ez.Text,{className:"flex-1 font-medium text-gray-900",children:e}),(0,l.jsx)(ez.Text,{className:"flex-1",children:(0,l.jsxs)("span",{className:`inline-flex items-center gap-1.5 ${"MASK"===t?"text-blue-600":"text-red-600"}`,children:["MASK"===t?(0,l.jsx)(eS.default,{}):(0,l.jsx)(ek.StopOutlined,{}),String(t)]})})]},e))})]})]}),d.litellm_params?.guardrail==="tool_permission"&&(0,l.jsx)(eG.Card,{className:"mt-6",children:(0,l.jsx)(eJ,{value:L,disabled:!0})}),d.litellm_params?.guardrail==="custom_code"&&d.litellm_params?.custom_code&&(0,l.jsxs)(eG.Card,{className:"mt-6",children:[(0,l.jsxs)("div",{className:"flex justify-between items-center mb-4",children:[(0,l.jsxs)("div",{className:"flex items-center gap-2",children:[(0,l.jsx)(tc.CodeOutlined,{className:"text-blue-500"}),(0,l.jsx)(ez.Text,{className:"font-medium text-lg",children:"Custom Code"})]}),s&&!Y&&(0,l.jsx)(c.Button,{size:"small",icon:(0,l.jsx)(tc.CodeOutlined,{}),onClick:()=>M(!0),children:"Edit Code"})]}),(0,l.jsx)("div",{className:"relative rounded-lg overflow-hidden border border-gray-700 bg-[#1e1e1e]",children:(0,l.jsx)("pre",{className:"p-4 text-sm text-gray-200 overflow-x-auto",style:{fontFamily:"'Fira Code', 'Monaco', 'Consolas', monospace"},children:(0,l.jsx)("code",{children:d.litellm_params.custom_code})})})]}),(0,l.jsx)(tk,{guardrailData:d,guardrailSettings:S,isEditing:!1,accessToken:a})]}),s&&(0,l.jsx)(th.TabPanel,{children:(0,l.jsxs)(eG.Card,{children:[(0,l.jsxs)("div",{className:"flex justify-between items-center mb-4",children:[(0,l.jsx)(ty.Title,{children:"Guardrail Settings"}),Y&&(0,l.jsx)(eb.Tooltip,{title:"Guardrail is defined in the config file and cannot be edited.",children:(0,l.jsx)(eq.InfoCircleOutlined,{})}),!j&&!Y&&(d.litellm_params?.guardrail==="custom_code"?(0,l.jsx)(c.Button,{icon:(0,l.jsx)(tc.CodeOutlined,{}),onClick:()=>M(!0),children:"Edit Code"}):(0,l.jsx)(c.Button,{onClick:()=>_(!0),children:"Edit Settings"}))]}),j?(0,l.jsxs)(r.Form,{form:b,onFinish:H,initialValues:{guardrail_name:d.guardrail_name,...(o={...d.litellm_params||{}},delete o.skip_system_message_in_guardrail,delete o.skip_tool_message_in_guardrail,o),skip_system_message_choice:ec(d.litellm_params?.skip_system_message_in_guardrail),skip_tool_message_choice:eu(d.litellm_params?.skip_tool_message_in_guardrail),guardrail_info:d.guardrail_info?JSON.stringify(d.guardrail_info,null,2):"",...d.litellm_params?.optional_params&&{optional_params:d.litellm_params.optional_params}},layout:"vertical",children:[(0,l.jsx)(r.Form.Item,{label:"Guardrail Name",name:"guardrail_name",rules:[{required:!0,message:"Please input a guardrail name"}],children:(0,l.jsx)(i.Input,{placeholder:"Enter guardrail name"})}),(0,l.jsx)(r.Form.Item,{label:"Default On",name:"default_on",children:(0,l.jsxs)(n.Select,{children:[(0,l.jsx)(n.Select.Option,{value:!0,children:"Yes"}),(0,l.jsx)(n.Select.Option,{value:!1,children:"No"})]})}),(0,l.jsx)(r.Form.Item,{label:"Skip system messages in guardrail",name:"skip_system_message_choice",tooltip:"Unified guardrails: omit role: system from guardrail input (LLM still gets full messages). Use global default follows litellm_settings.skip_system_message_in_guardrail.",children:(0,l.jsxs)(n.Select,{children:[(0,l.jsx)(n.Select.Option,{value:"inherit",children:"Use global default"}),(0,l.jsx)(n.Select.Option,{value:"yes",children:"Yes — exclude from guardrail scan"}),(0,l.jsx)(n.Select.Option,{value:"no",children:"No — always include in scan"})]})}),(0,l.jsx)(r.Form.Item,{label:"Skip tool messages in guardrail",name:"skip_tool_message_choice",tooltip:"Unified guardrails: omit role: tool from guardrail input (LLM still gets full messages). Use global default follows litellm_settings.skip_tool_message_in_guardrail.",children:(0,l.jsxs)(n.Select,{children:[(0,l.jsx)(n.Select.Option,{value:"inherit",children:"Use global default"}),(0,l.jsx)(n.Select.Option,{value:"yes",children:"Yes — exclude from guardrail scan"}),(0,l.jsx)(n.Select.Option,{value:"no",children:"No — always include in scan"})]})}),d.litellm_params?.guardrail==="presidio"&&(0,l.jsxs)(l.Fragment,{children:[(0,l.jsx)(eD.Divider,{orientation:"left",children:"PII Protection"}),(0,l.jsx)("div",{className:"mb-6",children:S&&(0,l.jsx)(eR,{entities:S.supported_entities,actions:S.supported_actions,selectedEntities:v,selectedActions:N,onEntitySelect:e=>{w(t=>t.includes(e)?t.filter(t=>t!==e):[...t,e])},onActionSelect:(e,t)=>{C(a=>({...a,[e]:t}))},entityCategories:S.pii_entity_categories})})]}),(0,l.jsx)(tk,{guardrailData:d,guardrailSettings:S,isEditing:!0,accessToken:a,onDataChange:G,onUnsavedChanges:T}),(d.litellm_params?.guardrail==="tool_permission"||x)&&(0,l.jsx)(eD.Divider,{orientation:"left",children:"Provider Settings"}),d.litellm_params?.guardrail==="tool_permission"?(0,l.jsx)(eJ,{value:L,onChange:B}):(0,l.jsxs)(l.Fragment,{children:[(0,l.jsx)(e_,{selectedProvider:Object.keys(ea).find(e=>ea[e]===d.litellm_params?.guardrail)||null,accessToken:a,providerParams:x,value:d.litellm_params}),x&&(()=>{let e=Object.keys(ea).find(e=>ea[e]===d.litellm_params?.guardrail);if(!e)return null;let t=x[ea[e]?.toLowerCase()];return t&&t.optional_params?(0,l.jsx)(ef,{optionalParams:t.optional_params,parentFieldKey:"optional_params",values:d.litellm_params}):null})()]}),(0,l.jsx)(eD.Divider,{orientation:"left",children:"Advanced Settings"}),(0,l.jsx)(r.Form.Item,{label:"Guardrail Information",name:"guardrail_info",children:(0,l.jsx)(i.Input.TextArea,{rows:5})}),(0,l.jsxs)("div",{className:"flex justify-end gap-2 mt-6",children:[(0,l.jsx)(c.Button,{onClick:()=>{_(!1),T(!1),q()},children:"Cancel"}),(0,l.jsx)(c.Button,{type:"primary",htmlType:"submit",children:"Save Changes"})]})]}):(0,l.jsxs)("div",{className:"space-y-4",children:[(0,l.jsxs)("div",{children:[(0,l.jsx)(ez.Text,{className:"font-medium",children:"Guardrail ID"}),(0,l.jsx)("div",{className:"font-mono",children:d.guardrail_id})]}),(0,l.jsxs)("div",{children:[(0,l.jsx)(ez.Text,{className:"font-medium",children:"Guardrail Name"}),(0,l.jsx)("div",{children:d.guardrail_name||"Unnamed Guardrail"})]}),(0,l.jsxs)("div",{children:[(0,l.jsx)(ez.Text,{className:"font-medium",children:"Provider"}),(0,l.jsx)("div",{children:W})]}),(0,l.jsxs)("div",{children:[(0,l.jsx)(ez.Text,{className:"font-medium",children:"Mode"}),(0,l.jsx)("div",{children:d.litellm_params?.mode||"-"})]}),(0,l.jsxs)("div",{children:[(0,l.jsx)(ez.Text,{className:"font-medium",children:"Default On"}),(0,l.jsx)(te.Badge,{color:d.litellm_params?.default_on?"green":"gray",children:d.litellm_params?.default_on?"Yes":"No"})]}),d.litellm_params?.pii_entities_config&&Object.keys(d.litellm_params.pii_entities_config).length>0&&(0,l.jsxs)("div",{children:[(0,l.jsx)(ez.Text,{className:"font-medium",children:"PII Protection"}),(0,l.jsx)("div",{className:"mt-2",children:(0,l.jsxs)(te.Badge,{color:"blue",children:[Object.keys(d.litellm_params.pii_entities_config).length," PII entities configured"]})})]}),(0,l.jsxs)("div",{children:[(0,l.jsx)(ez.Text,{className:"font-medium",children:"Created At"}),(0,l.jsx)("div",{children:J(d.created_at)})]}),(0,l.jsxs)("div",{children:[(0,l.jsx)(ez.Text,{className:"font-medium",children:"Last Updated"}),(0,l.jsx)("div",{children:J(d.updated_at)})]}),d.litellm_params?.guardrail==="tool_permission"&&(0,l.jsx)(eJ,{value:L,disabled:!0})]})]})})]})]}),(0,l.jsx)(tD,{visible:E,onClose:()=>M(!1),onSuccess:()=>{M(!1),z()},accessToken:a,editData:d?{guardrail_id:d.guardrail_id,guardrail_name:d.guardrail_name,litellm_params:d.litellm_params}:null})]})}],969641);var tK=e.i(573421),tq=e.i(19732),tH=e.i(928685),tJ=e.i(166406),tU=e.i(637235),tW=e.i(755151),tV=e.i(240647);let{Text:tY}=d.Typography,tQ=function({results:e,errors:t}){let[a,r]=(0,m.useState)(new Set),i=e=>{let t=new Set(a);t.has(e)?t.delete(e):t.add(e),r(t)},s=async e=>{try{if(navigator.clipboard&&window.isSecureContext)return await navigator.clipboard.writeText(e),!0;{let t=document.createElement("textarea");t.value=e,t.style.position="fixed",t.style.opacity="0",document.body.appendChild(t),t.focus(),t.select();let a=document.execCommand("copy");if(document.body.removeChild(t),!a)throw Error("execCommand failed");return!0}}catch(e){return console.error("Copy failed:",e),!1}};return e||t?(0,l.jsxs)("div",{className:"space-y-3 pt-4 border-t border-gray-200",children:[(0,l.jsx)("h3",{className:"text-sm font-semibold text-gray-900",children:"Results"}),e&&e.map(e=>{let t=a.has(e.guardrailName);return(0,l.jsx)(eG.Card,{className:"bg-green-50 border-green-200",children:(0,l.jsxs)("div",{className:"space-y-3",children:[(0,l.jsxs)("div",{className:"flex items-center justify-between",children:[(0,l.jsxs)("div",{className:"flex items-center space-x-2 cursor-pointer flex-1",onClick:()=>i(e.guardrailName),children:[t?(0,l.jsx)(tV.RightOutlined,{className:"text-gray-500 text-xs"}):(0,l.jsx)(tW.DownOutlined,{className:"text-gray-500 text-xs"}),(0,l.jsx)(tA.CheckCircleOutlined,{className:"text-green-600 text-lg"}),(0,l.jsx)("span",{className:"text-sm font-medium text-green-800",children:e.guardrailName})]}),(0,l.jsxs)("div",{className:"flex items-center gap-3",children:[(0,l.jsxs)("div",{className:"flex items-center space-x-1 text-xs text-gray-600",children:[(0,l.jsx)(tU.ClockCircleOutlined,{}),(0,l.jsxs)("span",{className:"font-medium",children:[e.latency,"ms"]})]}),!t&&(0,l.jsx)(e6.Button,{size:"xs",variant:"secondary",icon:tJ.CopyOutlined,onClick:async()=>{await s(e.response_text)?u.default.success("Result copied to clipboard"):u.default.fromBackend("Failed to copy result")},children:"Copy"})]})]}),!t&&(0,l.jsxs)(l.Fragment,{children:[(0,l.jsxs)("div",{className:"bg-white border border-green-200 rounded p-3",children:[(0,l.jsx)("label",{className:"text-xs font-medium text-gray-600 mb-2 block",children:"Output Text"}),(0,l.jsx)("div",{className:"font-mono text-sm text-gray-900 whitespace-pre-wrap break-words",children:e.response_text})]}),(0,l.jsxs)("div",{className:"text-xs text-gray-600",children:[(0,l.jsx)("span",{className:"font-medium",children:"Characters:"})," ",e.response_text.length]})]})]})},e.guardrailName)}),t&&t.map(e=>{let t=a.has(e.guardrailName);return(0,l.jsx)(eG.Card,{className:"bg-red-50 border-red-200",children:(0,l.jsxs)("div",{className:"flex items-start space-x-2",children:[(0,l.jsx)("div",{className:"cursor-pointer mt-0.5",onClick:()=>i(e.guardrailName),children:t?(0,l.jsx)(tV.RightOutlined,{className:"text-gray-500 text-xs"}):(0,l.jsx)(tW.DownOutlined,{className:"text-gray-500 text-xs"})}),(0,l.jsx)("div",{className:"text-red-600 mt-0.5",children:(0,l.jsx)("svg",{className:"w-5 h-5",fill:"currentColor",viewBox:"0 0 20 20",children:(0,l.jsx)("path",{fillRule:"evenodd",d:"M10 18a8 8 0 100-16 8 8 0 000 16zM8.707 7.293a1 1 0 00-1.414 1.414L8.586 10l-1.293 1.293a1 1 0 101.414 1.414L10 11.414l1.293 1.293a1 1 0 001.414-1.414L11.414 10l1.293-1.293a1 1 0 00-1.414-1.414L10 8.586 8.707 7.293z",clipRule:"evenodd"})})}),(0,l.jsxs)("div",{className:"flex-1",children:[(0,l.jsxs)("div",{className:"flex items-center justify-between mb-1",children:[(0,l.jsxs)("p",{className:"text-sm font-medium text-red-800 cursor-pointer",onClick:()=>i(e.guardrailName),children:[e.guardrailName," - Error"]}),(0,l.jsxs)("div",{className:"flex items-center space-x-1 text-xs text-gray-600",children:[(0,l.jsx)(tU.ClockCircleOutlined,{}),(0,l.jsxs)("span",{className:"font-medium",children:[e.latency,"ms"]})]})]}),!t&&(0,l.jsx)("p",{className:"text-sm text-red-700 mt-1",children:e.error.message})]})]})},e.guardrailName)})]}):null},{TextArea:tX}=i.Input,{Text:tZ}=d.Typography,t0=function({guardrailNames:e,onSubmit:t,isLoading:a,results:r,errors:i,onClose:s}){let[n,o]=(0,m.useState)(""),d=()=>{n.trim()?t(n):u.default.fromBackend("Please enter text to test")},c=async e=>{try{if(navigator.clipboard&&window.isSecureContext)return await navigator.clipboard.writeText(e),!0;{let t=document.createElement("textarea");t.value=e,t.style.position="fixed",t.style.opacity="0",document.body.appendChild(t),t.focus(),t.select();let a=document.execCommand("copy");if(document.body.removeChild(t),!a)throw Error("execCommand failed");return!0}}catch(e){return console.error("Copy failed:",e),!1}},p=async()=>{await c(n)?u.default.success("Input copied to clipboard"):u.default.fromBackend("Failed to copy input")};return(0,l.jsxs)("div",{className:"space-y-4 h-full flex flex-col",children:[(0,l.jsx)("div",{className:"flex items-center justify-between pb-3 border-b border-gray-200",children:(0,l.jsx)("div",{className:"flex items-center space-x-3",children:(0,l.jsxs)("div",{className:"flex-1 min-w-0",children:[(0,l.jsxs)("div",{className:"flex items-center space-x-2 mb-1",children:[(0,l.jsx)("h2",{className:"text-lg font-semibold text-gray-900",children:"Test Guardrails:"}),(0,l.jsx)("div",{className:"flex flex-wrap gap-2",children:e.map(e=>(0,l.jsx)("div",{className:"inline-flex items-center space-x-1 bg-blue-50 px-3 py-1 rounded-md border border-blue-200",children:(0,l.jsx)("span",{className:"font-mono text-blue-700 font-medium text-sm",children:e})},e))})]}),(0,l.jsxs)("p",{className:"text-sm text-gray-500",children:["Test ",e.length>1?"guardrails":"guardrail"," and compare results"]})]})})}),(0,l.jsxs)("div",{className:"flex-1 overflow-auto space-y-4",children:[(0,l.jsxs)("div",{className:"space-y-3",children:[(0,l.jsxs)("div",{children:[(0,l.jsxs)("div",{className:"flex justify-between items-center mb-2",children:[(0,l.jsxs)("div",{className:"flex items-center gap-2",children:[(0,l.jsx)("label",{className:"text-sm font-medium text-gray-700",children:"Input Text"}),(0,l.jsx)(eb.Tooltip,{title:"Press Enter to submit. Use Shift+Enter for new line.",children:(0,l.jsx)(eq.InfoCircleOutlined,{className:"text-gray-400 cursor-help"})})]}),n&&(0,l.jsx)(e6.Button,{size:"xs",variant:"secondary",icon:tJ.CopyOutlined,onClick:p,children:"Copy Input"})]}),(0,l.jsx)(tX,{value:n,onChange:e=>o(e.target.value),onKeyDown:e=>{"Enter"!==e.key||e.shiftKey||e.ctrlKey||e.metaKey||(e.preventDefault(),d())},placeholder:"Enter text to test with guardrails...",rows:8,className:"font-mono text-sm"}),(0,l.jsxs)("div",{className:"flex justify-between items-center mt-1",children:[(0,l.jsxs)(tZ,{className:"text-xs text-gray-500",children:["Press ",(0,l.jsx)("kbd",{className:"px-1 py-0.5 bg-gray-100 border border-gray-300 rounded text-xs",children:"Enter"})," to submit • ",(0,l.jsx)("kbd",{className:"px-1 py-0.5 bg-gray-100 border border-gray-300 rounded text-xs",children:"Shift+Enter"})," for new line"]}),(0,l.jsxs)(tZ,{className:"text-xs text-gray-500",children:["Characters: ",n.length]})]})]}),(0,l.jsx)("div",{className:"pt-2",children:(0,l.jsx)(e6.Button,{onClick:d,loading:a,disabled:!n.trim(),className:"w-full",children:a?`Testing ${e.length} guardrail${e.length>1?"s":""}...`:`Test ${e.length} guardrail${e.length>1?"s":""}`})})]}),(0,l.jsx)(tQ,{results:r,errors:i})]})]})};e.s(["default",0,({guardrailsList:e,isLoading:t,accessToken:a,onClose:r})=>{let[s,n]=(0,m.useState)(new Set),[o,c]=(0,m.useState)(""),[g,x]=(0,m.useState)([]),[f,y]=(0,m.useState)([]),[j,_]=(0,m.useState)(!1),b=e.filter(e=>e.guardrail_name?.toLowerCase().includes(o.toLowerCase())),v=async e=>{if(0===s.size||!a)return;_(!0),x([]),y([]);let t=[],l=[];await Promise.all(Array.from(s).map(async r=>{let i=Date.now();try{let l=await (0,p.applyGuardrail)(a,r,e,null,null),s=Date.now()-i;t.push({guardrailName:r,response_text:l.response_text,latency:s})}catch(t){let e=Date.now()-i;console.error(`Error testing guardrail ${r}:`,t),l.push({guardrailName:r,error:t,latency:e})}})),x(t),y(l),_(!1),t.length>0&&u.default.success(`${t.length} guardrail${t.length>1?"s":""} applied successfully`),l.length>0&&u.default.fromBackend(`${l.length} guardrail${l.length>1?"s":""} failed`)};return(0,l.jsx)("div",{className:"w-full h-[calc(100vh-200px)]",children:(0,l.jsx)(h.Card,{className:"h-full",styles:{body:{padding:0,height:"100%"}},children:(0,l.jsxs)("div",{className:"flex h-full",children:[(0,l.jsxs)("div",{className:"w-1/4 border-r border-gray-200 flex flex-col overflow-hidden",children:[(0,l.jsx)("div",{className:"p-4 border-b border-gray-200",children:(0,l.jsxs)("div",{className:"mb-3",children:[(0,l.jsx)("h3",{className:"text-lg font-semibold mb-3",children:"Guardrails"}),(0,l.jsx)(i.Input,{prefix:(0,l.jsx)(tH.SearchOutlined,{}),placeholder:"Search guardrails...",value:o,onChange:e=>c(e.target.value)})]})}),(0,l.jsx)("div",{className:"flex-1 overflow-auto",children:t?(0,l.jsx)("div",{className:"flex items-center justify-center h-32",children:(0,l.jsx)(ey.Spin,{})}):0===b.length?(0,l.jsx)("div",{className:"p-4",children:(0,l.jsx)(eK.Empty,{description:o?"No guardrails match your search":"No guardrails available"})}):(0,l.jsx)(tK.List,{dataSource:b,renderItem:e=>(0,l.jsx)(tK.List.Item,{onClick:()=>{var t;let a;e.guardrail_name&&(t=e.guardrail_name,(a=new Set(s)).has(t)?a.delete(t):a.add(t),n(a))},style:{paddingLeft:24,paddingRight:16},className:`cursor-pointer hover:bg-gray-50 transition-colors ${s.has(e.guardrail_name||"")?"bg-blue-50 border-l-4 border-l-blue-500":"border-l-4 border-l-transparent"}`,children:(0,l.jsx)(tK.List.Item.Meta,{title:(0,l.jsxs)("div",{className:"flex items-center space-x-2",children:[(0,l.jsx)(tq.ExperimentOutlined,{className:"text-gray-400"}),(0,l.jsx)("span",{className:"font-medium text-gray-900",children:e.guardrail_name})]}),description:(0,l.jsxs)("div",{className:"text-xs space-y-1 mt-1",children:[(0,l.jsxs)("div",{children:[(0,l.jsx)("span",{className:"font-medium",children:"Type: "}),(0,l.jsx)("span",{className:"text-gray-600",children:e.litellm_params.guardrail})]}),(0,l.jsxs)("div",{children:[(0,l.jsx)("span",{className:"font-medium",children:"Mode: "}),(0,l.jsx)("span",{className:"text-gray-600",children:e.litellm_params.mode})]})]})})})})}),(0,l.jsx)("div",{className:"p-3 border-t border-gray-200 bg-gray-50",children:(0,l.jsxs)(d.Typography.Text,{className:"text-xs text-gray-600",children:[s.size," of ",b.length," selected"]})})]}),(0,l.jsxs)("div",{className:"w-3/4 flex flex-col bg-white",children:[(0,l.jsx)("div",{className:"p-4 border-b border-gray-200 flex justify-between items-center",children:(0,l.jsx)(d.Typography.Title,{level:2,className:"text-xl font-semibold mb-0",children:"Guardrail Testing Playground"})}),(0,l.jsx)("div",{className:"flex-1 overflow-auto p-4",children:0===s.size?(0,l.jsxs)("div",{className:"h-full flex flex-col items-center justify-center text-gray-400",children:[(0,l.jsx)(tq.ExperimentOutlined,{style:{fontSize:"48px",marginBottom:"16px"}}),(0,l.jsx)(d.Typography.Paragraph,{className:"text-lg font-medium text-gray-600 mb-2",children:"Select Guardrails to Test"}),(0,l.jsx)(d.Typography.Paragraph,{className:"text-center text-gray-500 max-w-md",children:"Choose one or more guardrails from the left sidebar to start testing and comparing results."})]}):(0,l.jsx)("div",{className:"h-full",children:(0,l.jsx)(t0,{guardrailNames:Array.from(s),onSubmit:v,results:g.length>0?g:null,errors:f.length>0?f:null,isLoading:j,onClose:()=>n(new Set)})})})]})]})})})}],476993),e.s([],824296),e.s(["CustomCodeModal",0,tD],64352);let t1="../ui/assets/logos/",t2=[{id:"cf_denied_financial",name:"Denied Financial Advice",description:"Detects requests for personalized financial advice, investment recommendations, or financial planning.",category:"litellm",subcategory:"Content Category",logo:`${t1}litellm_logo.jpg`,tags:["Content Category","Topic Blocker"],eval:{f1:100,precision:100,recall:100,testCases:207,latency:"<0.1ms"}},{id:"cf_denied_insults",name:"Insults & Personal Attacks",description:"Detects insults, name-calling, and personal attacks directed at the chatbot, staff, or other people.",category:"litellm",subcategory:"Content Category",logo:`${t1}litellm_logo.jpg`,tags:["Content Category","Topic Blocker"],eval:{f1:100,precision:100,recall:100,testCases:299,latency:"<0.1ms"}},{id:"cf_denied_legal",name:"Denied Legal Advice",description:"Detects requests for unauthorized legal advice, case analysis, or legal recommendations.",category:"litellm",subcategory:"Content Category",logo:`${t1}litellm_logo.jpg`,tags:["Content Category","Topic Blocker"]},{id:"cf_denied_medical",name:"Denied Medical Advice",description:"Detects requests for medical diagnosis, treatment recommendations, or health advice.",category:"litellm",subcategory:"Content Category",logo:`${t1}litellm_logo.jpg`,tags:["Content Category","Topic Blocker"]},{id:"cf_harmful_violence",name:"Harmful Violence",description:"Detects content related to violence, criminal planning, attacks, and violent threats.",category:"litellm",subcategory:"Content Category",logo:`${t1}litellm_logo.jpg`,tags:["Content Category","Safety"]},{id:"cf_harmful_self_harm",name:"Harmful Self-Harm",description:"Detects content related to self-harm, suicide, and dangerous self-destructive behavior.",category:"litellm",subcategory:"Content Category",logo:`${t1}litellm_logo.jpg`,tags:["Content Category","Safety"]},{id:"cf_harmful_child_safety",name:"Harmful Child Safety",description:"Detects content that could endanger child safety or exploit minors.",category:"litellm",subcategory:"Content Category",logo:`${t1}litellm_logo.jpg`,tags:["Content Category","Safety"]},{id:"cf_harmful_illegal_weapons",name:"Harmful Illegal Weapons",description:"Detects content related to illegal weapons manufacturing, distribution, or acquisition.",category:"litellm",subcategory:"Content Category",logo:`${t1}litellm_logo.jpg`,tags:["Content Category","Safety"]},{id:"cf_bias_gender",name:"Bias: Gender",description:"Detects gender-based discrimination, stereotypes, and biased language.",category:"litellm",subcategory:"Content Category",logo:`${t1}litellm_logo.jpg`,tags:["Content Category","Bias"]},{id:"cf_bias_racial",name:"Bias: Racial",description:"Detects racial discrimination, stereotypes, and racially biased content.",category:"litellm",subcategory:"Content Category",logo:`${t1}litellm_logo.jpg`,tags:["Content Category","Bias"]},{id:"cf_bias_religious",name:"Bias: Religious",description:"Detects religious discrimination, intolerance, and religiously biased content.",category:"litellm",subcategory:"Content Category",logo:`${t1}litellm_logo.jpg`,tags:["Content Category","Bias"]},{id:"cf_bias_sexual_orientation",name:"Bias: Sexual Orientation",description:"Detects discrimination based on sexual orientation and related biased content.",category:"litellm",subcategory:"Content Category",logo:`${t1}litellm_logo.jpg`,tags:["Content Category","Bias"]},{id:"cf_prompt_injection_jailbreak",name:"Prompt Injection: Jailbreak",description:"Detects jailbreak attempts designed to bypass AI safety guidelines and restrictions.",category:"litellm",subcategory:"Content Category",logo:`${t1}litellm_logo.jpg`,tags:["Content Category","Prompt Injection"]},{id:"cf_prompt_injection_data_exfil",name:"Prompt Injection: Data Exfiltration",description:"Detects attempts to extract sensitive data through prompt manipulation.",category:"litellm",subcategory:"Content Category",logo:`${t1}litellm_logo.jpg`,tags:["Content Category","Prompt Injection"]},{id:"cf_prompt_injection_sql",name:"Prompt Injection: SQL",description:"Detects SQL injection attempts embedded in prompts.",category:"litellm",subcategory:"Content Category",logo:`${t1}litellm_logo.jpg`,tags:["Content Category","Prompt Injection"]},{id:"cf_prompt_injection_malicious_code",name:"Prompt Injection: Malicious Code",description:"Detects attempts to inject malicious code through prompts.",category:"litellm",subcategory:"Content Category",logo:`${t1}litellm_logo.jpg`,tags:["Content Category","Prompt Injection"]},{id:"cf_prompt_injection_system_prompt",name:"Prompt Injection: System Prompt",description:"Detects attempts to extract or override system prompts.",category:"litellm",subcategory:"Content Category",logo:`${t1}litellm_logo.jpg`,tags:["Content Category","Prompt Injection"]},{id:"cf_toxic_abuse",name:"Toxic & Abusive Language",description:"Detects toxic, abusive, and hateful language across multiple languages (EN, AU, DE, ES, FR).",category:"litellm",subcategory:"Content Category",logo:`${t1}litellm_logo.jpg`,tags:["Content Category","Toxicity"]},{id:"cf_patterns",name:"Pattern Matching",description:"Detect and block sensitive data patterns like SSNs, credit card numbers, API keys, and custom regex patterns.",category:"litellm",subcategory:"Patterns",logo:`${t1}litellm_logo.jpg`,tags:["PII","Regex","Data Protection"]},{id:"cf_keywords",name:"Keyword Blocking",description:"Block or mask content containing specific keywords or phrases. Upload custom word lists or add individual terms.",category:"litellm",subcategory:"Keywords",logo:`${t1}litellm_logo.jpg`,tags:["Keywords","Blocklist"]},{id:"block_code_execution",name:"Block Code Execution",description:"Detects markdown fenced code blocks in requests and responses. Block or mask executable code (e.g. Python, JavaScript, Bash) by language with configurable confidence.",category:"litellm",subcategory:"Code Safety",logo:`${t1}litellm_logo.jpg`,tags:["Code","Safety","Prompt Injection"]},{id:"cf_competitor_intent",name:"Competitor Name Blocking",description:"Block or reframe competitor comparison and ranking intent. Detect when users ask to compare or recommend competitors (airline or generic competitor lists).",category:"litellm",subcategory:"Content Category",logo:`${t1}litellm_logo.jpg`,tags:["Content Category","Competitor","Topic Blocker"]},{id:"presidio",name:"Presidio PII",description:"Microsoft Presidio for PII detection and anonymization. Supports 30+ entity types with configurable actions.",category:"partner",logo:`${t1}microsoft_azure.svg`,tags:["PII","Microsoft"],providerKey:"PresidioPII"},{id:"bedrock",name:"Bedrock Guardrail",description:"AWS Bedrock Guardrails for content filtering, topic avoidance, and sensitive information detection.",category:"partner",logo:`${t1}bedrock.svg`,tags:["AWS","Content Safety"],providerKey:"Bedrock"},{id:"lakera",name:"Lakera",description:"AI security platform protecting against prompt injections, data leakage, and harmful content.",category:"partner",logo:`${t1}lakeraai.jpeg`,tags:["Security","Prompt Injection"],providerKey:"Lakera"},{id:"openai_moderation",name:"OpenAI Moderation",description:"OpenAI's content moderation API for detecting harmful content across multiple categories.",category:"partner",logo:`${t1}openai_small.svg`,tags:["Content Moderation","OpenAI"]},{id:"google_model_armor",name:"Google Cloud Model Armor",description:"Google Cloud's model protection service for safe and responsible AI deployments.",category:"partner",logo:`${t1}google.svg`,tags:["Google Cloud","Safety"]},{id:"guardrails_ai",name:"Guardrails AI",description:"Open-source framework for adding structural, type, and quality guarantees to LLM outputs.",category:"partner",logo:`${t1}guardrails_ai.jpeg`,tags:["Open Source","Validation"]},{id:"zscaler",name:"Zscaler AI Guard",description:"Enterprise AI security from Zscaler for monitoring and protecting AI/ML workloads.",category:"partner",logo:`${t1}zscaler.svg`,tags:["Enterprise","Security"]},{id:"panw",name:"PANW Prisma AIRS",description:"Palo Alto Networks Prisma AI Runtime Security for securing AI applications in production.",category:"partner",logo:`${t1}palo_alto_networks.jpeg`,tags:["Enterprise","Security"]},{id:"noma",name:"Noma Security",description:"AI security platform for detecting and preventing AI-specific threats and vulnerabilities.",category:"partner",logo:`${t1}noma_security.png`,tags:["Security","Threat Detection"]},{id:"aporia",name:"Aporia AI",description:"Real-time AI guardrails for hallucination detection, topic control, and policy enforcement.",category:"partner",logo:`${t1}aporia.png`,tags:["Hallucination","Policy"]},{id:"aim",name:"AIM Guardrail",description:"AIM Security guardrails for comprehensive AI threat detection and mitigation.",category:"partner",logo:`${t1}aim_security.jpeg`,tags:["Security","Threat Detection"]},{id:"prompt_security",name:"Prompt Security",description:"Protect against prompt injection attacks, data leakage, and other LLM security threats.",category:"partner",logo:`${t1}prompt_security.png`,tags:["Prompt Injection","Security"]},{id:"lasso",name:"Lasso Guardrail",description:"Content moderation and safety guardrails for responsible AI deployments.",category:"partner",logo:`${t1}lasso.png`,tags:["Content Moderation"]},{id:"pangea",name:"Pangea Guardrail",description:"Pangea's AI guardrails for secure, compliant, and trustworthy AI applications.",category:"partner",logo:`${t1}pangea.png`,tags:["Compliance","Security"]},{id:"enkryptai",name:"EnkryptAI",description:"AI security and governance platform for enterprise AI safety and compliance.",category:"partner",logo:`${t1}enkrypt_ai.avif`,tags:["Enterprise","Governance"]},{id:"javelin",name:"Javelin Guardrails",description:"AI gateway with built-in guardrails for secure and compliant AI operations.",category:"partner",logo:`${t1}javelin.png`,tags:["Gateway","Security"]},{id:"pillar",name:"Pillar Guardrail",description:"AI safety platform for monitoring, testing, and securing AI systems.",category:"partner",logo:`${t1}pillar.jpeg`,tags:["Monitoring","Safety"]},{id:"akto",name:"Akto Guardrail",description:"AI security platform from Akto.io with automatic monitoring and guardrails for AI/ML applications.",category:"partner",logo:`${t1}akto.svg`,tags:["Security","Safety","Monitoring"]},{id:"promptguard",name:"PromptGuard",description:"AI security gateway with prompt injection detection, PII redaction, topic filtering, entity blocklists, and hallucination detection. Self-hostable with drop-in proxy integration.",category:"partner",logo:`${t1}promptguard.svg`,tags:["Security","Prompt Injection","PII"],providerKey:"Promptguard",eval:{f1:94.9,precision:100,recall:90.4,testCases:5384,latency:"~150ms"}},{id:"xecguard",name:"XecGuard",description:"CyCraft XecGuard AI security gateway. Multi-policy scanning (prompt injection, harmful content, PII, system-prompt enforcement) plus RAG context grounding.",category:"partner",logo:`${t1}xecguard.svg`,tags:["Security","Policy","Grounding","RAG"],providerKey:"Xecguard"},{id:"deepkeep",name:"DeepKeep AI Firewall",description:"DeepKeep AI Firewall for comprehensive LLM security — prompt injection detection, PII protection, content moderation, and policy enforcement with configurable guardrail pipelines.",category:"partner",logo:`${t1}deepkeep.svg`,tags:["Security","Prompt Injection","PII","Firewall"],providerKey:"Deepkeep"}];e.s(["ALL_CARDS",0,t2],230312)},826910,e=>{"use strict";var t=e.i(201072);e.s(["CheckCircleFilled",()=>t.default])},487304,168118,e=>{"use strict";var t=e.i(843476),a=e.i(271645),l=e.i(464571),r=e.i(326373),i=e.i(653496),s=e.i(755151),n=e.i(646563),o=e.i(245094),d=e.i(764205),c=e.i(185357),m=e.i(782719),u=e.i(708347),p=e.i(969641),g=e.i(476993),x=e.i(727749),h=e.i(127952),f=e.i(180766);e.i(824296);var y=e.i(64352),j=e.i(311451),_=e.i(928685),b=e.i(266537),v=e.i(230312),w=e.i(826910);let N=({src:e,name:l})=>{let[r,i]=(0,a.useState)(!1);return r||!e?(0,t.jsx)("div",{style:{width:28,height:28,borderRadius:6,backgroundColor:"#e5e7eb",display:"flex",alignItems:"center",justifyContent:"center",fontSize:13,fontWeight:600,color:"#6b7280",flexShrink:0},children:l?.charAt(0)||"?"}):(0,t.jsx)("img",{src:e,alt:"",style:{width:28,height:28,borderRadius:6,objectFit:"contain",flexShrink:0},onError:()=>i(!0)})},C=({card:e,onClick:l})=>{let[r,i]=(0,a.useState)(!1);return(0,t.jsxs)("div",{onClick:l,onMouseEnter:()=>i(!0),onMouseLeave:()=>i(!1),style:{borderRadius:12,border:r?"1px solid #93c5fd":"1px solid #e5e7eb",backgroundColor:"#ffffff",padding:"20px 20px 16px 20px",cursor:"pointer",transition:"border-color 0.15s, box-shadow 0.15s",display:"flex",flexDirection:"column",minHeight:170,boxShadow:r?"0 1px 6px rgba(59,130,246,0.08)":"none"},children:[(0,t.jsxs)("div",{style:{display:"flex",alignItems:"center",gap:10,marginBottom:10},children:[(0,t.jsx)(N,{src:e.logo,name:e.name}),(0,t.jsx)("span",{style:{fontSize:14,fontWeight:600,color:"#111827",lineHeight:1.3},children:e.name})]}),(0,t.jsx)("p",{className:"line-clamp-3",style:{fontSize:12,color:"#6b7280",lineHeight:1.6,margin:0,flex:1},children:e.description}),e.eval&&(0,t.jsxs)("div",{style:{marginTop:10,display:"flex",alignItems:"center",gap:4},children:[(0,t.jsx)(w.CheckCircleFilled,{style:{color:"#16a34a",fontSize:12}}),(0,t.jsxs)("span",{style:{fontSize:11,color:"#16a34a",fontWeight:500},children:["F1: ",e.eval.f1,"% · ",e.eval.testCases," test cases"]})]})]})};var S=e.i(447566);let k={cf_denied_financial:{provider:"LitellmContentFilter",categoryName:"denied_financial_advice",guardrailNameSuggestion:"Denied Financial Advice",mode:"pre_call",defaultOn:!1},cf_denied_legal:{provider:"LitellmContentFilter",categoryName:"denied_legal_advice",guardrailNameSuggestion:"Denied Legal Advice",mode:"pre_call",defaultOn:!1},cf_denied_medical:{provider:"LitellmContentFilter",categoryName:"denied_medical_advice",guardrailNameSuggestion:"Denied Medical Advice",mode:"pre_call",defaultOn:!1},cf_denied_insults:{provider:"LitellmContentFilter",categoryName:"denied_insults",guardrailNameSuggestion:"Insults & Personal Attacks",mode:"pre_call",defaultOn:!1},cf_harmful_violence:{provider:"LitellmContentFilter",categoryName:"harmful_violence",guardrailNameSuggestion:"Harmful Violence",mode:"pre_call",defaultOn:!1},cf_harmful_self_harm:{provider:"LitellmContentFilter",categoryName:"harmful_self_harm",guardrailNameSuggestion:"Harmful Self-Harm",mode:"pre_call",defaultOn:!1},cf_harmful_child_safety:{provider:"LitellmContentFilter",categoryName:"harmful_child_safety",guardrailNameSuggestion:"Harmful Child Safety",mode:"pre_call",defaultOn:!1},cf_harmful_illegal_weapons:{provider:"LitellmContentFilter",categoryName:"harmful_illegal_weapons",guardrailNameSuggestion:"Harmful Illegal Weapons",mode:"pre_call",defaultOn:!1},cf_bias_gender:{provider:"LitellmContentFilter",categoryName:"bias_gender",guardrailNameSuggestion:"Bias: Gender",mode:"pre_call",defaultOn:!1},cf_bias_racial:{provider:"LitellmContentFilter",categoryName:"bias_racial",guardrailNameSuggestion:"Bias: Racial",mode:"pre_call",defaultOn:!1},cf_bias_religious:{provider:"LitellmContentFilter",categoryName:"bias_religious",guardrailNameSuggestion:"Bias: Religious",mode:"pre_call",defaultOn:!1},cf_bias_sexual_orientation:{provider:"LitellmContentFilter",categoryName:"bias_sexual_orientation",guardrailNameSuggestion:"Bias: Sexual Orientation",mode:"pre_call",defaultOn:!1},cf_prompt_injection_jailbreak:{provider:"LitellmContentFilter",categoryName:"prompt_injection_jailbreak",guardrailNameSuggestion:"Prompt Injection: Jailbreak",mode:"pre_call",defaultOn:!1},cf_prompt_injection_data_exfil:{provider:"LitellmContentFilter",categoryName:"prompt_injection_data_exfiltration",guardrailNameSuggestion:"Prompt Injection: Data Exfiltration",mode:"pre_call",defaultOn:!1},cf_prompt_injection_sql:{provider:"LitellmContentFilter",categoryName:"prompt_injection_sql",guardrailNameSuggestion:"Prompt Injection: SQL",mode:"pre_call",defaultOn:!1},cf_prompt_injection_malicious_code:{provider:"LitellmContentFilter",categoryName:"prompt_injection_malicious_code",guardrailNameSuggestion:"Prompt Injection: Malicious Code",mode:"pre_call",defaultOn:!1},cf_prompt_injection_system_prompt:{provider:"LitellmContentFilter",categoryName:"prompt_injection_system_prompt",guardrailNameSuggestion:"Prompt Injection: System Prompt",mode:"pre_call",defaultOn:!1},cf_toxic_abuse:{provider:"LitellmContentFilter",categoryName:"harm_toxic_abuse",guardrailNameSuggestion:"Toxic & Abusive Language",mode:"pre_call",defaultOn:!1},cf_patterns:{provider:"LitellmContentFilter",guardrailNameSuggestion:"Pattern Matching",mode:"pre_call",defaultOn:!1},cf_keywords:{provider:"LitellmContentFilter",guardrailNameSuggestion:"Keyword Blocking",mode:"pre_call",defaultOn:!1},block_code_execution:{provider:"BlockCodeExecution",guardrailNameSuggestion:"Block Code Execution",mode:"pre_call",defaultOn:!1},cf_competitor_intent:{provider:"LitellmContentFilter",guardrailNameSuggestion:"Competitor Name Blocking",mode:"pre_call",defaultOn:!1},presidio:{provider:"PresidioPII",guardrailNameSuggestion:"Presidio PII",mode:"pre_call",defaultOn:!1},bedrock:{provider:"Bedrock",guardrailNameSuggestion:"Bedrock Guardrail",mode:"pre_call",defaultOn:!1},lakera:{provider:"Lakera",guardrailNameSuggestion:"Lakera",mode:"pre_call",defaultOn:!1},openai_moderation:{provider:"OpenaiModeration",guardrailNameSuggestion:"OpenAI Moderation",mode:"pre_call",defaultOn:!1},google_model_armor:{provider:"ModelArmor",guardrailNameSuggestion:"Google Cloud Model Armor",mode:"pre_call",defaultOn:!1},guardrails_ai:{provider:"GuardrailsAi",guardrailNameSuggestion:"Guardrails AI",mode:"pre_call",defaultOn:!1},zscaler:{provider:"ZscalerAiGuard",guardrailNameSuggestion:"Zscaler AI Guard",mode:"pre_call",defaultOn:!1},panw:{provider:"PanwPrismaAirs",guardrailNameSuggestion:"PANW Prisma AIRS",mode:"pre_call",defaultOn:!1},noma:{provider:"Noma",guardrailNameSuggestion:"Noma Security",mode:"pre_call",defaultOn:!1},aporia:{provider:"AporiaAi",guardrailNameSuggestion:"Aporia AI",mode:"pre_call",defaultOn:!1},aim:{provider:"Aim",guardrailNameSuggestion:"AIM Guardrail",mode:"pre_call",defaultOn:!1},prompt_security:{provider:"PromptSecurity",guardrailNameSuggestion:"Prompt Security",mode:"pre_call",defaultOn:!1},lasso:{provider:"Lasso",guardrailNameSuggestion:"Lasso Guardrail",mode:"pre_call",defaultOn:!1},pangea:{provider:"Pangea",guardrailNameSuggestion:"Pangea Guardrail",mode:"pre_call",defaultOn:!1},enkryptai:{provider:"Enkryptai",guardrailNameSuggestion:"EnkryptAI",mode:"pre_call",defaultOn:!1},javelin:{provider:"Javelin",guardrailNameSuggestion:"Javelin Guardrails",mode:"pre_call",defaultOn:!1},pillar:{provider:"Pillar",guardrailNameSuggestion:"Pillar Guardrail",mode:"pre_call",defaultOn:!1},akto:{provider:"Akto",guardrailNameSuggestion:"Akto Guardrail",mode:"pre_call",defaultOn:!1},promptguard:{provider:"Promptguard",guardrailNameSuggestion:"PromptGuard",mode:"pre_call",defaultOn:!1},xecguard:{provider:"Xecguard",guardrailNameSuggestion:"XecGuard",mode:"pre_call",defaultOn:!1},deepkeep:{provider:"Deepkeep",guardrailNameSuggestion:"DeepKeep AI Firewall",mode:"pre_call",defaultOn:!1}},I=({card:e,onBack:r,accessToken:i,onGuardrailCreated:s})=>{let[n,o]=(0,a.useState)(!1),[d,m]=(0,a.useState)("overview"),u=[{property:"Provider",value:"litellm"===e.category?"LiteLLM Content Filter":"Partner Guardrail"},...e.subcategory?[{property:"Subcategory",value:e.subcategory}]:[],..."litellm"===e.category?[{property:"Cost",value:"$0 / request"}]:[],..."litellm"===e.category?[{property:"External Dependencies",value:"None"}]:[],..."litellm"===e.category?[{property:"Latency",value:e.eval?.latency||"<1ms"}]:[]],p=e.eval?[{metric:"Precision",value:`${e.eval.precision}%`},{metric:"Recall",value:`${e.eval.recall}%`},{metric:"F1 Score",value:`${e.eval.f1}%`},{metric:"Test Cases",value:String(e.eval.testCases)},{metric:"False Positives",value:"0"},{metric:"False Negatives",value:"0"},{metric:"Latency (p50)",value:e.eval.latency}]:[],g=[{key:"overview",label:"Overview"},...e.eval?[{key:"eval",label:"Eval Results"}]:[]];return(0,t.jsxs)("div",{style:{maxWidth:960,margin:"0 auto"},children:[(0,t.jsxs)("div",{onClick:r,style:{display:"inline-flex",alignItems:"center",gap:6,color:"#5f6368",cursor:"pointer",fontSize:14,marginBottom:24},children:[(0,t.jsx)(S.ArrowLeftOutlined,{style:{fontSize:11}}),(0,t.jsx)("span",{children:e.name})]}),(0,t.jsxs)("div",{style:{display:"flex",alignItems:"center",gap:16,marginBottom:8},children:[(0,t.jsx)("img",{src:e.logo,alt:"",style:{width:40,height:40,borderRadius:8,objectFit:"contain"},onError:e=>{e.target.style.display="none"}}),(0,t.jsx)("h1",{style:{fontSize:28,fontWeight:400,color:"#202124",margin:0,lineHeight:1.2},children:e.name})]}),(0,t.jsx)("p",{style:{fontSize:14,color:"#5f6368",margin:"0 0 20px 0",lineHeight:1.6},children:e.description}),(0,t.jsx)("div",{style:{display:"flex",gap:10,marginBottom:32},children:(0,t.jsx)(l.Button,{onClick:()=>o(!0),style:{borderRadius:20,padding:"4px 20px",height:36,borderColor:"#dadce0",color:"#1a73e8",fontWeight:500,fontSize:14},children:"Create Guardrail"})}),(0,t.jsx)("div",{style:{borderBottom:"1px solid #dadce0",marginBottom:28},children:(0,t.jsx)("div",{style:{display:"flex",gap:0},children:g.map(e=>(0,t.jsx)("div",{onClick:()=>m(e.key),style:{padding:"12px 20px",fontSize:14,color:d===e.key?"#1a73e8":"#5f6368",borderBottom:d===e.key?"3px solid #1a73e8":"3px solid transparent",cursor:"pointer",fontWeight:d===e.key?500:400,marginBottom:-1},children:e.label},e.key))})}),"overview"===d&&(0,t.jsxs)("div",{style:{display:"flex",gap:64},children:[(0,t.jsxs)("div",{style:{flex:1,minWidth:0},children:[(0,t.jsx)("h2",{style:{fontSize:18,fontWeight:400,color:"#202124",margin:"0 0 12px 0"},children:"Overview"}),(0,t.jsx)("p",{style:{fontSize:14,color:"#3c4043",lineHeight:1.7,margin:"0 0 32px 0"},children:e.description}),(0,t.jsx)("h2",{style:{fontSize:18,fontWeight:400,color:"#202124",margin:"0 0 4px 0"},children:"Guardrail Details"}),(0,t.jsx)("p",{style:{fontSize:13,color:"#5f6368",margin:"0 0 16px 0"},children:"Details are as follows"}),(0,t.jsxs)("table",{style:{width:"100%",borderCollapse:"collapse",fontSize:14},children:[(0,t.jsx)("thead",{children:(0,t.jsxs)("tr",{style:{borderBottom:"1px solid #dadce0"},children:[(0,t.jsx)("th",{style:{textAlign:"left",padding:"12px 0",color:"#5f6368",fontWeight:500,width:200},children:"Property"}),(0,t.jsx)("th",{style:{textAlign:"left",padding:"12px 0",color:"#5f6368",fontWeight:500},children:e.name})]})}),(0,t.jsx)("tbody",{children:u.map((e,a)=>(0,t.jsxs)("tr",{style:{borderBottom:"1px solid #f1f3f4"},children:[(0,t.jsx)("td",{style:{padding:"12px 0",color:"#3c4043"},children:e.property}),(0,t.jsx)("td",{style:{padding:"12px 0",color:"#202124"},children:e.value})]},a))})]})]}),(0,t.jsxs)("div",{style:{width:240,flexShrink:0},children:[(0,t.jsxs)("div",{style:{marginBottom:28},children:[(0,t.jsx)("div",{style:{fontSize:12,color:"#5f6368",marginBottom:4},children:"Guardrail ID"}),(0,t.jsxs)("div",{style:{fontSize:13,color:"#202124",wordBreak:"break-all"},children:["litellm/",e.id]})]}),(0,t.jsxs)("div",{style:{marginBottom:28},children:[(0,t.jsx)("div",{style:{fontSize:12,color:"#5f6368",marginBottom:4},children:"Type"}),(0,t.jsx)("div",{style:{fontSize:13,color:"#202124"},children:"litellm"===e.category?"Content Filter":"Partner"})]}),e.tags.length>0&&(0,t.jsxs)("div",{style:{marginBottom:28},children:[(0,t.jsx)("div",{style:{fontSize:12,color:"#5f6368",marginBottom:8},children:"Tags"}),(0,t.jsx)("div",{style:{display:"flex",flexWrap:"wrap",gap:6},children:e.tags.map(e=>(0,t.jsx)("span",{style:{fontSize:12,padding:"4px 12px",borderRadius:16,border:"1px solid #dadce0",color:"#3c4043",backgroundColor:"#fff"},children:e},e))})]})]})]}),"eval"===d&&(0,t.jsxs)("div",{children:[(0,t.jsx)("h2",{style:{fontSize:18,fontWeight:400,color:"#202124",margin:"0 0 16px 0"},children:"Eval Results"}),(0,t.jsxs)("table",{style:{width:"100%",maxWidth:560,borderCollapse:"collapse",fontSize:14},children:[(0,t.jsx)("thead",{children:(0,t.jsxs)("tr",{style:{backgroundColor:"#f8f9fa",borderBottom:"1px solid #dadce0"},children:[(0,t.jsx)("th",{style:{textAlign:"left",padding:"12px 16px",color:"#5f6368",fontWeight:500},children:"Metric"}),(0,t.jsx)("th",{style:{textAlign:"left",padding:"12px 16px",color:"#5f6368",fontWeight:500},children:"Value"})]})}),(0,t.jsx)("tbody",{children:p.map((e,a)=>(0,t.jsxs)("tr",{style:{borderBottom:"1px solid #f1f3f4"},children:[(0,t.jsx)("td",{style:{padding:"12px 16px",color:"#3c4043"},children:e.metric}),(0,t.jsx)("td",{style:{padding:"12px 16px",color:"#202124",fontWeight:500},children:e.value})]},a))})]})]}),(0,t.jsx)(c.default,{visible:n,onClose:()=>o(!1),accessToken:i,onSuccess:()=>{o(!1),s()},preset:k[e.id]})]})},A=({accessToken:e,onGuardrailCreated:l})=>{let[r,i]=(0,a.useState)(""),[s,n]=(0,a.useState)(null),[o,d]=(0,a.useState)(!1),c=v.ALL_CARDS.filter(e=>{if(!r)return!0;let t=r.toLowerCase();return e.name.toLowerCase().includes(t)||e.description.toLowerCase().includes(t)||e.tags.some(e=>e.toLowerCase().includes(t))}),m=c.filter(e=>"litellm"===e.category),u=c.filter(e=>"partner"===e.category);return s?(0,t.jsx)(I,{card:s,onBack:()=>n(null),accessToken:e,onGuardrailCreated:l}):(0,t.jsxs)("div",{children:[(0,t.jsx)("div",{style:{marginBottom:24},children:(0,t.jsx)(j.Input,{size:"large",placeholder:"Search guardrails",prefix:(0,t.jsx)(_.SearchOutlined,{style:{color:"#9ca3af"}}),value:r,onChange:e=>i(e.target.value),style:{borderRadius:8}})}),(0,t.jsxs)("div",{style:{marginBottom:40},children:[(0,t.jsxs)("div",{style:{display:"flex",alignItems:"center",justifyContent:"space-between",marginBottom:4},children:[(0,t.jsx)("h2",{style:{fontSize:20,fontWeight:600,color:"#111827",margin:0},children:"LiteLLM Content Filter"}),(0,t.jsx)("span",{style:{display:"inline-flex",alignItems:"center",gap:6,fontSize:14,color:"#1a73e8",cursor:"pointer"},onClick:()=>d(!o),children:o?(0,t.jsx)(t.Fragment,{children:"Show less"}):(0,t.jsxs)(t.Fragment,{children:[(0,t.jsx)(b.ArrowRightOutlined,{style:{fontSize:12}}),`Show all (${m.length})`]})})]}),(0,t.jsx)("p",{style:{fontSize:13,color:"#6b7280",margin:"4px 0 20px 0"},children:"Built-in guardrails powered by LiteLLM. Zero latency, no external dependencies, no additional cost."}),(0,t.jsx)("div",{style:{display:"grid",gridTemplateColumns:"repeat(auto-fill, minmax(220px, 1fr))",gap:16},children:(o?m:m.slice(0,10)).map(e=>(0,t.jsx)(C,{card:e,onClick:()=>n(e)},e.id))})]}),(0,t.jsxs)("div",{style:{marginBottom:40},children:[(0,t.jsx)("h2",{style:{fontSize:20,fontWeight:600,color:"#111827",margin:"0 0 4px 0"},children:"Partner Guardrails"}),(0,t.jsx)("p",{style:{fontSize:13,color:"#6b7280",margin:"4px 0 20px 0"},children:"Third-party guardrail integrations from leading AI security providers."}),(0,t.jsx)("div",{style:{display:"grid",gridTemplateColumns:"repeat(auto-fill, minmax(220px, 1fr))",gap:16},children:u.map(e=>(0,t.jsx)(C,{card:e,onClick:()=>n(e)},e.id))})]})]})};var O=e.i(988846),T=e.i(837007),P=e.i(409797),L=e.i(54131),B=e.i(995926),F=e.i(678784),$=e.i(634831),E=e.i(438100),M=e.i(302202),R=e.i(328196),G=e.i(879664);e.s(["InfoIcon",()=>G.default],168118);var G=G,z=e.i(212931),D=e.i(808613),K=e.i(199133),q=e.i(663435),H=e.i(954616),J=e.i(912598),U=e.i(135214),W=e.i(243652);let V=async(e,t)=>{let a=(0,d.getProxyBaseUrl)(),l=`${a}/guardrails/register`,r=await fetch(l,{method:"POST",headers:{[(0,d.getGlobalLitellmHeaderName)()]:`Bearer ${e}`,"Content-Type":"application/json"},body:JSON.stringify(t)});if(!r.ok){let e=await r.json().catch(()=>({})),t=(0,d.deriveErrorMessage)(e);throw(0,d.handleError)(t),Error(t)}return r.json()},Y=(0,W.createQueryKeys)("guardrails");function Q(e){var t;let a=e.litellm_params??{},l=e.guardrail_info??{},r=a.headers,i=Array.isArray(r)?r.map(e=>({key:(e.key??e.name??"").toString(),value:String(e.value??"")})):"object"==typeof r&&null!==r?Object.entries(r).map(([e,t])=>({key:e,value:String(t??"")})):[],s=a.api_base??a.url??"",n=l.model??a.model??"—",o=a.forward_api_key??!0,d=Array.isArray(a.extra_headers)?a.extra_headers.filter(e=>"string"==typeof e):[];return{id:e.guardrail_id,team:e.team_id??"—",name:e.guardrail_name,endpoint:s,status:"pending_review"===(t=e.status)?"pending":"active"===t||"rejected"===t?t:"active",model:n,forwardKey:o,description:l.description??"",method:a.method??"POST",customHeaders:i,extraHeaders:d,submittedAt:function(e){if(!e)return"—";try{let t=new Date(e);return isNaN(t.getTime())?e:t.toISOString().slice(0,10)}catch{return e}}(e.submitted_at),submittedBy:e.submitted_by_email??e.submitted_by_user_id??"—",mode:a.mode,unreachable_fallback:a.unreachable_fallback,additionalProviderParams:a.additional_provider_specific_params,guardrailType:a.guardrail}}let X={active:{label:"Active",bg:"bg-green-50",text:"text-green-700",dot:"bg-green-500"},pending:{label:"Pending Review",bg:"bg-yellow-50",text:"text-yellow-700",dot:"bg-yellow-500"},rejected:{label:"Rejected",bg:"bg-red-50",text:"text-red-700",dot:"bg-red-500"}},Z={"ML Platform":"bg-purple-100 text-purple-700","Data Science":"bg-blue-100 text-blue-700",Security:"bg-red-100 text-red-700","Customer Success":"bg-orange-100 text-orange-700",Legal:"bg-gray-100 text-gray-700",Finance:"bg-green-100 text-green-700"};function ee({label:e,value:a,color:l}){return(0,t.jsxs)("div",{className:"bg-white border border-gray-200 rounded-lg px-4 py-3",children:[(0,t.jsx)("div",{className:`text-2xl font-bold ${l}`,children:a}),(0,t.jsx)("div",{className:"text-xs text-gray-500 mt-0.5",children:e})]})}function et({enabled:e,onToggle:a}){return(0,t.jsx)("button",{type:"button",onClick:a,role:"switch","aria-checked":e,className:`relative inline-flex h-5 w-9 items-center rounded-full transition-colors focus:outline-none focus:ring-2 focus:ring-blue-500 focus:ring-offset-1 ${e?"bg-blue-500":"bg-gray-200"}`,children:(0,t.jsx)("span",{className:`inline-block h-3.5 w-3.5 transform rounded-full bg-white shadow transition-transform ${e?"translate-x-4":"translate-x-0.5"}`})})}function ea({guardrail:e,isSelected:a,isHeadersExpanded:l,onSelect:r,onToggleForwardKey:i,onToggleHeaders:s,onApprove:n,onReject:o}){let d=X[e.status],c=Z[e.team]??"bg-gray-100 text-gray-700";return(0,t.jsxs)("div",{className:`bg-white border rounded-lg p-4 transition-all ${a?"border-blue-400 ring-1 ring-blue-200":"border-gray-200"}`,children:[(0,t.jsxs)("div",{className:"flex items-start justify-between gap-4",children:[(0,t.jsxs)("div",{className:"flex-1 min-w-0",children:[(0,t.jsxs)("div",{className:"flex items-center gap-2 mb-1.5 flex-wrap",children:[(0,t.jsxs)("span",{className:`text-xs font-medium px-2 py-0.5 rounded-full ${c}`,children:["Team: ",e.team]}),(0,t.jsxs)("span",{className:`inline-flex items-center gap-1.5 text-xs font-medium px-2 py-0.5 rounded-full ${d.bg} ${d.text}`,children:[(0,t.jsx)("span",{className:`w-1.5 h-1.5 rounded-full ${d.dot}`}),d.label]})]}),(0,t.jsx)("h3",{className:"text-sm font-semibold text-gray-900 mb-1",children:e.name}),(0,t.jsx)("p",{className:"text-xs text-gray-500 mb-2 line-clamp-1",children:e.description}),(0,t.jsxs)("div",{className:"flex items-center gap-1.5 mb-2",children:[(0,t.jsx)(M.ServerIcon,{className:"h-3.5 w-3.5 text-gray-400 flex-shrink-0"}),(0,t.jsx)("code",{className:"text-xs text-gray-500 font-mono truncate",children:e.endpoint})]}),(0,t.jsxs)("div",{className:"flex items-center gap-4 text-xs text-gray-500",children:[(0,t.jsxs)("span",{children:["Model: ",(0,t.jsx)("span",{className:"font-medium text-gray-700",children:e.model})]}),(0,t.jsxs)("span",{children:["Submitted:"," ",(0,t.jsx)("span",{className:"font-medium text-gray-700",children:e.submittedAt})]})]})]}),(0,t.jsxs)("div",{className:"flex flex-col items-end gap-2 flex-shrink-0",children:[(0,t.jsxs)("div",{className:"flex items-center gap-2",children:[(0,t.jsx)("span",{className:"text-xs text-gray-500 whitespace-nowrap",children:"Forward API Key"}),(0,t.jsx)(et,{enabled:e.forwardKey,onToggle:i})]}),(0,t.jsxs)("div",{className:"flex items-center gap-2 mt-1",children:[(0,t.jsx)("button",{type:"button",onClick:r,className:"text-xs border border-gray-300 text-gray-600 hover:bg-gray-50 px-3 py-1.5 rounded-md transition-colors font-medium",children:a?"Close":"Review"}),"pending"===e.status&&(0,t.jsxs)(t.Fragment,{children:[(0,t.jsx)("button",{type:"button",onClick:n,className:"text-xs bg-green-500 hover:bg-green-600 text-white px-3 py-1.5 rounded-md transition-colors font-medium",children:"Approve"}),(0,t.jsx)("button",{type:"button",onClick:o,className:"text-xs border border-red-300 text-red-600 hover:bg-red-50 px-3 py-1.5 rounded-md transition-colors font-medium",children:"Reject"})]})]})]})]}),(0,t.jsxs)("div",{className:"mt-3 pt-3 border-t border-gray-100",children:[(0,t.jsxs)("button",{type:"button",onClick:s,className:"flex items-center gap-1.5 text-xs text-gray-500 hover:text-gray-700 transition-colors",children:[l?(0,t.jsx)(L.ChevronUpIcon,{className:"h-3.5 w-3.5"}):(0,t.jsx)(P.ChevronDownIcon,{className:"h-3.5 w-3.5"}),"Static headers",e.customHeaders.length>0&&(0,t.jsx)("span",{className:"ml-1 bg-gray-100 text-gray-600 rounded-full px-1.5 py-0.5 text-xs",children:e.customHeaders.length})]}),l&&(0,t.jsx)("div",{className:"mt-2",children:0===e.customHeaders.length?(0,t.jsx)("p",{className:"text-xs text-gray-400 italic",children:"No static headers configured."}):(0,t.jsx)("div",{className:"space-y-1",children:e.customHeaders.map((e,a)=>(0,t.jsxs)("div",{className:"flex items-center gap-2 text-xs font-mono",children:[(0,t.jsx)("span",{className:"text-gray-500 bg-gray-50 border border-gray-200 rounded px-2 py-0.5",children:e.key}),(0,t.jsx)("span",{className:"text-gray-400",children:":"}),(0,t.jsx)("span",{className:"text-gray-700 bg-gray-50 border border-gray-200 rounded px-2 py-0.5",children:e.value})]},`${e.key}-${a}`))})})]})]})}function el({label:e,children:a}){return(0,t.jsxs)("div",{children:[(0,t.jsx)("div",{className:"text-xs font-semibold text-gray-500 mb-1",children:e}),(0,t.jsx)("div",{children:a})]})}function er({guardrail:e,onClose:l,onApprove:r,onReject:i,onToggleForwardKey:s,onUpdateCustomHeaders:n,onUpdateExtraHeaders:o}){let[d,c]=(0,a.useState)(!1),[m,u]=(0,a.useState)(""),[p,g]=(0,a.useState)(""),[x,h]=(0,a.useState)(""),f=X[e.status],y=Z[e.team]??"bg-gray-100 text-gray-700";return(0,t.jsx)("div",{className:"w-96 flex-shrink-0 bg-white overflow-auto",children:(0,t.jsxs)("div",{className:"p-5",children:[(0,t.jsxs)("div",{className:"flex items-start justify-between mb-4",children:[(0,t.jsxs)("div",{children:[(0,t.jsxs)("div",{className:"flex items-center gap-2 mb-1",children:[(0,t.jsxs)("span",{className:`text-xs font-medium px-2 py-0.5 rounded-full ${y}`,children:["Team: ",e.team]}),(0,t.jsxs)("span",{className:`inline-flex items-center gap-1.5 text-xs font-medium px-2 py-0.5 rounded-full ${f.bg} ${f.text}`,children:[(0,t.jsx)("span",{className:`w-1.5 h-1.5 rounded-full ${f.dot}`}),f.label]})]}),(0,t.jsx)("h2",{className:"text-base font-semibold text-gray-900",children:e.name}),(0,t.jsxs)("p",{className:"text-xs text-gray-500 mt-0.5",children:["Submitted by ",e.submittedBy," on ",e.submittedAt]})]}),(0,t.jsx)("button",{type:"button",onClick:l,className:"text-gray-400 hover:text-gray-600 transition-colors","aria-label":"Close detail panel",children:(0,t.jsx)(B.XIcon,{className:"h-4 w-4"})})]}),(0,t.jsx)("p",{className:"text-sm text-gray-600 mb-5",children:e.description}),(0,t.jsxs)("div",{className:"space-y-4",children:[(0,t.jsx)(el,{label:"Endpoint",children:(0,t.jsxs)("div",{className:"flex items-center gap-1.5",children:[(0,t.jsx)("code",{className:"text-xs font-mono text-gray-700 break-all",children:e.endpoint}),(0,t.jsx)("a",{href:e.endpoint,target:"_blank",rel:"noopener noreferrer",className:"text-gray-400 hover:text-blue-500 flex-shrink-0",children:(0,t.jsx)($.ExternalLinkIcon,{className:"h-3.5 w-3.5"})})]})}),(0,t.jsx)(el,{label:"Method",children:(0,t.jsx)("span",{className:"text-xs font-mono font-medium text-gray-700 bg-gray-100 px-2 py-0.5 rounded",children:e.method})}),(0,t.jsxs)("div",{className:"border border-blue-100 bg-blue-50 rounded-lg p-3",children:[(0,t.jsxs)("div",{className:"flex items-center justify-between mb-2",children:[(0,t.jsxs)("div",{className:"flex items-center gap-1.5",children:[(0,t.jsx)(E.KeyIcon,{className:"h-3.5 w-3.5 text-blue-500"}),(0,t.jsx)("span",{className:"text-xs font-semibold text-blue-800",children:"Forward LiteLLM API Key"})]}),(0,t.jsx)(et,{enabled:e.forwardKey,onToggle:s})]}),(0,t.jsxs)("p",{className:"text-xs text-blue-700 leading-relaxed",children:["When enabled, the caller's LiteLLM API key is forwarded as an"," ",(0,t.jsx)("code",{className:"font-mono bg-blue-100 px-1 rounded",children:"Authorization"})," ","header to your guardrail endpoint. This allows your guardrail to authenticate model calls using the original caller's credentials."]})]}),(0,t.jsxs)("div",{children:[(0,t.jsxs)("div",{className:"flex items-center gap-1.5 mb-2",children:[(0,t.jsx)("span",{className:"text-xs font-semibold text-gray-700",children:"Static headers"}),e.customHeaders.length>0&&(0,t.jsx)("span",{className:"bg-gray-100 text-gray-600 rounded-full px-1.5 py-0.5 text-xs",children:e.customHeaders.length})]}),(0,t.jsx)("p",{className:"text-xs text-gray-400 mb-2",children:"Sent with every request to the guardrail."}),0===e.customHeaders.length?(0,t.jsx)("p",{className:"text-xs text-gray-400 italic mb-2",children:"No static headers configured."}):(0,t.jsx)("ul",{className:"list-none space-y-1 mb-2",children:e.customHeaders.map((a,l)=>(0,t.jsxs)("li",{className:"flex items-center justify-between gap-2 text-xs font-mono bg-gray-50 border border-gray-200 rounded px-2 py-1.5",children:[(0,t.jsxs)("span",{className:"text-gray-700 truncate",children:[a.key,": ",a.value]}),(0,t.jsx)("button",{type:"button",onClick:()=>n(e.customHeaders.filter((e,t)=>t!==l)),className:"text-gray-400 hover:text-red-600 flex-shrink-0","aria-label":`Remove ${a.key}`,children:(0,t.jsx)(B.XIcon,{className:"h-3.5 w-3.5"})})]},`${a.key}-${l}`))}),(0,t.jsxs)("div",{className:"flex flex-col gap-2 sm:flex-row sm:items-end",children:[(0,t.jsx)("input",{type:"text",value:p,onChange:e=>g(e.target.value),placeholder:"Header name (e.g. X-API-Key)",className:"flex-1 min-w-0 text-xs font-mono border border-gray-200 rounded px-2 py-1.5 text-gray-700 placeholder-gray-400 focus:outline-none focus:ring-1 focus:ring-blue-500",onKeyDown:t=>{if("Enter"===t.key){t.preventDefault();let a=p.trim(),l=x.trim();a&&!e.customHeaders.some(e=>e.key.toLowerCase()===a.toLowerCase())&&(n([...e.customHeaders,{key:a,value:l}]),g(""),h(""))}}}),(0,t.jsx)("input",{type:"text",value:x,onChange:e=>h(e.target.value),placeholder:"Value",className:"flex-1 min-w-0 text-xs font-mono border border-gray-200 rounded px-2 py-1.5 text-gray-700 placeholder-gray-400 focus:outline-none focus:ring-1 focus:ring-blue-500",onKeyDown:t=>{if("Enter"===t.key){t.preventDefault();let a=p.trim(),l=x.trim();a&&!e.customHeaders.some(e=>e.key.toLowerCase()===a.toLowerCase())&&(n([...e.customHeaders,{key:a,value:l}]),g(""),h(""))}}}),(0,t.jsx)("button",{type:"button",onClick:()=>{let t=p.trim(),a=x.trim();t&&!e.customHeaders.some(e=>e.key.toLowerCase()===t.toLowerCase())&&(n([...e.customHeaders,{key:t,value:a}]),g(""),h(""))},className:"text-xs font-medium text-blue-600 hover:text-blue-700 border border-blue-200 bg-blue-50 hover:bg-blue-100 px-2 py-1.5 rounded transition-colors flex-shrink-0",children:"Add"})]})]}),(0,t.jsxs)("div",{children:[(0,t.jsxs)("div",{className:"flex items-center gap-1.5 mb-2",children:[(0,t.jsx)("span",{className:"text-xs font-semibold text-gray-700",children:"Forward client headers"}),e.extraHeaders.length>0&&(0,t.jsx)("span",{className:"bg-gray-100 text-gray-600 rounded-full px-1.5 py-0.5 text-xs",children:e.extraHeaders.length})]}),(0,t.jsx)("p",{className:"text-xs text-gray-400 mb-2",children:"Allowed header names to forward from the client request to the guardrail (e.g. x-request-id)."}),0===e.extraHeaders.length?(0,t.jsx)("p",{className:"text-xs text-gray-400 italic mb-2",children:"No forward client headers configured."}):(0,t.jsx)("ul",{className:"list-none space-y-1 mb-2",children:e.extraHeaders.map((a,l)=>(0,t.jsxs)("li",{className:"flex items-center justify-between gap-2 text-xs font-mono bg-gray-50 border border-gray-200 rounded px-2 py-1.5",children:[(0,t.jsx)("span",{className:"text-gray-700 truncate",children:a}),(0,t.jsx)("button",{type:"button",onClick:()=>o(e.extraHeaders.filter((e,t)=>t!==l)),className:"text-gray-400 hover:text-red-600 flex-shrink-0","aria-label":`Remove ${a}`,children:(0,t.jsx)(B.XIcon,{className:"h-3.5 w-3.5"})})]},`${a}-${l}`))}),(0,t.jsxs)("div",{className:"flex gap-2",children:[(0,t.jsx)("input",{type:"text",value:m,onChange:e=>u(e.target.value),placeholder:"e.g. x-request-id",className:"flex-1 min-w-0 text-xs font-mono border border-gray-200 rounded px-2 py-1.5 text-gray-700 placeholder-gray-400 focus:outline-none focus:ring-1 focus:ring-blue-500",onKeyDown:t=>{if("Enter"===t.key){t.preventDefault();let a=m.trim().toLowerCase();a&&!e.extraHeaders.map(e=>e.toLowerCase()).includes(a)&&(o([...e.extraHeaders,a]),u(""))}}}),(0,t.jsx)("button",{type:"button",onClick:()=>{let t=m.trim().toLowerCase();t&&!e.extraHeaders.map(e=>e.toLowerCase()).includes(t)&&(o([...e.extraHeaders,t]),u(""))},className:"text-xs font-medium text-blue-600 hover:text-blue-700 border border-blue-200 bg-blue-50 hover:bg-blue-100 px-2 py-1.5 rounded transition-colors",children:"Add"})]})]}),(0,t.jsxs)("div",{className:"border border-gray-200 rounded-lg overflow-hidden",children:[(0,t.jsxs)("button",{type:"button",onClick:()=>c(!d),className:"w-full flex items-center justify-between px-3 py-2 text-left text-xs font-semibold text-gray-700 bg-gray-50 hover:bg-gray-100 transition-colors",children:[(0,t.jsx)("span",{children:"Equivalent config"}),d?(0,t.jsx)(L.ChevronUpIcon,{className:"h-3.5 w-3.5 text-gray-500"}):(0,t.jsx)(P.ChevronDownIcon,{className:"h-3.5 w-3.5 text-gray-500"})]}),d&&(0,t.jsx)("pre",{className:"p-3 text-xs font-mono text-gray-700 bg-white border-t border-gray-200 overflow-x-auto whitespace-pre-wrap break-all",children:function(e){let t=["litellm_settings:"," guardrails:",` - guardrail_name: "${e.name.replace(/\\/g,"\\\\").replace(/"/g,'\\"')}"`," litellm_params:",` guardrail: ${e.guardrailType??"generic_guardrail_api"}`,` mode: ${e.mode??"pre_call"} # or post_call, during_call`,` api_base: ${e.endpoint||"https://your-guardrail-api.com"}`," api_key: os.environ/YOUR_GUARDRAIL_API_KEY # optional",` unreachable_fallback: ${e.unreachable_fallback??"fail_closed"} # default: fail_closed. Set to fail_open to proceed if the guardrail endpoint is unreachable.`,` forward_api_key: ${e.forwardKey}`];if(e.model&&"—"!==e.model&&t.push(` model: "${e.model}" # LLM model name sent to the guardrail for context`),e.customHeaders.length>0)for(let a of(t.push(" headers: # static headers (sent with every request)"),e.customHeaders))t.push(` ${a.key}: "${String(a.value).replace(/\\/g,"\\\\").replace(/"/g,'\\"')}"`);if(e.extraHeaders.length>0)for(let a of(t.push(" extra_headers: # forward these client request headers to the guardrail"),e.extraHeaders))t.push(` - ${a}`);if(e.additionalProviderParams&&Object.keys(e.additionalProviderParams).length>0)for(let[a,l]of(t.push(" additional_provider_specific_params:"),Object.entries(e.additionalProviderParams))){let e="string"==typeof l?`"${l}"`:String(l);t.push(` ${a}: ${e}`)}return t.join("\n")}(e)})]}),(0,t.jsxs)("div",{className:"flex items-start gap-2 bg-gray-50 border border-gray-200 rounded-lg p-3",children:[(0,t.jsx)(G.default,{className:"h-3.5 w-3.5 text-gray-400 flex-shrink-0 mt-0.5"}),(0,t.jsxs)("p",{className:"text-xs text-gray-500 leading-relaxed",children:["This guardrail runs on a separate instance. It receives the user request and forwards the result to the next step in the pipeline. See"," ",(0,t.jsx)("a",{href:"https://docs.litellm.ai/docs/adding_provider/generic_guardrail_api",target:"_blank",rel:"noopener noreferrer",className:"text-blue-500 hover:underline",children:"LiteLLM Generic Guardrail API docs"})," ","for configuration details."]})]})]}),(0,t.jsxs)("div",{className:"mt-5 pt-4 border-t border-gray-100 space-y-2",children:[(0,t.jsxs)("button",{type:"button",className:"w-full flex items-center justify-center gap-2 border border-gray-300 text-gray-700 hover:bg-gray-50 text-sm font-medium py-2 rounded-md transition-colors",children:[(0,t.jsx)($.ExternalLinkIcon,{className:"h-4 w-4"}),"Test Endpoint"]}),"pending"===e.status&&(0,t.jsxs)("div",{className:"flex gap-2",children:[(0,t.jsxs)("button",{type:"button",onClick:r,className:"flex-1 flex items-center justify-center gap-1.5 bg-green-500 hover:bg-green-600 text-white text-sm font-medium py-2 rounded-md transition-colors",children:[(0,t.jsx)(F.CheckIcon,{className:"h-4 w-4"}),"Approve"]}),(0,t.jsxs)("button",{type:"button",onClick:i,className:"flex-1 flex items-center justify-center gap-1.5 border border-red-300 text-red-600 hover:bg-red-50 text-sm font-medium py-2 rounded-md transition-colors",children:[(0,t.jsx)(B.XIcon,{className:"h-4 w-4"}),"Reject"]})]})]})]})})}function ei({action:e,guardrailName:a,onConfirm:l,onCancel:r}){let i="approve"===e;return(0,t.jsx)("div",{className:"fixed inset-0 bg-black/30 flex items-center justify-center z-50",children:(0,t.jsxs)("div",{className:"bg-white rounded-xl shadow-xl p-6 max-w-sm w-full mx-4",children:[(0,t.jsx)("div",{className:`w-10 h-10 rounded-full flex items-center justify-center mb-4 ${i?"bg-green-100":"bg-red-100"}`,children:i?(0,t.jsx)(F.CheckIcon,{className:"h-5 w-5 text-green-600"}):(0,t.jsx)(R.AlertCircleIcon,{className:"h-5 w-5 text-red-600"})}),(0,t.jsx)("h3",{className:"text-base font-semibold text-gray-900 mb-1",children:i?"Approve Guardrail":"Reject Guardrail"}),(0,t.jsxs)("p",{className:"text-sm text-gray-500 mb-5",children:["Are you sure you want to ",e," ",(0,t.jsxs)("span",{className:"font-medium text-gray-700",children:['"',a,'"']}),"?"," ",i?"This will make it active and available for use.":"This will mark it as rejected and notify the team."]}),(0,t.jsxs)("div",{className:"flex gap-3",children:[(0,t.jsx)("button",{type:"button",onClick:r,className:"flex-1 border border-gray-300 text-gray-700 hover:bg-gray-50 text-sm font-medium py-2 rounded-md transition-colors",children:"Cancel"}),(0,t.jsx)("button",{type:"button",onClick:l,className:`flex-1 text-white text-sm font-medium py-2 rounded-md transition-colors ${i?"bg-green-500 hover:bg-green-600":"bg-red-500 hover:bg-red-600"}`,children:i?"Approve":"Reject"})]})]})})}function es({accessToken:e}){let[l,r]=(0,a.useState)([]),[i,s]=(0,a.useState)({total:0,pending_review:0,active:0,rejected:0}),[n,o]=(0,a.useState)(""),[c,m]=(0,a.useState)("all"),[u,p]=(0,a.useState)(null),[g,h]=(0,a.useState)(new Set),[f,y]=(0,a.useState)(null),[_,b]=(0,a.useState)(!0),[v,w]=(0,a.useState)(null),[N,C]=(0,a.useState)(""),[S,k]=(0,a.useState)(!1),[I]=D.Form.useForm(),A=(()=>{let{accessToken:e}=(0,U.default)(),t=(0,J.useQueryClient)();return(0,H.useMutation)({mutationFn:async t=>{if(!e)throw Error("Access token is required");return V(e,t)},onSuccess:()=>{t.invalidateQueries({queryKey:Y.all})}})})();(0,a.useEffect)(()=>{let e=setTimeout(()=>C(n),300);return()=>clearTimeout(e)},[n]);let P=(0,a.useCallback)(async()=>{if(!e)return void b(!1);b(!0),w(null);try{let t="all"===c?void 0:"pending"===c?"pending_review":c,a=await (0,d.listGuardrailSubmissions)(e,{status:t,search:N.trim()||void 0});r(a.submissions.map(Q)),s(a.summary)}catch(e){w(e instanceof Error?e.message:"Failed to load submissions"),r([])}finally{b(!1)}},[e,c,N]);(0,a.useEffect)(()=>{P()},[P]);let L=l.find(e=>e.id===u)??null,B=i.total,F=i.pending_review,$=i.active,E=i.rejected;async function M(t){if(!e)return;let a=l.find(e=>e.id===t);if(!a)return;let i=!a.forwardKey;try{await (0,d.updateGuardrailCall)(e,t,{litellm_params:{forward_api_key:i}}),r(e=>e.map(e=>e.id===t?{...e,forwardKey:i}:e)),x.default.success(i?"Forward API key enabled":"Forward API key disabled")}catch{x.default.fromBackend("Failed to update forward API key")}}async function R(t,a){if(!e)return;let l={};for(let{key:e,value:t}of a)e.trim()&&(l[e.trim()]=t);try{await (0,d.updateGuardrailCall)(e,t,{litellm_params:{headers:l}}),r(e=>e.map(e=>e.id===t?{...e,customHeaders:a.filter(e=>e.key.trim())}:e)),x.default.success("Static headers updated")}catch{x.default.fromBackend("Failed to update static headers")}}async function G(t,a){if(e)try{await (0,d.updateGuardrailCall)(e,t,{litellm_params:{extra_headers:a}}),r(e=>e.map(e=>e.id===t?{...e,extraHeaders:a}:e)),x.default.success("Forward client headers updated")}catch{x.default.fromBackend("Failed to update forward client headers")}}async function W(t){if(e)try{await (0,d.approveGuardrailSubmission)(e,t),y(null),u===t&&p(null),await P(),x.default.success("Guardrail approved")}catch{x.default.fromBackend("Failed to approve guardrail")}}async function X(t){if(e)try{await (0,d.rejectGuardrailSubmission)(e,t),y(null),u===t&&p(null),await P(),x.default.success("Guardrail rejected")}catch{x.default.fromBackend("Failed to reject guardrail")}}return(0,t.jsxs)("div",{className:"flex h-full",children:[(0,t.jsxs)("div",{className:`flex-1 min-w-0 p-6 overflow-auto ${L?"border-r border-gray-200":""}`,children:[(0,t.jsxs)("div",{className:"grid grid-cols-4 gap-4 mb-6",children:[(0,t.jsx)(ee,{label:"Total Submitted",value:B,color:"text-gray-900"}),(0,t.jsx)(ee,{label:"Pending Review",value:F,color:"text-yellow-600"}),(0,t.jsx)(ee,{label:"Active",value:$,color:"text-green-600"}),(0,t.jsx)(ee,{label:"Rejected",value:E,color:"text-red-600"})]}),(0,t.jsxs)("div",{className:"flex items-center gap-3 mb-5",children:[(0,t.jsxs)("div",{className:"relative flex-1 max-w-xs",children:[(0,t.jsx)(O.SearchIcon,{className:"absolute left-3 top-1/2 -translate-y-1/2 h-4 w-4 text-gray-400"}),(0,t.jsx)("input",{type:"text",placeholder:"Search guardrails...",value:n,onChange:e=>o(e.target.value),className:"w-full pl-9 pr-4 py-2 border border-gray-200 rounded-md text-sm text-gray-700 placeholder-gray-400 focus:outline-none focus:ring-1 focus:ring-blue-500 focus:border-blue-500"})]}),(0,t.jsxs)("select",{value:c,onChange:e=>m(e.target.value),className:"border border-gray-200 rounded-md px-3 py-2 text-sm text-gray-700 focus:outline-none focus:ring-1 focus:ring-blue-500 focus:border-blue-500 bg-white",children:[(0,t.jsx)("option",{value:"all",children:"All Status"}),(0,t.jsx)("option",{value:"pending",children:"Pending Review"}),(0,t.jsx)("option",{value:"active",children:"Active"}),(0,t.jsx)("option",{value:"rejected",children:"Rejected"})]}),(0,t.jsxs)("button",{type:"button",onClick:()=>k(!0),className:"ml-auto flex items-center gap-2 bg-blue-500 hover:bg-blue-600 text-white text-sm font-medium px-4 py-2 rounded-md transition-colors",children:[(0,t.jsx)(T.PlusIcon,{className:"h-4 w-4"}),"Add Guardrail"]})]}),(0,t.jsxs)("div",{className:"space-y-3",children:[_&&(0,t.jsx)("div",{className:"text-center py-12 text-gray-500 text-sm",children:"Loading submissions…"}),v&&(0,t.jsx)("div",{className:"text-center py-12 text-red-600 text-sm",children:v}),!_&&!v&&0===l.length&&(0,t.jsx)("div",{className:"text-center py-12 text-gray-400 text-sm",children:"No guardrails match your filters."}),!_&&!v&&l.map(e=>(0,t.jsx)(ea,{guardrail:e,isSelected:u===e.id,isHeadersExpanded:g.has(e.id),onSelect:()=>p(u===e.id?null:e.id),onToggleForwardKey:()=>M(e.id),onToggleHeaders:()=>{var t;return t=e.id,void h(e=>{let a=new Set(e);return a.has(t)?a.delete(t):a.add(t),a})},onApprove:()=>y({id:e.id,action:"approve"}),onReject:()=>y({id:e.id,action:"reject"})},e.id))]})]}),L&&(0,t.jsx)(er,{guardrail:L,onClose:()=>p(null),onApprove:()=>y({id:L.id,action:"approve"}),onReject:()=>y({id:L.id,action:"reject"}),onToggleForwardKey:()=>M(L.id),onUpdateCustomHeaders:e=>R(L.id,e),onUpdateExtraHeaders:e=>G(L.id,e)}),f&&(0,t.jsx)(ei,{action:f.action,guardrailName:l.find(e=>e.id===f.id)?.name??"",onConfirm:()=>"approve"===f.action?W(f.id):X(f.id),onCancel:()=>y(null)}),(0,t.jsxs)(z.Modal,{title:"Submit Guardrail for Review",open:S,onCancel:()=>{k(!1),I.resetFields()},onOk:()=>I.submit(),okText:"Submit for Review",children:[(0,t.jsx)("div",{className:"rounded-md bg-blue-50 border border-blue-200 px-4 py-3 text-sm text-blue-800 mb-4",children:"Your guardrail will be sent for admin review before it becomes active."}),(0,t.jsxs)(D.Form,{form:I,layout:"vertical",initialValues:{mode:"pre_call"},onFinish:async e=>{let t={...e.extra_litellm_params?JSON.parse(e.extra_litellm_params):{},guardrail:"generic_guardrail_api",mode:e.mode,api_base:e.api_base};try{await A.mutateAsync({team_id:e.team_id,guardrail_name:e.guardrail_name,litellm_params:t,guardrail_info:e.guardrail_info?JSON.parse(e.guardrail_info):void 0}),x.default.success("Guardrail submitted for review"),k(!1),I.resetFields(),P()}catch{}},children:[(0,t.jsx)(D.Form.Item,{label:"Team",name:"team_id",rules:[{required:!0,message:"Select a team"}],children:(0,t.jsx)(q.default,{})}),(0,t.jsx)(D.Form.Item,{label:"Guardrail Name",name:"guardrail_name",rules:[{required:!0,message:"Enter a guardrail name"}],children:(0,t.jsx)(j.Input,{placeholder:"e.g. pii-detection"})}),(0,t.jsx)(D.Form.Item,{label:"Mode",name:"mode",rules:[{required:!0,message:"Select a mode"}],children:(0,t.jsxs)(K.Select,{children:[(0,t.jsx)(K.Select.Option,{value:"pre_call",children:"Pre Call"}),(0,t.jsx)(K.Select.Option,{value:"post_call",children:"Post Call"}),(0,t.jsx)(K.Select.Option,{value:"during_call",children:"During Call"})]})}),(0,t.jsx)(D.Form.Item,{label:"API Base URL",name:"api_base",rules:[{required:!0,message:"Enter the API base URL"},{type:"url",message:"Must be a valid URL"}],children:(0,t.jsx)(j.Input,{placeholder:"https://your-guardrail-api.com/v1/check",className:"font-mono"})}),(0,t.jsx)(D.Form.Item,{label:"Additional litellm_params (optional)",name:"extra_litellm_params",tooltip:"JSON object merged into litellm_params. e.g. forward_api_key, headers, model, unreachable_fallback",rules:[{validator:(e,t)=>{if(!t)return Promise.resolve();try{let e=JSON.parse(t);if("object"!=typeof e||Array.isArray(e))return Promise.reject("Must be a JSON object");return Promise.resolve()}catch{return Promise.reject("Invalid JSON")}}}],children:(0,t.jsx)(j.Input.TextArea,{rows:3,className:"font-mono text-xs",placeholder:'{"forward_api_key": true, "headers": {"X-Custom": "value"}}'})}),(0,t.jsx)(D.Form.Item,{label:"Guardrail Info (optional)",name:"guardrail_info",rules:[{validator:(e,t)=>{if(!t)return Promise.resolve();try{return JSON.parse(t),Promise.resolve()}catch{return Promise.reject("Invalid JSON")}}}],children:(0,t.jsx)(j.Input.TextArea,{rows:3,className:"font-mono text-xs",placeholder:'{"description": "Detects PII in requests"}'})})]})]})]})}e.s(["default",0,({accessToken:e,userRole:j})=>{let[_,b]=(0,a.useState)([]),[v,w]=(0,a.useState)(!1),[N,C]=(0,a.useState)(!1),[S,k]=(0,a.useState)(!1),[I,O]=(0,a.useState)(!1),[T,P]=(0,a.useState)(null),[L,B]=(0,a.useState)(!1),[F,$]=(0,a.useState)(null),E=!!j&&(0,u.isAdminRole)(j),M=async()=>{if(e){k(!0);try{let t=await (0,d.getGuardrailsList)(e);console.log(`guardrails: ${JSON.stringify(t)}`),b(t.guardrails)}catch(e){console.error("Error fetching guardrails:",e)}finally{k(!1)}}};(0,a.useEffect)(()=>{M()},[e]);let R=()=>{M()},G=async()=>{if(T&&e){O(!0);try{await (0,d.deleteGuardrailCall)(e,T.guardrail_id),x.default.success(`Guardrail "${T.guardrail_name}" deleted successfully`),await M()}catch(e){console.error("Error deleting guardrail:",e),x.default.fromBackend("Failed to delete guardrail")}finally{O(!1),B(!1),P(null)}}},z=T&&T.litellm_params?(0,f.getGuardrailLogoAndName)(T.litellm_params.guardrail).displayName:void 0;return(0,t.jsx)("div",{className:"w-full mx-auto flex-auto overflow-y-auto m-8 p-2",children:(0,t.jsx)(i.Tabs,{defaultActiveKey:"submitted",items:[...E?[{key:"garden",label:"Guardrail Garden",children:(0,t.jsx)(A,{accessToken:e,onGuardrailCreated:R})},{key:"guardrails",label:"Guardrails",children:(0,t.jsxs)(t.Fragment,{children:[(0,t.jsx)("div",{className:"flex justify-between items-center mb-4",children:(0,t.jsx)(r.Dropdown,{menu:{items:[{key:"provider",icon:(0,t.jsx)(n.PlusOutlined,{}),label:"Add Provider Guardrail",onClick:()=>{F&&$(null),w(!0)}},{key:"custom_code",icon:(0,t.jsx)(o.CodeOutlined,{}),label:"Create Custom Code Guardrail",onClick:()=>{F&&$(null),C(!0)}}]},trigger:["click"],disabled:!e,children:(0,t.jsxs)(l.Button,{disabled:!e,children:["+ Add New Guardrail ",(0,t.jsx)(s.DownOutlined,{className:"ml-2"})]})})}),F?(0,t.jsx)(p.default,{guardrailId:F,onClose:()=>$(null),accessToken:e,isAdmin:E}):(0,t.jsx)(m.default,{guardrailsList:_,isLoading:S,onDeleteClick:(e,t)=>{P(_.find(t=>t.guardrail_id===e)||null),B(!0)},accessToken:e,onGuardrailUpdated:M,isAdmin:E,onGuardrailClick:e=>$(e)}),(0,t.jsx)(c.default,{visible:v,onClose:()=>{w(!1)},accessToken:e,onSuccess:R}),(0,t.jsx)(y.CustomCodeModal,{visible:N,onClose:()=>{C(!1)},accessToken:e,onSuccess:R}),(0,t.jsx)(h.default,{isOpen:L,title:"Delete Guardrail",message:`Are you sure you want to delete guardrail: ${T?.guardrail_name}? This action cannot be undone.`,resourceInformationTitle:"Guardrail Information",resourceInformation:[{label:"Name",value:T?.guardrail_name},{label:"ID",value:T?.guardrail_id,code:!0},{label:"Provider",value:z},{label:"Mode",value:T?.litellm_params.mode},{label:"Default On",value:T?.litellm_params.default_on?"Yes":"No"}],onCancel:()=>{B(!1),P(null)},onOk:G,confirmLoading:I})]})},{key:"playground",label:"Test Playground",disabled:!e,children:(0,t.jsx)(g.default,{guardrailsList:_,isLoading:S,accessToken:e,onClose:()=>{}})}]:[],{key:"submitted",label:"Submitted Guardrails",children:(0,t.jsx)(es,{accessToken:e})}]})})}],487304)}]); \ No newline at end of file diff --git a/litellm/proxy/_experimental/out/_next/static/chunks/0.75_v4.brf7u.js b/litellm/proxy/_experimental/out/_next/static/chunks/0.75_v4.brf7u.js new file mode 100644 index 00000000000..ee955aa2b47 --- /dev/null +++ b/litellm/proxy/_experimental/out/_next/static/chunks/0.75_v4.brf7u.js @@ -0,0 +1,3 @@ +(globalThis.TURBOPACK||(globalThis.TURBOPACK=[])).push(["object"==typeof document?document.currentScript:void 0,721369,e=>{"use strict";e.i(247167);var t=e.i(271645),n=e.i(864517),a=e.i(867384),o=e.i(959013),i=e.i(343794),r=e.i(931067),l=e.i(211577),c=e.i(209428),u=e.i(392221),d=e.i(410160),s=e.i(703923),f=e.i(914949),v=e.i(614761);let b=(0,t.createContext)(null);var p=e.i(8211),m=e.i(430073),h=e.i(175066),g=e.i(611935),$=e.i(963188);let y=function(e){var n=e.activeTabOffset,a=e.horizontal,o=e.rtl,i=e.indicator,r=void 0===i?{}:i,l=r.size,c=r.align,d=void 0===c?"center":c,s=(0,t.useState)(),f=(0,u.default)(s,2),v=f[0],b=f[1],p=(0,t.useRef)(),m=t.default.useCallback(function(e){return"function"==typeof l?l(e):"number"==typeof l?l:e},[l]);function h(){$.default.cancel(p.current)}return(0,t.useEffect)(function(){var e={};if(n)if(a){e.width=m(n.width);var t=o?"right":"left";"start"===d&&(e[t]=n[t]),"center"===d&&(e[t]=n[t]+n.width/2,e.transform=o?"translateX(50%)":"translateX(-50%)"),"end"===d&&(e[t]=n[t]+n.width,e.transform="translateX(-100%)")}else e.height=m(n.height),"start"===d&&(e.top=n.top),"center"===d&&(e.top=n.top+n.height/2,e.transform="translateY(-50%)"),"end"===d&&(e.top=n.top+n.height,e.transform="translateY(-100%)");return h(),p.current=(0,$.default)(function(){v&&e&&Object.keys(e).every(function(t){var n=e[t],a=v[t];return"number"==typeof n&&"number"==typeof a?Math.round(n)===Math.round(a):n===a})||b(e)}),h},[JSON.stringify(n),a,o,d,m]),{style:v}};var k={width:0,height:0,left:0,top:0};function w(e,n){var a=t.useRef(e),o=t.useState({}),i=(0,u.default)(o,2)[1];return[a.current,function(e){var t="function"==typeof e?e(a.current):e;t!==a.current&&n(t,a.current),a.current=t,i({})}]}var x=e.i(174428);function _(e){var n=(0,t.useState)(0),a=(0,u.default)(n,2),o=a[0],i=a[1],r=(0,t.useRef)(0),l=(0,t.useRef)();return l.current=e,(0,x.useLayoutUpdateEffect)(function(){var e;null==(e=l.current)||e.call(l)},[o]),function(){r.current===o&&(r.current+=1,i(r.current))}}var S={width:0,height:0,left:0,top:0,right:0};function E(e){var t;return e instanceof Map?(t={},e.forEach(function(e,n){t[n]=e})):t=e,JSON.stringify(t)}function C(e){return String(e).replace(/"/g,"TABS_DQ")}function R(e,t,n,a){return!!n&&!a&&!1!==e&&(void 0!==e||!1!==t&&null!==t)}var T=t.forwardRef(function(e,n){var a=e.prefixCls,o=e.editable,i=e.locale,r=e.style;return o&&!1!==o.showAdd?t.createElement("button",{ref:n,type:"button",className:"".concat(a,"-nav-add"),style:r,"aria-label":(null==i?void 0:i.addAriaLabel)||"Add tab",onClick:function(e){o.onEdit("add",{event:e})}},o.addIcon||"+"):null}),P=t.forwardRef(function(e,n){var a,o=e.position,i=e.prefixCls,r=e.extra;if(!r)return null;var l={};return"object"!==(0,d.default)(r)||t.isValidElement(r)?l.right=r:l=r,"right"===o&&(a=l.right),"left"===o&&(a=l.left),a?t.createElement("div",{className:"".concat(i,"-extra-content"),ref:n},a):null}),I=e.i(878081),M=e.i(375565),O=e.i(452741),O=O,L=e.i(404948),B=t.forwardRef(function(e,n){var a=e.prefixCls,o=e.id,c=e.tabs,d=e.locale,s=e.mobile,f=e.more,v=void 0===f?{}:f,b=e.style,p=e.className,m=e.editable,h=e.tabBarGutter,g=e.rtl,$=e.removeAriaLabel,y=e.onTabClick,k=e.getPopupContainer,w=e.popupClassName,x=(0,t.useState)(!1),_=(0,u.default)(x,2),S=_[0],E=_[1],C=(0,t.useState)(null),P=(0,u.default)(C,2),B=P[0],D=P[1],z=v.icon,N="".concat(o,"-more-popup"),j="".concat(a,"-dropdown"),H=null!==B?"".concat(N,"-").concat(B):null,A=null==d?void 0:d.dropdownAriaLabel,G=t.createElement(M.default,{onClick:function(e){y(e.key,e.domEvent),E(!1)},prefixCls:"".concat(j,"-menu"),id:N,tabIndex:-1,role:"listbox","aria-activedescendant":H,selectedKeys:[B],"aria-label":void 0!==A?A:"expanded dropdown"},c.map(function(e){var n=e.closable,a=e.disabled,i=e.closeIcon,r=e.key,l=e.label,c=R(n,i,m,a);return t.createElement(O.default,{key:r,id:"".concat(N,"-").concat(r),role:"option","aria-controls":o&&"".concat(o,"-panel-").concat(r),disabled:a},t.createElement("span",null,l),c&&t.createElement("button",{type:"button","aria-label":$||"remove",tabIndex:0,className:"".concat(j,"-menu-item-remove"),onClick:function(e){e.stopPropagation(),e.preventDefault(),e.stopPropagation(),m.onEdit("remove",{key:r,event:e})}},i||m.removeIcon||"×"))}));function W(e){for(var t=c.filter(function(e){return!e.disabled}),n=t.findIndex(function(e){return e.key===B})||0,a=t.length,o=0;oMath.abs(l-n)?[l,c,u-t.left,d-t.top]:[n,a,i,o]},j=function(e){var t=e.current||{},n=t.offsetWidth,a=void 0===n?0:n,o=t.offsetHeight;if(e.current){var i=e.current.getBoundingClientRect(),r=i.width,l=i.height;if(1>Math.abs(r-a))return[r,l]}return[a,void 0===o?0:o]},H=function(e,t){return e[+!t]},A=t.forwardRef(function(e,n){var a,o,d,s,f,v,$,x,I,M,O,L,B,A,G,W,X,K,F,U,q,V,Y,J,Q,Z,ee,et,en,ea,eo,ei,er,el,ec,eu,ed,es,ef,ev=e.className,eb=e.style,ep=e.id,em=e.animated,eh=e.activeKey,eg=e.rtl,e$=e.extra,ey=e.editable,ek=e.locale,ew=e.tabPosition,ex=e.tabBarGutter,e_=e.children,eS=e.onTabClick,eE=e.onTabScroll,eC=e.indicator,eR=t.useContext(b),eT=eR.prefixCls,eP=eR.tabs,eI=(0,t.useRef)(null),eM=(0,t.useRef)(null),eO=(0,t.useRef)(null),eL=(0,t.useRef)(null),eB=(0,t.useRef)(null),eD=(0,t.useRef)(null),ez=(0,t.useRef)(null),eN="top"===ew||"bottom"===ew,ej=w(0,function(e,t){eN&&eE&&eE({direction:e>t?"left":"right"})}),eH=(0,u.default)(ej,2),eA=eH[0],eG=eH[1],eW=w(0,function(e,t){!eN&&eE&&eE({direction:e>t?"top":"bottom"})}),eX=(0,u.default)(eW,2),eK=eX[0],eF=eX[1],eU=(0,t.useState)([0,0]),eq=(0,u.default)(eU,2),eV=eq[0],eY=eq[1],eJ=(0,t.useState)([0,0]),eQ=(0,u.default)(eJ,2),eZ=eQ[0],e0=eQ[1],e1=(0,t.useState)([0,0]),e2=(0,u.default)(e1,2),e4=e2[0],e6=e2[1],e9=(0,t.useState)([0,0]),e3=(0,u.default)(e9,2),e5=e3[0],e7=e3[1],e8=(a=new Map,o=(0,t.useRef)([]),d=(0,t.useState)({}),s=(0,u.default)(d,2)[1],f=(0,t.useRef)("function"==typeof a?a():a),v=_(function(){var e=f.current;o.current.forEach(function(t){e=t(e)}),o.current=[],f.current=e,s({})}),[f.current,function(e){o.current.push(e),v()}]),te=(0,u.default)(e8,2),tt=te[0],tn=te[1],ta=($=eZ[0],(0,t.useMemo)(function(){for(var e=new Map,t=tt.get(null==(o=eP[0])?void 0:o.key)||k,n=t.left+t.width,a=0;atf?tf:e}eN&&eg?(ts=0,tf=Math.max(0,ti-tu)):(ts=Math.min(0,tu-ti),tf=0);var tb=(0,t.useRef)(null),tp=(0,t.useState)(),tm=(0,u.default)(tp,2),th=tm[0],tg=tm[1];function t$(){tg(Date.now())}function ty(){tb.current&&clearTimeout(tb.current)}x=function(e,t){function n(e,t){e(function(e){return tv(e+t)})}return!!tc&&(eN?n(eG,e):n(eF,t),ty(),t$(),!0)},I=(0,t.useState)(),O=(M=(0,u.default)(I,2))[0],L=M[1],B=(0,t.useState)(0),G=(A=(0,u.default)(B,2))[0],W=A[1],X=(0,t.useState)(0),F=(K=(0,u.default)(X,2))[0],U=K[1],q=(0,t.useState)(),Y=(V=(0,u.default)(q,2))[0],J=V[1],Q=(0,t.useRef)(),Z=(0,t.useRef)(),(ee=(0,t.useRef)(null)).current={onTouchStart:function(e){var t=e.touches[0];L({x:t.screenX,y:t.screenY}),window.clearInterval(Q.current)},onTouchMove:function(e){if(O){var t=e.touches[0],n=t.screenX,a=t.screenY;L({x:n,y:a});var o=n-O.x,i=a-O.y;x(o,i);var r=Date.now();W(r),U(r-G),J({x:o,y:i})}},onTouchEnd:function(){if(O&&(L(null),J(null),Y)){var e=Y.x/F,t=Y.y/F;if(!(.1>Math.max(Math.abs(e),Math.abs(t)))){var n=e,a=t;Q.current=window.setInterval(function(){.01>Math.abs(n)&&.01>Math.abs(a)?window.clearInterval(Q.current):(n*=.9046104802746175,a*=.9046104802746175,x(20*n,20*a))},20)}}},onWheel:function(e){var t=e.deltaX,n=e.deltaY,a=0,o=Math.abs(t),i=Math.abs(n);o===i?a="x"===Z.current?t:n:o>i?(a=t,Z.current="x"):(a=n,Z.current="y"),x(-a,-a)&&e.preventDefault()}},t.useEffect(function(){function e(e){ee.current.onTouchMove(e)}function t(e){ee.current.onTouchEnd(e)}return document.addEventListener("touchmove",e,{passive:!1}),document.addEventListener("touchend",t,{passive:!0}),eL.current.addEventListener("touchstart",function(e){ee.current.onTouchStart(e)},{passive:!0}),eL.current.addEventListener("wheel",function(e){ee.current.onWheel(e)},{passive:!1}),function(){document.removeEventListener("touchmove",e),document.removeEventListener("touchend",t)}},[]),(0,t.useEffect)(function(){return ty(),th&&(tb.current=setTimeout(function(){tg(0)},100)),ty},[th]);var tk=(et=eN?eA:eK,er=(en=(0,c.default)((0,c.default)({},e),{},{tabs:eP})).tabs,el=en.tabPosition,ec=en.rtl,["top","bottom"].includes(el)?(ea="width",eo=ec?"right":"left",ei=Math.abs(et)):(ea="height",eo="top",ei=-et),(0,t.useMemo)(function(){if(!er.length)return[0,0];for(var e=er.length,t=e,n=0;nMath.floor(ei+tu)){t=n-1;break}}for(var o=0,i=e-1;i>=0;i-=1)if((ta.get(er[i].key)||S)[eo]t?[0,-1]:[o,t]},[ta,tu,ti,tr,tl,ei,el,er.map(function(e){return e.key}).join("_"),ec])),tw=(0,u.default)(tk,2),tx=tw[0],t_=tw[1],tS=(0,h.default)(function(){var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:eh,t=ta.get(e)||{width:0,height:0,left:0,right:0,top:0};if(eN){var n=eA;eg?t.righteA+tu&&(n=t.right+t.width-tu):t.left<-eA?n=-t.left:t.left+t.width>-eA+tu&&(n=-(t.left+t.width-tu)),eF(0),eG(tv(n))}else{var a=eK;t.top<-eK?a=-t.top:t.top+t.height>-eK+tu&&(a=-(t.top+t.height-tu)),eG(0),eF(tv(a))}}),tE=(0,t.useState)(),tC=(0,u.default)(tE,2),tR=tC[0],tT=tC[1],tP=(0,t.useState)(!1),tI=(0,u.default)(tP,2),tM=tI[0],tO=tI[1],tL=eP.filter(function(e){return!e.disabled}).map(function(e){return e.key}),tB=function(e){var t=tL.indexOf(tR||eh),n=tL.length;tT(tL[(t+e+n)%n])},tD=function(e,t){var n=tL.indexOf(e),a=eP.find(function(t){return t.key===e});R(null==a?void 0:a.closable,null==a?void 0:a.closeIcon,ey,null==a?void 0:a.disabled)&&(t.preventDefault(),t.stopPropagation(),ey.onEdit("remove",{key:e,event:t}),n===tL.length-1?tB(-1):tB(1))},tz=function(e,t){tO(!0),1===t.button&&tD(e,t)},tN=function(e){var t=e.code,n=eg&&eN,a=tL[0],o=tL[tL.length-1];switch(t){case"ArrowLeft":eN&&tB(n?1:-1);break;case"ArrowRight":eN&&tB(n?-1:1);break;case"ArrowUp":e.preventDefault(),eN||tB(-1);break;case"ArrowDown":e.preventDefault(),eN||tB(1);break;case"Home":e.preventDefault(),tT(a);break;case"End":e.preventDefault(),tT(o);break;case"Enter":case"Space":e.preventDefault(),eS(null!=tR?tR:eh,e);break;case"Backspace":case"Delete":tD(tR,e)}},tj={};eN?tj[eg?"marginRight":"marginLeft"]=ex:tj.marginTop=ex;var tH=eP.map(function(e,n){var a=e.key;return t.createElement(z,{id:ep,prefixCls:eT,key:a,tab:e,style:0===n?void 0:tj,closable:e.closable,editable:ey,active:a===eh,focus:a===tR,renderWrapper:e_,removeAriaLabel:null==ek?void 0:ek.removeAriaLabel,tabCount:tL.length,currentPosition:n+1,onClick:function(e){eS(a,e)},onKeyDown:tN,onFocus:function(){tM||tT(a),tS(a),t$(),eL.current&&(eg||(eL.current.scrollLeft=0),eL.current.scrollTop=0)},onBlur:function(){tT(void 0)},onMouseDown:function(e){return tz(a,e)},onMouseUp:function(){tO(!1)}})}),tA=function(){return tn(function(){var e,t=new Map,n=null==(e=eB.current)?void 0:e.getBoundingClientRect();return eP.forEach(function(e){var a,o=e.key,i=null==(a=eB.current)?void 0:a.querySelector('[data-node-key="'.concat(C(o),'"]'));if(i){var r=N(i,n),l=(0,u.default)(r,4),c=l[0],d=l[1],s=l[2],f=l[3];t.set(o,{width:c,height:d,left:s,top:f})}}),t})};(0,t.useEffect)(function(){tA()},[eP.map(function(e){return e.key}).join("_")]);var tG=_(function(){var e=j(eI),t=j(eM),n=j(eO);eY([e[0]-t[0]-n[0],e[1]-t[1]-n[1]]);var a=j(ez);e6(a),e7(j(eD));var o=j(eB);e0([o[0]-a[0],o[1]-a[1]]),tA()}),tW=eP.slice(0,tx),tX=eP.slice(t_+1),tK=[].concat((0,p.default)(tW),(0,p.default)(tX)),tF=ta.get(eh),tU=y({activeTabOffset:tF,horizontal:eN,indicator:eC,rtl:eg}).style;(0,t.useEffect)(function(){tS()},[eh,ts,tf,E(tF),E(ta),eN]),(0,t.useEffect)(function(){tG()},[eg]);var tq=!!tK.length,tV="".concat(eT,"-nav-wrap");return eN?eg?(ed=eA>0,eu=eA!==tf):(eu=eA<0,ed=eA!==ts):(es=eK<0,ef=eK!==ts),t.createElement(m.default,{onResize:tG},t.createElement("div",{ref:(0,g.useComposeRef)(n,eI),role:"tablist","aria-orientation":eN?"horizontal":"vertical",className:(0,i.default)("".concat(eT,"-nav"),ev),style:eb,onKeyDown:function(){t$()}},t.createElement(P,{ref:eM,position:"left",extra:e$,prefixCls:eT}),t.createElement(m.default,{onResize:tG},t.createElement("div",{className:(0,i.default)(tV,(0,l.default)((0,l.default)((0,l.default)((0,l.default)({},"".concat(tV,"-ping-left"),eu),"".concat(tV,"-ping-right"),ed),"".concat(tV,"-ping-top"),es),"".concat(tV,"-ping-bottom"),ef)),ref:eL},t.createElement(m.default,{onResize:tG},t.createElement("div",{ref:eB,className:"".concat(eT,"-nav-list"),style:{transform:"translate(".concat(eA,"px, ").concat(eK,"px)"),transition:th?"none":void 0}},tH,t.createElement(T,{ref:ez,prefixCls:eT,locale:ek,editable:ey,style:(0,c.default)((0,c.default)({},0===tH.length?void 0:tj),{},{visibility:tq?"hidden":null})}),t.createElement("div",{className:(0,i.default)("".concat(eT,"-ink-bar"),(0,l.default)({},"".concat(eT,"-ink-bar-animated"),em.inkBar)),style:tU}))))),t.createElement(D,(0,r.default)({},e,{removeAriaLabel:null==ek?void 0:ek.removeAriaLabel,ref:eD,prefixCls:eT,tabs:tK,className:!tq&&td,tabMoving:!!th})),t.createElement(P,{ref:eO,position:"right",extra:e$,prefixCls:eT})))}),G=t.forwardRef(function(e,n){var a=e.prefixCls,o=e.className,r=e.style,l=e.id,c=e.active,u=e.tabKey,d=e.children;return t.createElement("div",{id:l&&"".concat(l,"-panel-").concat(u),role:"tabpanel",tabIndex:c?0:-1,"aria-labelledby":l&&"".concat(l,"-tab-").concat(u),"aria-hidden":!c,style:r,className:(0,i.default)(a,c&&"".concat(a,"-active"),o),ref:n},d)}),W=["renderTabBar"],X=["label","key"];let K=function(e){var n=e.renderTabBar,a=(0,s.default)(e,W),o=t.useContext(b).tabs;return n?n((0,c.default)((0,c.default)({},a),{},{panes:o.map(function(e){var n=e.label,a=e.key,o=(0,s.default)(e,X);return t.createElement(G,(0,r.default)({tab:n,key:a,tabKey:a},o))})}),A):t.createElement(A,a)};var F=e.i(361275),U=["key","forceRender","style","className","destroyInactiveTabPane"];let q=function(e){var n=e.id,a=e.activeKey,o=e.animated,u=e.tabPosition,d=e.destroyInactiveTabPane,f=t.useContext(b),v=f.prefixCls,p=f.tabs,m=o.tabPane,h="".concat(v,"-tabpane");return t.createElement("div",{className:(0,i.default)("".concat(v,"-content-holder"))},t.createElement("div",{className:(0,i.default)("".concat(v,"-content"),"".concat(v,"-content-").concat(u),(0,l.default)({},"".concat(v,"-content-animated"),m))},p.map(function(e){var l=e.key,u=e.forceRender,f=e.style,v=e.className,b=e.destroyInactiveTabPane,p=(0,s.default)(e,U),g=l===a;return t.createElement(F.default,(0,r.default)({key:l,visible:g,forceRender:u,removeOnLeave:!!(d||b),leavedClassName:"".concat(h,"-hidden")},o.tabPaneMotion),function(e,a){var o=e.style,u=e.className;return t.createElement(G,(0,r.default)({},p,{prefixCls:h,id:n,tabKey:l,animated:m,active:g,style:(0,c.default)((0,c.default)({},f),o),className:(0,i.default)(v,u),ref:a}))})})))};e.i(883110);var V=["id","prefixCls","className","items","direction","activeKey","defaultActiveKey","editable","animated","tabPosition","tabBarGutter","tabBarStyle","tabBarExtraContent","locale","more","destroyInactiveTabPane","renderTabBar","onChange","onTabClick","onTabScroll","getPopupContainer","popupClassName","indicator"],Y=0,J=t.forwardRef(function(e,n){var a=e.id,o=e.prefixCls,p=void 0===o?"rc-tabs":o,m=e.className,h=e.items,g=e.direction,$=e.activeKey,y=e.defaultActiveKey,k=e.editable,w=e.animated,x=e.tabPosition,_=void 0===x?"top":x,S=e.tabBarGutter,E=e.tabBarStyle,C=e.tabBarExtraContent,R=e.locale,T=e.more,P=e.destroyInactiveTabPane,I=e.renderTabBar,M=e.onChange,O=e.onTabClick,L=e.onTabScroll,B=e.getPopupContainer,D=e.popupClassName,z=e.indicator,N=(0,s.default)(e,V),j=t.useMemo(function(){return(h||[]).filter(function(e){return e&&"object"===(0,d.default)(e)&&"key"in e})},[h]),H="rtl"===g,A=function(){var e,t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{inkBar:!0,tabPane:!1};return(e=!1===t?{inkBar:!1,tabPane:!1}:!0===t?{inkBar:!0,tabPane:!1}:(0,c.default)({inkBar:!0},"object"===(0,d.default)(t)?t:{})).tabPaneMotion&&void 0===e.tabPane&&(e.tabPane=!0),!e.tabPaneMotion&&e.tabPane&&(e.tabPane=!1),e}(w),G=(0,t.useState)(!1),W=(0,u.default)(G,2),X=W[0],F=W[1];(0,t.useEffect)(function(){F((0,v.default)())},[]);var U=(0,f.default)(function(){var e;return null==(e=j[0])?void 0:e.key},{value:$,defaultValue:y}),J=(0,u.default)(U,2),Q=J[0],Z=J[1],ee=(0,t.useState)(function(){return j.findIndex(function(e){return e.key===Q})}),et=(0,u.default)(ee,2),en=et[0],ea=et[1];(0,t.useEffect)(function(){var e,t=j.findIndex(function(e){return e.key===Q});-1===t&&(t=Math.max(0,Math.min(en,j.length-1)),Z(null==(e=j[t])?void 0:e.key)),ea(t)},[j.map(function(e){return e.key}).join("_"),Q,en]);var eo=(0,f.default)(null,{value:a}),ei=(0,u.default)(eo,2),er=ei[0],el=ei[1];(0,t.useEffect)(function(){a||(el("rc-tabs-".concat(Y)),Y+=1)},[]);var ec={id:er,activeKey:Q,animated:A,tabPosition:_,rtl:H,mobile:X},eu=(0,c.default)((0,c.default)({},ec),{},{editable:k,locale:R,more:T,tabBarGutter:S,onTabClick:function(e,t){null==O||O(e,t);var n=e!==Q;Z(e),n&&(null==M||M(e))},onTabScroll:L,extra:C,style:E,panes:null,getPopupContainer:B,popupClassName:D,indicator:z});return t.createElement(b.Provider,{value:{tabs:j,prefixCls:p}},t.createElement("div",(0,r.default)({ref:n,id:a,className:(0,i.default)(p,"".concat(p,"-").concat(_),(0,l.default)((0,l.default)((0,l.default)({},"".concat(p,"-mobile"),X),"".concat(p,"-editable"),k),"".concat(p,"-rtl"),H),m)},N),t.createElement(K,(0,r.default)({},eu,{renderTabBar:I})),t.createElement(q,(0,r.default)({destroyInactiveTabPane:P},ec,{animated:A}))))}),Q=e.i(242064),Z=e.i(321883),ee=e.i(517455),et=e.i(613541);let en={motionAppear:!1,motionEnter:!0,motionLeave:!0};var ea=e.i(876556),eo=function(e,t){var n={};for(var a in e)Object.prototype.hasOwnProperty.call(e,a)&&0>t.indexOf(a)&&(n[a]=e[a]);if(null!=e&&"function"==typeof Object.getOwnPropertySymbols)for(var o=0,a=Object.getOwnPropertySymbols(e);ot.indexOf(a[o])&&Object.prototype.propertyIsEnumerable.call(e,a[o])&&(n[a[o]]=e[a[o]]);return n};e.i(296059);var ei=e.i(915654),er=e.i(183293),el=e.i(246422),ec=e.i(838378),eu=e.i(664142);let ed=(0,el.genStyleHooks)("Tabs",e=>{let t=(0,ec.mergeToken)(e,{tabsCardPadding:e.cardPadding,dropdownEdgeChildVerticalPadding:e.paddingXXS,tabsActiveTextShadow:"0 0 0.25px currentcolor",tabsDropdownHeight:200,tabsDropdownWidth:120,tabsHorizontalItemMargin:`0 0 0 ${(0,ei.unit)(e.horizontalItemGutter)}`,tabsHorizontalItemMarginRTL:`0 0 0 ${(0,ei.unit)(e.horizontalItemGutter)}`});return[(e=>{let{componentCls:t,cardPaddingSM:n,cardPaddingLG:a,cardHeightSM:o,cardHeightLG:i,horizontalItemPaddingSM:r,horizontalItemPaddingLG:l}=e;return{[t]:{"&-small":{[`> ${t}-nav`]:{[`${t}-tab`]:{padding:r,fontSize:e.titleFontSizeSM}}},"&-large":{[`> ${t}-nav`]:{[`${t}-tab`]:{padding:l,fontSize:e.titleFontSizeLG,lineHeight:e.lineHeightLG}}}},[`${t}-card`]:{[`&${t}-small`]:{[`> ${t}-nav`]:{[`${t}-tab`]:{padding:n},[`${t}-nav-add`]:{minWidth:o,minHeight:o}},[`&${t}-bottom`]:{[`> ${t}-nav ${t}-tab`]:{borderRadius:`0 0 ${(0,ei.unit)(e.borderRadius)} ${(0,ei.unit)(e.borderRadius)}`}},[`&${t}-top`]:{[`> ${t}-nav ${t}-tab`]:{borderRadius:`${(0,ei.unit)(e.borderRadius)} ${(0,ei.unit)(e.borderRadius)} 0 0`}},[`&${t}-right`]:{[`> ${t}-nav ${t}-tab`]:{borderRadius:{_skip_check_:!0,value:`0 ${(0,ei.unit)(e.borderRadius)} ${(0,ei.unit)(e.borderRadius)} 0`}}},[`&${t}-left`]:{[`> ${t}-nav ${t}-tab`]:{borderRadius:{_skip_check_:!0,value:`${(0,ei.unit)(e.borderRadius)} 0 0 ${(0,ei.unit)(e.borderRadius)}`}}}},[`&${t}-large`]:{[`> ${t}-nav`]:{[`${t}-tab`]:{padding:a},[`${t}-nav-add`]:{minWidth:i,minHeight:i}}}}}})(t),(e=>{let{componentCls:t,tabsHorizontalItemMarginRTL:n,iconCls:a,cardGutter:o,calc:i}=e;return{[`${t}-rtl`]:{direction:"rtl",[`${t}-nav`]:{[`${t}-tab`]:{margin:{_skip_check_:!0,value:n},[`${t}-tab:last-of-type`]:{marginLeft:{_skip_check_:!0,value:0}},[a]:{marginRight:{_skip_check_:!0,value:0},marginLeft:{_skip_check_:!0,value:(0,ei.unit)(e.marginSM)}},[`${t}-tab-remove`]:{marginRight:{_skip_check_:!0,value:(0,ei.unit)(e.marginXS)},marginLeft:{_skip_check_:!0,value:(0,ei.unit)(i(e.marginXXS).mul(-1).equal())},[a]:{margin:0}}}},[`&${t}-left`]:{[`> ${t}-nav`]:{order:1},[`> ${t}-content-holder`]:{order:0}},[`&${t}-right`]:{[`> ${t}-nav`]:{order:0},[`> ${t}-content-holder`]:{order:1}},[`&${t}-card${t}-top, &${t}-card${t}-bottom`]:{[`> ${t}-nav, > div > ${t}-nav`]:{[`${t}-tab + ${t}-tab`]:{marginRight:{_skip_check_:!0,value:o},marginLeft:{_skip_check_:!0,value:0}}}}},[`${t}-dropdown-rtl`]:{direction:"rtl"},[`${t}-menu-item`]:{[`${t}-dropdown-rtl`]:{textAlign:{_skip_check_:!0,value:"right"}}}}})(t),(e=>{let{componentCls:t,margin:n,colorBorderSecondary:a,horizontalMargin:o,verticalItemPadding:i,verticalItemMargin:r,calc:l}=e;return{[`${t}-top, ${t}-bottom`]:{flexDirection:"column",[`> ${t}-nav, > div > ${t}-nav`]:{margin:o,"&::before":{position:"absolute",right:{_skip_check_:!0,value:0},left:{_skip_check_:!0,value:0},borderBottom:`${(0,ei.unit)(e.lineWidth)} ${e.lineType} ${a}`,content:"''"},[`${t}-ink-bar`]:{height:e.lineWidthBold,"&-animated":{transition:`width ${e.motionDurationSlow}, left ${e.motionDurationSlow}, + right ${e.motionDurationSlow}`}},[`${t}-nav-wrap`]:{"&::before, &::after":{top:0,bottom:0,width:e.controlHeight},"&::before":{left:{_skip_check_:!0,value:0},boxShadow:e.boxShadowTabsOverflowLeft},"&::after":{right:{_skip_check_:!0,value:0},boxShadow:e.boxShadowTabsOverflowRight},[`&${t}-nav-wrap-ping-left::before`]:{opacity:1},[`&${t}-nav-wrap-ping-right::after`]:{opacity:1}}}},[`${t}-top`]:{[`> ${t}-nav, + > div > ${t}-nav`]:{"&::before":{bottom:0},[`${t}-ink-bar`]:{bottom:0}}},[`${t}-bottom`]:{[`> ${t}-nav, > div > ${t}-nav`]:{order:1,marginTop:n,marginBottom:0,"&::before":{top:0},[`${t}-ink-bar`]:{top:0}},[`> ${t}-content-holder, > div > ${t}-content-holder`]:{order:0}},[`${t}-left, ${t}-right`]:{[`> ${t}-nav, > div > ${t}-nav`]:{flexDirection:"column",minWidth:l(e.controlHeight).mul(1.25).equal(),[`${t}-tab`]:{padding:i,textAlign:"center"},[`${t}-tab + ${t}-tab`]:{margin:r},[`${t}-nav-wrap`]:{flexDirection:"column","&::before, &::after":{right:{_skip_check_:!0,value:0},left:{_skip_check_:!0,value:0},height:e.controlHeight},"&::before":{top:0,boxShadow:e.boxShadowTabsOverflowTop},"&::after":{bottom:0,boxShadow:e.boxShadowTabsOverflowBottom},[`&${t}-nav-wrap-ping-top::before`]:{opacity:1},[`&${t}-nav-wrap-ping-bottom::after`]:{opacity:1}},[`${t}-ink-bar`]:{width:e.lineWidthBold,"&-animated":{transition:`height ${e.motionDurationSlow}, top ${e.motionDurationSlow}`}},[`${t}-nav-list, ${t}-nav-operations`]:{flex:"1 0 auto",flexDirection:"column"}}},[`${t}-left`]:{[`> ${t}-nav, > div > ${t}-nav`]:{[`${t}-ink-bar`]:{right:{_skip_check_:!0,value:0}}},[`> ${t}-content-holder, > div > ${t}-content-holder`]:{marginLeft:{_skip_check_:!0,value:(0,ei.unit)(l(e.lineWidth).mul(-1).equal())},borderLeft:{_skip_check_:!0,value:`${(0,ei.unit)(e.lineWidth)} ${e.lineType} ${e.colorBorder}`},[`> ${t}-content > ${t}-tabpane`]:{paddingLeft:{_skip_check_:!0,value:e.paddingLG}}}},[`${t}-right`]:{[`> ${t}-nav, > div > ${t}-nav`]:{order:1,[`${t}-ink-bar`]:{left:{_skip_check_:!0,value:0}}},[`> ${t}-content-holder, > div > ${t}-content-holder`]:{order:0,marginRight:{_skip_check_:!0,value:l(e.lineWidth).mul(-1).equal()},borderRight:{_skip_check_:!0,value:`${(0,ei.unit)(e.lineWidth)} ${e.lineType} ${e.colorBorder}`},[`> ${t}-content > ${t}-tabpane`]:{paddingRight:{_skip_check_:!0,value:e.paddingLG}}}}}})(t),(e=>{let{componentCls:t,itemHoverColor:n,dropdownEdgeChildVerticalPadding:a}=e;return{[`${t}-dropdown`]:Object.assign(Object.assign({},(0,er.resetComponent)(e)),{position:"absolute",top:-9999,left:{_skip_check_:!0,value:-9999},zIndex:e.zIndexPopup,display:"block","&-hidden":{display:"none"},[`${t}-dropdown-menu`]:{maxHeight:e.tabsDropdownHeight,margin:0,padding:`${(0,ei.unit)(a)} 0`,overflowX:"hidden",overflowY:"auto",textAlign:{_skip_check_:!0,value:"left"},listStyleType:"none",backgroundColor:e.colorBgContainer,backgroundClip:"padding-box",borderRadius:e.borderRadiusLG,outline:"none",boxShadow:e.boxShadowSecondary,"&-item":Object.assign(Object.assign({},er.textEllipsis),{display:"flex",alignItems:"center",minWidth:e.tabsDropdownWidth,margin:0,padding:`${(0,ei.unit)(e.paddingXXS)} ${(0,ei.unit)(e.paddingSM)}`,color:e.colorText,fontWeight:"normal",fontSize:e.fontSize,lineHeight:e.lineHeight,cursor:"pointer",transition:`all ${e.motionDurationSlow}`,"> span":{flex:1,whiteSpace:"nowrap"},"&-remove":{flex:"none",marginLeft:{_skip_check_:!0,value:e.marginSM},color:e.colorIcon,fontSize:e.fontSizeSM,background:"transparent",border:0,cursor:"pointer","&:hover":{color:n}},"&:hover":{background:e.controlItemBgHover},"&-disabled":{"&, &:hover":{color:e.colorTextDisabled,background:"transparent",cursor:"not-allowed"}}})}})}})(t),(e=>{let{componentCls:t,tabsCardPadding:n,cardBg:a,cardGutter:o,colorBorderSecondary:i,itemSelectedColor:r}=e;return{[`${t}-card`]:{[`> ${t}-nav, > div > ${t}-nav`]:{[`${t}-tab`]:{margin:0,padding:n,background:a,border:`${(0,ei.unit)(e.lineWidth)} ${e.lineType} ${i}`,transition:`all ${e.motionDurationSlow} ${e.motionEaseInOut}`},[`${t}-tab-active`]:{color:r,background:e.colorBgContainer},[`${t}-tab-focus:has(${t}-tab-btn:focus-visible)`]:(0,er.genFocusOutline)(e,-3),[`& ${t}-tab${t}-tab-focus ${t}-tab-btn:focus-visible`]:{outline:"none"},[`${t}-ink-bar`]:{visibility:"hidden"}},[`&${t}-top, &${t}-bottom`]:{[`> ${t}-nav, > div > ${t}-nav`]:{[`${t}-tab + ${t}-tab`]:{marginLeft:{_skip_check_:!0,value:(0,ei.unit)(o)}}}},[`&${t}-top`]:{[`> ${t}-nav, > div > ${t}-nav`]:{[`${t}-tab`]:{borderRadius:`${(0,ei.unit)(e.borderRadiusLG)} ${(0,ei.unit)(e.borderRadiusLG)} 0 0`},[`${t}-tab-active`]:{borderBottomColor:e.colorBgContainer}}},[`&${t}-bottom`]:{[`> ${t}-nav, > div > ${t}-nav`]:{[`${t}-tab`]:{borderRadius:`0 0 ${(0,ei.unit)(e.borderRadiusLG)} ${(0,ei.unit)(e.borderRadiusLG)}`},[`${t}-tab-active`]:{borderTopColor:e.colorBgContainer}}},[`&${t}-left, &${t}-right`]:{[`> ${t}-nav, > div > ${t}-nav`]:{[`${t}-tab + ${t}-tab`]:{marginTop:(0,ei.unit)(o)}}},[`&${t}-left`]:{[`> ${t}-nav, > div > ${t}-nav`]:{[`${t}-tab`]:{borderRadius:{_skip_check_:!0,value:`${(0,ei.unit)(e.borderRadiusLG)} 0 0 ${(0,ei.unit)(e.borderRadiusLG)}`}},[`${t}-tab-active`]:{borderRightColor:{_skip_check_:!0,value:e.colorBgContainer}}}},[`&${t}-right`]:{[`> ${t}-nav, > div > ${t}-nav`]:{[`${t}-tab`]:{borderRadius:{_skip_check_:!0,value:`0 ${(0,ei.unit)(e.borderRadiusLG)} ${(0,ei.unit)(e.borderRadiusLG)} 0`}},[`${t}-tab-active`]:{borderLeftColor:{_skip_check_:!0,value:e.colorBgContainer}}}}}}})(t),(e=>{let{componentCls:t,tabsCardPadding:n,cardHeight:a,cardGutter:o,itemHoverColor:i,itemActiveColor:r,colorBorderSecondary:l}=e;return{[t]:Object.assign(Object.assign(Object.assign(Object.assign({},(0,er.resetComponent)(e)),{display:"flex",[`> ${t}-nav, > div > ${t}-nav`]:{position:"relative",display:"flex",flex:"none",alignItems:"center",[`${t}-nav-wrap`]:{position:"relative",display:"flex",flex:"auto",alignSelf:"stretch",overflow:"hidden",whiteSpace:"nowrap",transform:"translate(0)","&::before, &::after":{position:"absolute",zIndex:1,opacity:0,transition:`opacity ${e.motionDurationSlow}`,content:"''",pointerEvents:"none"}},[`${t}-nav-list`]:{position:"relative",display:"flex",transition:`opacity ${e.motionDurationSlow}`},[`${t}-nav-operations`]:{display:"flex",alignSelf:"stretch"},[`${t}-nav-operations-hidden`]:{position:"absolute",visibility:"hidden",pointerEvents:"none"},[`${t}-nav-more`]:{position:"relative",padding:n,background:"transparent",border:0,color:e.colorText,"&::after":{position:"absolute",right:{_skip_check_:!0,value:0},bottom:0,left:{_skip_check_:!0,value:0},height:e.calc(e.controlHeightLG).div(8).equal(),transform:"translateY(100%)",content:"''"}},[`${t}-nav-add`]:Object.assign({minWidth:a,minHeight:a,marginLeft:{_skip_check_:!0,value:o},background:"transparent",border:`${(0,ei.unit)(e.lineWidth)} ${e.lineType} ${l}`,borderRadius:`${(0,ei.unit)(e.borderRadiusLG)} ${(0,ei.unit)(e.borderRadiusLG)} 0 0`,outline:"none",cursor:"pointer",color:e.colorText,transition:`all ${e.motionDurationSlow} ${e.motionEaseInOut}`,"&:hover":{color:i},"&:active, &:focus:not(:focus-visible)":{color:r}},(0,er.genFocusStyle)(e,-3))},[`${t}-extra-content`]:{flex:"none"},[`${t}-ink-bar`]:{position:"absolute",background:e.inkBarColor,pointerEvents:"none"}}),(e=>{let{componentCls:t,itemActiveColor:n,itemHoverColor:a,iconCls:o,tabsHorizontalItemMargin:i,horizontalItemPadding:r,itemSelectedColor:l,itemColor:c}=e,u=`${t}-tab`;return{[u]:{position:"relative",WebkitTouchCallout:"none",WebkitTapHighlightColor:"transparent",display:"inline-flex",alignItems:"center",padding:r,fontSize:e.titleFontSize,background:"transparent",border:0,outline:"none",cursor:"pointer",color:c,"&-btn, &-remove":{"&:focus:not(:focus-visible), &:active":{color:n}},"&-btn":{outline:"none",transition:`all ${e.motionDurationSlow}`,[`${u}-icon:not(:last-child)`]:{marginInlineEnd:e.marginSM}},"&-remove":Object.assign({flex:"none",lineHeight:1,marginRight:{_skip_check_:!0,value:e.calc(e.marginXXS).mul(-1).equal()},marginLeft:{_skip_check_:!0,value:e.marginXS},color:e.colorIcon,fontSize:e.fontSizeSM,background:"transparent",border:"none",outline:"none",cursor:"pointer",transition:`all ${e.motionDurationSlow}`,"&:hover":{color:e.colorTextHeading}},(0,er.genFocusStyle)(e)),"&:hover":{color:a},[`&${u}-active ${u}-btn`]:{color:l,textShadow:e.tabsActiveTextShadow},[`&${u}-focus ${u}-btn:focus-visible`]:(0,er.genFocusOutline)(e),[`&${u}-disabled`]:{color:e.colorTextDisabled,cursor:"not-allowed"},[`&${u}-disabled ${u}-btn, &${u}-disabled ${t}-remove`]:{"&:focus, &:active":{color:e.colorTextDisabled}},[`& ${u}-remove ${o}`]:{margin:0,verticalAlign:"middle"},[`${o}:not(:last-child)`]:{marginRight:{_skip_check_:!0,value:e.marginSM}}},[`${u} + ${u}`]:{margin:{_skip_check_:!0,value:i}}}})(e)),{[`${t}-content`]:{position:"relative",width:"100%"},[`${t}-content-holder`]:{flex:"auto",minWidth:0,minHeight:0},[`${t}-tabpane`]:Object.assign(Object.assign({},(0,er.genFocusStyle)(e)),{"&-hidden":{display:"none"}})}),[`${t}-centered`]:{[`> ${t}-nav, > div > ${t}-nav`]:{[`${t}-nav-wrap`]:{[`&:not([class*='${t}-nav-wrap-ping']) > ${t}-nav-list`]:{margin:"auto"}}}}}})(t),(e=>{let{componentCls:t,motionDurationSlow:n}=e;return[{[t]:{[`${t}-switch`]:{"&-appear, &-enter":{transition:"none","&-start":{opacity:0},"&-active":{opacity:1,transition:`opacity ${n}`}},"&-leave":{position:"absolute",transition:"none",inset:0,"&-start":{opacity:1},"&-active":{opacity:0,transition:`opacity ${n}`}}}}},[(0,eu.initSlideMotion)(e,"slide-up"),(0,eu.initSlideMotion)(e,"slide-down")]]})(t)]},e=>{let{cardHeight:t,cardHeightSM:n,cardHeightLG:a,controlHeight:o,controlHeightLG:i}=e,r=t||i,l=n||o,c=a||i+8;return{zIndexPopup:e.zIndexPopupBase+50,cardBg:e.colorFillAlter,cardHeight:r,cardHeightSM:l,cardHeightLG:c,cardPadding:`${(r-e.fontHeight)/2-e.lineWidth}px ${e.padding}px`,cardPaddingSM:`${(l-e.fontHeight)/2-e.lineWidth}px ${e.paddingXS}px`,cardPaddingLG:`${(c-e.fontHeightLG)/2-e.lineWidth}px ${e.padding}px`,titleFontSize:e.fontSize,titleFontSizeLG:e.fontSizeLG,titleFontSizeSM:e.fontSize,inkBarColor:e.colorPrimary,horizontalMargin:`0 0 ${e.margin}px 0`,horizontalItemGutter:32,horizontalItemMargin:"",horizontalItemMarginRTL:"",horizontalItemPadding:`${e.paddingSM}px 0`,horizontalItemPaddingSM:`${e.paddingXS}px 0`,horizontalItemPaddingLG:`${e.padding}px 0`,verticalItemPadding:`${e.paddingXS}px ${e.paddingLG}px`,verticalItemMargin:`${e.margin}px 0 0 0`,itemColor:e.colorText,itemSelectedColor:e.colorPrimary,itemHoverColor:e.colorPrimaryHover,itemActiveColor:e.colorPrimaryActive,cardGutter:e.marginXXS/2}});var es=function(e,t){var n={};for(var a in e)Object.prototype.hasOwnProperty.call(e,a)&&0>t.indexOf(a)&&(n[a]=e[a]);if(null!=e&&"function"==typeof Object.getOwnPropertySymbols)for(var o=0,a=Object.getOwnPropertySymbols(e);ot.indexOf(a[o])&&Object.prototype.propertyIsEnumerable.call(e,a[o])&&(n[a[o]]=e[a[o]]);return n};let ef=t.forwardRef((e,r)=>{var l,c,u,d,s,f,v,b,p,m,h,g,$;let y,{type:k,className:w,rootClassName:x,size:_,onEdit:S,hideAdd:E,centered:C,addIcon:R,removeIcon:T,moreIcon:P,more:I,popupClassName:M,children:O,items:L,animated:B,style:D,indicatorSize:z,indicator:N,destroyInactiveTabPane:j,destroyOnHidden:H}=e,A=es(e,["type","className","rootClassName","size","onEdit","hideAdd","centered","addIcon","removeIcon","moreIcon","more","popupClassName","children","items","animated","style","indicatorSize","indicator","destroyInactiveTabPane","destroyOnHidden"]),{prefixCls:G}=A,{direction:W,tabs:X,getPrefixCls:K,getPopupContainer:F}=t.useContext(Q.ConfigContext),U=K("tabs",G),q=(0,Z.default)(U),[V,Y,ei]=ed(U,q),er=t.useRef(null);t.useImperativeHandle(r,()=>({nativeElement:er.current})),"editable-card"===k&&(y={onEdit:(e,{key:t,event:n})=>{null==S||S("add"===e?n:t,e)},removeIcon:null!=(l=null!=T?T:null==X?void 0:X.removeIcon)?l:t.createElement(n.default,null),addIcon:(null!=R?R:null==X?void 0:X.addIcon)||t.createElement(o.default,null),showAdd:!0!==E});let el=K(),ec=(0,ee.default)(_),eu=(g=L,$=O,g?g.map(e=>{var t;let n=null!=(t=e.destroyOnHidden)?t:e.destroyInactiveTabPane;return Object.assign(Object.assign({},e),{destroyInactiveTabPane:n})}):(0,ea.default)($).map(e=>{if(t.isValidElement(e)){let{key:t,props:n}=e,a=n||{},{tab:o}=a,i=eo(a,["tab"]);return Object.assign(Object.assign({key:String(t)},i),{label:o})}return null}).filter(e=>e)),ef=function(e,t={inkBar:!0,tabPane:!1}){let n;return(n=!1===t?{inkBar:!1,tabPane:!1}:!0===t?{inkBar:!0,tabPane:!0}:Object.assign({inkBar:!0},"object"==typeof t?t:{})).tabPane&&(n.tabPaneMotion=Object.assign(Object.assign({},en),{motionName:(0,et.getTransitionName)(e,"switch")})),n}(U,B),ev=Object.assign(Object.assign({},null==X?void 0:X.style),D),eb={align:null!=(c=null==N?void 0:N.align)?c:null==(u=null==X?void 0:X.indicator)?void 0:u.align,size:null!=(v=null!=(s=null!=(d=null==N?void 0:N.size)?d:z)?s:null==(f=null==X?void 0:X.indicator)?void 0:f.size)?v:null==X?void 0:X.indicatorSize};return V(t.createElement(J,Object.assign({ref:er,direction:W,getPopupContainer:F},A,{items:eu,className:(0,i.default)({[`${U}-${ec}`]:ec,[`${U}-card`]:["card","editable-card"].includes(k),[`${U}-editable-card`]:"editable-card"===k,[`${U}-centered`]:C},null==X?void 0:X.className,w,x,Y,ei,q),popupClassName:(0,i.default)(M,Y,ei,q),style:ev,editable:y,more:Object.assign({icon:null!=(h=null!=(m=null!=(p=null==(b=null==X?void 0:X.more)?void 0:b.icon)?p:null==X?void 0:X.moreIcon)?m:P)?h:t.createElement(a.default,null),transitionName:`${el}-slide-up`},I),prefixCls:U,animated:ef,indicator:eb,destroyInactiveTabPane:null!=H?H:j})))});ef.TabPane=()=>null,e.s(["default",0,ef],721369)}]); \ No newline at end of file diff --git a/litellm/proxy/_experimental/out/_next/static/chunks/0.d2a_5_6-5pn.js b/litellm/proxy/_experimental/out/_next/static/chunks/0.d2a_5_6-5pn.js new file mode 100644 index 00000000000..9086fa0c010 --- /dev/null +++ b/litellm/proxy/_experimental/out/_next/static/chunks/0.d2a_5_6-5pn.js @@ -0,0 +1,2 @@ +(globalThis.TURBOPACK||(globalThis.TURBOPACK=[])).push(["object"==typeof document?document.currentScript:void 0,220508,e=>{"use strict";var t=e.i(271645);let r=t.forwardRef(function(e,r){return t.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",fill:"none",viewBox:"0 0 24 24",strokeWidth:2,stroke:"currentColor","aria-hidden":"true",ref:r},e),t.createElement("path",{strokeLinecap:"round",strokeLinejoin:"round",d:"M9 12l2 2 4-4m6 2a9 9 0 11-18 0 9 9 0 0118 0z"}))});e.s(["CheckCircleIcon",0,r],220508)},503269,214520,601893,694421,140721,942803,35889,722678,e=>{"use strict";var t=e.i(271645),r=e.i(914189);e.s(["useControllable",0,function(e,n,i){let[o,s]=(0,t.useState)(i),a=void 0!==e,l=(0,t.useRef)(a),u=(0,t.useRef)(!1),c=(0,t.useRef)(!1);return!a||l.current||u.current?a||!l.current||c.current||(c.current=!0,l.current=a,console.error("A component is changing from controlled to uncontrolled. This may be caused by the value changing from a defined value to undefined, which should not happen.")):(u.current=!0,l.current=a,console.error("A component is changing from uncontrolled to controlled. This may be caused by the value changing from undefined to a defined value, which should not happen.")),[a?e:o,(0,r.useEvent)(e=>(a||s(e),null==n?void 0:n(e)))]}],503269),e.s(["useDefaultValue",0,function(e){let[r]=(0,t.useState)(e);return r}],214520);let n=(0,t.createContext)(void 0);function i(){return(0,t.useContext)(n)}e.s(["useDisabled",0,i],601893);var o=e.i(174080),s=e.i(746725);function a(e={},t=null,r=[]){for(let[n,i]of Object.entries(e))!function e(t,r,n){if(Array.isArray(n))for(let[i,o]of n.entries())e(t,l(r,i.toString()),o);else n instanceof Date?t.push([r,n.toISOString()]):"boolean"==typeof n?t.push([r,n?"1":"0"]):"string"==typeof n?t.push([r,n]):"number"==typeof n?t.push([r,`${n}`]):null==n?t.push([r,""]):a(n,r,t)}(r,l(t,n),i);return r}function l(e,t){return e?e+"["+t+"]":t}e.s(["attemptSubmit",0,function(e){var t,r;let n=null!=(t=null==e?void 0:e.form)?t:e.closest("form");if(n){for(let t of n.elements)if(t!==e&&("INPUT"===t.tagName&&"submit"===t.type||"BUTTON"===t.tagName&&"submit"===t.type||"INPUT"===t.nodeName&&"image"===t.type))return void t.click();null==(r=n.requestSubmit)||r.call(n)}},"objectToFormEntries",0,a],694421);var u=e.i(700020),c=e.i(2788);let d=(0,t.createContext)(null);function f({children:e}){let r=(0,t.useContext)(d);if(!r)return t.default.createElement(t.default.Fragment,null,e);let{target:n}=r;return n?(0,o.createPortal)(t.default.createElement(t.default.Fragment,null,e),n):null}function p({setForm:e,formId:r}){return(0,t.useEffect)(()=>{if(r){let t=document.getElementById(r);t&&e(t)}},[e,r]),r?null:t.default.createElement(c.Hidden,{features:c.HiddenFeatures.Hidden,as:"input",type:"hidden",hidden:!0,readOnly:!0,ref:t=>{if(!t)return;let r=t.closest("form");r&&e(r)}})}e.s(["FormFields",0,function({data:e,form:r,disabled:n,onReset:i,overrides:o}){let[l,d]=(0,t.useState)(null),h=(0,s.useDisposables)();return(0,t.useEffect)(()=>{if(i&&l)return h.addEventListener(l,"reset",i)},[l,r,i]),t.default.createElement(f,null,t.default.createElement(p,{setForm:d,formId:r}),a(e).map(([e,i])=>t.default.createElement(c.Hidden,{features:c.HiddenFeatures.Hidden,...(0,u.compact)({key:e,as:"input",type:"hidden",hidden:!0,readOnly:!0,form:r,disabled:n,name:e,value:i,...o})})))}],140721);let h=(0,t.createContext)(void 0);function m(){return(0,t.useContext)(h)}e.s(["useProvidedId",0,m],942803);var g=e.i(835696),v=e.i(294316);let b=(0,t.createContext)(null);b.displayName="DescriptionContext";let y=Object.assign((0,u.forwardRefWithAs)(function(e,r){let n=(0,t.useId)(),o=i(),{id:s=`headlessui-description-${n}`,...a}=e,l=function e(){let r=(0,t.useContext)(b);if(null===r){let t=Error("You used a component, but it is not inside a relevant parent.");throw Error.captureStackTrace&&Error.captureStackTrace(t,e),t}return r}(),c=(0,v.useSyncRefs)(r);(0,g.useIsoMorphicEffect)(()=>l.register(s),[s,l.register]);let d=o||!1,f=(0,t.useMemo)(()=>({...l.slot,disabled:d}),[l.slot,d]),p={ref:c,...l.props,id:s};return(0,u.useRender)()({ourProps:p,theirProps:a,slot:f,defaultTag:"p",name:l.name||"Description"})}),{});e.s(["Description",0,y,"useDescribedBy",0,function(){var e,r;return null!=(r=null==(e=(0,t.useContext)(b))?void 0:e.value)?r:void 0},"useDescriptions",0,function(){let[e,n]=(0,t.useState)([]);return[e.length>0?e.join(" "):void 0,(0,t.useMemo)(()=>function(e){let i=(0,r.useEvent)(e=>(n(t=>[...t,e]),()=>n(t=>{let r=t.slice(),n=r.indexOf(e);return -1!==n&&r.splice(n,1),r}))),o=(0,t.useMemo)(()=>({register:i,slot:e.slot,name:e.name,props:e.props,value:e.value}),[i,e.slot,e.name,e.props,e.value]);return t.default.createElement(b.Provider,{value:o},e.children)},[n])]}],35889);let _=(0,t.createContext)(null);function E(e){var r,n,i;let o=null!=(n=null==(r=(0,t.useContext)(_))?void 0:r.value)?n:void 0;return(null!=(i=null==e?void 0:e.length)?i:0)>0?[o,...e].filter(Boolean).join(" "):o}_.displayName="LabelContext";let C=Object.assign((0,u.forwardRefWithAs)(function(e,n){var o;let s=(0,t.useId)(),a=function e(){let r=(0,t.useContext)(_);if(null===r){let t=Error("You used a