diff --git a/.github/workflows/llm-translation-testing.yml b/.github/workflows/llm-translation-testing.yml new file mode 100644 index 00000000000..83566d6eb6a --- /dev/null +++ b/.github/workflows/llm-translation-testing.yml @@ -0,0 +1,157 @@ +name: LLM Translation Test Results for Release Candidates + +on: + workflow_dispatch: + inputs: + release_candidate_tag: + description: 'Release candidate tag/version' + required: true + type: string + push: + tags: + - 'v*-rc*' # Triggers on release candidate tags like v1.0.0-rc1 + +jobs: + fetch-llm-translation-results: + runs-on: ubuntu-latest + timeout-minutes: 30 + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ github.event.inputs.release_candidate_tag || github.ref }} + + - name: Install CircleCI CLI + run: | + curl -fLSs https://raw.githubusercontent.com/CircleCI-Public/circleci-cli/main/install.sh | sudo bash + + - name: Create test results directory + run: mkdir -p test-results + + - name: Fetch LLM Translation Test Results from CircleCI + env: + CIRCLE_TOKEN: ${{ secrets.CIRCLE_TOKEN }} + run: | + # Get the latest CircleCI pipeline for this commit + COMMIT_SHA="${{ github.sha }}" + echo "Fetching CircleCI results for commit: $COMMIT_SHA" + + # Get pipeline info + PIPELINE_INFO=$(curl -s -H "Circle-Token: $CIRCLE_TOKEN" \ + "https://circleci.com/api/v2/project/github/BerriAI/litellm/pipeline?branch=main" | \ + jq -r ".items[] | select(.vcs.revision == \"$COMMIT_SHA\") | .id" | head -1) + + if [ -z "$PIPELINE_INFO" ]; then + echo "No CircleCI pipeline found for commit $COMMIT_SHA" + echo "Creating placeholder test results..." + echo '' > test-results/junit.xml + echo '' >> test-results/junit.xml + echo 'No CircleCI results found for this commit' >> test-results/junit.xml + echo '' >> test-results/junit.xml + exit 0 + fi + + echo "Found pipeline: $PIPELINE_INFO" + + # Get workflow info + WORKFLOW_ID=$(curl -s -H "Circle-Token: $CIRCLE_TOKEN" \ + "https://circleci.com/api/v2/pipeline/$PIPELINE_INFO/workflow" | \ + jq -r '.items[] | select(.name | contains("test")) | .id' | head -1) + + if [ -z "$WORKFLOW_ID" ]; then + echo "No test workflow found in pipeline" + exit 1 + fi + + echo "Found workflow: $WORKFLOW_ID" + + # Get job info for llm_translation tests + JOB_INFO=$(curl -s -H "Circle-Token: $CIRCLE_TOKEN" \ + "https://circleci.com/api/v2/workflow/$WORKFLOW_ID/job" | \ + jq -r '.items[] | select(.name | contains("llm_translation")) | select(.status == "success" or .status == "failed") | .job_number' | head -1) + + if [ -z "$JOB_INFO" ]; then + echo "No completed llm_translation job found" + exit 1 + fi + + echo "Found job: $JOB_INFO" + + # Download artifacts + curl -s -H "Circle-Token: $CIRCLE_TOKEN" \ + "https://circleci.com/api/v2/project/github/BerriAI/litellm/$JOB_INFO/artifacts" | \ + jq -r '.items[] | select(.path | contains("junit") or contains("coverage") or contains("report")) | .url' | \ + while read -r artifact_url; do + filename=$(basename "$artifact_url" | sed 's/[?&].*//') + echo "Downloading artifact: $filename" + curl -s -H "Circle-Token: $CIRCLE_TOKEN" -o "test-results/$filename" "$artifact_url" + done + + # If no artifacts found, create placeholder + if [ ! -f "test-results/junit.xml" ]; then + echo "No test artifacts found, creating placeholder..." + echo '' > test-results/junit.xml + echo '' >> test-results/junit.xml + echo 'Test artifacts not available from CircleCI' >> test-results/junit.xml + echo '' >> test-results/junit.xml + fi + continue-on-error: true + + - name: Generate test summary + run: | + echo "# LLM Translation Testing Results" > test-results/summary.md + echo "" >> test-results/summary.md + echo "**Release Candidate:** ${{ github.event.inputs.release_candidate_tag || github.ref_name }}" >> test-results/summary.md + echo "**Fetched Date:** $(date)" >> test-results/summary.md + echo "**Commit:** ${{ github.sha }}" >> test-results/summary.md + echo "**Source:** CircleCI Pipeline" >> test-results/summary.md + echo "" >> test-results/summary.md + + # Parse junit.xml for test statistics if it exists + if [ -f "test-results/junit.xml" ]; then + python -c " + import xml.etree.ElementTree as ET + try: + tree = ET.parse('test-results/junit.xml') + root = tree.getroot() + tests = root.get('tests', '0') + failures = root.get('failures', '0') + errors = root.get('errors', '0') + skipped = root.get('skipped', '0') + time = root.get('time', '0') + + print(f'**Total Tests:** {tests}') + print(f'**Passed:** {int(tests) - int(failures) - int(errors) - int(skipped)}') + print(f'**Failed:** {failures}') + print(f'**Errors:** {errors}') + print(f'**Skipped:** {skipped}') + print(f'**Duration:** {time} seconds') + except Exception as e: + print(f'Could not parse test results: {e}') + " >> test-results/summary.md + fi + + echo "" >> test-results/summary.md + echo "## Test Files Covered" >> test-results/summary.md + ls tests/llm_translation/*.py | sed 's/^/- /' >> test-results/summary.md + + - name: Upload test results + uses: actions/upload-artifact@v4 + if: always() + with: + name: llm-translation-test-results-${{ github.event.inputs.release_candidate_tag || github.ref_name }} + path: | + test-results/ + coverage.xml + htmlcov/ + .coverage + retention-days: 30 + + - name: Upload JUnit test results + uses: actions/upload-artifact@v4 + if: always() + with: + name: junit-xml-${{ github.event.inputs.release_candidate_tag || github.ref_name }} + path: test-results/junit.xml + retention-days: 30 \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000000..50bed6e43e2 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,89 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Development Commands + +### Installation +- `make install-dev` - Install core development dependencies +- `make install-proxy-dev` - Install proxy development dependencies with full feature set +- `make install-test-deps` - Install all test dependencies + +### Testing +- `make test` - Run all tests +- `make test-unit` - Run unit tests (tests/test_litellm) with 4 parallel workers +- `make test-integration` - Run integration tests (excludes unit tests) +- `pytest tests/` - Direct pytest execution + +### Code Quality +- `make lint` - Run all linting (Ruff, MyPy, Black, circular imports, import safety) +- `make format` - Apply Black code formatting +- `make lint-ruff` - Run Ruff linting only +- `make lint-mypy` - Run MyPy type checking only + +### Single Test Files +- `poetry run pytest tests/path/to/test_file.py -v` - Run specific test file +- `poetry run pytest tests/path/to/test_file.py::test_function -v` - Run specific test + +## Architecture Overview + +LiteLLM is a unified interface for 100+ LLM providers with two main components: + +### Core Library (`litellm/`) +- **Main entry point**: `litellm/main.py` - Contains core completion() function +- **Provider implementations**: `litellm/llms/` - Each provider has its own subdirectory +- **Router system**: `litellm/router.py` + `litellm/router_utils/` - Load balancing and fallback logic +- **Type definitions**: `litellm/types/` - Pydantic models and type hints +- **Integrations**: `litellm/integrations/` - Third-party observability, caching, logging +- **Caching**: `litellm/caching/` - Multiple cache backends (Redis, in-memory, S3, etc.) + +### Proxy Server (`litellm/proxy/`) +- **Main server**: `proxy_server.py` - FastAPI application +- **Authentication**: `auth/` - API key management, JWT, OAuth2 +- **Database**: `db/` - Prisma ORM with PostgreSQL/SQLite support +- **Management endpoints**: `management_endpoints/` - Admin APIs for keys, teams, models +- **Pass-through endpoints**: `pass_through_endpoints/` - Provider-specific API forwarding +- **Guardrails**: `guardrails/` - Safety and content filtering hooks +- **UI Dashboard**: Served from `_experimental/out/` (Next.js build) + +## Key Patterns + +### Provider Implementation +- Providers inherit from base classes in `litellm/llms/base.py` +- Each provider has transformation functions for input/output formatting +- Support both sync and async operations +- Handle streaming responses and function calling + +### Error Handling +- Provider-specific exceptions mapped to OpenAI-compatible errors +- Fallback logic handled by Router system +- Comprehensive logging through `litellm/_logging.py` + +### Configuration +- YAML config files for proxy server (see `proxy/example_config_yaml/`) +- Environment variables for API keys and settings +- Database schema managed via Prisma (`proxy/schema.prisma`) + +## Development Notes + +### Code Style +- Uses Black formatter, Ruff linter, MyPy type checker +- Pydantic v2 for data validation +- Async/await patterns throughout +- Type hints required for all public APIs + +### Testing Strategy +- Unit tests in `tests/test_litellm/` +- Integration tests for each provider in `tests/llm_translation/` +- Proxy tests in `tests/proxy_unit_tests/` +- Load tests in `tests/load_tests/` + +### Database Migrations +- Prisma handles schema migrations +- Migration files auto-generated with `prisma migrate dev` +- Always test migrations against both PostgreSQL and SQLite + +### Enterprise Features +- Enterprise-specific code in `enterprise/` directory +- Optional features enabled via environment variables +- Separate licensing and authentication for enterprise features \ No newline at end of file diff --git a/docs/my-website/docs/completion/input.md b/docs/my-website/docs/completion/input.md index fb0fc390ad0..b08941cde92 100644 --- a/docs/my-website/docs/completion/input.md +++ b/docs/my-website/docs/completion/input.md @@ -39,31 +39,32 @@ This is a list of openai params we translate across providers. Use `litellm.get_supported_openai_params()` for an updated list of params for each model + provider -| Provider | temperature | max_completion_tokens | max_tokens | top_p | stream | stream_options | stop | n | presence_penalty | frequency_penalty | functions | function_call | logit_bias | user | response_format | seed | tools | tool_choice | logprobs | top_logprobs | extra_headers | -|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---| -|Anthropic| ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ | | | | | | |✅ | ✅ | | ✅ | ✅ | | | ✅ | -|OpenAI| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ | ✅ | -|Azure OpenAI| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ | ✅ | -|xAI| ✅ | | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | -|Replicate | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | -|Anyscale | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -|Cohere| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | -|Huggingface| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | -|Openrouter| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | ✅ |✅ | | | | -|AI21| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | -|VertexAI| ✅ | ✅ | ✅ | | ✅ | ✅ | | | | | | | | | ✅ | ✅ | | | -|Bedrock| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | | | | | ✅ (model dependent) | | -|Sagemaker| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | -|TogetherAI| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | ✅ | | | ✅ | | ✅ | ✅ | | | | -|Sambanova| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | | | ✅ | | ✅ | ✅ | | | | -|AlephAlpha| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | -|NLP Cloud| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | -|Petals| ✅ | ✅ | | ✅ | ✅ | | | | | | -|Ollama| ✅ | ✅ | ✅ |✅ | ✅ | ✅ | | | ✅ | | | | | ✅ | | |✅| | | | | | | -|Databricks| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | | | | | | -|ClarifAI| ✅ | ✅ | ✅ | |✅ | ✅ | | | | | | | | | | | -|Github| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | ✅ |✅ (model dependent)|✅ (model dependent)| | | -|Novita AI| ✅ | ✅ | | ✅ | ✅ | ✅ | | ✅ | ✅ | ✅ | ✅ | | | ✅ | | | | | | | | +| Provider | temperature | max_completion_tokens | max_tokens | top_p | stream | stream_options | stop | n | presence_penalty | frequency_penalty | functions | function_call | logit_bias | user | response_format | seed| tools | tool_choice | logprobs | top_logprobs | extra_headers | +|--------------|-------------|------------------------|------------|-------|--------|----------------|------|-----|------------------|-------------------|-----------|----------------|-------------|------|------------------|-------------------|--------|--------------|----------|---------------|----------------------| +| Anthropic| ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | || | || | ✅ | ✅ | | ✅ | ✅ || | ✅| +| OpenAI | ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | ✅| ✅ | ✅| ✅| ✅ | ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | ✅| ✅| +| Azure OpenAI | ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | ✅| ✅ | ✅| ✅| ✅ | ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | ✅| ✅| +| xAI| ✅|| ✅ | ✅| ✅ | ✅ | ✅ | ✅| ✅ | ✅| || ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | ✅|| +| Replicate| ✅| ✅ | ✅ | ✅| ✅ | ✅ || || | || ||| |||| || +| Anyscale | ✅| ✅ | ✅ | ✅| ✅ | ✅ || || | || ||| |||| || +| Cohere | ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | ✅|| | || ||| |||| || +| Huggingface| ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | || | || ||| |||| || +| Openrouter | ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | ✅| ✅ | ✅| ✅|| ||| ✅| ✅ ||| || +| AI21 | ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | ✅|| | || ||| |||| || +| VertexAI | ✅| ✅ | ✅ | | ✅ | ✅ || || | || || ✅ | ✅|||| || +| Bedrock| ✅| ✅ | ✅ | ✅| ✅ | ✅ || || | || || ✅ (model dependent) | |||| || +| Sagemaker| ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | || | || ||| |||| || +| TogetherAI | ✅| ✅ | ✅ | ✅| ✅ | ✅ || || | ✅|| || ✅ | | ✅ | ✅ || || +| Sambanova| ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | || | || || ✅ | | ✅ | ✅ || || +| AlephAlpha | ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | || | || ||| |||| || +| NLP Cloud| ✅| ✅ | ✅ | ✅| ✅ | ✅ || || | || ||| |||| || +| Petals | ✅| ✅ || ✅| ✅ ||| || | || ||| |||| || +| Ollama | ✅| ✅ | ✅ | ✅| ✅ | ✅ || ✅|| | || ✅||| | ✅ ||| || +| Databricks | ✅| ✅ | ✅ | ✅| ✅ | ✅ || || | || ||| |||| || +| ClarifAI | ✅| ✅ | ✅ | | ✅ | ✅ || || | || ||| |||| || +| Github | ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | ✅| ✅ | ✅| ✅|| || ✅ | ✅ (model dependent) | ✅ (model dependent) || || +| Novita AI| ✅| ✅ || ✅| ✅ | ✅ | ✅ | ✅| ✅ | ✅| || ✅||| |||| || + :::note By default, LiteLLM raises an exception if the openai param being passed in isn't supported. diff --git a/docs/my-website/docs/completion/knowledgebase.md b/docs/my-website/docs/completion/knowledgebase.md index 033dccea200..a1c926274cd 100644 --- a/docs/my-website/docs/completion/knowledgebase.md +++ b/docs/my-website/docs/completion/knowledgebase.md @@ -17,6 +17,8 @@ LiteLLM integrates with vector stores, allowing your models to access your organ ## Supported Vector Stores - [Bedrock Knowledge Bases](https://aws.amazon.com/bedrock/knowledge-bases/) +- [OpenAI Vector Stores](https://platform.openai.com/docs/api-reference/vector-stores/search) +- [Azure Vector Stores](https://learn.microsoft.com/en-us/azure/ai-services/openai/how-to/file-search?tabs=python#vector-stores ## Quick Start diff --git a/docs/my-website/docs/completion/web_search.md b/docs/my-website/docs/completion/web_search.md index b0c77debe3a..fe49be852a7 100644 --- a/docs/my-website/docs/completion/web_search.md +++ b/docs/my-website/docs/completion/web_search.md @@ -8,7 +8,7 @@ Use web search with litellm | Feature | Details | |---------|---------| | Supported Endpoints | - `/chat/completions`
- `/responses` | -| Supported Providers | `openai`, `xai`, `vertex_ai`, `gemini` | +| Supported Providers | `openai`, `xai`, `vertex_ai`, `gemini`, `perplexity` | | LiteLLM Cost Tracking | ✅ Supported | | LiteLLM Version | `v1.71.0+` | diff --git a/docs/my-website/docs/providers/azure/azure_responses.md b/docs/my-website/docs/providers/azure/azure_responses.md index b17ef8a2853..34ec0e194f7 100644 --- a/docs/my-website/docs/providers/azure/azure_responses.md +++ b/docs/my-website/docs/providers/azure/azure_responses.md @@ -233,3 +233,63 @@ for event in response: + +## Calling via `/chat/completions` + +You can also call the Azure Responses API via the `/chat/completions` endpoint. + + + + + +```python showLineNumbers +from litellm import completion +import os + +os.environ["AZURE_API_BASE"] = "https://my-endpoint-sweden-berri992.openai.azure.com/" +os.environ["AZURE_API_VERSION"] = "2023-03-15-preview" +os.environ["AZURE_API_KEY"] = "my-api-key" + +response = completion( + model="azure/responses/my-custom-o1-pro", + messages=[{"role": "user", "content": "Hello world"}], +) + +print(response) +``` + + + +1. Setup config.yaml + +```yaml showLineNumbers +model_list: + - model_name: my-custom-o1-pro + litellm_params: + model: azure/responses/my-custom-o1-pro + api_key: os.environ/AZURE_API_KEY + api_base: https://my-endpoint-sweden-berri992.openai.azure.com/ + api_version: 2023-03-15-preview +``` + +2. Start LiteLLM proxy +```bash +litellm --config /path/to/config.yaml + +# RUNNING on http://0.0.0.0:4000 +``` + +3. Test it! + +```bash +curl http://localhost:4000/v1/chat/completions \ + -X POST \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $LITELLM_API_KEY" \ + -d '{ + "model": "my-custom-o1-pro", + "messages": [{"role": "user", "content": "Hello world"}] + }' +``` + + \ No newline at end of file diff --git a/docs/my-website/docs/providers/custom_llm_server.md b/docs/my-website/docs/providers/custom_llm_server.md index 055b6906a9f..61099d1a358 100644 --- a/docs/my-website/docs/providers/custom_llm_server.md +++ b/docs/my-website/docs/providers/custom_llm_server.md @@ -13,11 +13,12 @@ Call your custom torch-serve / internal LLM APIs via LiteLLM ::: Supported Routes: -- `/v1/chat/completions` -> `litellm.completion` -- `/v1/completions` -> `litellm.text_completion` -- `/v1/embeddings` -> `litellm.embedding` -- `/v1/images/generations` -> `litellm.image_generation` +- `/v1/chat/completions` -> `litellm.acompletion` +- `/v1/completions` -> `litellm.atext_completion` +- `/v1/embeddings` -> `litellm.aembedding` +- `/v1/images/generations` -> `litellm.aimage_generation` +- `/v1/messages` -> `litellm.acompletion` ## Quick Start @@ -262,6 +263,102 @@ Expected Response } ``` +## Anthropic `/v1/messages` + +- Write the integration for .acompletion +- litellm will transform it to /v1/messages + +1. Setup your `custom_handler.py` file + +```python +import litellm +from litellm import CustomLLM, completion, get_llm_provider + + +class MyCustomLLM(CustomLLM): + async def acompletion(self, *args, **kwargs) -> litellm.ModelResponse: + return litellm.completion( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "Hello world"}], + mock_response="Hi!", + ) # type: ignore + + +my_custom_llm = MyCustomLLM() +``` + +2. Add to `config.yaml` + +In the config below, we pass + +python_filename: `custom_handler.py` +custom_handler_instance_name: `my_custom_llm`. This is defined in Step 1 + +custom_handler: `custom_handler.my_custom_llm` + +```yaml +model_list: + - model_name: "test-model" + litellm_params: + model: "openai/text-embedding-ada-002" + - model_name: "my-custom-model" + litellm_params: + model: "my-custom-llm/my-model" + +litellm_settings: + custom_provider_map: + - {"provider": "my-custom-llm", "custom_handler": custom_handler.my_custom_llm} +``` + +```bash +litellm --config /path/to/config.yaml +``` + +3. Test it! + +```bash +curl -L -X POST 'http://0.0.0.0:4000/v1/messages' \ +-H 'anthropic-version: 2023-06-01' \ +-H 'content-type: application/json' \ +-H 'Authorization: Bearer sk-1234' \ +-d '{ + "model": "my-custom-model", + "max_tokens": 1024, + "messages": [{ + "role": "user", + "content": [ + { + "type": "text", + "text": "What are the key findings in this document 12?" + }] + }] +}' +``` + +Expected Response + +```json +{ + "id": "chatcmpl-Bm4qEp4h4vCe7Zi4Gud1MAxTWgibO", + "type": "message", + "role": "assistant", + "model": "gpt-3.5-turbo-0125", + "stop_sequence": null, + "usage": { + "input_tokens": 18, + "output_tokens": 44 + }, + "content": [ + { + "type": "text", + "text": "Without the specific document being provided, it is not possible to determine the key findings within it. If you can provide the content or a summary of document 12, I would be happy to help identify the key findings." + } + ], + "stop_reason": "end_turn" +} +``` + + ## Additional Parameters Additional parameters are passed inside `optional_params` key in the `completion` or `image_generation` function. diff --git a/docs/my-website/docs/proxy/dynamic_logging.md b/docs/my-website/docs/proxy/dynamic_logging.md new file mode 100644 index 00000000000..3bc9f72b033 --- /dev/null +++ b/docs/my-website/docs/proxy/dynamic_logging.md @@ -0,0 +1,214 @@ +import Image from '@theme/IdealImage'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + + +# Dynamic Callback Management + +:::info + +✨ This is an enterprise feature. + +[Get started with LiteLLM Enterprise](https://www.litellm.ai/enterprise) + +::: + +LiteLLM's dynamic callback management enables teams to control logging behavior on a per-request basis without requiring central infrastructure changes. This is essential for organizations managing large-scale service ecosystems where: + +- **Teams manage their own compliance** - Services can handle sensitive data appropriately without central oversight +- **Decentralized responsibility** - Each team controls their data handling while using shared infrastructure + +You can disable callbacks by passing the `x-litellm-disable-callbacks` header with your requests, giving teams granular control over where their data is logged. + +## Getting Started: List and Disable Callbacks + +Managing callbacks is a two-step process: + +1. **First, list your active callbacks** to see what's currently enabled +2. **Then, disable specific callbacks** as needed for your requests + + + +## 1. List Active Callbacks + +Start by viewing all currently enabled callbacks on your proxy to see what's available to disable. + +#### Request + +```bash +curl -X 'GET' \ + 'http://localhost:4000/callbacks/list' \ + -H 'accept: application/json' \ + -H 'x-litellm-api-key: sk-1234' +``` + +#### Response + +```json +{ + "success": [ + "deployment_callback_on_success", + "sync_deployment_callback_on_success" + ], + "failure": [ + "async_deployment_callback_on_failure", + "deployment_callback_on_failure" + ], + "success_and_failure": [ + "langfuse", + "datadog" + ] +} +``` + +#### Response Fields + +The response contains three arrays that categorize your active callbacks: +- **`success`** - Callbacks that only execute when requests complete successfully. These callbacks receive data from successful LLM responses. +- **`failure`** - Callbacks that only execute when requests fail or encounter errors. These callbacks receive error information and failed request data. +- **`success_and_failure`** - Callbacks that execute for both successful and failed requests. These are typically logging/observability tools that need to capture all request data regardless of outcome. + +--- + +## 2. Disable Callbacks + +Now that you know which callbacks are active, you can selectively disable them using the `x-litellm-disable-callbacks` header. You can reference any callback name from the list response above. + +### Disable a Single Callback + +Use the `x-litellm-disable-callbacks` header to disable specific callbacks for individual requests. + + + + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Content-Type: application/json' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'x-litellm-disable-callbacks: langfuse' \ + --data '{ + "model": "claude-sonnet-4-20250514", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ] +}' +``` + + + + +```python +import openai + +client = openai.OpenAI( + api_key="sk-1234", + base_url="http://0.0.0.0:4000" +) + +response = client.chat.completions.create( + model="claude-sonnet-4-20250514", + messages=[ + { + "role": "user", + "content": "what llm are you" + } + ], + extra_headers={ + "x-litellm-disable-callbacks": "langfuse" + } +) + +print(response) +``` + + + + +### Disable Multiple Callbacks + +You can disable multiple callbacks by providing a comma-separated list in the header. Use any combination of callback names from your `/callbacks/list` response. + + + + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Content-Type: application/json' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'x-litellm-disable-callbacks: langfuse,datadog,prometheus' \ + --data '{ + "model": "claude-sonnet-4-20250514", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ] +}' +``` + + + + +```python +import openai + +client = openai.OpenAI( + api_key="sk-1234", + base_url="http://0.0.0.0:4000" +) + +response = client.chat.completions.create( + model="claude-sonnet-4-20250514", + messages=[ + { + "role": "user", + "content": "what llm are you" + } + ], + extra_headers={ + "x-litellm-disable-callbacks": "langfuse,datadog,prometheus" + } +) + +print(response) +``` + + + + +## Header Format and Case Sensitivity + +### Expected Header Format + +The `x-litellm-disable-callbacks` header accepts callback names in the following formats (use the exact names returned by `/callbacks/list`): + +- **Single callback**: `x-litellm-disable-callbacks: langfuse` +- **Multiple callbacks**: `x-litellm-disable-callbacks: langfuse,datadog,prometheus` + +When specifying multiple callbacks, use comma-separated values without spaces around the commas. + +### Case Sensitivity + +**Callback name checks are case insensitive.** This means all of the following are equivalent: + +```bash +# These are all equivalent +x-litellm-disable-callbacks: langfuse +x-litellm-disable-callbacks: LANGFUSE +x-litellm-disable-callbacks: LangFuse +x-litellm-disable-callbacks: langFUSE +``` + +This applies to both single and multiple callback specifications: + +```bash +# Case insensitive for multiple callbacks +x-litellm-disable-callbacks: LANGFUSE,datadog,PROMETHEUS +x-litellm-disable-callbacks: langfuse,DATADOG,prometheus +``` + + diff --git a/docs/my-website/docs/proxy/logging.md b/docs/my-website/docs/proxy/logging.md index 3bf5ed12300..7ec9080dfdd 100644 --- a/docs/my-website/docs/proxy/logging.md +++ b/docs/my-website/docs/proxy/logging.md @@ -56,27 +56,6 @@ components in your system, including in logging tools. ## Logging Features -### Conditional Logging by Virtual Keys, Teams - -Use this to: -1. Conditionally enable logging for some virtual keys/teams -2. Set different logging providers for different virtual keys/teams - -[👉 **Get Started** - Team/Key Based Logging](team_logging) - - -### Redacting UserAPIKeyInfo - -Redact information about the user api key (hashed token, user_id, team id, etc.), from logs. - -Currently supported for Langfuse, OpenTelemetry, Logfire, ArizeAI logging. - -```yaml -litellm_settings: - callbacks: ["langfuse"] - redact_user_api_key_info: true -``` - ### Redact Messages, Response Content @@ -172,6 +151,18 @@ curl -L -X POST 'http://0.0.0.0:4000/v1/chat/completions' \ +### Redacting UserAPIKeyInfo + +Redact information about the user api key (hashed token, user_id, team id, etc.), from logs. + +Currently supported for Langfuse, OpenTelemetry, Logfire, ArizeAI logging. + +```yaml +litellm_settings: + callbacks: ["langfuse"] + redact_user_api_key_info: true +``` + ### Disable Message Redaction If you have `litellm.turn_on_message_logging` turned on, you can override it for specific requests by @@ -269,6 +260,81 @@ print(response) LiteLLM.Info: "no-log request, skipping logging" ``` +### ✨ Dynamically Disable specific callbacks + +:::info + +This is an enterprise feature. + +[Proceed with LiteLLM Enterprise](https://www.litellm.ai/enterprise) + +::: + +For some use cases, you may want to disable specific callbacks for a request. You can do this by passing `x-litellm-disable-callbacks: ` in the request headers. + +Send the list of callbacks to disable in the request header `x-litellm-disable-callbacks`. + + + + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Content-Type: application/json' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'x-litellm-disable-callbacks: langfuse' \ + --data '{ + "model": "claude-sonnet-4-20250514", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ] +}' +``` + + + + +```python +import openai + +client = openai.OpenAI( + api_key="sk-1234", + base_url="http://0.0.0.0:4000" +) + +response = client.chat.completions.create( + model="claude-sonnet-4-20250514", + messages=[ + { + "role": "user", + "content": "what llm are you" + } + ], + extra_headers={ + "x-litellm-disable-callbacks": "langfuse" + } +) + +print(response) +``` + + + + + +### ✨ Conditional Logging by Virtual Keys, Teams + +Use this to: +1. Conditionally enable logging for some virtual keys/teams +2. Set different logging providers for different virtual keys/teams + +[👉 **Get Started** - Team/Key Based Logging](team_logging) + + + + ## What gets logged? diff --git a/docs/my-website/docs/proxy/prod.md b/docs/my-website/docs/proxy/prod.md index 7cbaf145552..a618e530796 100644 --- a/docs/my-website/docs/proxy/prod.md +++ b/docs/my-website/docs/proxy/prod.md @@ -49,7 +49,21 @@ Need Help or want dedicated support ? Talk to a founder [here]: (https://calendl ::: -## 2. On Kubernetes - Use 1 Uvicorn worker [Suggested CMD] +## 2. Recommended Machine Specifications + +For optimal performance in production, we recommend the following minimum machine specifications: + +| Resource | Recommended Value | +|----------|------------------| +| CPU | 2 vCPU | +| Memory | 4 GB RAM | + +These specifications provide: +- Sufficient compute power for handling concurrent requests +- Adequate memory for request processing and caching + + +## 3. On Kubernetes - Use 1 Uvicorn worker [Suggested CMD] Use this Docker `CMD`. This will start the proxy with 1 Uvicorn Async Worker @@ -59,7 +73,7 @@ CMD ["--port", "4000", "--config", "./proxy_server_config.yaml"] ``` -## 3. Use Redis 'port','host', 'password'. NOT 'redis_url' +## 4. Use Redis 'port','host', 'password'. NOT 'redis_url' If you decide to use Redis, DO NOT use 'redis_url'. We recommend using redis port, host, and password params. @@ -92,13 +106,13 @@ litellm_settings: password: os.environ/REDIS_PASSWORD ``` -## 4. Disable 'load_dotenv' +## 5. Disable 'load_dotenv' Set `export LITELLM_MODE="PRODUCTION"` This disables the load_dotenv() functionality, which will automatically load your environment credentials from the local `.env`. -## 5. If running LiteLLM on VPC, gracefully handle DB unavailability +## 6. If running LiteLLM on VPC, gracefully handle DB unavailability When running LiteLLM on a VPC (and inaccessible from the public internet), you can enable graceful degradation so that request processing continues even if the database is temporarily unavailable. @@ -125,7 +139,7 @@ When `allow_requests_on_db_unavailable` is set to `true`, LiteLLM will handle er | LiteLLM Budget Errors or Model Errors | ❌ Request will be blocked | Triggered when the DB is reachable but the authentication token is invalid, lacks access, or exceeds budget limits. | -## 6. Disable spend_logs & error_logs if not using the LiteLLM UI +## 7. Disable spend_logs & error_logs if not using the LiteLLM UI By default, LiteLLM writes several types of logs to the database: - Every LLM API request to the `LiteLLM_SpendLogs` table @@ -141,7 +155,7 @@ general_settings: [More information about what the Database is used for here](db_info) -## 7. Use Helm PreSync Hook for Database Migrations [BETA] +## 8. Use Helm PreSync Hook for Database Migrations [BETA] To ensure only one service manages database migrations, use our [Helm PreSync hook for Database Migrations](https://github.com/BerriAI/litellm/blob/main/deploy/charts/litellm-helm/templates/migrations-job.yaml). This ensures migrations are handled during `helm upgrade` or `helm install`, while LiteLLM pods explicitly disable migrations. @@ -169,7 +183,7 @@ To ensure only one service manages database migrations, use our [Helm PreSync ho ``` -## 8. Set LiteLLM Salt Key +## 9. Set LiteLLM Salt Key If you plan on using the DB, set a salt key for encrypting/decrypting variables in the DB. @@ -184,7 +198,7 @@ export LITELLM_SALT_KEY="sk-1234" [**See Code**](https://github.com/BerriAI/litellm/blob/036a6821d588bd36d170713dcf5a72791a694178/litellm/proxy/common_utils/encrypt_decrypt_utils.py#L15) -## 9. Use `prisma migrate deploy` +## 10. Use `prisma migrate deploy` Use this to handle db migrations across LiteLLM versions in production diff --git a/docs/my-website/docs/proxy/self_serve.md b/docs/my-website/docs/proxy/self_serve.md index 39db0fffdbd..e7860b42478 100644 --- a/docs/my-website/docs/proxy/self_serve.md +++ b/docs/my-website/docs/proxy/self_serve.md @@ -161,6 +161,11 @@ Here's the available UI roles for a LiteLLM Internal User: - `internal_user`: can login, view/create/delete their own keys, view their spend. **Cannot** add new users. - `internal_user_viewer`: can login, view their own keys, view their own spend. **Cannot** create/delete keys, add new users. +**Team Roles:** + - `admin`: can add new members to the team, can control Team Permissions, can add team-only models (useful for onboarding a team's finetuned models). + - `user`: can login, view their own keys, view their own spend. **Cannot** create/delete keys (controllable via Team Permissions), add new users. + + ## Auto-add SSO users to teams This walks through setting up sso auto-add for **Okta, Google SSO** @@ -273,6 +278,65 @@ This budget does not apply to keys created under non-default teams. [**Go Here**](./team_budgets.md) +### Default Team + + + + +Go to `Internal Users` -> `Default User Settings` and set the default team to the team you just created. + +Let's also set the default models to `no-default-models`. This means a user can only create keys within a team. + + + + + + +:::info +Team must be created before setting it as the default team. +::: + +```yaml +litellm_settings: + default_internal_user_params: # Default Params used when a new user signs in Via SSO + user_role: "internal_user" # one of "internal_user", "internal_user_viewer", + models: ["no-default-models"] # Optional[List[str]], optional): models to be used by the user + teams: # Optional[List[NewUserRequestTeam]], optional): teams to be used by the user + - team_id: "team_id_1" # Required[str]: team_id to be used by the user + user_role: "user" # Optional[str], optional): Default role in the team. Values: "user" or "admin". Defaults to "user" +``` + + + + +### Team Member Budgets + +Set a max budget for a team member. + +You can do this when creating a new team, or by updating an existing team. + + + + + + + + + +```bash +curl -X POST '/team/new' \ +-H 'Authorization: Bearer ' \ +-H 'Content-Type: application/json' \ +-D '{ + "team_alias": "team_1", + "budget_duration": "10d", + "team_member_budget": 10 +}' +``` + + + + ### Set default params for new teams When you connect litellm to your SSO provider, litellm can auto-create teams. Use this to set the default `models`, `max_budget`, `budget_duration` for these auto-created teams. diff --git a/docs/my-website/docs/tutorials/claude_responses_api.md b/docs/my-website/docs/tutorials/claude_responses_api.md index d95f75c7d74..9fe92cf11c9 100644 --- a/docs/my-website/docs/tutorials/claude_responses_api.md +++ b/docs/my-website/docs/tutorials/claude_responses_api.md @@ -47,7 +47,7 @@ curl -X POST http://0.0.0.0:4000/v1/messages \ - Setup environment variables ```bash -export ANTHROPIC_API_BASE="http://0.0.0.0:4000" +export ANTHROPIC_BASE_URL="http://0.0.0.0:4000" export ANTHROPIC_API_KEY="sk-1234" # replace with your LiteLLM key ``` diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 0ba5cc5109c..2539b7af510 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -198,7 +198,8 @@ const sidebars = { items: [ "proxy/logging", "proxy/logging_spec", - "proxy/team_logging" + "proxy/team_logging", + "proxy/dynamic_logging" ], }, diff --git a/enterprise/dist/litellm_enterprise-0.1.9-py3-none-any.whl b/enterprise/dist/litellm_enterprise-0.1.9-py3-none-any.whl new file mode 100644 index 00000000000..eb4b9d1083b Binary files /dev/null and b/enterprise/dist/litellm_enterprise-0.1.9-py3-none-any.whl differ diff --git a/enterprise/dist/litellm_enterprise-0.1.9.tar.gz b/enterprise/dist/litellm_enterprise-0.1.9.tar.gz new file mode 100644 index 00000000000..748ed2150ef Binary files /dev/null and b/enterprise/dist/litellm_enterprise-0.1.9.tar.gz differ diff --git a/enterprise/litellm_enterprise/enterprise_callbacks/callback_controls.py b/enterprise/litellm_enterprise/enterprise_callbacks/callback_controls.py new file mode 100644 index 00000000000..c4316012738 --- /dev/null +++ b/enterprise/litellm_enterprise/enterprise_callbacks/callback_controls.py @@ -0,0 +1,66 @@ +import litellm +from litellm._logging import verbose_logger +from litellm.constants import X_LITELLM_DISABLE_CALLBACKS +from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.llm_request_utils import ( + get_proxy_server_request_headers, +) +from litellm.proxy._types import CommonProxyErrors + + +class EnterpriseCallbackControls: + @staticmethod + def is_callback_disabled_via_headers( + callback: litellm.CALLBACK_TYPES, litellm_params: dict + ) -> bool: + """ + Check if a callback is disabled via the x-litellm-disable-callbacks header. + + Args: + callback: The callback to check (can be string, CustomLogger instance, or callable) + litellm_params: Parameters containing proxy server request info + + Returns: + bool: True if the callback should be disabled, False otherwise + """ + from litellm.litellm_core_utils.custom_logger_registry import ( + CustomLoggerRegistry, + ) + + try: + request_headers = get_proxy_server_request_headers(litellm_params) + disabled_callbacks = request_headers.get(X_LITELLM_DISABLE_CALLBACKS, None) + verbose_logger.debug(f"Dynamically disabled callbacks from {X_LITELLM_DISABLE_CALLBACKS}: {disabled_callbacks}") + verbose_logger.debug(f"Checking if {callback} is disabled via headers. Disable callbacks from headers: {disabled_callbacks}") + if disabled_callbacks is not None: + ######################################################### + # premium user check + ######################################################### + if not EnterpriseCallbackControls._premium_user_check(): + return False + ######################################################### + disabled_callbacks = set([cb.strip().lower() for cb in disabled_callbacks.split(",")]) + if isinstance(callback, str): + if callback.lower() in disabled_callbacks: + verbose_logger.debug(f"Not logging to {callback} because it is disabled via {X_LITELLM_DISABLE_CALLBACKS}") + return True + elif isinstance(callback, CustomLogger): + # get the string name of the callback + callback_str = CustomLoggerRegistry.get_callback_str_from_class_type(callback.__class__) + if callback_str is not None and callback_str.lower() in disabled_callbacks: + verbose_logger.debug(f"Not logging to {callback_str} because it is disabled via {X_LITELLM_DISABLE_CALLBACKS}") + return True + return False + except Exception as e: + verbose_logger.debug( + f"Error checking disabled callbacks header: {str(e)}" + ) + return False + + @staticmethod + def _premium_user_check(): + from litellm.proxy.proxy_server import premium_user + if premium_user: + return True + verbose_logger.warning(f"Disabling callbacks using request headers is an enterprise feature. {CommonProxyErrors.not_premium_user.value}") + return False \ No newline at end of file diff --git a/enterprise/litellm_enterprise/proxy/management_endpoints/key_management_endpoints.py b/enterprise/litellm_enterprise/proxy/management_endpoints/key_management_endpoints.py new file mode 100644 index 00000000000..19ce8090db7 --- /dev/null +++ b/enterprise/litellm_enterprise/proxy/management_endpoints/key_management_endpoints.py @@ -0,0 +1,30 @@ +from typing import Optional + +from litellm.proxy._types import GenerateKeyRequest, LiteLLM_TeamTable + + +def add_team_member_key_duration( + team_table: Optional[LiteLLM_TeamTable], + data: GenerateKeyRequest, +) -> GenerateKeyRequest: + if team_table is None: + return data + + if data.user_id is None: # only apply for team member keys, not service accounts + return data + + if ( + team_table.metadata is not None + and team_table.metadata.get("team_member_key_duration") is not None + ): + data.duration = team_table.metadata["team_member_key_duration"] + + return data + + +def apply_enterprise_key_management_params( + data: GenerateKeyRequest, + team_table: Optional[LiteLLM_TeamTable], +) -> GenerateKeyRequest: + data = add_team_member_key_duration(team_table, data) + return data diff --git a/enterprise/pyproject.toml b/enterprise/pyproject.toml index 28395b63eae..a650eda22b1 100644 --- a/enterprise/pyproject.toml +++ b/enterprise/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm-enterprise" -version = "0.1.8" +version = "0.1.9" description = "Package for LiteLLM Enterprise features" authors = ["BerriAI"] readme = "README.md" @@ -22,7 +22,7 @@ requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "0.1.8" +version = "0.1.9" version_files = [ "pyproject.toml:version", "../requirements.txt:litellm-enterprise==", diff --git a/litellm/__init__.py b/litellm/__init__.py index 614d6de8e7d..2921c0600d9 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -324,6 +324,7 @@ use_aiohttp_transport: bool = ( True # Older variable, aiohttp is now the default. use disable_aiohttp_transport instead. ) disable_aiohttp_transport: bool = False # Set this to true to use httpx instead +disable_aiohttp_trust_env: bool = False # When False, aiohttp will respect HTTP(S)_PROXY env vars force_ipv4: bool = ( False # when True, litellm will force ipv4 for all LLM requests. Some users have seen httpx ConnectionError when using ipv6. ) @@ -1140,6 +1141,7 @@ from .router import Router from .assistants.main import * from .batches.main import * from .images.main import * +from .vector_stores import * from .batch_completion.main import * # type: ignore from .rerank_api.main import * from .llms.anthropic.experimental_pass_through.messages.handler import * diff --git a/litellm/constants.py b/litellm/constants.py index c96e1dfaba8..61eeb615069 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -693,6 +693,9 @@ PROMETHEUS_BUDGET_METRICS_REFRESH_INTERVAL_MINUTES = int( MCP_TOOL_NAME_PREFIX = "mcp_tool" MAXIMUM_TRACEBACK_LINES_TO_LOG = int(os.getenv("MAXIMUM_TRACEBACK_LINES_TO_LOG", 100)) +# Headers to control callbacks +X_LITELLM_DISABLE_CALLBACKS = "x-litellm-disable-callbacks" + ########################### LiteLLM Proxy Specific Constants ########################### ######################################################################################## MAX_SPENDLOG_ROWS_TO_QUERY = int( diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 69a14a7aa7a..b55da01aff1 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -45,6 +45,9 @@ from litellm.llms.openai.cost_calculation import ( cost_per_second as openai_cost_per_second, ) from litellm.llms.openai.cost_calculation import cost_per_token as openai_cost_per_token +from litellm.llms.perplexity.cost_calculator import ( + cost_per_token as perplexity_cost_per_token, +) from litellm.llms.together_ai.cost_calculator import get_model_params_and_category from litellm.llms.vertex_ai.cost_calculator import ( cost_per_character as google_cost_per_character, @@ -329,6 +332,8 @@ def cost_per_token( # noqa: PLR0915 return gemini_cost_per_token(model=model, usage=usage_block) elif custom_llm_provider == "deepseek": return deepseek_cost_per_token(model=model, usage=usage_block) + elif custom_llm_provider == "perplexity": + return perplexity_cost_per_token(model=model, usage=usage_block) else: model_info = _cached_get_model_info_helper( model=model, custom_llm_provider=custom_llm_provider @@ -661,9 +666,9 @@ def completion_cost( # noqa: PLR0915 or isinstance(completion_response, dict) ): # tts returns a custom class if isinstance(completion_response, dict): - usage_obj: Optional[ - Union[dict, Usage] - ] = completion_response.get("usage", {}) + usage_obj: Optional[Union[dict, Usage]] = ( + completion_response.get("usage", {}) + ) else: usage_obj = getattr(completion_response, "usage", {}) if isinstance(usage_obj, BaseModel) and not _is_known_usage_objects( diff --git a/litellm/integrations/vector_stores/base_vector_store.py b/litellm/integrations/vector_store_integrations/base_vector_store.py similarity index 100% rename from litellm/integrations/vector_stores/base_vector_store.py rename to litellm/integrations/vector_store_integrations/base_vector_store.py diff --git a/litellm/integrations/vector_store_integrations/bedrock_vector_store.py b/litellm/integrations/vector_store_integrations/bedrock_vector_store.py new file mode 100644 index 00000000000..a00acefb6a3 --- /dev/null +++ b/litellm/integrations/vector_store_integrations/bedrock_vector_store.py @@ -0,0 +1,409 @@ +# +-------------------------------------------------------------+ +# +# Add Bedrock Knowledge Base Context to your LLM calls +# +# +-------------------------------------------------------------+ +# Thank you users! We ❤️ you! - Krrish & Ishaan + +import json +from datetime import datetime +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple + +import litellm +from litellm._logging import verbose_logger, verbose_proxy_logger +from litellm.integrations.custom_logger import CustomLogger +from litellm.integrations.vector_store_integrations.base_vector_store import ( + BaseVectorStore, +) +from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM +from litellm.llms.custom_httpx.http_handler import ( + get_async_httpx_client, + httpxSpecialProvider, +) +from litellm.types.integrations.rag.bedrock_knowledgebase import ( + BedrockKBContent, + BedrockKBGuardrailConfiguration, + BedrockKBRequest, + BedrockKBResponse, + BedrockKBRetrievalConfiguration, + BedrockKBRetrievalQuery, + BedrockKBRetrievalResult, +) +from litellm.types.llms.openai import AllMessageValues, ChatCompletionUserMessage +from litellm.types.utils import StandardLoggingVectorStoreRequest +from litellm.types.vector_stores import ( + VectorStoreResultContent, + VectorStoreSearchResponse, + VectorStoreSearchResult, +) + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import StandardCallbackDynamicParams +else: + StandardCallbackDynamicParams = Any + + +class BedrockVectorStore(BaseVectorStore, BaseAWSLLM): + CONTENT_PREFIX_STRING = "Context: \n\n" + CUSTOM_LLM_PROVIDER = "bedrock" + + def __init__( + self, + **kwargs, + ): + self.async_handler = get_async_httpx_client( + llm_provider=httpxSpecialProvider.LoggingCallback + ) + + # store kwargs as optional_params + self.optional_params = kwargs + + super().__init__(**kwargs) + BaseAWSLLM.__init__(self) + + async def async_get_chat_completion_prompt( + self, + model: str, + messages: List[AllMessageValues], + non_default_params: dict, + prompt_id: Optional[str], + prompt_variables: Optional[dict], + dynamic_callback_params: StandardCallbackDynamicParams, + litellm_logging_obj: LiteLLMLoggingObj, + tools: Optional[List[Dict]] = None, + prompt_label: Optional[str] = None, + ) -> Tuple[str, List[AllMessageValues], dict]: + """ + Retrieves the context from the Bedrock Knowledge Base and appends it to the messages. + """ + if litellm.vector_store_registry is None: + return model, messages, non_default_params + + vector_store_ids = litellm.vector_store_registry.pop_vector_store_ids_to_run( + non_default_params=non_default_params, tools=tools + ) + vector_store_request_metadata: List[StandardLoggingVectorStoreRequest] = [] + if vector_store_ids: + for vector_store_id in vector_store_ids: + start_time = datetime.now() + query = self._get_kb_query_from_messages(messages) + bedrock_kb_response = await self.make_bedrock_kb_retrieve_request( + knowledge_base_id=vector_store_id, + query=query, + non_default_params=non_default_params, + ) + verbose_logger.debug( + f"Bedrock Knowledge Base Response: {bedrock_kb_response}" + ) + + ( + context_message, + context_string, + ) = self.get_chat_completion_message_from_bedrock_kb_response( + bedrock_kb_response + ) + if context_message is not None: + messages.append(context_message) + + ################################################################################################# + ########## LOGGING for Standard Logging Payload, Langfuse, s3, LiteLLM DB etc. ################## + ################################################################################################# + vector_store_search_response: VectorStoreSearchResponse = ( + self.transform_bedrock_kb_response_to_vector_store_search_response( + bedrock_kb_response=bedrock_kb_response, query=query + ) + ) + vector_store_request_metadata.append( + StandardLoggingVectorStoreRequest( + vector_store_id=vector_store_id, + query=query, + vector_store_search_response=vector_store_search_response, + custom_llm_provider=self.CUSTOM_LLM_PROVIDER, + start_time=start_time.timestamp(), + end_time=datetime.now().timestamp(), + ) + ) + + litellm_logging_obj.model_call_details[ + "vector_store_request_metadata" + ] = vector_store_request_metadata + + return model, messages, non_default_params + + def transform_bedrock_kb_response_to_vector_store_search_response( + self, + bedrock_kb_response: BedrockKBResponse, + query: str, + ) -> VectorStoreSearchResponse: + """ + Transform a BedrockKBResponse to a VectorStoreSearchResponse + """ + retrieval_results: Optional[ + List[BedrockKBRetrievalResult] + ] = bedrock_kb_response.get("retrievalResults", None) + vector_store_search_response: VectorStoreSearchResponse = ( + VectorStoreSearchResponse(search_query=query, data=[]) + ) + if retrieval_results is None: + return vector_store_search_response + + vector_search_response_data: List[VectorStoreSearchResult] = [] + for retrieval_result in retrieval_results: + content: Optional[BedrockKBContent] = retrieval_result.get("content", None) + if content is None: + continue + content_text: Optional[str] = content.get("text", None) + if content_text is None: + continue + vector_store_search_result: VectorStoreSearchResult = ( + VectorStoreSearchResult( + score=retrieval_result.get("score", None), + content=[VectorStoreResultContent(text=content_text, type="text")], + ) + ) + vector_search_response_data.append(vector_store_search_result) + vector_store_search_response["data"] = vector_search_response_data + return vector_store_search_response + + def _get_kb_query_from_messages(self, messages: List[AllMessageValues]) -> str: + """ + Uses the text `content` field of the last message in the list of messages + """ + if len(messages) == 0: + return "" + last_message = messages[-1] + last_message_content = last_message.get("content", None) + if last_message_content is None: + return "" + if isinstance(last_message_content, str): + return last_message_content + elif isinstance(last_message_content, list): + return "\n".join([item.get("text", "") for item in last_message_content]) + return "" + + def _prepare_request( + self, + credentials: Any, + data: BedrockKBRequest, + optional_params: dict, + aws_region_name: str, + api_base: str, + extra_headers: Optional[dict] = None, + ) -> Any: + """ + Prepare a signed AWS request. + + Args: + credentials: AWS credentials + data: Request data + optional_params: Additional parameters + aws_region_name: AWS region name + api_base: Base API URL + extra_headers: Additional headers + + Returns: + AWSRequest: A signed AWS request + """ + try: + from botocore.auth import SigV4Auth + from botocore.awsrequest import AWSRequest + except ImportError: + raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.") + + sigv4 = SigV4Auth(credentials, "bedrock", aws_region_name) + + encoded_data = json.dumps(data).encode("utf-8") + headers = {"Content-Type": "application/json"} + if extra_headers is not None: + headers = {"Content-Type": "application/json", **extra_headers} + + request = AWSRequest( + method="POST", url=api_base, data=encoded_data, headers=headers + ) + sigv4.add_auth(request) + if extra_headers is not None and "Authorization" in extra_headers: + # prevent sigv4 from overwriting the auth header + request.headers["Authorization"] = extra_headers["Authorization"] + + return request.prepare() + + async def make_bedrock_kb_retrieve_request( + self, + knowledge_base_id: str, + query: str, + guardrail_id: Optional[str] = None, + guardrail_version: Optional[str] = None, + next_token: Optional[str] = None, + retrieval_configuration: Optional[BedrockKBRetrievalConfiguration] = None, + non_default_params: Optional[dict] = None, + ) -> BedrockKBResponse: + """ + Make a Bedrock Knowledge Base retrieve request. + + Args: + knowledge_base_id (str): The unique identifier of the knowledge base to query + query (str): The query text to search for + guardrail_id (Optional[str]): The guardrail ID to apply + guardrail_version (Optional[str]): The version of the guardrail to apply + next_token (Optional[str]): Token for pagination + retrieval_configuration (Optional[BedrockKBRetrievalConfiguration]): Configuration for the retrieval process + + Returns: + BedrockKBRetrievalResponse: A typed response object containing the retrieval results + """ + from fastapi import HTTPException + + non_default_params = non_default_params or {} + credentials_dict: Dict[str, Any] = {} + if litellm.vector_store_registry is not None: + credentials_dict = ( + litellm.vector_store_registry.get_credentials_for_vector_store( + knowledge_base_id + ) + ) + + credentials = self.get_credentials( + aws_access_key_id=credentials_dict.get( + "aws_access_key_id", non_default_params.get("aws_access_key_id", None) + ), + aws_secret_access_key=credentials_dict.get( + "aws_secret_access_key", + non_default_params.get("aws_secret_access_key", None), + ), + aws_session_token=credentials_dict.get( + "aws_session_token", non_default_params.get("aws_session_token", None) + ), + aws_region_name=credentials_dict.get( + "aws_region_name", non_default_params.get("aws_region_name", None) + ), + aws_session_name=credentials_dict.get( + "aws_session_name", non_default_params.get("aws_session_name", None) + ), + aws_profile_name=credentials_dict.get( + "aws_profile_name", non_default_params.get("aws_profile_name", None) + ), + aws_role_name=credentials_dict.get( + "aws_role_name", non_default_params.get("aws_role_name", None) + ), + aws_web_identity_token=credentials_dict.get( + "aws_web_identity_token", + non_default_params.get("aws_web_identity_token", None), + ), + aws_sts_endpoint=credentials_dict.get( + "aws_sts_endpoint", non_default_params.get("aws_sts_endpoint", None) + ), + ) + aws_region_name = self.get_aws_region_name_for_non_llm_api_calls( + aws_region_name=credentials_dict.get( + "aws_region_name", non_default_params.get("aws_region_name", None) + ), + ) + + # Prepare request data + request_data: BedrockKBRequest = BedrockKBRequest( + retrievalQuery=BedrockKBRetrievalQuery(text=query), + ) + if next_token: + request_data["nextToken"] = next_token + if retrieval_configuration: + request_data["retrievalConfiguration"] = retrieval_configuration + if guardrail_id and guardrail_version: + request_data["guardrailConfiguration"] = BedrockKBGuardrailConfiguration( + guardrailId=guardrail_id, guardrailVersion=guardrail_version + ) + verbose_logger.debug( + f"Request Data: {json.dumps(request_data, indent=4, default=str)}" + ) + + # Prepare the request + api_base = f"https://bedrock-agent-runtime.{aws_region_name}.amazonaws.com/knowledgebases/{knowledge_base_id}/retrieve" + + prepared_request = self._prepare_request( + credentials=credentials, + data=request_data, + optional_params=self.optional_params, + aws_region_name=aws_region_name, + api_base=api_base, + ) + + verbose_proxy_logger.debug( + "Bedrock Knowledge Base request body: %s, url %s, headers: %s", + request_data, + prepared_request.url, + prepared_request.headers, + ) + + response = await self.async_handler.post( + url=prepared_request.url, + data=prepared_request.body, # type: ignore + headers=prepared_request.headers, # type: ignore + ) + + verbose_proxy_logger.debug("Bedrock Knowledge Base response: %s", response.text) + + if response.status_code == 200: + response_data = response.json() + return BedrockKBResponse(**response_data) + else: + verbose_proxy_logger.error( + "Bedrock Knowledge Base: error in response. Status code: %s, response: %s", + response.status_code, + response.text, + ) + raise HTTPException( + status_code=response.status_code, + detail={ + "error": "Error calling Bedrock Knowledge Base", + "response": response.text, + }, + ) + + @staticmethod + def get_initialized_custom_logger() -> Optional[CustomLogger]: + from litellm.litellm_core_utils.litellm_logging import ( + _init_custom_logger_compatible_class, + ) + + return _init_custom_logger_compatible_class( + logging_integration="bedrock_vector_store", + internal_usage_cache=None, + llm_router=None, + ) + + @staticmethod + def get_chat_completion_message_from_bedrock_kb_response( + response: BedrockKBResponse, + ) -> Tuple[Optional[ChatCompletionUserMessage], str]: + """ + Retrieves the context from the Bedrock Knowledge Base response and returns a ChatCompletionUserMessage object. + """ + retrieval_results: Optional[List[BedrockKBRetrievalResult]] = response.get( + "retrievalResults", None + ) + if retrieval_results is None: + return None, "" + + # string to combine the context from the knowledge base + context_string: str = BedrockVectorStore.CONTENT_PREFIX_STRING + for retrieval_result in retrieval_results: + retrieval_result_content: Optional[BedrockKBContent] = ( + retrieval_result.get("content", None) or {} + ) + if retrieval_result_content is None: + continue + retrieval_result_text: Optional[str] = retrieval_result_content.get( + "text", None + ) + if retrieval_result_text is None: + continue + context_string += retrieval_result_text + message = ChatCompletionUserMessage( + role="user", + content=context_string, + ) + return message, context_string diff --git a/litellm/integrations/vector_stores/bedrock_vector_store.py b/litellm/integrations/vector_stores/bedrock_vector_store.py index 0523dac8edd..a00acefb6a3 100644 --- a/litellm/integrations/vector_stores/bedrock_vector_store.py +++ b/litellm/integrations/vector_stores/bedrock_vector_store.py @@ -12,7 +12,9 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple import litellm from litellm._logging import verbose_logger, verbose_proxy_logger from litellm.integrations.custom_logger import CustomLogger -from litellm.integrations.vector_stores.base_vector_store import BaseVectorStore +from litellm.integrations.vector_store_integrations.base_vector_store import ( + BaseVectorStore, +) from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, diff --git a/litellm/litellm_core_utils/custom_logger_registry.py b/litellm/litellm_core_utils/custom_logger_registry.py new file mode 100644 index 00000000000..1b75cc3e3df --- /dev/null +++ b/litellm/litellm_core_utils/custom_logger_registry.py @@ -0,0 +1,135 @@ +""" +Registry mapping the callback class string to the class type. + +This is used to get the class type from the callback class string. + +Example: + "datadog" -> DataDogLogger + "prometheus" -> PrometheusLogger +""" + +from litellm.integrations.agentops import AgentOps +from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook +from litellm.integrations.argilla import ArgillaLogger +from litellm.integrations.azure_storage.azure_storage import AzureBlobStorageLogger +from litellm.integrations.braintrust_logging import BraintrustLogger +from litellm.integrations.datadog.datadog import DataDogLogger +from litellm.integrations.datadog.datadog_llm_obs import DataDogLLMObsLogger +from litellm.integrations.deepeval import DeepEvalLogger +from litellm.integrations.galileo import GalileoObserve +from litellm.integrations.gcs_bucket.gcs_bucket import GCSBucketLogger +from litellm.integrations.gcs_pubsub.pub_sub import GcsPubSubLogger +from litellm.integrations.humanloop import HumanloopLogger +from litellm.integrations.lago import LagoLogger +from litellm.integrations.langfuse.langfuse_prompt_management import ( + LangfusePromptManagement, +) +from litellm.integrations.langsmith import LangsmithLogger +from litellm.integrations.literal_ai import LiteralAILogger +from litellm.integrations.mlflow import MlflowLogger +from litellm.integrations.openmeter import OpenMeterLogger +from litellm.integrations.opentelemetry import OpenTelemetry +from litellm.integrations.opik.opik import OpikLogger +from litellm.integrations.prometheus import PrometheusLogger +from litellm.integrations.s3_v2 import S3Logger +from litellm.integrations.vector_store_integrations.bedrock_vector_store import ( + BedrockVectorStore, +) +from litellm.proxy.hooks.dynamic_rate_limiter import _PROXY_DynamicRateLimitHandler + + +class CustomLoggerRegistry: + """ + Registry mapping the callback class string to the class type. + """ + CALLBACK_CLASS_STR_TO_CLASS_TYPE = { + "lago": LagoLogger, + "openmeter": OpenMeterLogger, + "braintrust": BraintrustLogger, + "galileo": GalileoObserve, + "langsmith": LangsmithLogger, + "literalai": LiteralAILogger, + "prometheus": PrometheusLogger, + "datadog": DataDogLogger, + "datadog_llm_observability": DataDogLLMObsLogger, + "gcs_bucket": GCSBucketLogger, + "opik": OpikLogger, + "argilla": ArgillaLogger, + "opentelemetry": OpenTelemetry, + "azure_storage": AzureBlobStorageLogger, + "humanloop": HumanloopLogger, + # OTEL compatible loggers + "logfire": OpenTelemetry, + "arize": OpenTelemetry, + "langfuse_otel": OpenTelemetry, + "arize_phoenix": OpenTelemetry, + "langtrace": OpenTelemetry, + "mlflow": MlflowLogger, + "langfuse": LangfusePromptManagement, + "otel": OpenTelemetry, + "gcs_pubsub": GcsPubSubLogger, + "anthropic_cache_control_hook": AnthropicCacheControlHook, + "agentops": AgentOps, + "bedrock_vector_store": BedrockVectorStore, + "deepeval": DeepEvalLogger, + "s3_v2": S3Logger, + "dynamic_rate_limiter": _PROXY_DynamicRateLimitHandler, + } + + try: + from litellm_enterprise.enterprise_callbacks.generic_api_callback import ( + GenericAPILogger, + ) + from litellm_enterprise.enterprise_callbacks.pagerduty.pagerduty import ( + PagerDutyAlerting, + ) + from litellm_enterprise.enterprise_callbacks.send_emails.resend_email import ( + ResendEmailLogger, + ) + from litellm_enterprise.enterprise_callbacks.send_emails.smtp_email import ( + SMTPEmailLogger, + ) + + enterprise_loggers = { + "pagerduty": PagerDutyAlerting, + "generic_api": GenericAPILogger, + "resend_email": ResendEmailLogger, + "smtp_email": SMTPEmailLogger, + } + CALLBACK_CLASS_STR_TO_CLASS_TYPE.update(enterprise_loggers) + except ImportError: + pass # enterprise not installed + + @classmethod + def get_callback_str_from_class_type(cls, class_type: type) -> str | None: + """ + Get the callback string from the class type. + + Args: + class_type: The class type to find the string for + + Returns: + str: The callback string, or None if not found + """ + for callback_str, callback_class in cls.CALLBACK_CLASS_STR_TO_CLASS_TYPE.items(): + if callback_class == class_type: + return callback_str + return None + + @classmethod + def get_all_callback_strs_from_class_type(cls, class_type: type) -> list[str]: + """ + Get all callback strings that map to the same class type. + Some class types (like OpenTelemetry) have multiple string mappings. + + Args: + class_type: The class type to find all strings for + + Returns: + list: List of callback strings that map to the class type + """ + callback_strs: list[str] = [] + for callback_str, callback_class in cls.CALLBACK_CLASS_STR_TO_CLASS_TYPE.items(): + if callback_class == class_type: + callback_strs.append(callback_str) + return callback_strs \ No newline at end of file diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 4523342a091..aa0280210d4 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -54,7 +54,9 @@ from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.deepeval.deepeval import DeepEvalLogger from litellm.integrations.mlflow import MlflowLogger -from litellm.integrations.vector_stores.bedrock_vector_store import BedrockVectorStore +from litellm.integrations.vector_store_integrations.bedrock_vector_store import ( + BedrockVectorStore, +) from litellm.litellm_core_utils.get_litellm_params import get_litellm_params from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import ( StandardBuiltInToolCostTracking, @@ -147,6 +149,9 @@ from .initialize_dynamic_callback_params import ( from .specialty_caches.dynamic_logging_cache import DynamicLoggingCache try: + from litellm_enterprise.enterprise_callbacks.callback_controls import ( + EnterpriseCallbackControls, + ) from litellm_enterprise.enterprise_callbacks.generic_api_callback import ( GenericAPILogger, ) @@ -174,6 +179,7 @@ except Exception as e: ResendEmailLogger = CustomLogger # type: ignore SMTPEmailLogger = CustomLogger # type: ignore PagerDutyAlerting = CustomLogger # type: ignore + EnterpriseCallbackControls = None # type: ignore EnterpriseStandardLoggingPayloadSetupVAR = None _in_memory_loggers: List[Any] = [] @@ -1079,6 +1085,18 @@ class Logging(LiteLLMLoggingBaseClass): used for consistent cost calculation across response headers + logging integrations. """ + if isinstance(result, BaseModel) and hasattr(result, "_hidden_params"): + hidden_params = getattr(result, "_hidden_params", {}) + if ( + "response_cost" in hidden_params + and hidden_params["response_cost"] is not None + ): # use cost if already calculated + return hidden_params["response_cost"] + elif ( + router_model_id is None and "model_id" in hidden_params + ): # use model_id if not already set + router_model_id = hidden_params["model_id"] + ## RESPONSE COST ## custom_pricing = use_custom_pricing_for_model( litellm_params=( @@ -1217,6 +1235,14 @@ class Logging(LiteLLMLoggingBaseClass): f"no-log request, skipping logging for {event_hook} event" ) return False + + # Check for dynamically disabled callbacks via headers + if EnterpriseCallbackControls is not None and EnterpriseCallbackControls.is_callback_disabled_via_headers(callback, litellm_params): + verbose_logger.debug( + f"Callback {callback} disabled via x-litellm-disable-callbacks header for {event_hook} event" + ) + return False + return True def _update_completion_start_time(self, completion_start_time: datetime.datetime): @@ -2246,6 +2272,14 @@ class Logging(LiteLLMLoggingBaseClass): self.has_run_logging(event_type="sync_failure") for callback in callbacks: try: + litellm_params = self.model_call_details.get("litellm_params", {}) + should_run = self.should_run_callback( + callback=callback, + litellm_params=litellm_params, + event_hook="failure_handler", + ) + if not should_run: + continue if callback == "lunary" and lunaryLogger is not None: print_verbose("reaches lunary for logging error!") @@ -2427,6 +2461,14 @@ class Logging(LiteLLMLoggingBaseClass): self.has_run_logging(event_type="async_failure") for callback in callbacks: try: + litellm_params = self.model_call_details.get("litellm_params", {}) + should_run = self.should_run_callback( + callback=callback, + litellm_params=litellm_params, + event_hook="async_failure_handler", + ) + if not should_run: + continue if isinstance(callback, CustomLogger): # custom logger class await callback.async_log_failure_event( kwargs=self.model_call_details, diff --git a/litellm/litellm_core_utils/llm_request_utils.py b/litellm/litellm_core_utils/llm_request_utils.py index 50dbdc5536e..89f5728979f 100644 --- a/litellm/litellm_core_utils/llm_request_utils.py +++ b/litellm/litellm_core_utils/llm_request_utils.py @@ -66,3 +66,18 @@ def pick_cheapest_chat_models_from_llm_provider(custom_llm_provider: str, n=1): # Return the top n cheapest models return [model for model, _ in model_costs[:n]] + +def get_proxy_server_request_headers(litellm_params: Optional[dict]) -> dict: + """ + Get the `proxy_server_request` headers from the litellm_params.\ + + Use this if you want to access the request headers made to LiteLLM proxy server. + """ + if litellm_params is None: + return {} + + proxy_request_headers = ( + litellm_params.get("proxy_server_request", {}).get("headers", {}) or {} + ) + + return proxy_request_headers \ No newline at end of file diff --git a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py index fb310bf33c7..54adef9c958 100644 --- a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py +++ b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py @@ -40,6 +40,34 @@ from litellm.types.utils import ( from .get_headers import get_response_headers +def _safe_convert_created_field(created_value) -> int: + """ + Safely convert a 'created' field value to an integer. + + Some providers (like SambaNova) return the 'created' field as a float + (Unix timestamp with fractional seconds), but LiteLLM expects an integer. + + Args: + created_value: The value from response_object["created"] + + Returns: + int: Unix timestamp as integer + """ + if created_value is None: + return int(time.time()) + elif isinstance(created_value, int): + return created_value + elif isinstance(created_value, float): + return int(created_value) + else: + # for strings, etc + try: + return int(float(created_value)) + except (ValueError, TypeError): + # Fallback to current time if conversion fails + return int(time.time()) + + def convert_tool_call_to_json_mode( tool_calls: List[ChatCompletionMessageToolCall], convert_tool_call_to_json_mode: bool, @@ -133,7 +161,7 @@ async def convert_to_streaming_response_async(response_object: Optional[dict] = model_response_object.id = response_object["id"] if "created" in response_object: - model_response_object.created = response_object["created"] + model_response_object.created = _safe_convert_created_field(response_object["created"]) if "system_fingerprint" in response_object: model_response_object.system_fingerprint = response_object["system_fingerprint"] @@ -181,7 +209,7 @@ def convert_to_streaming_response(response_object: Optional[dict] = None): model_response_object.id = response_object["id"] if "created" in response_object: - model_response_object.created = response_object["created"] + model_response_object.created = _safe_convert_created_field(response_object["created"]) if "system_fingerprint" in response_object: model_response_object.system_fingerprint = response_object["system_fingerprint"] @@ -578,9 +606,7 @@ def convert_to_model_response_object( # noqa: PLR0915 usage_object = litellm.Usage(**response_object["usage"]) setattr(model_response_object, "usage", usage_object) if "created" in response_object: - model_response_object.created = response_object["created"] or int( - time.time() - ) + model_response_object.created = _safe_convert_created_field(response_object["created"]) if "id" in response_object: model_response_object.id = response_object["id"] or str(uuid.uuid4()) diff --git a/litellm/litellm_core_utils/logging_callback_manager.py b/litellm/litellm_core_utils/logging_callback_manager.py index e1bddc65497..ab6c2ac7659 100644 --- a/litellm/litellm_core_utils/logging_callback_manager.py +++ b/litellm/litellm_core_utils/logging_callback_manager.py @@ -4,6 +4,7 @@ import litellm from litellm._logging import verbose_logger from litellm.integrations.additional_logging_utils import AdditionalLoggingUtils from litellm.integrations.custom_logger import CustomLogger +from litellm.types.utils import CallbacksByType class LoggingCallbackManager: @@ -275,3 +276,65 @@ class LoggingCallbackManager: isinstance(callback, callback_type) for callback in self._get_all_callbacks() ) + + def get_callbacks_by_type(self) -> CallbacksByType: + """ + Get all active callbacks categorized by their type (success, failure, success_and_failure). + + Returns: + CallbacksByType: Dict with keys 'success', 'failure', 'success_and_failure' containing lists of callback strings + """ + # Get callback lists + success_callbacks = set(litellm.success_callback + litellm._async_success_callback) + failure_callbacks = set(litellm.failure_callback + litellm._async_failure_callback) + general_callbacks = set(litellm.callbacks) + + # Get all unique callbacks + all_callbacks = success_callbacks | failure_callbacks | general_callbacks + + result: CallbacksByType = CallbacksByType( + success=[], + failure=[], + success_and_failure=[] + ) + + for callback in all_callbacks: + callback_str = self._get_callback_string(callback) + + is_in_success = callback in success_callbacks + is_in_failure = callback in failure_callbacks + is_in_general = callback in general_callbacks + + if is_in_general or (is_in_success and is_in_failure): + result["success_and_failure"].append(callback_str) + elif is_in_success: + result["success"].append(callback_str) + elif is_in_failure: + result["failure"].append(callback_str) + + + + # final de-duplication + result["success"] = list(set(result["success"])) + result["failure"] = list(set(result["failure"])) + result["success_and_failure"] = list(set(result["success_and_failure"])) + + return result + + def _get_callback_string( + self, + callback: Union[CustomLogger, Callable, str] + ) -> str: + from litellm.litellm_core_utils.custom_logger_registry import ( + CustomLoggerRegistry, + ) + """Convert a callback to its string representation""" + if isinstance(callback, str): + return callback + elif isinstance(callback, CustomLogger): + # Try to get the string representation from the registry + callback_str = CustomLoggerRegistry.get_callback_str_from_class_type(type(callback)) + return callback_str if callback_str is not None else type(callback).__name__ + elif callable(callback): + return getattr(callback, '__name__', str(callback)) + return str(callback) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index a4ba25fc466..626c8b7f297 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -489,38 +489,100 @@ def extract_file_data(file_data: FileTypes) -> ExtractedFileData: ) -def unpack_defs(schema, defs): - properties = schema.get("properties", None) - if properties is None: - return +# --------------------------------------------------------------------------- +# Generic, dependency-free implementation of `unpack_defs` +# --------------------------------------------------------------------------- - for name, value in properties.items(): - ref_key = value.get("$ref", None) - if ref_key is not None: - ref = defs[ref_key.split("defs/")[-1]] - unpack_defs(ref, defs) - properties[name] = ref + +def unpack_defs(schema: dict, defs: dict) -> None: + """Expand *all* ``$ref`` entries pointing into ``$defs`` / ``definitions``. + + This utility walks the entire schema tree (dicts and lists) so it naturally + resolves references hidden under any keyword – ``items``, ``allOf``, + ``anyOf``, ``oneOf``, ``additionalProperties``, etc. + + It mutates *schema* in-place and does **not** return anything. The helper + keeps memory overhead low by resolving nodes as it encounters them rather + than materialising a fully dereferenced copy first. + """ + + import copy + from collections import deque + + # Combine the defs handed down by the caller with defs/definitions found on + # the current node. Local keys shadow parent keys to match JSON-schema + # scoping rules. + root_defs: dict = { + **defs, + **schema.get("$defs", {}), + **schema.get("definitions", {}), + } + + # Use iterative approach with queue to avoid recursion + # Each item in queue is (node, parent_container, key/index, active_defs, seen_ids) + queue: deque[tuple[Any, Union[dict, list, None], Union[str, int, None], dict, set]] = deque([(schema, None, None, root_defs, set())]) + + while queue: + node, parent, key, active_defs, seen = queue.popleft() + + # Avoid infinite loops on self-referential schemas + if id(node) in seen: continue + seen = seen.copy() # Create new set for this branch + seen.add(id(node)) - anyof = value.get("anyOf", None) - if anyof is not None: - for i, atype in enumerate(anyof): - ref_key = atype.get("$ref", None) - if ref_key is not None: - ref = defs[ref_key.split("defs/")[-1]] - unpack_defs(ref, defs) - anyof[i] = ref - continue + # ----------------------------- dict ----------------------------- + if isinstance(node, dict): + # --- Case 1: this node *is* a reference --- + if "$ref" in node: + ref_name = node["$ref"].split("/")[-1] + target_schema = active_defs.get(ref_name) + # Unknown reference – leave untouched + if target_schema is None: + continue - items = value.get("items", None) - if items is not None: - ref_key = items.get("$ref", None) - if ref_key is not None: - ref = defs[ref_key.split("defs/")[-1]] - unpack_defs(ref, defs) - value["items"] = ref + # Merge defs from the target to capture nested definitions + child_defs = { + **active_defs, + **target_schema.get("$defs", {}), + **target_schema.get("definitions", {}), + } + + # Replace the reference with resolved copy + resolved = copy.deepcopy(target_schema) + if parent is not None and key is not None: + if isinstance(parent, dict) and isinstance(key, str): + parent[key] = resolved + elif isinstance(parent, list) and isinstance(key, int): + parent[key] = resolved + else: + # This is the root schema itself + schema.clear() + schema.update(resolved) + resolved = schema + + # Add resolved node to queue for further processing + queue.append((resolved, parent, key, child_defs, seen)) continue + # --- Case 2: regular dict – process its values --- + # Update defs with any nested $defs/definitions present *here*. + current_defs = { + **active_defs, + **node.get("$defs", {}), + **node.get("definitions", {}), + } + + # Add all dict values to queue + for k, v in node.items(): + queue.append((v, node, k, current_defs, seen)) + + # ---------------------------- list ------------------------------ + elif isinstance(node, list): + # Add all list items to queue + for idx, item in enumerate(node): + queue.append((item, node, idx, active_defs, seen)) + def _get_image_mime_type_from_url(url: str) -> Optional[str]: """ diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 079cef4631e..529e7c8399b 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -85,9 +85,9 @@ class CustomStreamWrapper: self.system_fingerprint: Optional[str] = None self.received_finish_reason: Optional[str] = None - self.intermittent_finish_reason: Optional[ - str - ] = None # finish reasons that show up mid-stream + self.intermittent_finish_reason: Optional[str] = ( + None # finish reasons that show up mid-stream + ) self.special_tokens = [ "<|assistant|>", "<|system|>", @@ -643,6 +643,7 @@ class CustomStreamWrapper: model_response._hidden_params = { **model_response._hidden_params, **self._hidden_params, + "response_cost": None, } if ( @@ -1322,9 +1323,9 @@ class CustomStreamWrapper: _json_delta = delta.model_dump() print_verbose(f"_json_delta: {_json_delta}") if "role" not in _json_delta or _json_delta["role"] is None: - _json_delta[ - "role" - ] = "assistant" # mistral's api returns role as None + _json_delta["role"] = ( + "assistant" # mistral's api returns role as None + ) if "tool_calls" in _json_delta and isinstance( _json_delta["tool_calls"], list ): @@ -1715,9 +1716,9 @@ class CustomStreamWrapper: chunk = next(self.completion_stream) if chunk is not None and chunk != b"": print_verbose(f"PROCESSED CHUNK PRE CHUNK CREATOR: {chunk}") - processed_chunk: Optional[ - ModelResponseStream - ] = self.chunk_creator(chunk=chunk) + processed_chunk: Optional[ModelResponseStream] = ( + self.chunk_creator(chunk=chunk) + ) print_verbose( f"PROCESSED CHUNK POST CHUNK CREATOR: {processed_chunk}" ) diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py index ddb3eaed80b..37fd839b3c5 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py @@ -119,6 +119,8 @@ def anthropic_messages_handler( """ Makes Anthropic `/v1/messages` API calls In the Anthropic API Spec """ + from litellm.types.utils import LlmProviders + local_vars = locals() is_async = kwargs.pop("is_async", False) # Use provided client or create a new one @@ -141,12 +143,17 @@ def anthropic_messages_handler( api_key=litellm_params.api_key, ) - anthropic_messages_provider_config: Optional[ - BaseAnthropicMessagesConfig - ] = ProviderConfigManager.get_provider_anthropic_messages_config( - model=model, - provider=litellm.LlmProviders(custom_llm_provider), - ) + anthropic_messages_provider_config: Optional[BaseAnthropicMessagesConfig] = None + + if custom_llm_provider is not None and custom_llm_provider in [ + provider.value for provider in LlmProviders + ]: + anthropic_messages_provider_config = ( + ProviderConfigManager.get_provider_anthropic_messages_config( + model=model, + provider=litellm.LlmProviders(custom_llm_provider), + ) + ) if anthropic_messages_provider_config is None: # Handle non-Anthropic models using the adapter return ( diff --git a/litellm/llms/azure/chat/o_series_transformation.py b/litellm/llms/azure/chat/o_series_transformation.py index 69fb941ca58..767f2d46df3 100644 --- a/litellm/llms/azure/chat/o_series_transformation.py +++ b/litellm/llms/azure/chat/o_series_transformation.py @@ -17,7 +17,7 @@ from typing import List, Optional import litellm from litellm import verbose_logger from litellm.types.llms.openai import AllMessageValues -from litellm.utils import get_model_info +from litellm.utils import get_model_info, supports_reasoning from ...openai.chat.o_series_transformation import OpenAIOSeriesConfig @@ -38,7 +38,9 @@ class AzureOpenAIO1Config(OpenAIOSeriesConfig): "top_logprobs", ] - o_series_only_param = ["reasoning_effort"] + o_series_only_param = [] + if supports_reasoning(model): + o_series_only_param.append("reasoning_effort") all_openai_params.extend(o_series_only_param) return [ param for param in all_openai_params if param not in non_supported_params diff --git a/litellm/llms/azure/common_utils.py b/litellm/llms/azure/common_utils.py index ee154d2c8a1..f2a8defe13f 100644 --- a/litellm/llms/azure/common_utils.py +++ b/litellm/llms/azure/common_utils.py @@ -1,6 +1,6 @@ import json import os -from typing import Any, Callable, Dict, Optional, Union +from typing import Any, Callable, Dict, Literal, Optional, Union, cast import httpx from openai import AsyncAzureOpenAI, AzureOpenAI @@ -14,6 +14,8 @@ from litellm.secret_managers.get_azure_ad_token_provider import ( get_azure_ad_token_provider, ) from litellm.secret_managers.main import get_secret_str +from litellm.types.router import GenericLiteLLMParams +from litellm.utils import _add_path_to_api_base azure_ad_cache = DualCache() @@ -264,6 +266,126 @@ def select_azure_base_url_or_endpoint(azure_client_params: dict): return azure_client_params +def get_azure_ad_token( + litellm_params: GenericLiteLLMParams, +) -> Optional[str]: + """ + Get Azure AD token from various authentication methods. + + This function tries different methods to obtain an Azure AD token: + 1. From an existing token provider + 2. From Entra ID using tenant_id, client_id, and client_secret + 3. From username and password + 4. From OIDC token + 5. From a service principal with secret workflow + + Args: + litellm_params: Dictionary containing authentication parameters + - azure_ad_token_provider: Optional callable that returns a token + - azure_ad_token: Optional existing token + - tenant_id: Optional Azure tenant ID + - client_id: Optional Azure client ID + - client_secret: Optional Azure client secret + - azure_username: Optional Azure username + - azure_password: Optional Azure password + + Returns: + Azure AD token as string if successful, None otherwise + """ + # Extract parameters + azure_ad_token_provider = litellm_params.get("azure_ad_token_provider") + azure_ad_token = litellm_params.get("azure_ad_token", None) or get_secret_str( + "AZURE_AD_TOKEN" + ) + tenant_id = litellm_params.get("tenant_id", os.getenv("AZURE_TENANT_ID")) + client_id = litellm_params.get("client_id", os.getenv("AZURE_CLIENT_ID")) + client_secret = litellm_params.get( + "client_secret", os.getenv("AZURE_CLIENT_SECRET") + ) + azure_username = litellm_params.get("azure_username", os.getenv("AZURE_USERNAME")) + azure_password = litellm_params.get("azure_password", os.getenv("AZURE_PASSWORD")) + scope = litellm_params.get( + "azure_scope", + os.getenv("AZURE_SCOPE", "https://cognitiveservices.azure.com/.default"), + ) + if scope is None: + scope = "https://cognitiveservices.azure.com/.default" + + # Try to get token provider from Entra ID + if azure_ad_token_provider is None and tenant_id and client_id and client_secret: + verbose_logger.debug( + "Using Azure AD Token Provider from Entra ID for Azure Auth" + ) + azure_ad_token_provider = get_azure_ad_token_from_entra_id( + tenant_id=tenant_id, + client_id=client_id, + client_secret=client_secret, + scope=scope, + ) + + # Try to get token provider from username and password + if ( + azure_ad_token_provider is None + and azure_username + and azure_password + and client_id + ): + verbose_logger.debug("Using Azure Username and Password for Azure Auth") + azure_ad_token_provider = get_azure_ad_token_from_username_password( + azure_username=azure_username, + azure_password=azure_password, + client_id=client_id, + scope=scope, + ) + + # Try to get token from OIDC + if ( + client_id + and tenant_id + and azure_ad_token + and azure_ad_token.startswith("oidc/") + ): + verbose_logger.debug("Using Azure OIDC Token for Azure Auth") + azure_ad_token = get_azure_ad_token_from_oidc( + azure_ad_token=azure_ad_token, + azure_client_id=client_id, + azure_tenant_id=tenant_id, + scope=scope, + ) + # Try to get token provider from service principal + elif ( + azure_ad_token_provider is None + and litellm.enable_azure_ad_token_refresh is True + ): + verbose_logger.debug( + "Using Azure AD token provider based on Service Principal with Secret workflow for Azure Auth" + ) + try: + azure_ad_token_provider = get_azure_ad_token_provider(azure_scope=scope) + except ValueError: + verbose_logger.debug("Azure AD Token Provider could not be used.") + + # Execute the token provider to get the token if available + if azure_ad_token_provider and callable(azure_ad_token_provider): + try: + token = azure_ad_token_provider() + if not isinstance(token, str): + verbose_logger.error( + f"Azure AD token provider returned non-string value: {type(token)}" + ) + raise TypeError(f"Azure AD token must be a string, got {type(token)}") + else: + azure_ad_token = token + except TypeError: + # Re-raise TypeError directly + raise + except Exception as e: + verbose_logger.error(f"Error calling Azure AD token provider: {str(e)}") + raise RuntimeError(f"Failed to get Azure AD token: {str(e)}") from e + + return azure_ad_token + + class BaseAzureLLM(BaseOpenAILLM): def get_azure_openai_client( self, @@ -492,3 +614,78 @@ class BaseAzureLLM(BaseOpenAILLM): else: client = AzureOpenAI(**azure_client_params) # type: ignore return client + + @staticmethod + def _base_validate_azure_environment( + headers: dict, litellm_params: Optional[GenericLiteLLMParams] + ) -> dict: + litellm_params = litellm_params or GenericLiteLLMParams() + api_key = ( + litellm_params.api_key + or litellm.api_key + or litellm.azure_key + or get_secret_str("AZURE_OPENAI_API_KEY") + or get_secret_str("AZURE_API_KEY") + ) + + if api_key: + headers["api-key"] = api_key + return headers + + ### Fallback to Azure AD token-based authentication if no API key is available + ### Retrieves Azure AD token and adds it to the Authorization header + azure_ad_token = get_azure_ad_token(litellm_params) + if azure_ad_token: + headers["Authorization"] = f"Bearer {azure_ad_token}" + + return headers + + @staticmethod + def _get_base_azure_url( + api_base: Optional[str], + litellm_params: Optional[Union[GenericLiteLLMParams, Dict[str, Any]]], + route: Literal["/openai/responses", "/openai/vector_stores"] + ) -> str: + api_base = api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") + if api_base is None: + raise ValueError( + f"api_base is required for Azure AI Studio. Please set the api_base parameter. Passed `api_base={api_base}`" + ) + original_url = httpx.URL(api_base) + + # Extract api_version or use default + litellm_params = litellm_params or {} + api_version = cast(Optional[str], litellm_params.get("api_version")) + + # Create a new dictionary with existing params + query_params = dict(original_url.params) + + # Add api_version if needed + if "api-version" not in query_params and api_version: + query_params["api-version"] = api_version + + # Add the path to the base URL + if route not in api_base: + new_url = _add_path_to_api_base( + api_base=api_base, ending_path=route + ) + else: + new_url = api_base + + if BaseAzureLLM._is_azure_v1_api_version(api_version): + # ensure the request go to /openai/v1 and not just /openai + if "/openai/v1" not in new_url: + parsed_url = httpx.URL(new_url) + new_url = str(parsed_url.copy_with(path=parsed_url.path.replace("/openai", "/openai/v1"))) + + + # Use the new query_params dictionary + final_url = httpx.URL(new_url).copy_with(params=query_params) + + return str(final_url) + + @staticmethod + def _is_azure_v1_api_version(api_version: Optional[str]) -> bool: + if api_version is None: + return False + return api_version == "preview" or api_version == "latest" diff --git a/litellm/llms/azure/responses/transformation.py b/litellm/llms/azure/responses/transformation.py index da80aa3735e..e6f48179e49 100644 --- a/litellm/llms/azure/responses/transformation.py +++ b/litellm/llms/azure/responses/transformation.py @@ -1,15 +1,11 @@ -from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, cast +from typing import TYPE_CHECKING, Any, Dict, Literal, Optional, Tuple -import httpx - -import litellm from litellm._logging import verbose_logger +from litellm.llms.azure.common_utils import BaseAzureLLM from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig -from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import * from litellm.types.responses.main import * from litellm.types.router import GenericLiteLLMParams -from litellm.utils import _add_path_to_api_base if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj @@ -21,26 +17,13 @@ else: class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): def validate_environment( - self, - headers: dict, - model: str, - api_key: Optional[str] = None, + self, headers: dict, model: str, litellm_params: Optional[GenericLiteLLMParams] ) -> dict: - api_key = ( - api_key - or litellm.api_key - or litellm.azure_key - or get_secret_str("AZURE_OPENAI_API_KEY") - or get_secret_str("AZURE_API_KEY") + return BaseAzureLLM._base_validate_azure_environment( + headers=headers, + litellm_params=litellm_params ) - headers.update( - { - "Authorization": f"Bearer {api_key}", - } - ) - return headers - def get_complete_url( self, api_base: Optional[str], @@ -62,47 +45,12 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): - A complete URL string, e.g., "https://litellm8397336933.openai.azure.com/openai/responses?api-version=2024-05-01-preview" """ - api_base = api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") - if api_base is None: - raise ValueError( - f"api_base is required for Azure AI Studio. Please set the api_base parameter. Passed `api_base={api_base}`" - ) - original_url = httpx.URL(api_base) - - # Extract api_version or use default - api_version = cast(Optional[str], litellm_params.get("api_version")) - - # Create a new dictionary with existing params - query_params = dict(original_url.params) - - # Add api_version if needed - if "api-version" not in query_params and api_version: - query_params["api-version"] = api_version - - # Add the path to the base URL - if "/openai/responses" not in api_base: - new_url = _add_path_to_api_base( - api_base=api_base, ending_path="/openai/responses" - ) - else: - new_url = api_base - - if self._is_azure_v1_api_version(api_version): - # ensure the request go to /openai/v1 and not just /openai - if "/openai/v1" not in new_url: - parsed_url = httpx.URL(new_url) - new_url = str(parsed_url.copy_with(path=parsed_url.path.replace("/openai", "/openai/v1"))) - - - # Use the new query_params dictionary - final_url = httpx.URL(new_url).copy_with(params=query_params) - - return str(final_url) + return BaseAzureLLM._get_base_azure_url( + api_base=api_base, + litellm_params=litellm_params, + route="/openai/responses" + ) - def _is_azure_v1_api_version(self, api_version: Optional[str]) -> bool: - if api_version is None: - return False - return api_version == "preview" or api_version == "latest" ######################################################### ########## DELETE RESPONSE API TRANSFORMATION ############## diff --git a/litellm/llms/azure/vector_stores/transformation.py b/litellm/llms/azure/vector_stores/transformation.py new file mode 100644 index 00000000000..f1cd81b2bf2 --- /dev/null +++ b/litellm/llms/azure/vector_stores/transformation.py @@ -0,0 +1,27 @@ +from typing import Optional + +from litellm.llms.azure.common_utils import BaseAzureLLM +from litellm.llms.openai.vector_stores.transformation import OpenAIVectorStoreConfig +from litellm.types.router import GenericLiteLLMParams + + +class AzureOpenAIVectorStoreConfig(OpenAIVectorStoreConfig): + def get_complete_url( + self, + api_base: Optional[str], + litellm_params: dict, + ) -> str: + return BaseAzureLLM._get_base_azure_url( + api_base=api_base, + litellm_params=litellm_params, + route="/openai/vector_stores" + ) + + + def validate_environment( + self, headers: dict, litellm_params: Optional[GenericLiteLLMParams] + ) -> dict: + return BaseAzureLLM._base_validate_azure_environment( + headers=headers, + litellm_params=litellm_params + ) \ No newline at end of file diff --git a/litellm/llms/base_llm/responses/transformation.py b/litellm/llms/base_llm/responses/transformation.py index b2a555086d8..e2f89da5e86 100644 --- a/litellm/llms/base_llm/responses/transformation.py +++ b/litellm/llms/base_llm/responses/transformation.py @@ -63,10 +63,7 @@ class BaseResponsesAPIConfig(ABC): @abstractmethod def validate_environment( - self, - headers: dict, - model: str, - api_key: Optional[str] = None, + self, headers: dict, model: str, litellm_params: Optional[GenericLiteLLMParams] ) -> dict: return {} diff --git a/litellm/llms/base_llm/vector_store/transformation.py b/litellm/llms/base_llm/vector_store/transformation.py new file mode 100644 index 00000000000..6ec9d25ae59 --- /dev/null +++ b/litellm/llms/base_llm/vector_store/transformation.py @@ -0,0 +1,86 @@ +from abc import abstractmethod +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union + +import httpx + +from litellm.types.router import GenericLiteLLMParams +from litellm.types.vector_stores import ( + VectorStoreCreateOptionalRequestParams, + VectorStoreCreateResponse, + VectorStoreSearchOptionalRequestParams, + VectorStoreSearchResponse, +) + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + from ..chat.transformation import BaseLLMException as _BaseLLMException + + LiteLLMLoggingObj = _LiteLLMLoggingObj + BaseLLMException = _BaseLLMException +else: + LiteLLMLoggingObj = Any + BaseLLMException = Any + +class BaseVectorStoreConfig: + @abstractmethod + def transform_search_vector_store_request( + self, + vector_store_id: str, + query: Union[str, List[str]], + vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams, + api_base: str, + ) -> Tuple[str, Dict]: + pass + + @abstractmethod + def transform_search_vector_store_response(self, response: httpx.Response) -> VectorStoreSearchResponse: + pass + + @abstractmethod + def transform_create_vector_store_request( + self, + vector_store_create_optional_params: VectorStoreCreateOptionalRequestParams, + api_base: str, + ) -> Tuple[str, Dict]: + pass + + @abstractmethod + def transform_create_vector_store_response(self, response: httpx.Response) -> VectorStoreCreateResponse: + pass + + @abstractmethod + def validate_environment( + self, headers: dict, litellm_params: Optional[GenericLiteLLMParams] + ) -> dict: + return {} + + @abstractmethod + def get_complete_url( + self, + api_base: Optional[str], + litellm_params: dict, + ) -> str: + """ + OPTIONAL + + Get the complete url for the request + + Some providers need `model` in `api_base` + """ + if api_base is None: + raise ValueError("api_base is required") + return api_base + + + def get_error_class( + self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] + ) -> BaseLLMException: + from ..chat.transformation import BaseLLMException + + raise BaseLLMException( + status_code=status_code, + message=error_message, + headers=headers, + ) + diff --git a/litellm/llms/custom_httpx/aiohttp_transport.py b/litellm/llms/custom_httpx/aiohttp_transport.py index 279cf2e9f45..3ed7d04bde6 100644 --- a/litellm/llms/custom_httpx/aiohttp_transport.py +++ b/litellm/llms/custom_httpx/aiohttp_transport.py @@ -1,6 +1,8 @@ import asyncio import contextlib +import os import typing +import urllib.request from typing import Callable, Dict, Union import aiohttp @@ -9,7 +11,9 @@ import aiohttp.http_exceptions import httpx from aiohttp.client import ClientResponse, ClientSession +import litellm from litellm._logging import verbose_logger +from litellm.secret_managers.main import str_to_bool AIOHTTP_EXC_MAP: Dict = { # Order matters here, most specific exception first @@ -182,7 +186,7 @@ class LiteLLMAiohttpTransport(AiohttpTransport): self.client = ClientSession() return self.client - + async def handle_async_request( self, request: httpx.Request, @@ -196,6 +200,9 @@ class LiteLLMAiohttpTransport(AiohttpTransport): # Use helper to ensure we have a valid session for the current event loop client_session = self._get_valid_client_session() + # Resolve proxy settings from environment variables + proxy = await self._get_proxy_settings(request) + with map_aiohttp_exceptions(): try: data = request.content @@ -215,6 +222,7 @@ class LiteLLMAiohttpTransport(AiohttpTransport): sock_read=timeout.get("read"), connect=timeout.get("pool"), ), + proxy=proxy, server_hostname=sni_hostname, ).__aenter__() @@ -224,3 +232,29 @@ class LiteLLMAiohttpTransport(AiohttpTransport): content=AiohttpResponseStream(response), request=request, ) + + + async def _get_proxy_settings(self, request: httpx.Request): + proxy = None + if not ( + litellm.disable_aiohttp_trust_env + or str_to_bool(os.getenv("DISABLE_AIOHTTP_TRUST_ENV", "False")) + ): + try: + proxy = self._proxy_from_env(request.url) + except Exception as e: # pragma: no cover - best effort + verbose_logger.debug(f"Error reading proxy env: {e}") + + return proxy + + + def _proxy_from_env(self, url: httpx.URL) -> typing.Optional[str]: + """Return proxy URL from env for the given request URL.""" + proxies = urllib.request.getproxies() + if urllib.request.proxy_bypass(url.host): + return None + + proxy = proxies.get(url.scheme) or proxies.get("all") + if proxy and "://" not in proxy: + proxy = f"http://{proxy}" + return proxy diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 3a5490eee49..92f9d6d958b 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -35,6 +35,7 @@ from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig +from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, HTTPHandler, @@ -60,6 +61,12 @@ from litellm.types.rerank import OptionalRerankParams, RerankResponse from litellm.types.responses.main import DeleteResponseResult from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import EmbeddingResponse, FileTypes, TranscriptionResponse +from litellm.types.vector_stores import ( + VectorStoreCreateOptionalRequestParams, + VectorStoreCreateResponse, + VectorStoreSearchOptionalRequestParams, + VectorStoreSearchResponse, +) from litellm.utils import ( CustomStreamWrapper, ImageResponse, @@ -1108,18 +1115,21 @@ class BaseLLMHTTPHandler: ) # Prepare the request - headers, complete_url, binary_data, json_data = ( - self._prepare_audio_transcription_request( - model=model, - audio_file=audio_file, - optional_params=optional_params, - litellm_params=litellm_params, - logging_obj=logging_obj, - api_key=api_key, - api_base=api_base, - headers=headers, - provider_config=provider_config, - ) + ( + headers, + complete_url, + binary_data, + json_data, + ) = self._prepare_audio_transcription_request( + model=model, + audio_file=audio_file, + optional_params=optional_params, + litellm_params=litellm_params, + logging_obj=logging_obj, + api_key=api_key, + api_base=api_base, + headers=headers, + provider_config=provider_config, ) if client is None or not isinstance(client, HTTPHandler): @@ -1170,18 +1180,21 @@ class BaseLLMHTTPHandler: ) # Prepare the request - headers, complete_url, binary_data, json_data = ( - self._prepare_audio_transcription_request( - model=model, - audio_file=audio_file, - optional_params=optional_params, - litellm_params=litellm_params, - logging_obj=logging_obj, - api_key=api_key, - api_base=api_base, - headers=headers, - provider_config=provider_config, - ) + ( + headers, + complete_url, + binary_data, + json_data, + ) = self._prepare_audio_transcription_request( + model=model, + audio_file=audio_file, + optional_params=optional_params, + litellm_params=litellm_params, + logging_obj=logging_obj, + api_key=api_key, + api_base=api_base, + headers=headers, + provider_config=provider_config, ) if client is None or not isinstance(client, AsyncHTTPHandler): @@ -1437,9 +1450,9 @@ class BaseLLMHTTPHandler: sync_httpx_client = client headers = responses_api_provider_config.validate_environment( - api_key=litellm_params.api_key, headers=response_api_optional_request_params.get("extra_headers", {}) or {}, model=model, + litellm_params=litellm_params, ) if extra_headers: @@ -1557,9 +1570,9 @@ class BaseLLMHTTPHandler: async_httpx_client = client headers = responses_api_provider_config.validate_environment( - api_key=litellm_params.api_key, headers=response_api_optional_request_params.get("extra_headers", {}) or {}, model=model, + litellm_params=litellm_params, ) if extra_headers: @@ -1678,9 +1691,7 @@ class BaseLLMHTTPHandler: async_httpx_client = client headers = responses_api_provider_config.validate_environment( - api_key=litellm_params.api_key, - headers=extra_headers or {}, - model="None", + headers=extra_headers or {}, model="None", litellm_params=litellm_params ) if extra_headers: @@ -1762,9 +1773,7 @@ class BaseLLMHTTPHandler: sync_httpx_client = client headers = responses_api_provider_config.validate_environment( - api_key=litellm_params.api_key, - headers=extra_headers or {}, - model="None", + headers=extra_headers or {}, model="None", litellm_params=litellm_params ) if extra_headers: @@ -1847,9 +1856,7 @@ class BaseLLMHTTPHandler: sync_httpx_client = client headers = responses_api_provider_config.validate_environment( - api_key=litellm_params.api_key, - headers=extra_headers or {}, - model="None", + headers=extra_headers or {}, model="None", litellm_params=litellm_params ) if extra_headers: @@ -1915,9 +1922,7 @@ class BaseLLMHTTPHandler: async_httpx_client = client headers = responses_api_provider_config.validate_environment( - api_key=litellm_params.api_key, - headers=extra_headers or {}, - model="None", + headers=extra_headers or {}, model="None", litellm_params=litellm_params ) if extra_headers: @@ -2008,9 +2013,7 @@ class BaseLLMHTTPHandler: sync_httpx_client = client headers = responses_api_provider_config.validate_environment( - api_key=litellm_params.api_key, - headers=extra_headers or {}, - model="None", + headers=extra_headers or {}, model="None", litellm_params=litellm_params ) if extra_headers: @@ -2078,9 +2081,7 @@ class BaseLLMHTTPHandler: async_httpx_client = client headers = responses_api_provider_config.validate_environment( - api_key=litellm_params.api_key, - headers=extra_headers or {}, - model="None", + headers=extra_headers or {}, model="None", litellm_params=litellm_params ) if extra_headers: @@ -2348,7 +2349,7 @@ class BaseLLMHTTPHandler: self, e: Exception, provider_config: Union[ - BaseConfig, BaseRerankConfig, BaseResponsesAPIConfig, BaseImageEditConfig + BaseConfig, BaseRerankConfig, BaseResponsesAPIConfig, BaseImageEditConfig, BaseVectorStoreConfig ], ): status_code = getattr(e, "status_code", 500) @@ -2447,10 +2448,7 @@ class BaseLLMHTTPHandler: _is_async: bool = False, fake_stream: bool = False, litellm_metadata: Optional[Dict[str, Any]] = None, - ) -> Union[ - ImageResponse, - Coroutine[Any, Any, ImageResponse], - ]: + ) -> Union[ImageResponse, Coroutine[Any, Any, ImageResponse],]: """ Handles image edit requests. @@ -2622,3 +2620,271 @@ class BaseLLMHTTPHandler: raw_response=response, logging_obj=logging_obj, ) + + ###### VECTOR STORE HANDLER ###### + async def async_vector_store_search_handler( + self, + vector_store_id: str, + query: Union[str, List[str]], + vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams, + vector_store_provider_config: BaseVectorStoreConfig, + custom_llm_provider: str, + litellm_params: GenericLiteLLMParams, + logging_obj: LiteLLMLoggingObj, + extra_headers: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, + client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + _is_async: bool = False, + ) -> VectorStoreSearchResponse: + if client is None or not isinstance(client, AsyncHTTPHandler): + async_httpx_client = get_async_httpx_client( + llm_provider=litellm.LlmProviders(custom_llm_provider), + params={"ssl_verify": litellm_params.get("ssl_verify", None)}, + ) + else: + async_httpx_client = client + + headers = vector_store_provider_config.validate_environment( + headers=extra_headers or {}, + litellm_params=litellm_params + ) + + if extra_headers: + headers.update(extra_headers) + + api_base = vector_store_provider_config.get_complete_url( + api_base=litellm_params.api_base, + litellm_params=dict(litellm_params), + ) + + url, request_body = vector_store_provider_config.transform_search_vector_store_request( + vector_store_id=vector_store_id, + query=query, + vector_store_search_optional_params=vector_store_search_optional_params, + api_base=api_base, + ) + + logging_obj.pre_call( + input="", + api_key="", + additional_args={ + "complete_input_dict": request_body, + "api_base": api_base, + "headers": headers, + }, + ) + + try: + response = await async_httpx_client.post(url=url, headers=headers, json=request_body, timeout=timeout) + except Exception as e: + raise self._handle_error(e=e, provider_config=vector_store_provider_config) + + return vector_store_provider_config.transform_search_vector_store_response( + response=response, + ) + + def vector_store_search_handler( + self, + vector_store_id: str, + query: Union[str, List[str]], + vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams, + vector_store_provider_config: BaseVectorStoreConfig, + custom_llm_provider: str, + litellm_params: GenericLiteLLMParams, + logging_obj: LiteLLMLoggingObj, + extra_headers: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, + client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + _is_async: bool = False, + ) -> Union[VectorStoreSearchResponse, Coroutine[Any, Any, VectorStoreSearchResponse]]: + if _is_async: + return self.async_vector_store_search_handler( + vector_store_id=vector_store_id, + query=query, + vector_store_search_optional_params=vector_store_search_optional_params, + vector_store_provider_config=vector_store_provider_config, + litellm_params=litellm_params, + logging_obj=logging_obj, + custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, + extra_body=extra_body, + timeout=timeout, + client=client, + ) + + if client is None or not isinstance(client, HTTPHandler): + sync_httpx_client = _get_httpx_client( + params={"ssl_verify": litellm_params.get("ssl_verify", None)} + ) + else: + sync_httpx_client = client + + headers = vector_store_provider_config.validate_environment( + headers=extra_headers or {}, + litellm_params=litellm_params + ) + + if extra_headers: + headers.update(extra_headers) + + api_base = vector_store_provider_config.get_complete_url( + api_base=litellm_params.api_base, + litellm_params=dict(litellm_params), + ) + + url, request_body = vector_store_provider_config.transform_search_vector_store_request( + vector_store_id=vector_store_id, + query=query, + vector_store_search_optional_params=vector_store_search_optional_params, + api_base=api_base, + ) + + logging_obj.pre_call( + input="", + api_key="", + additional_args={ + "complete_input_dict": request_body, + "api_base": api_base, + "headers": headers, + }, + ) + + try: + response = sync_httpx_client.post(url=url, headers=headers, json=request_body) + except Exception as e: + raise self._handle_error(e=e, provider_config=vector_store_provider_config) + + return vector_store_provider_config.transform_search_vector_store_response( + response=response, + ) + + async def async_vector_store_create_handler( + self, + vector_store_create_optional_params: VectorStoreCreateOptionalRequestParams, + vector_store_provider_config: BaseVectorStoreConfig, + custom_llm_provider: str, + litellm_params: GenericLiteLLMParams, + logging_obj: LiteLLMLoggingObj, + extra_headers: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, + client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + _is_async: bool = False, + ) -> VectorStoreCreateResponse: + if client is None or not isinstance(client, AsyncHTTPHandler): + async_httpx_client = get_async_httpx_client( + llm_provider=litellm.LlmProviders(custom_llm_provider), + params={"ssl_verify": litellm_params.get("ssl_verify", None)}, + ) + else: + async_httpx_client = client + + headers = vector_store_provider_config.validate_environment( + headers=extra_headers or {}, + litellm_params=litellm_params + ) + + if extra_headers: + headers.update(extra_headers) + + api_base = vector_store_provider_config.get_complete_url( + api_base=litellm_params.api_base, + litellm_params=dict(litellm_params), + ) + + url, request_body = vector_store_provider_config.transform_create_vector_store_request( + vector_store_create_optional_params=vector_store_create_optional_params, + api_base=api_base, + ) + + logging_obj.pre_call( + input="", + api_key="", + additional_args={ + "complete_input_dict": request_body, + "api_base": api_base, + "headers": headers, + }, + ) + + try: + response = await async_httpx_client.post(url=url, headers=headers, json=request_body, timeout=timeout) + except Exception as e: + raise self._handle_error(e=e, provider_config=vector_store_provider_config) + + return vector_store_provider_config.transform_create_vector_store_response( + response=response, + ) + + def vector_store_create_handler( + self, + vector_store_create_optional_params: VectorStoreCreateOptionalRequestParams, + vector_store_provider_config: BaseVectorStoreConfig, + custom_llm_provider: str, + litellm_params: GenericLiteLLMParams, + logging_obj: LiteLLMLoggingObj, + extra_headers: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, + client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + _is_async: bool = False, + ) -> Union[VectorStoreCreateResponse, Coroutine[Any, Any, VectorStoreCreateResponse]]: + if _is_async: + return self.async_vector_store_create_handler( + vector_store_create_optional_params=vector_store_create_optional_params, + vector_store_provider_config=vector_store_provider_config, + litellm_params=litellm_params, + logging_obj=logging_obj, + custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, + extra_body=extra_body, + timeout=timeout, + client=client, + ) + + if client is None or not isinstance(client, HTTPHandler): + sync_httpx_client = _get_httpx_client( + params={"ssl_verify": litellm_params.get("ssl_verify", None)} + ) + else: + sync_httpx_client = client + + headers = vector_store_provider_config.validate_environment( + headers=extra_headers or {}, + litellm_params=litellm_params + ) + + if extra_headers: + headers.update(extra_headers) + + api_base = vector_store_provider_config.get_complete_url( + api_base=litellm_params.api_base, + litellm_params=dict(litellm_params), + ) + + url, request_body = vector_store_provider_config.transform_create_vector_store_request( + vector_store_create_optional_params=vector_store_create_optional_params, + api_base=api_base, + ) + + logging_obj.pre_call( + input="", + api_key="", + additional_args={ + "complete_input_dict": request_body, + "api_base": api_base, + "headers": headers, + }, + ) + + try: + response = sync_httpx_client.post(url=url, headers=headers, json=request_body) + except Exception as e: + raise self._handle_error(e=e, provider_config=vector_store_provider_config) + + return vector_store_provider_config.transform_create_vector_store_response( + response=response, + ) + diff --git a/litellm/llms/mistral/mistral_chat_transformation.py b/litellm/llms/mistral/mistral_chat_transformation.py index 871fd8c7ea1..e281e055537 100644 --- a/litellm/llms/mistral/mistral_chat_transformation.py +++ b/litellm/llms/mistral/mistral_chat_transformation.py @@ -6,7 +6,7 @@ Why separate file? Make it easy to see how transformation works Docs - https://docs.mistral.ai/api/ """ -from typing import Any, Coroutine, List, Literal, Optional, Tuple, Union, overload, cast +from typing import Any, Coroutine, List, Literal, Optional, Tuple, Union, cast, overload from litellm.litellm_core_utils.prompt_templates.common_utils import ( handle_messages_with_content_list_to_str_conversion, @@ -107,15 +107,28 @@ class MistralConfig(OpenAIGPTConfig): def _get_mistral_reasoning_system_prompt() -> str: """ Returns the system prompt for Mistral reasoning models. - Based on Mistral's documentation: https://docs.mistral.ai/capabilities/reasoning/ + Based on Mistral's documentation: https://huggingface.co/mistralai/Magistral-Small-2506 + + Mistral recommends the following system prompt for reasoning: """ - return """When solving problems, think step-by-step in tags before providing your final answer. Use the following format: + return """ + [SYSTEM_PROMPT]system_prompt + A user will ask you to solve a task. You should first draft your thinking process (inner monologue) until you have derived the final answer. Afterwards, write a self-contained summary of your thoughts (i.e. your summary should be succinct but contain all the critical steps you needed to reach the conclusion). You should use Markdown to format your response. Write both your thoughts and summary in the same language as the task posed by the user. NEVER use \boxed{} in your response. - -Your step-by-step reasoning process. Be thorough and work through the problem carefully. - + Your thinking process must follow the template below: + + Your thoughts or/and draft, like working through an exercise on scratch paper. Be as casual and as long as you want until you are confident to generate a correct answer. + -Then provide a clear, concise answer based on your reasoning.""" + Here, provide a concise summary that reflects your reasoning and presents a clear final answer to the user. Don't mention that this is a summary. + + Problem: + + [/SYSTEM_PROMPT][INST]user_message[/INST] + reasoning_traces + + assistant_response[INST]user_message[/INST] + """ def map_openai_params( self, diff --git a/litellm/llms/nvidia_nim/chat/transformation.py b/litellm/llms/nvidia_nim/chat/transformation.py index 20478afb59f..e687229949b 100644 --- a/litellm/llms/nvidia_nim/chat/transformation.py +++ b/litellm/llms/nvidia_nim/chat/transformation.py @@ -91,6 +91,7 @@ class NvidiaNimConfig(OpenAIGPTConfig): "tools", "tool_choice", "parallel_tool_calls", + "response_format", ] def map_openai_params( diff --git a/litellm/llms/ollama/completion/handler.py b/litellm/llms/ollama/completion/handler.py index 9f507aded0f..9e6497e66ab 100644 --- a/litellm/llms/ollama/completion/handler.py +++ b/litellm/llms/ollama/completion/handler.py @@ -5,15 +5,15 @@ Ollama /chat/completion calls handled in llm_http_handler.py """ from typing import Any, Dict, List + import litellm from litellm.types.utils import EmbeddingResponse + def _prepare_ollama_embedding_payload( - model: str, - prompts: List[str], - optional_params: Dict[str, Any] + model: str, prompts: List[str], optional_params: Dict[str, Any] ) -> Dict[str, Any]: - + data: Dict[str, Any] = {"model": model, "input": prompts} special_optional_params = ["truncate", "options", "keep_alive"] @@ -26,13 +26,14 @@ def _prepare_ollama_embedding_payload( data["options"].update({k: v}) return data + def _process_ollama_embedding_response( response_json: dict, prompts: List[str], model: str, model_response: EmbeddingResponse, logging_obj: Any, - encoding: Any + encoding: Any, ) -> EmbeddingResponse: output_data = [] embeddings: List[List[float]] = response_json["embeddings"] @@ -46,11 +47,15 @@ def _process_ollama_embedding_response( if encoding is not None: input_tokens = len(encoding.encode("".join(prompts))) if logging_obj: - logging_obj.debug("Ollama response missing prompt_eval_count; estimated with encoding.") + logging_obj.debug( + "Ollama response missing prompt_eval_count; estimated with encoding." + ) else: input_tokens = 0 if logging_obj: - logging_obj.warning("Missing prompt_eval_count and no encoding provided; defaulted to 0.") + logging_obj.warning( + "Missing prompt_eval_count and no encoding provided; defaulted to 0." + ) model_response.object = "list" model_response.data = output_data @@ -64,6 +69,7 @@ def _process_ollama_embedding_response( ) return model_response + async def ollama_aembeddings( api_base: str, model: str, @@ -79,7 +85,7 @@ async def ollama_aembeddings( data = _prepare_ollama_embedding_payload(model, prompts, optional_params) response = await litellm.module_level_aclient.post(url=api_base, json=data) - response_json = await response.json() + response_json = response.json() return _process_ollama_embedding_response( response_json=response_json, @@ -87,9 +93,10 @@ async def ollama_aembeddings( model=model, model_response=model_response, logging_obj=logging_obj, - encoding=encoding + encoding=encoding, ) + def ollama_embeddings( api_base: str, model: str, @@ -113,5 +120,5 @@ def ollama_embeddings( model=model, model_response=model_response, logging_obj=logging_obj, - encoding=encoding + encoding=encoding, ) diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 871662eccaf..cf742bc52cb 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -92,13 +92,11 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): return ResponsesAPIResponse(**raw_response_json) def validate_environment( - self, - headers: dict, - model: str, - api_key: Optional[str] = None, + self, headers: dict, model: str, litellm_params: Optional[GenericLiteLLMParams] ) -> dict: + litellm_params = litellm_params or GenericLiteLLMParams() api_key = ( - api_key + litellm_params.api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY") diff --git a/litellm/llms/openai/vector_stores/transformation.py b/litellm/llms/openai/vector_stores/transformation.py new file mode 100644 index 00000000000..11d76937ab4 --- /dev/null +++ b/litellm/llms/openai/vector_stores/transformation.py @@ -0,0 +1,140 @@ +from typing import Dict, List, Optional, Tuple, Union, cast + +import httpx + +import litellm +from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig +from litellm.secret_managers.main import get_secret_str +from litellm.types.router import GenericLiteLLMParams +from litellm.types.vector_stores import ( + VectorStoreCreateOptionalRequestParams, + VectorStoreCreateRequest, + VectorStoreCreateResponse, + VectorStoreSearchOptionalRequestParams, + VectorStoreSearchRequest, + VectorStoreSearchResponse, +) + + +class OpenAIVectorStoreConfig(BaseVectorStoreConfig): + ASSISTANTS_HEADER_KEY = "OpenAI-Beta" + ASSISTANTS_HEADER_VALUE = "assistants=v2" + + def validate_environment( + self, headers: dict, litellm_params: Optional[GenericLiteLLMParams] + ) -> dict: + litellm_params = litellm_params or GenericLiteLLMParams() + api_key = ( + litellm_params.api_key + or litellm.api_key + or litellm.openai_key + or get_secret_str("OPENAI_API_KEY") + ) + headers.update( + { + "Authorization": f"Bearer {api_key}", + } + ) + + ######################################################### + # Ensure OpenAI Assistants header is includes + ######################################################### + if self.ASSISTANTS_HEADER_KEY not in headers: + headers.update( + { + self.ASSISTANTS_HEADER_KEY: self.ASSISTANTS_HEADER_VALUE, + } + ) + + return headers + + def get_complete_url( + self, + api_base: Optional[str], + litellm_params: dict, + ) -> str: + """ + Get the Base endpoint for OpenAI Vector Stores API + """ + api_base = ( + api_base + or litellm.api_base + or get_secret_str("OPENAI_BASE_URL") + or get_secret_str("OPENAI_API_BASE") + or "https://api.openai.com/v1" + ) + + # Remove trailing slashes + api_base = api_base.rstrip("/") + + return f"{api_base}/vector_stores" + + + def transform_search_vector_store_request( + self, + vector_store_id: str, + query: Union[str, List[str]], + vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams, + api_base: str, + ) -> Tuple[str, Dict]: + url = f"{api_base}/{vector_store_id}/search" + typed_request_body = VectorStoreSearchRequest( + query=query, + filters=vector_store_search_optional_params.get("filters", None), + max_num_results=vector_store_search_optional_params.get("max_num_results", None), + ranking_options=vector_store_search_optional_params.get("ranking_options", None), + rewrite_query=vector_store_search_optional_params.get("rewrite_query", None), + ) + + dict_request_body = cast(dict, typed_request_body) + return url, dict_request_body + + + + def transform_search_vector_store_response(self, response: httpx.Response) -> VectorStoreSearchResponse: + try: + response_json = response.json() + return VectorStoreSearchResponse( + **response_json + ) + except Exception as e: + raise self.get_error_class( + error_message=str(e), + status_code=response.status_code, + headers=response.headers + ) + + def transform_create_vector_store_request( + self, + vector_store_create_optional_params: VectorStoreCreateOptionalRequestParams, + api_base: str, + ) -> Tuple[str, Dict]: + url = api_base # Base URL for creating vector stores + typed_request_body = VectorStoreCreateRequest( + name=vector_store_create_optional_params.get("name", None), + file_ids=vector_store_create_optional_params.get("file_ids", None), + expires_after=vector_store_create_optional_params.get("expires_after", None), + chunking_strategy=vector_store_create_optional_params.get("chunking_strategy", None), + metadata=vector_store_create_optional_params.get("metadata", None), + ) + + dict_request_body = cast(dict, typed_request_body) + return url, dict_request_body + + def transform_create_vector_store_response(self, response: httpx.Response) -> VectorStoreCreateResponse: + try: + response_json = response.json() + return VectorStoreCreateResponse( + **response_json + ) + except Exception as e: + raise self.get_error_class( + error_message=str(e), + status_code=response.status_code, + headers=response.headers + ) + + + + + \ No newline at end of file diff --git a/litellm/llms/perplexity/chat/transformation.py b/litellm/llms/perplexity/chat/transformation.py index 4ce2df51b6e..955fdff0818 100644 --- a/litellm/llms/perplexity/chat/transformation.py +++ b/litellm/llms/perplexity/chat/transformation.py @@ -2,13 +2,17 @@ Translate from OpenAI's `/v1/chat/completions` to Perplexity's `/v1/chat/completions` """ -from typing import Optional, Tuple +from typing import Any, List, Optional, Tuple +import httpx import litellm from litellm._logging import verbose_logger from litellm.secret_managers.main import get_secret_str - -from ...openai.chat.gpt_transformation import OpenAIGPTConfig +from litellm.types.llms.openai import AllMessageValues +from litellm.types.utils import Usage, PromptTokensDetailsWrapper +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig +from litellm.types.utils import ModelResponse class PerplexityChatConfig(OpenAIGPTConfig): @@ -55,4 +59,105 @@ class PerplexityChatConfig(OpenAIGPTConfig): base_openai_params.append("reasoning_effort") except Exception as e: verbose_logger.debug(f"Error checking if model supports reasoning: {e}") + + try: + if litellm.supports_web_search( + model=model, custom_llm_provider=self.custom_llm_provider + ): + base_openai_params.append("web_search_options") + except Exception as e: + verbose_logger.debug(f"Error checking if model supports web search: {e}") + return base_openai_params + + def transform_response( + self, + model: str, + raw_response: httpx.Response, + model_response: ModelResponse, + logging_obj: LiteLLMLoggingObj, + request_data: dict, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + encoding: Any, + api_key: Optional[str] = None, + json_mode: Optional[bool] = None, + ) -> ModelResponse: + # Call the parent transform_response first to handle the standard transformation + model_response = super().transform_response( + model=model, + raw_response=raw_response, + model_response=model_response, + logging_obj=logging_obj, + request_data=request_data, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + encoding=encoding, + api_key=api_key, + json_mode=json_mode, + ) + + # Extract and enhance usage with Perplexity-specific fields + try: + raw_response_json = raw_response.json() + self._enhance_usage_with_perplexity_fields(model_response, raw_response_json) + except Exception as e: + verbose_logger.debug(f"Error extracting Perplexity-specific usage fields: {e}") + + return model_response + + def _enhance_usage_with_perplexity_fields( + self, model_response: ModelResponse, raw_response_json: dict + ) -> None: + """ + Extract citation tokens and search queries from Perplexity API response + and add them to the usage object using standard LiteLLM fields. + """ + if not hasattr(model_response, "usage") or model_response.usage is None: + # Create a usage object if it doesn't exist (when usage was None) + model_response.usage = Usage( # type: ignore[attr-defined] + prompt_tokens=0, + completion_tokens=0, + total_tokens=0 + ) + + usage = model_response.usage # type: ignore[attr-defined] + + # Extract citation tokens count + citations = raw_response_json.get("citations", []) + citation_tokens = 0 + if citations: + # Count total characters in citations as a proxy for citation tokens + # This is an estimation - in practice, you might want to use proper tokenization + total_citation_chars = sum(len(str(citation)) for citation in citations if citation) + # Rough estimation: ~4 characters per token (OpenAI's general rule) + if total_citation_chars > 0: + citation_tokens = max(1, total_citation_chars // 4) + + # Extract search queries count from usage or response metadata + # Perplexity might include this in the usage object or as separate metadata + perplexity_usage = raw_response_json.get("usage", {}) + + # Try to extract search queries from usage field first, then root level + num_search_queries = perplexity_usage.get("num_search_queries") + if num_search_queries is None: + num_search_queries = raw_response_json.get("num_search_queries") + if num_search_queries is None: + num_search_queries = perplexity_usage.get("search_queries") + if num_search_queries is None: + num_search_queries = raw_response_json.get("search_queries") + + # Create or update prompt_tokens_details to include web search requests and citation tokens + if citation_tokens > 0 or (num_search_queries is not None and num_search_queries > 0): + if usage.prompt_tokens_details is None: + usage.prompt_tokens_details = PromptTokensDetailsWrapper() + + # Store citation tokens count for cost calculation + if citation_tokens > 0: + setattr(usage, "citation_tokens", citation_tokens) + + # Store search queries count in the standard web_search_requests field + if num_search_queries is not None and num_search_queries > 0: + usage.prompt_tokens_details.web_search_requests = num_search_queries diff --git a/litellm/llms/perplexity/cost_calculator.py b/litellm/llms/perplexity/cost_calculator.py new file mode 100644 index 00000000000..c8fd2a682a8 --- /dev/null +++ b/litellm/llms/perplexity/cost_calculator.py @@ -0,0 +1,79 @@ +""" +Helper util for handling perplexity-specific cost calculation +- e.g.: citation tokens, search queries +""" + +from typing import Tuple, Union + +from litellm.types.utils import Usage +from litellm.utils import get_model_info + + +def cost_per_token(model: str, usage: Usage) -> Tuple[float, float]: + """ + Calculates the cost per token for a given model, prompt tokens, and completion tokens. + + Input: + - model: str, the model name without provider prefix + - usage: LiteLLM Usage block, containing perplexity-specific usage information + + Returns: + Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd + """ + ## GET MODEL INFO + model_info = get_model_info(model=model, custom_llm_provider="perplexity") + + def _safe_float_cast(value: Union[str, int, float, None, object], default: float = 0.0) -> float: + """Safely cast a value to float with proper type handling for mypy.""" + if value is None: + return default + try: + return float(value) # type: ignore + except (ValueError, TypeError): + return default + + ## CALCULATE INPUT COST + input_cost_per_token = _safe_float_cast(model_info.get("input_cost_per_token")) + prompt_cost: float = (usage.prompt_tokens or 0) * input_cost_per_token + + ## ADD CITATION TOKENS COST (if present) + citation_tokens = getattr(usage, "citation_tokens", 0) or 0 + citation_cost_value = model_info.get("citation_cost_per_token") + if citation_tokens > 0 and citation_cost_value is not None: + citation_cost_per_token = _safe_float_cast(citation_cost_value) + prompt_cost += citation_tokens * citation_cost_per_token + + ## CALCULATE OUTPUT COST + output_cost_per_token = _safe_float_cast(model_info.get("output_cost_per_token")) + completion_cost: float = (usage.completion_tokens or 0) * output_cost_per_token + + ## ADD REASONING TOKENS COST (if present) + reasoning_tokens = getattr(usage, "reasoning_tokens", 0) or 0 + # Also check completion_tokens_details if reasoning_tokens is not directly available + if reasoning_tokens == 0 and hasattr(usage, "completion_tokens_details") and usage.completion_tokens_details: + reasoning_tokens = getattr(usage.completion_tokens_details, "reasoning_tokens", 0) or 0 + + reasoning_cost_value = model_info.get("output_cost_per_reasoning_token") + if reasoning_tokens > 0 and reasoning_cost_value is not None: + reasoning_cost_per_token = _safe_float_cast(reasoning_cost_value) + completion_cost += reasoning_tokens * reasoning_cost_per_token + + ## ADD SEARCH QUERIES COST (if present) + num_search_queries = 0 + if hasattr(usage, "prompt_tokens_details") and usage.prompt_tokens_details: + num_search_queries = getattr(usage.prompt_tokens_details, "web_search_requests", 0) or 0 + + # Check both possible keys for search cost (legacy and current) + search_cost_value = model_info.get("search_queries_cost_per_query") or model_info.get("search_context_cost_per_query") + if num_search_queries > 0 and search_cost_value is not None: + # Handle both dict and float formats + if isinstance(search_cost_value, dict): + # Use the "low" size as default - tests expect 0.005 / 1000 + search_cost_per_query = _safe_float_cast(search_cost_value.get("search_context_size_low", 0)) / 1000 + else: + search_cost_per_query = _safe_float_cast(search_cost_value) + search_cost = num_search_queries * search_cost_per_query + # Add search cost to completion cost (similar to how other providers handle it) + completion_cost += search_cost + + return prompt_cost, completion_cost \ No newline at end of file diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py index 23facabbf89..cceac0ea794 100644 --- a/litellm/llms/vertex_ai/common_utils.py +++ b/litellm/llms/vertex_ai/common_utils.py @@ -204,6 +204,7 @@ def _build_vertex_schema(parameters: dict, add_property_ordering: bool = False): add_object_type(parameters) # Postprocessing # Filter out fields that don't exist in Schema + parameters = filter_schema_fields(parameters, valid_schema_fields) if add_property_ordering: @@ -318,6 +319,11 @@ def filter_schema_fields( k: filter_schema_fields(v, valid_fields, processed) for k, v in value.items() } + elif key == "format": + if value in {"enum", "date-time"}: + result[key] = value + else: + continue elif key == "items" and isinstance(value, dict): result[key] = filter_schema_fields(value, valid_fields, processed) elif key == "anyOf" and isinstance(value, list): diff --git a/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py b/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py index 5cfb9141a55..33a480aa6bb 100644 --- a/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py +++ b/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py @@ -205,6 +205,7 @@ class ContextCachingEndpoints(VertexBase): def check_and_create_cache( self, messages: List[AllMessageValues], # receives openai format messages + optional_params: dict, # cache the tools if present, in case cache content exists in messages api_key: str, api_base: Optional[str], model: str, @@ -213,7 +214,7 @@ class ContextCachingEndpoints(VertexBase): logging_obj: Logging, extra_headers: Optional[dict] = None, cached_content: Optional[str] = None, - ) -> Tuple[List[AllMessageValues], Optional[str]]: + ) -> Tuple[List[AllMessageValues], dict, Optional[str]]: """ Receives - messages: List of dict - messages in the openai format @@ -225,7 +226,16 @@ class ContextCachingEndpoints(VertexBase): Follows - https://ai.google.dev/api/caching#request-body """ if cached_content is not None: - return messages, cached_content + return messages, optional_params, cached_content + + cached_messages, non_cached_messages = separate_cached_messages( + messages=messages + ) + + if len(cached_messages) == 0: + return messages, optional_params, None + + tools = optional_params.pop("tools", None) ## AUTHORIZATION ## token, url = self._get_token_and_url_context_caching( @@ -252,15 +262,10 @@ class ContextCachingEndpoints(VertexBase): else: client = client - cached_messages, non_cached_messages = separate_cached_messages( - messages=messages - ) - - if len(cached_messages) == 0: - return messages, None - ## CHECK IF CACHED ALREADY - generated_cache_key = local_cache_obj.get_cache_key(messages=cached_messages) + generated_cache_key = local_cache_obj.get_cache_key( + messages=cached_messages, tools=tools + ) google_cache_name = self.check_cache( cache_key=generated_cache_key, client=client, @@ -270,7 +275,7 @@ class ContextCachingEndpoints(VertexBase): logging_obj=logging_obj, ) if google_cache_name: - return non_cached_messages, google_cache_name + return non_cached_messages, optional_params, google_cache_name ## TRANSFORM REQUEST cached_content_request_body = ( @@ -279,6 +284,8 @@ class ContextCachingEndpoints(VertexBase): ) ) + cached_content_request_body["tools"] = tools + ## LOGGING logging_obj.pre_call( input=messages, @@ -305,11 +312,16 @@ class ContextCachingEndpoints(VertexBase): cached_content_response_obj = VertexAICachedContentResponseObject( name=raw_response_cached.get("name"), model=raw_response_cached.get("model") ) - return (non_cached_messages, cached_content_response_obj["name"]) + return ( + non_cached_messages, + optional_params, + cached_content_response_obj["name"], + ) async def async_check_and_create_cache( self, messages: List[AllMessageValues], # receives openai format messages + optional_params: dict, # cache the tools if present, in case cache content exists in messages api_key: str, api_base: Optional[str], model: str, @@ -318,7 +330,7 @@ class ContextCachingEndpoints(VertexBase): logging_obj: Logging, extra_headers: Optional[dict] = None, cached_content: Optional[str] = None, - ) -> Tuple[List[AllMessageValues], Optional[str]]: + ) -> Tuple[List[AllMessageValues], dict, Optional[str]]: """ Receives - messages: List of dict - messages in the openai format @@ -330,14 +342,16 @@ class ContextCachingEndpoints(VertexBase): Follows - https://ai.google.dev/api/caching#request-body """ if cached_content is not None: - return messages, cached_content + return messages, optional_params, cached_content cached_messages, non_cached_messages = separate_cached_messages( messages=messages ) if len(cached_messages) == 0: - return messages, None + return messages, optional_params, None + + tools = optional_params.pop("tools", None) ## AUTHORIZATION ## token, url = self._get_token_and_url_context_caching( @@ -362,7 +376,9 @@ class ContextCachingEndpoints(VertexBase): client = client ## CHECK IF CACHED ALREADY - generated_cache_key = local_cache_obj.get_cache_key(messages=cached_messages) + generated_cache_key = local_cache_obj.get_cache_key( + messages=cached_messages, tools=tools + ) google_cache_name = await self.async_check_cache( cache_key=generated_cache_key, client=client, @@ -371,8 +387,9 @@ class ContextCachingEndpoints(VertexBase): api_base=api_base, logging_obj=logging_obj, ) + if google_cache_name: - return non_cached_messages, google_cache_name + return non_cached_messages, optional_params, google_cache_name ## TRANSFORM REQUEST cached_content_request_body = ( @@ -381,6 +398,8 @@ class ContextCachingEndpoints(VertexBase): ) ) + cached_content_request_body["tools"] = tools + ## LOGGING logging_obj.pre_call( input=messages, @@ -407,7 +426,11 @@ class ContextCachingEndpoints(VertexBase): cached_content_response_obj = VertexAICachedContentResponseObject( name=raw_response_cached.get("name"), model=raw_response_cached.get("model") ) - return (non_cached_messages, cached_content_response_obj["name"]) + return ( + non_cached_messages, + optional_params, + cached_content_response_obj["name"], + ) def get_cache(self): pass diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index 39edb9642e2..85e3f15364b 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -1,5 +1,5 @@ """ -Transformation logic from OpenAI format to Gemini format. +Transformation logic from OpenAI format to Gemini format. Why separate file? Make it easy to see how transformation works """ @@ -402,16 +402,19 @@ def sync_transform_request_body( context_caching_endpoints = ContextCachingEndpoints() if gemini_api_key is not None: - messages, cached_content = context_caching_endpoints.check_and_create_cache( - messages=messages, - api_key=gemini_api_key, - api_base=api_base, - model=model, - client=client, - timeout=timeout, - extra_headers=extra_headers, - cached_content=optional_params.pop("cached_content", None), - logging_obj=logging_obj, + messages, optional_params, cached_content = ( + context_caching_endpoints.check_and_create_cache( + messages=messages, + optional_params=optional_params, + api_key=gemini_api_key, + api_base=api_base, + model=model, + client=client, + timeout=timeout, + extra_headers=extra_headers, + cached_content=optional_params.pop("cached_content", None), + logging_obj=logging_obj, + ) ) else: # [TODO] implement context caching for gemini as well cached_content = optional_params.pop("cached_content", None) @@ -446,9 +449,11 @@ async def async_transform_request_body( if gemini_api_key is not None: ( messages, + optional_params, cached_content, ) = await context_caching_endpoints.async_check_and_create_cache( messages=messages, + optional_params=optional_params, api_key=gemini_api_key, api_base=api_base, model=model, diff --git a/litellm/llms/volcengine.py b/litellm/llms/volcengine.py index c569475b11c..58d2371af53 100644 --- a/litellm/llms/volcengine.py +++ b/litellm/llms/volcengine.py @@ -63,3 +63,26 @@ class VolcEngineConfig(OpenAILikeChatConfig): "extra_headers", "thinking", ] # works across all models + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + replace_max_completion_tokens_with_max_tokens: bool = True, + ) -> dict: + optional_params = super().map_openai_params( + non_default_params, + optional_params, + model, + drop_params, + replace_max_completion_tokens_with_max_tokens, + ) + + if "thinking" in optional_params: + optional_params.setdefault("extra_body", {})["thinking"] = ( + optional_params.pop("thinking") + ) + + return optional_params diff --git a/litellm/llms/watsonx/chat/handler.py b/litellm/llms/watsonx/chat/handler.py index 45378c55292..5c19757fecb 100644 --- a/litellm/llms/watsonx/chat/handler.py +++ b/litellm/llms/watsonx/chat/handler.py @@ -52,7 +52,7 @@ class WatsonXChatHandler(OpenAILikeChatHandler): litellm_params=litellm_params, ) - ## UPDATE PAYLOAD (optional params) + ## UPDATE PAYLOAD (optional params and special cases for models deployed in spaces) watsonx_auth_payload = watsonx_chat_transformation._prepare_payload( model=model, api_params=api_params, @@ -70,7 +70,7 @@ class WatsonXChatHandler(OpenAILikeChatHandler): ) return super().completion( - model=model, + model=watsonx_auth_payload.get("model_id", None), messages=messages, api_base=api_base, custom_llm_provider=custom_llm_provider, diff --git a/litellm/llms/watsonx/chat/transformation.py b/litellm/llms/watsonx/chat/transformation.py index 3c2d1c6f0bf..71d8bba4ef4 100644 --- a/litellm/llms/watsonx/chat/transformation.py +++ b/litellm/llms/watsonx/chat/transformation.py @@ -7,7 +7,7 @@ Docs: https://cloud.ibm.com/apidocs/watsonx-ai#text-chat from typing import List, Optional, Tuple, Union from litellm.secret_managers.main import get_secret_str -from litellm.types.llms.watsonx import WatsonXAIEndpoint +from litellm.types.llms.watsonx import WatsonXAIEndpoint, WatsonXAPIParams from ....utils import _remove_additional_properties, _remove_strict_from_schema from ...openai.chat.gpt_transformation import OpenAIGPTConfig @@ -108,3 +108,15 @@ class IBMWatsonXChatConfig(IBMWatsonXMixin, OpenAIGPTConfig): url=url, api_version=optional_params.pop("api_version", None) ) return url + + def _prepare_payload(self, model: str, api_params: WatsonXAPIParams) -> dict: + """ + Prepare payload for deployment models. + Deployment models cannot have 'model_id' or 'model' in the request body. + """ + payload: dict = {} + payload["model_id"] = None if model.startswith("deployment/") else model + payload["project_id"] = ( + None if model.startswith("deployment/") else api_params["project_id"] + ) + return payload diff --git a/litellm/main.py b/litellm/main.py index d27411ad3b8..2a2ed517ebe 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1297,6 +1297,12 @@ def completion( # type: ignore # noqa: PLR0915 except Exception as e: verbose_logger.debug("Error getting model info: {}".format(e)) model_info = {} + if model.startswith( + "responses/" + ): # handle azure models - `azure/responses/` + model = model.split("/")[1] + mode = "responses" + model_info["mode"] = mode if model_info.get("mode") == "responses": from litellm.completion_extras import responses_api_bridge diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 9d72f852de1..8d72806c32b 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -2156,6 +2156,66 @@ "supports_tool_choice": true, "supports_native_streaming": true }, + "azure/o3-pro": { + "max_tokens": 100000, + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "input_cost_per_token": 2e-05, + "output_cost_per_token": 8e-05, + "input_cost_per_token_batches": 1e-05, + "output_cost_per_token_batches": 4e-05, + "litellm_provider": "azure", + "mode": "chat", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": false, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, + "azure/o3-pro-2025-06-10": { + "max_tokens": 100000, + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "input_cost_per_token": 2e-05, + "output_cost_per_token": 8e-05, + "input_cost_per_token_batches": 1e-05, + "output_cost_per_token_batches": 4e-05, + "litellm_provider": "azure", + "mode": "chat", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": false, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, "azure/o3": { "max_tokens": 100000, "max_input_tokens": 200000, @@ -3986,7 +4046,8 @@ "litellm_provider": "mistral", "mode": "chat", "supports_assistant_prefill": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_response_schema": true }, "mistral/mistral-small": { "max_tokens": 8191, @@ -3998,7 +4059,8 @@ "supports_function_calling": true, "mode": "chat", "supports_assistant_prefill": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_response_schema": true }, "mistral/mistral-small-latest": { "max_tokens": 8191, @@ -4010,7 +4072,8 @@ "supports_function_calling": true, "mode": "chat", "supports_assistant_prefill": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_response_schema": true }, "mistral/mistral-medium": { "max_tokens": 8191, @@ -4021,7 +4084,8 @@ "litellm_provider": "mistral", "mode": "chat", "supports_assistant_prefill": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_response_schema": true }, "mistral/mistral-medium-latest": { "max_tokens": 8191, @@ -4033,7 +4097,8 @@ "mode": "chat", "supports_function_calling": true, "supports_assistant_prefill": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_response_schema": true }, "mistral/mistral-medium-2505": { "max_tokens": 8191, @@ -4045,7 +4110,8 @@ "mode": "chat", "supports_function_calling": true, "supports_assistant_prefill": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_response_schema": true }, "mistral/mistral-medium-2312": { "max_tokens": 8191, @@ -4056,7 +4122,8 @@ "litellm_provider": "mistral", "mode": "chat", "supports_assistant_prefill": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_response_schema": true }, "mistral/mistral-large-latest": { "max_tokens": 128000, @@ -4068,7 +4135,8 @@ "mode": "chat", "supports_function_calling": true, "supports_assistant_prefill": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_response_schema": true }, "mistral/mistral-large-2411": { "max_tokens": 128000, @@ -4080,7 +4148,8 @@ "mode": "chat", "supports_function_calling": true, "supports_assistant_prefill": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_response_schema": true }, "mistral/mistral-large-2402": { "max_tokens": 8191, @@ -4092,7 +4161,8 @@ "mode": "chat", "supports_function_calling": true, "supports_assistant_prefill": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_response_schema": true }, "mistral/mistral-large-2407": { "max_tokens": 128000, @@ -4104,7 +4174,8 @@ "mode": "chat", "supports_function_calling": true, "supports_assistant_prefill": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_response_schema": true }, "mistral/pixtral-large-latest": { "max_tokens": 128000, @@ -4117,7 +4188,8 @@ "supports_function_calling": true, "supports_assistant_prefill": true, "supports_vision": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_response_schema": true }, "mistral/pixtral-large-2411": { "max_tokens": 128000, @@ -4130,7 +4202,8 @@ "supports_function_calling": true, "supports_assistant_prefill": true, "supports_vision": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_response_schema": true }, "mistral/pixtral-12b-2409": { "max_tokens": 128000, @@ -4143,7 +4216,8 @@ "supports_function_calling": true, "supports_assistant_prefill": true, "supports_vision": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_response_schema": true }, "mistral/open-mistral-7b": { "max_tokens": 8191, @@ -4154,7 +4228,8 @@ "litellm_provider": "mistral", "mode": "chat", "supports_assistant_prefill": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_response_schema": true }, "mistral/open-mixtral-8x7b": { "max_tokens": 8191, @@ -4166,7 +4241,8 @@ "mode": "chat", "supports_function_calling": true, "supports_assistant_prefill": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_response_schema": true }, "mistral/open-mixtral-8x22b": { "max_tokens": 8191, @@ -4178,7 +4254,8 @@ "mode": "chat", "supports_function_calling": true, "supports_assistant_prefill": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_response_schema": true }, "mistral/codestral-latest": { "max_tokens": 8191, @@ -4189,7 +4266,8 @@ "litellm_provider": "mistral", "mode": "chat", "supports_assistant_prefill": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_response_schema": true }, "mistral/codestral-2405": { "max_tokens": 8191, @@ -4200,7 +4278,8 @@ "litellm_provider": "mistral", "mode": "chat", "supports_assistant_prefill": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_response_schema": true }, "mistral/open-mistral-nemo": { "max_tokens": 128000, @@ -4212,7 +4291,8 @@ "mode": "chat", "source": "https://mistral.ai/technology/", "supports_assistant_prefill": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_response_schema": true }, "mistral/open-mistral-nemo-2407": { "max_tokens": 128000, @@ -4224,7 +4304,8 @@ "mode": "chat", "source": "https://mistral.ai/technology/", "supports_assistant_prefill": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_response_schema": true }, "mistral/open-codestral-mamba": { "max_tokens": 256000, @@ -4261,7 +4342,8 @@ "source": "https://mistral.ai/news/devstral", "supports_function_calling": true, "supports_assistant_prefill": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_response_schema": true }, "mistral/magistral-medium-latest": { "max_tokens": 40000, @@ -4275,7 +4357,8 @@ "supports_function_calling": true, "supports_assistant_prefill": true, "supports_tool_choice": true, - "supports_reasoning": true + "supports_reasoning": true, + "supports_response_schema": true }, "mistral/magistral-medium-2506": { "max_tokens": 40000, @@ -4289,7 +4372,8 @@ "supports_function_calling": true, "supports_assistant_prefill": true, "supports_tool_choice": true, - "supports_reasoning": true + "supports_reasoning": true, + "supports_response_schema": true }, "mistral/magistral-small-latest": { "max_tokens": 40000, @@ -4303,7 +4387,8 @@ "supports_function_calling": true, "supports_assistant_prefill": true, "supports_tool_choice": true, - "supports_reasoning": true + "supports_reasoning": true, + "supports_response_schema": true }, "mistral/magistral-small-2506": { "max_tokens": 40000, @@ -4317,7 +4402,8 @@ "supports_function_calling": true, "supports_assistant_prefill": true, "supports_tool_choice": true, - "supports_reasoning": true + "supports_reasoning": true, + "supports_response_schema": true }, "mistral/mistral-embed": { "max_tokens": 8192, @@ -9995,7 +10081,15 @@ "mode": "chat", "supports_tool_choice": true }, - "mistralai/mistral-small-3.1-24b-instruct": { + "openrouter/mistralai/mistral-small-3.1-24b-instruct": { + "max_tokens": 32000, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "supports_tool_choice": true + }, + "openrouter/mistralai/mistral-small-3.2-24b-instruct": { "max_tokens": 32000, "input_cost_per_token": 1e-07, "output_cost_per_token": 3e-07, @@ -13647,13 +13741,14 @@ "input_cost_per_token": 2e-06, "output_cost_per_token": 8e-06, "output_cost_per_reasoning_token": 3e-06, - "litellm_provider": "perplexity", - "mode": "chat", + "citation_cost_per_token": 2e-06, "search_context_cost_per_query": { "search_context_size_low": 0.005, "search_context_size_medium": 0.005, "search_context_size_high": 0.005 }, + "litellm_provider": "perplexity", + "mode": "chat", "supports_reasoning": true, "supports_web_search": true }, diff --git a/litellm/proxy/_experimental/out/_next/static/chunks/250-f07ccab57fe599d4.js b/litellm/proxy/_experimental/out/_next/static/chunks/250-f07ccab57fe599d4.js index 0770ae40d66..55794ed7607 100644 --- a/litellm/proxy/_experimental/out/_next/static/chunks/250-f07ccab57fe599d4.js +++ b/litellm/proxy/_experimental/out/_next/static/chunks/250-f07ccab57fe599d4.js @@ -1 +1 @@ -"use strict";(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[250],{19250:function(e,t,o){o.d(t,{$D:function(){return eJ},$I:function(){return Q},$o:function(){return tF},AZ:function(){return X},Au:function(){return e_},BL:function(){return eZ},Br:function(){return b},Bw:function(){return tx},E9:function(){return eX},EB:function(){return th},EG:function(){return eW},EY:function(){return e0},Eb:function(){return N},FC:function(){return eh},Gh:function(){return eG},H1:function(){return I},H2:function(){return a},Hx:function(){return ej},I1:function(){return S},It:function(){return B},J$:function(){return ec},JO:function(){return O},K8:function(){return h},K_:function(){return eQ},Ko:function(){return tN},LY:function(){return eV},Lp:function(){return eR},MO:function(){return p},Mx:function(){return tf},N3:function(){return eb},N8:function(){return er},NL:function(){return e9},NV:function(){return m},Nc:function(){return eP},Nz:function(){return e1},O3:function(){return eD},OD:function(){return eC},OU:function(){return eu},Of:function(){return F},Og:function(){return y},Ou:function(){return ty},Ov:function(){return C},Oz:function(){return tB},PC:function(){return e6},PT:function(){return K},PY:function(){return tv},Pj:function(){return tt},Pv:function(){return tm},Qg:function(){return ex},RQ:function(){return j},Rg:function(){return et},Sb:function(){return ez},So:function(){return ea},TF:function(){return tu},Tj:function(){return e2},Tx:function(){return tO},U8:function(){return te},UM:function(){return tl},VA:function(){return A},Vt:function(){return eY},W_:function(){return M},X:function(){return es},XB:function(){return tg},XO:function(){return T},Xd:function(){return eS},Xm:function(){return x},YU:function(){return eH},Yi:function(){return tj},Yo:function(){return U},Z9:function(){return V},Zr:function(){return k},a6:function(){return P},aC:function(){return tw},ao:function(){return eK},b1:function(){return ew},cq:function(){return R},cu:function(){return eI},e2:function(){return eT},eH:function(){return W},eW:function(){return tC},eZ:function(){return eB},fE:function(){return td},fP:function(){return eo},fk:function(){return tS},g:function(){return e4},gX:function(){return eO},gl:function(){return to},h3:function(){return ed},hT:function(){return ev},hy:function(){return f},ix:function(){return Y},j2:function(){return ei},jA:function(){return e$},jE:function(){return eM},jr:function(){return tT},kK:function(){return g},kn:function(){return $},lP:function(){return w},lU:function(){return ta},lg:function(){return eN},mC:function(){return ti},mR:function(){return en},mY:function(){return tc},m_:function(){return D},mp:function(){return eq},n$:function(){return em},n9:function(){return ts},nJ:function(){return tb},nd:function(){return e7},o6:function(){return ee},oC:function(){return eF},ol:function(){return L},pf:function(){return eL},pu:function(){return tk},qI:function(){return _},qW:function(){return t_},qd:function(){return tE},qk:function(){return e8},qm:function(){return u},r1:function(){return tp},r6:function(){return G},rs:function(){return v},s0:function(){return Z},sN:function(){return eA},t$:function(){return J},t0:function(){return eE},t3:function(){return e5},tB:function(){return tn},tN:function(){return ep},u5:function(){return el},v9:function(){return ek},vh:function(){return eU},wX:function(){return E},wd:function(){return eg},xA:function(){return ey},xO:function(){return e3},xX:function(){return z},xZ:function(){return tr},zX:function(){return c},zg:function(){return ef}});var r=o(41021);let a=null;console.log=function(){};let n=function(e){let t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:null,o=window.location.origin,r=t||o;console.log("proxyBaseUrl:",a),console.log("serverRootPath:",e),e.length>0&&!r.endsWith(e)&&"/"!=e&&(r+=e,a=r),console.log("Updated proxyBaseUrl:",a)},c=()=>a||window.location.origin,s={GET:"GET",DELETE:"DELETE"},i=0,l=async e=>{let t=Date.now();t-i>6e4?(e.includes("Authentication Error - Expired Key")&&(r.ZP.info("UI Session Expired. Logging out."),i=t,document.cookie="token=; expires=Thu, 01 Jan 1970 00:00:00 UTC; path=/;",window.location.href=window.location.pathname),i=t):console.log("Error suppressed to prevent spam:",e)},d="Authorization";function h(){let e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:"Authorization";console.log("setGlobalLitellmHeaderName: ".concat(e)),d=e}let p=async()=>{console.log("Getting UI config");let e=await fetch("/litellm/.well-known/litellm-ui-config"),t=await e.json();return console.log("jsonData in getUiConfig:",t),n(t.server_root_path,t.proxy_base_url),t},w=async()=>{let e=a?"".concat(a,"/openapi.json"):"/openapi.json",t=await fetch(e);return await t.json()},u=async e=>{try{let t=a?"".concat(a,"/get/litellm_model_cost_map"):"/get/litellm_model_cost_map",o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}}),r=await o.json();return console.log("received litellm model cost data: ".concat(r)),r}catch(e){throw console.error("Failed to get model cost map:",e),e}},g=async(e,t)=>{try{let o=a?"".concat(a,"/model/new"):"/model/new",n=await fetch(o,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({...t})});if(!n.ok){let e=await n.text()||"Network response was not ok";throw r.ZP.error(e),Error(e)}let c=await n.json();return console.log("API Response:",c),r.ZP.destroy(),r.ZP.success("Model ".concat(t.model_name," created successfully"),2),c}catch(e){throw console.error("Failed to create key:",e),e}},f=async e=>{try{let t=a?"".concat(a,"/model/settings"):"/model/settings",o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error("Network response was not ok")}return await o.json()}catch(e){console.error("Failed to get model settings:",e)}},y=async(e,t)=>{console.log("model_id in model delete call: ".concat(t));try{let o=a?"".concat(a,"/model/delete"):"/model/delete",r=await fetch(o,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({id:t})});if(!r.ok){let e=await r.text();throw l(e),console.error("Error response from the server:",e),Error("Network response was not ok")}let n=await r.json();return console.log("API Response:",n),n}catch(e){throw console.error("Failed to create key:",e),e}},m=async(e,t)=>{if(console.log("budget_id in budget delete call: ".concat(t)),null!=e)try{let o=a?"".concat(a,"/budget/delete"):"/budget/delete",r=await fetch(o,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({id:t})});if(!r.ok){let e=await r.text();throw l(e),console.error("Error response from the server:",e),Error("Network response was not ok")}let n=await r.json();return console.log("API Response:",n),n}catch(e){throw console.error("Failed to create key:",e),e}},k=async(e,t)=>{try{console.log("Form Values in budgetCreateCall:",t),console.log("Form Values after check:",t);let o=a?"".concat(a,"/budget/new"):"/budget/new",r=await fetch(o,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({...t})});if(!r.ok){let e=await r.text();throw l(e),console.error("Error response from the server:",e),Error("Network response was not ok")}let n=await r.json();return console.log("API Response:",n),n}catch(e){throw console.error("Failed to create key:",e),e}},_=async(e,t)=>{try{console.log("Form Values in budgetUpdateCall:",t),console.log("Form Values after check:",t);let o=a?"".concat(a,"/budget/update"):"/budget/update",r=await fetch(o,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({...t})});if(!r.ok){let e=await r.text();throw l(e),console.error("Error response from the server:",e),Error("Network response was not ok")}let n=await r.json();return console.log("API Response:",n),n}catch(e){throw console.error("Failed to create key:",e),e}},T=async(e,t)=>{try{let o=a?"".concat(a,"/invitation/new"):"/invitation/new",r=await fetch(o,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({user_id:t})});if(!r.ok){let e=await r.text();throw l(e),console.error("Error response from the server:",e),Error("Network response was not ok")}let n=await r.json();return console.log("API Response:",n),n}catch(e){throw console.error("Failed to create key:",e),e}},j=async e=>{try{let t=a?"".concat(a,"/alerting/settings"):"/alerting/settings",o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error("Network response was not ok")}return await o.json()}catch(e){throw console.error("Failed to get callbacks:",e),e}},E=async(e,t,o)=>{try{if(console.log("Form Values in keyCreateCall:",o),o.description&&(o.metadata||(o.metadata={}),o.metadata.description=o.description,delete o.description,o.metadata=JSON.stringify(o.metadata)),o.metadata){console.log("formValues.metadata:",o.metadata);try{o.metadata=JSON.parse(o.metadata)}catch(e){throw Error("Failed to parse metadata: "+e)}}console.log("Form Values after check:",o);let r=a?"".concat(a,"/key/generate"):"/key/generate",n=await fetch(r,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({user_id:t,...o})});if(!n.ok){let e=await n.text();throw l(e),console.error("Error response from the server:",e),Error(e)}let c=await n.json();return console.log("API Response:",c),c}catch(e){throw console.error("Failed to create key:",e),e}},C=async(e,t,o)=>{try{if(console.log("Form Values in keyCreateCall:",o),o.description&&(o.metadata||(o.metadata={}),o.metadata.description=o.description,delete o.description,o.metadata=JSON.stringify(o.metadata)),o.auto_create_key=!1,o.metadata){console.log("formValues.metadata:",o.metadata);try{o.metadata=JSON.parse(o.metadata)}catch(e){throw Error("Failed to parse metadata: "+e)}}console.log("Form Values after check:",o);let r=a?"".concat(a,"/user/new"):"/user/new",n=await fetch(r,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({user_id:t,...o})});if(!n.ok){let e=await n.text();throw l(e),console.error("Error response from the server:",e),Error(e)}let c=await n.json();return console.log("API Response:",c),c}catch(e){throw console.error("Failed to create key:",e),e}},S=async(e,t)=>{try{let o=a?"".concat(a,"/key/delete"):"/key/delete";console.log("in keyDeleteCall:",t);let r=await fetch(o,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({keys:[t]})});if(!r.ok){let e=await r.text();throw l(e),Error("Network response was not ok")}let n=await r.json();return console.log(n),n}catch(e){throw console.error("Failed to create key:",e),e}},N=async(e,t)=>{try{let o=a?"".concat(a,"/user/delete"):"/user/delete";console.log("in userDeleteCall:",t);let r=await fetch(o,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({user_ids:t})});if(!r.ok){let e=await r.text();throw l(e),Error("Network response was not ok")}let n=await r.json();return console.log(n),n}catch(e){throw console.error("Failed to delete user(s):",e),e}},v=async(e,t)=>{try{let o=a?"".concat(a,"/team/delete"):"/team/delete";console.log("in teamDeleteCall:",t);let r=await fetch(o,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({team_ids:[t]})});if(!r.ok){let e=await r.text();throw l(e),Error("Network response was not ok")}let n=await r.json();return console.log(n),n}catch(e){throw console.error("Failed to delete key:",e),e}},F=async function(e){let t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:null,o=arguments.length>2&&void 0!==arguments[2]?arguments[2]:null,r=arguments.length>3&&void 0!==arguments[3]?arguments[3]:null,n=arguments.length>4&&void 0!==arguments[4]?arguments[4]:null,c=arguments.length>5&&void 0!==arguments[5]?arguments[5]:null,s=arguments.length>6&&void 0!==arguments[6]?arguments[6]:null,i=arguments.length>7&&void 0!==arguments[7]?arguments[7]:null,h=arguments.length>8&&void 0!==arguments[8]?arguments[8]:null,p=arguments.length>9&&void 0!==arguments[9]?arguments[9]:null;try{let w=a?"".concat(a,"/user/list"):"/user/list";console.log("in userListCall");let u=new URLSearchParams;if(t&&t.length>0){let e=t.join(",");u.append("user_ids",e)}o&&u.append("page",o.toString()),r&&u.append("page_size",r.toString()),n&&u.append("user_email",n),c&&u.append("role",c),s&&u.append("team",s),i&&u.append("sso_user_ids",i),h&&u.append("sort_by",h),p&&u.append("sort_order",p);let g=u.toString();g&&(w+="?".concat(g));let f=await fetch(w,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!f.ok){let e=await f.text();throw l(e),Error("Network response was not ok")}let y=await f.json();return console.log("/user/list API Response:",y),y}catch(e){throw console.error("Failed to create key:",e),e}},b=async function(e,t,o){let r=arguments.length>3&&void 0!==arguments[3]&&arguments[3],n=arguments.length>4?arguments[4]:void 0,c=arguments.length>5?arguments[5]:void 0,s=arguments.length>6&&void 0!==arguments[6]&&arguments[6];console.log("userInfoCall: ".concat(t,", ").concat(o,", ").concat(r,", ").concat(n,", ").concat(c,", ").concat(s));try{let i;if(r){i=a?"".concat(a,"/user/list"):"/user/list";let e=new URLSearchParams;null!=n&&e.append("page",n.toString()),null!=c&&e.append("page_size",c.toString()),i+="?".concat(e.toString())}else i=a?"".concat(a,"/user/info"):"/user/info",("Admin"!==o&&"Admin Viewer"!==o||s)&&t&&(i+="?user_id=".concat(t));console.log("Requesting user data from:",i);let h=await fetch(i,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!h.ok){let e=await h.text();throw l(e),Error("Network response was not ok")}let p=await h.json();return console.log("API Response:",p),p}catch(e){throw console.error("Failed to fetch user data:",e),e}},x=async(e,t)=>{try{let o=a?"".concat(a,"/team/info"):"/team/info";t&&(o="".concat(o,"?team_id=").concat(t)),console.log("in teamInfoCall");let r=await fetch(o,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!r.ok){let e=await r.text();throw l(e),Error("Network response was not ok")}let n=await r.json();return console.log("API Response:",n),n}catch(e){throw console.error("Failed to create key:",e),e}},O=async function(e,t){let o=arguments.length>2&&void 0!==arguments[2]?arguments[2]:null,r=arguments.length>3&&void 0!==arguments[3]?arguments[3]:null,n=arguments.length>4&&void 0!==arguments[4]?arguments[4]:null;arguments.length>5&&void 0!==arguments[5]&&arguments[5],arguments.length>6&&void 0!==arguments[6]&&arguments[6],arguments.length>7&&void 0!==arguments[7]&&arguments[7],arguments.length>8&&void 0!==arguments[8]&&arguments[8];try{let c=a?"".concat(a,"/v2/team/list"):"/v2/team/list";console.log("in teamInfoCall");let s=new URLSearchParams;o&&s.append("user_id",o.toString()),t&&s.append("organization_id",t.toString()),r&&s.append("team_id",r.toString()),n&&s.append("team_alias",n.toString());let i=s.toString();i&&(c+="?".concat(i));let h=await fetch(c,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!h.ok){let e=await h.text();throw l(e),Error("Network response was not ok")}let p=await h.json();return console.log("/v2/team/list API Response:",p),p}catch(e){throw console.error("Failed to create key:",e),e}},B=async function(e,t){let o=arguments.length>2&&void 0!==arguments[2]?arguments[2]:null,r=arguments.length>3&&void 0!==arguments[3]?arguments[3]:null,n=arguments.length>4&&void 0!==arguments[4]?arguments[4]:null;try{let c=a?"".concat(a,"/team/list"):"/team/list";console.log("in teamInfoCall");let s=new URLSearchParams;o&&s.append("user_id",o.toString()),t&&s.append("organization_id",t.toString()),r&&s.append("team_id",r.toString()),n&&s.append("team_alias",n.toString());let i=s.toString();i&&(c+="?".concat(i));let h=await fetch(c,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!h.ok){let e=await h.text();throw l(e),Error("Network response was not ok")}let p=await h.json();return console.log("/team/list API Response:",p),p}catch(e){throw console.error("Failed to create key:",e),e}},P=async e=>{try{let t=a?"".concat(a,"/team/available"):"/team/available";console.log("in availableTeamListCall");let o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error("Network response was not ok")}let r=await o.json();return console.log("/team/available_teams API Response:",r),r}catch(e){throw e}},G=async e=>{try{let t=a?"".concat(a,"/organization/list"):"/organization/list",o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error("Network response was not ok")}return await o.json()}catch(e){throw console.error("Failed to create key:",e),e}},J=async(e,t)=>{try{let o=a?"".concat(a,"/organization/info"):"/organization/info";t&&(o="".concat(o,"?organization_id=").concat(t)),console.log("in teamInfoCall");let r=await fetch(o,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!r.ok){let e=await r.text();throw l(e),Error("Network response was not ok")}let n=await r.json();return console.log("API Response:",n),n}catch(e){throw console.error("Failed to create key:",e),e}},I=async(e,t)=>{try{if(console.log("Form Values in organizationCreateCall:",t),t.metadata){console.log("formValues.metadata:",t.metadata);try{t.metadata=JSON.parse(t.metadata)}catch(e){throw console.error("Failed to parse metadata:",e),Error("Failed to parse metadata: "+e)}}let o=a?"".concat(a,"/organization/new"):"/organization/new",r=await fetch(o,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({...t})});if(!r.ok){let e=await r.text();throw l(e),console.error("Error response from the server:",e),Error("Network response was not ok")}let n=await r.json();return console.log("API Response:",n),n}catch(e){throw console.error("Failed to create key:",e),e}},A=async(e,t)=>{try{console.log("Form Values in organizationUpdateCall:",t);let o=a?"".concat(a,"/organization/update"):"/organization/update",r=await fetch(o,{method:"PATCH",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({...t})});if(!r.ok){let e=await r.text();throw l(e),console.error("Error response from the server:",e),Error("Network response was not ok")}let n=await r.json();return console.log("Update Team Response:",n),n}catch(e){throw console.error("Failed to create key:",e),e}},R=async(e,t)=>{try{let o=a?"".concat(a,"/organization/delete"):"/organization/delete",r=await fetch(o,{method:"DELETE",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({organization_ids:[t]})});if(!r.ok){let e=await r.text();throw l(e),Error("Error deleting organization: ".concat(e))}return await r.json()}catch(e){throw console.error("Failed to delete organization:",e),e}},U=async(e,t)=>{try{let o=a?"".concat(a,"/utils/transform_request"):"/utils/transform_request",r=await fetch(o,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify(t)});if(!r.ok){let e=await r.text();throw l(e),Error("Network response was not ok")}return await r.json()}catch(e){throw console.error("Failed to create key:",e),e}},z=async function(e,t,o){let r=arguments.length>3&&void 0!==arguments[3]?arguments[3]:1;try{let n=a?"".concat(a,"/user/daily/activity"):"/user/daily/activity",c=new URLSearchParams;c.append("start_date",t.toISOString()),c.append("end_date",o.toISOString()),c.append("page_size","1000"),c.append("page",r.toString());let s=c.toString();s&&(n+="?".concat(s));let i=await fetch(n,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!i.ok){let e=await i.text();throw l(e),Error("Network response was not ok")}return await i.json()}catch(e){throw console.error("Failed to create key:",e),e}},V=async function(e,t,o){let r=arguments.length>3&&void 0!==arguments[3]?arguments[3]:1,n=arguments.length>4&&void 0!==arguments[4]?arguments[4]:null;try{let c=a?"".concat(a,"/tag/daily/activity"):"/tag/daily/activity",s=new URLSearchParams;s.append("start_date",t.toISOString()),s.append("end_date",o.toISOString()),s.append("page_size","1000"),s.append("page",r.toString()),n&&s.append("tags",n.join(","));let i=s.toString();i&&(c+="?".concat(i));let h=await fetch(c,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!h.ok){let e=await h.text();throw l(e),Error("Network response was not ok")}return await h.json()}catch(e){throw console.error("Failed to create key:",e),e}},L=async function(e,t,o){let r=arguments.length>3&&void 0!==arguments[3]?arguments[3]:1,n=arguments.length>4&&void 0!==arguments[4]?arguments[4]:null;try{let c=a?"".concat(a,"/team/daily/activity"):"/team/daily/activity",s=new URLSearchParams;s.append("start_date",t.toISOString()),s.append("end_date",o.toISOString()),s.append("page_size","1000"),s.append("page",r.toString()),n&&s.append("team_ids",n.join(",")),s.append("exclude_team_ids","litellm-dashboard");let i=s.toString();i&&(c+="?".concat(i));let h=await fetch(c,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!h.ok){let e=await h.text();throw l(e),Error("Network response was not ok")}return await h.json()}catch(e){throw console.error("Failed to create key:",e),e}},M=async e=>{try{let t=a?"".concat(a,"/onboarding/get_token"):"/onboarding/get_token";t+="?invite_link=".concat(e);let o=await fetch(t,{method:"GET",headers:{"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error("Network response was not ok")}return await o.json()}catch(e){throw console.error("Failed to create key:",e),e}},D=async(e,t,o,r)=>{let n=a?"".concat(a,"/onboarding/claim_token"):"/onboarding/claim_token";try{let a=await fetch(n,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({invitation_link:t,user_id:o,password:r})});if(!a.ok){let e=await a.text();throw l(e),Error("Network response was not ok")}let c=await a.json();return console.log(c),c}catch(e){throw console.error("Failed to delete key:",e),e}},Z=async(e,t,o)=>{try{let r=a?"".concat(a,"/key/").concat(t,"/regenerate"):"/key/".concat(t,"/regenerate"),n=await fetch(r,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify(o)});if(!n.ok){let e=await n.text();throw l(e),Error("Network response was not ok")}let c=await n.json();return console.log("Regenerate key Response:",c),c}catch(e){throw console.error("Failed to regenerate key:",e),e}},H=!1,q=null,X=async(e,t,o)=>{try{console.log("modelInfoCall:",e,t,o);let n=a?"".concat(a,"/v2/model/info"):"/v2/model/info",c=new URLSearchParams;c.append("include_team_models","true"),c.toString()&&(n+="?".concat(c.toString()));let s=await fetch(n,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!s.ok){let e=await s.text();throw e+="error shown=".concat(H),H||(e.includes("No model list passed")&&(e="No Models Exist. Click Add Model to get started."),r.ZP.info(e,10),H=!0,q&&clearTimeout(q),q=setTimeout(()=>{H=!1},1e4)),Error("Network response was not ok")}let i=await s.json();return console.log("modelInfoCall:",i),i}catch(e){throw console.error("Failed to create key:",e),e}},Y=async(e,t)=>{try{let o=a?"".concat(a,"/v1/model/info"):"/v1/model/info";o+="?litellm_model_id=".concat(t);let r=await fetch(o,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!r.ok)throw await r.text(),Error("Network response was not ok");let n=await r.json();return console.log("modelInfoV1Call:",n),n}catch(e){throw console.error("Failed to create key:",e),e}},$=async e=>{try{let t=a?"".concat(a,"/model_group/info"):"/model_group/info",o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok)throw await o.text(),Error("Network response was not ok");let r=await o.json();return console.log("modelHubCall:",r),r}catch(e){throw console.error("Failed to create key:",e),e}},K=async e=>{try{let t=a?"".concat(a,"/get/allowed_ips"):"/get/allowed_ips",o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw Error("Network response was not ok: ".concat(e))}let r=await o.json();return console.log("getAllowedIPs:",r),r.data}catch(e){throw console.error("Failed to get allowed IPs:",e),e}},W=async(e,t)=>{try{let o=a?"".concat(a,"/add/allowed_ip"):"/add/allowed_ip",r=await fetch(o,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({ip:t})});if(!r.ok){let e=await r.text();throw Error("Network response was not ok: ".concat(e))}let n=await r.json();return console.log("addAllowedIP:",n),n}catch(e){throw console.error("Failed to add allowed IP:",e),e}},Q=async(e,t)=>{try{let o=a?"".concat(a,"/delete/allowed_ip"):"/delete/allowed_ip",r=await fetch(o,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({ip:t})});if(!r.ok){let e=await r.text();throw Error("Network response was not ok: ".concat(e))}let n=await r.json();return console.log("deleteAllowedIP:",n),n}catch(e){throw console.error("Failed to delete allowed IP:",e),e}},ee=async(e,t,o,r,n,c,s,i)=>{try{let t=a?"".concat(a,"/model/metrics"):"/model/metrics";r&&(t="".concat(t,"?_selected_model_group=").concat(r,"&startTime=").concat(n,"&endTime=").concat(c,"&api_key=").concat(s,"&customer=").concat(i));let o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error("Network response was not ok")}return await o.json()}catch(e){throw console.error("Failed to create key:",e),e}},et=async(e,t,o,r)=>{try{let n=a?"".concat(a,"/model/streaming_metrics"):"/model/streaming_metrics";t&&(n="".concat(n,"?_selected_model_group=").concat(t,"&startTime=").concat(o,"&endTime=").concat(r));let c=await fetch(n,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!c.ok){let e=await c.text();throw l(e),Error("Network response was not ok")}return await c.json()}catch(e){throw console.error("Failed to create key:",e),e}},eo=async(e,t,o,r,n,c,s,i)=>{try{let t=a?"".concat(a,"/model/metrics/slow_responses"):"/model/metrics/slow_responses";r&&(t="".concat(t,"?_selected_model_group=").concat(r,"&startTime=").concat(n,"&endTime=").concat(c,"&api_key=").concat(s,"&customer=").concat(i));let o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error("Network response was not ok")}return await o.json()}catch(e){throw console.error("Failed to create key:",e),e}},er=async(e,t,o,r,n,c,s,i)=>{try{let t=a?"".concat(a,"/model/metrics/exceptions"):"/model/metrics/exceptions";r&&(t="".concat(t,"?_selected_model_group=").concat(r,"&startTime=").concat(n,"&endTime=").concat(c,"&api_key=").concat(s,"&customer=").concat(i));let o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error("Network response was not ok")}return await o.json()}catch(e){throw console.error("Failed to create key:",e),e}},ea=async function(e,t,o){let r=arguments.length>3&&void 0!==arguments[3]&&arguments[3],n=arguments.length>4&&void 0!==arguments[4]?arguments[4]:null,c=(arguments.length>5&&void 0!==arguments[5]&&arguments[5],arguments.length>6&&void 0!==arguments[6]&&arguments[6]);console.log("in /models calls, globalLitellmHeaderName",d);try{let t=a?"".concat(a,"/models"):"/models",o=new URLSearchParams;o.append("include_model_access_groups","True"),!0===r&&o.append("return_wildcard_routes","True"),!0===c&&o.append("only_model_access_groups","True"),n&&o.append("team_id",n.toString()),o.toString()&&(t+="?".concat(o.toString()));let s=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!s.ok){let e=await s.text();throw l(e),Error("Network response was not ok")}return await s.json()}catch(e){throw console.error("Failed to create key:",e),e}},en=async e=>{try{let t=a?"".concat(a,"/global/spend/teams"):"/global/spend/teams";console.log("in teamSpendLogsCall:",t);let o=await fetch("".concat(t),{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error("Network response was not ok")}let r=await o.json();return console.log(r),r}catch(e){throw console.error("Failed to create key:",e),e}},ec=async(e,t,o,r)=>{try{let n=a?"".concat(a,"/global/spend/tags"):"/global/spend/tags";t&&o&&(n="".concat(n,"?start_date=").concat(t,"&end_date=").concat(o)),r&&(n+="".concat(n,"&tags=").concat(r.join(","))),console.log("in tagsSpendLogsCall:",n);let c=await fetch("".concat(n),{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!c.ok)throw await c.text(),Error("Network response was not ok");let s=await c.json();return console.log(s),s}catch(e){throw console.error("Failed to create key:",e),e}},es=async e=>{try{let t=a?"".concat(a,"/global/spend/all_tag_names"):"/global/spend/all_tag_names";console.log("in global/spend/all_tag_names call",t);let o=await fetch("".concat(t),{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok)throw await o.text(),Error("Network response was not ok");let r=await o.json();return console.log(r),r}catch(e){throw console.error("Failed to create key:",e),e}},ei=async e=>{try{let t=a?"".concat(a,"/global/all_end_users"):"/global/all_end_users";console.log("in global/all_end_users call",t);let o=await fetch("".concat(t),{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok)throw await o.text(),Error("Network response was not ok");let r=await o.json();return console.log(r),r}catch(e){throw console.error("Failed to create key:",e),e}},el=async(e,t)=>{try{let o=a?"".concat(a,"/user/filter/ui"):"/user/filter/ui";t.get("user_email")&&(o+="?user_email=".concat(t.get("user_email"))),t.get("user_id")&&(o+="?user_id=".concat(t.get("user_id")));let r=await fetch(o,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!r.ok){let e=await r.text();throw l(e),Error("Network response was not ok")}return await r.json()}catch(e){throw console.error("Failed to create key:",e),e}},ed=async(e,t,o,r,n,c,s,i,h,p,w)=>{try{let u=a?"".concat(a,"/spend/logs/ui"):"/spend/logs/ui",g=new URLSearchParams;t&&g.append("api_key",t),o&&g.append("team_id",o),r&&g.append("request_id",r),n&&g.append("start_date",n),c&&g.append("end_date",c),s&&g.append("page",s.toString()),i&&g.append("page_size",i.toString()),h&&g.append("user_id",h),p&&g.append("status_filter",p),w&&g.append("model",w);let f=g.toString();f&&(u+="?".concat(f));let y=await fetch(u,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!y.ok){let e=await y.text();throw l(e),Error("Network response was not ok")}let m=await y.json();return console.log("Spend Logs Response:",m),m}catch(e){throw console.error("Failed to fetch spend logs:",e),e}},eh=async e=>{try{let t=a?"".concat(a,"/global/spend/logs"):"/global/spend/logs",o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error("Network response was not ok")}let r=await o.json();return console.log(r),r}catch(e){throw console.error("Failed to create key:",e),e}},ep=async e=>{try{let t=a?"".concat(a,"/global/spend/keys?limit=5"):"/global/spend/keys?limit=5",o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error("Network response was not ok")}let r=await o.json();return console.log(r),r}catch(e){throw console.error("Failed to create key:",e),e}},ew=async(e,t,o,r)=>{try{let n=a?"".concat(a,"/global/spend/end_users"):"/global/spend/end_users",c="";c=t?JSON.stringify({api_key:t,startTime:o,endTime:r}):JSON.stringify({startTime:o,endTime:r});let s={method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:c},i=await fetch(n,s);if(!i.ok){let e=await i.text();throw l(e),Error("Network response was not ok")}let h=await i.json();return console.log(h),h}catch(e){throw console.error("Failed to create key:",e),e}},eu=async(e,t,o,r)=>{try{let n=a?"".concat(a,"/global/spend/provider"):"/global/spend/provider";o&&r&&(n+="?start_date=".concat(o,"&end_date=").concat(r)),t&&(n+="&api_key=".concat(t));let c={method:"GET",headers:{[d]:"Bearer ".concat(e)}},s=await fetch(n,c);if(!s.ok){let e=await s.text();throw l(e),Error("Network response was not ok")}let i=await s.json();return console.log(i),i}catch(e){throw console.error("Failed to fetch spend data:",e),e}},eg=async(e,t,o)=>{try{let r=a?"".concat(a,"/global/activity"):"/global/activity";t&&o&&(r+="?start_date=".concat(t,"&end_date=").concat(o));let n={method:"GET",headers:{[d]:"Bearer ".concat(e)}},c=await fetch(r,n);if(!c.ok)throw await c.text(),Error("Network response was not ok");let s=await c.json();return console.log(s),s}catch(e){throw console.error("Failed to fetch spend data:",e),e}},ef=async(e,t,o)=>{try{let r=a?"".concat(a,"/global/activity/cache_hits"):"/global/activity/cache_hits";t&&o&&(r+="?start_date=".concat(t,"&end_date=").concat(o));let n={method:"GET",headers:{[d]:"Bearer ".concat(e)}},c=await fetch(r,n);if(!c.ok)throw await c.text(),Error("Network response was not ok");let s=await c.json();return console.log(s),s}catch(e){throw console.error("Failed to fetch spend data:",e),e}},ey=async(e,t,o)=>{try{let r=a?"".concat(a,"/global/activity/model"):"/global/activity/model";t&&o&&(r+="?start_date=".concat(t,"&end_date=").concat(o));let n={method:"GET",headers:{[d]:"Bearer ".concat(e)}},c=await fetch(r,n);if(!c.ok)throw await c.text(),Error("Network response was not ok");let s=await c.json();return console.log(s),s}catch(e){throw console.error("Failed to fetch spend data:",e),e}},em=async(e,t,o,r)=>{try{let n=a?"".concat(a,"/global/activity/exceptions"):"/global/activity/exceptions";t&&o&&(n+="?start_date=".concat(t,"&end_date=").concat(o)),r&&(n+="&model_group=".concat(r));let c={method:"GET",headers:{[d]:"Bearer ".concat(e)}},s=await fetch(n,c);if(!s.ok)throw await s.text(),Error("Network response was not ok");let i=await s.json();return console.log(i),i}catch(e){throw console.error("Failed to fetch spend data:",e),e}},ek=async(e,t,o,r)=>{try{let n=a?"".concat(a,"/global/activity/exceptions/deployment"):"/global/activity/exceptions/deployment";t&&o&&(n+="?start_date=".concat(t,"&end_date=").concat(o)),r&&(n+="&model_group=".concat(r));let c={method:"GET",headers:{[d]:"Bearer ".concat(e)}},s=await fetch(n,c);if(!s.ok)throw await s.text(),Error("Network response was not ok");let i=await s.json();return console.log(i),i}catch(e){throw console.error("Failed to fetch spend data:",e),e}},e_=async e=>{try{let t=a?"".concat(a,"/global/spend/models?limit=5"):"/global/spend/models?limit=5",o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error("Network response was not ok")}let r=await o.json();return console.log(r),r}catch(e){throw console.error("Failed to create key:",e),e}},eT=async(e,t)=>{try{let o=a?"".concat(a,"/v2/key/info"):"/v2/key/info",r=await fetch(o,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({keys:t})});if(!r.ok){let e=await r.text();if(e.includes("Invalid proxy server token passed"))throw Error("Invalid proxy server token passed");throw l(e),Error("Network response was not ok")}let n=await r.json();return console.log(n),n}catch(e){throw console.error("Failed to create key:",e),e}},ej=async(e,t,o)=>{try{console.log("Sending model connection test request:",JSON.stringify(t));let n=a?"".concat(a,"/health/test_connection"):"/health/test_connection",c=await fetch(n,{method:"POST",headers:{"Content-Type":"application/json",[d]:"Bearer ".concat(e)},body:JSON.stringify({litellm_params:t,mode:o})}),s=c.headers.get("content-type");if(!s||!s.includes("application/json")){let e=await c.text();throw console.error("Received non-JSON response:",e),Error("Received non-JSON response (".concat(c.status,": ").concat(c.statusText,"). Check network tab for details."))}let i=await c.json();if(!c.ok||"error"===i.status){if("error"===i.status);else{var r;return{status:"error",message:(null===(r=i.error)||void 0===r?void 0:r.message)||"Connection test failed: ".concat(c.status," ").concat(c.statusText)}}}return i}catch(e){throw console.error("Model connection test error:",e),e}},eE=async(e,t)=>{try{console.log("entering keyInfoV1Call");let o=a?"".concat(a,"/key/info"):"/key/info";o="".concat(o,"?key=").concat(t);let n=await fetch(o,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(console.log("response",n),!n.ok){let e=await n.text();l(e),r.ZP.error("Failed to fetch key info - "+e)}let c=await n.json();return console.log("data",c),c}catch(e){throw console.error("Failed to fetch key info:",e),e}},eC=async function(e,t,o,r,n,c,s,i){let h=arguments.length>8&&void 0!==arguments[8]?arguments[8]:null,p=arguments.length>9&&void 0!==arguments[9]?arguments[9]:null;try{let w=a?"".concat(a,"/key/list"):"/key/list";console.log("in keyListCall");let u=new URLSearchParams;o&&u.append("team_id",o.toString()),t&&u.append("organization_id",t.toString()),r&&u.append("key_alias",r),c&&u.append("key_hash",c),n&&u.append("user_id",n.toString()),s&&u.append("page",s.toString()),i&&u.append("size",i.toString()),h&&u.append("sort_by",h),p&&u.append("sort_order",p),u.append("return_full_object","true"),u.append("include_team_keys","true");let g=u.toString();g&&(w+="?".concat(g));let f=await fetch(w,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!f.ok){let e=await f.text();throw l(e),Error("Network response was not ok")}let y=await f.json();return console.log("/team/list API Response:",y),y}catch(e){throw console.error("Failed to create key:",e),e}},eS=async(e,t)=>{try{let o=a?"".concat(a,"/user/get_users?role=").concat(t):"/user/get_users?role=".concat(t);console.log("in userGetAllUsersCall:",o);let r=await fetch(o,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!r.ok){let e=await r.text();throw l(e),Error("Network response was not ok")}let n=await r.json();return console.log(n),n}catch(e){throw console.error("Failed to get requested models:",e),e}},eN=async e=>{try{let t=a?"".concat(a,"/user/available_roles"):"/user/available_roles",o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok)throw await o.text(),Error("Network response was not ok");let r=await o.json();return console.log("response from user/available_role",r),r}catch(e){throw e}},ev=async(e,t)=>{try{if(console.log("Form Values in teamCreateCall:",t),t.metadata){console.log("formValues.metadata:",t.metadata);try{t.metadata=JSON.parse(t.metadata)}catch(e){throw Error("Failed to parse metadata: "+e)}}let o=a?"".concat(a,"/team/new"):"/team/new",r=await fetch(o,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({...t})});if(!r.ok){let e=await r.text();throw l(e),console.error("Error response from the server:",e),Error("Network response was not ok")}let n=await r.json();return console.log("API Response:",n),n}catch(e){throw console.error("Failed to create key:",e),e}},eF=async(e,t)=>{try{if(console.log("Form Values in credentialCreateCall:",t),t.metadata){console.log("formValues.metadata:",t.metadata);try{t.metadata=JSON.parse(t.metadata)}catch(e){throw Error("Failed to parse metadata: "+e)}}let o=a?"".concat(a,"/credentials"):"/credentials",r=await fetch(o,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({...t})});if(!r.ok){let e=await r.text();throw l(e),console.error("Error response from the server:",e),Error("Network response was not ok")}let n=await r.json();return console.log("API Response:",n),n}catch(e){throw console.error("Failed to create key:",e),e}},eb=async e=>{try{let t=a?"".concat(a,"/credentials"):"/credentials";console.log("in credentialListCall");let o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error("Network response was not ok")}let r=await o.json();return console.log("/credentials API Response:",r),r}catch(e){throw console.error("Failed to create key:",e),e}},ex=async(e,t,o)=>{try{let r=a?"".concat(a,"/credentials"):"/credentials";t?r+="/by_name/".concat(t):o&&(r+="/by_model/".concat(o)),console.log("in credentialListCall");let n=await fetch(r,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!n.ok){let e=await n.text();throw l(e),Error("Network response was not ok")}let c=await n.json();return console.log("/credentials API Response:",c),c}catch(e){throw console.error("Failed to create key:",e),e}},eO=async(e,t)=>{try{let o=a?"".concat(a,"/credentials/").concat(t):"/credentials/".concat(t);console.log("in credentialDeleteCall:",t);let r=await fetch(o,{method:"DELETE",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!r.ok){let e=await r.text();throw l(e),Error("Network response was not ok")}let n=await r.json();return console.log(n),n}catch(e){throw console.error("Failed to delete key:",e),e}},eB=async(e,t,o)=>{try{if(console.log("Form Values in credentialUpdateCall:",o),o.metadata){console.log("formValues.metadata:",o.metadata);try{o.metadata=JSON.parse(o.metadata)}catch(e){throw Error("Failed to parse metadata: "+e)}}let r=a?"".concat(a,"/credentials/").concat(t):"/credentials/".concat(t),n=await fetch(r,{method:"PATCH",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({...o})});if(!n.ok){let e=await n.text();throw l(e),console.error("Error response from the server:",e),Error("Network response was not ok")}let c=await n.json();return console.log("API Response:",c),c}catch(e){throw console.error("Failed to create key:",e),e}},eP=async(e,t)=>{try{if(console.log("Form Values in keyUpdateCall:",t),t.model_tpm_limit){console.log("formValues.model_tpm_limit:",t.model_tpm_limit);try{t.model_tpm_limit=JSON.parse(t.model_tpm_limit)}catch(e){throw Error("Failed to parse model_tpm_limit: "+e)}}if(t.model_rpm_limit){console.log("formValues.model_rpm_limit:",t.model_rpm_limit);try{t.model_rpm_limit=JSON.parse(t.model_rpm_limit)}catch(e){throw Error("Failed to parse model_rpm_limit: "+e)}}let o=a?"".concat(a,"/key/update"):"/key/update",r=await fetch(o,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({...t})});if(!r.ok){let e=await r.text();throw l(e),console.error("Error response from the server:",e),Error("Network response was not ok")}let n=await r.json();return console.log("Update key Response:",n),n}catch(e){throw console.error("Failed to create key:",e),e}},eG=async(e,t)=>{try{console.log("Form Values in teamUpateCall:",t);let o=a?"".concat(a,"/team/update"):"/team/update",n=await fetch(o,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({...t})});if(!n.ok){let e=await n.text();throw l(e),console.error("Error response from the server:",e),r.ZP.error("Failed to update team settings: "+e),Error(e)}let c=await n.json();return console.log("Update Team Response:",c),c}catch(e){throw console.error("Failed to update team:",e),e}},eJ=async(e,t,o)=>{try{console.log("Form Values in modelUpateCall:",t);let r=a?"".concat(a,"/model/").concat(o,"/update"):"/model/".concat(o,"/update"),n=await fetch(r,{method:"PATCH",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({...t})});if(!n.ok){let e=await n.text();throw l(e),console.error("Error update from the server:",e),Error("Network response was not ok")}let c=await n.json();return console.log("Update model Response:",c),c}catch(e){throw console.error("Failed to update model:",e),e}},eI=async(e,t,o)=>{try{console.log("Form Values in teamMemberAddCall:",o);let n=a?"".concat(a,"/team/member_add"):"/team/member_add",c=await fetch(n,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({team_id:t,member:o})});if(!c.ok){var r;let e=await c.text(),t={};try{t=JSON.parse(e)}catch(t){console.warn("Failed to parse error body as JSON:",e)}let o=(null==t?void 0:null===(r=t.detail)||void 0===r?void 0:r.error)||"Failed to add team member",a=Error(o);throw a.raw=t,a}let s=await c.json();return console.log("API Response:",s),s}catch(e){throw console.error("Failed to create key:",e),e}},eA=async(e,t,o)=>{try{console.log("Form Values in teamMemberUpdateCall:",o);let n=a?"".concat(a,"/team/member_update"):"/team/member_update",c=await fetch(n,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({team_id:t,role:o.role,user_id:o.user_id})});if(!c.ok){var r;let e=await c.text(),t={};try{t=JSON.parse(e)}catch(t){console.warn("Failed to parse error body as JSON:",e)}let o=(null==t?void 0:null===(r=t.detail)||void 0===r?void 0:r.error)||"Failed to add team member",a=Error(o);throw a.raw=t,a}let s=await c.json();return console.log("API Response:",s),s}catch(e){throw console.error("Failed to update team member:",e),e}},eR=async(e,t,o)=>{try{console.log("Form Values in teamMemberAddCall:",o);let r=a?"".concat(a,"/team/member_delete"):"/team/member_delete",n=await fetch(r,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({team_id:t,...void 0!==o.user_email&&{user_email:o.user_email},...void 0!==o.user_id&&{user_id:o.user_id}})});if(!n.ok){let e=await n.text();throw l(e),console.error("Error response from the server:",e),Error("Network response was not ok")}let c=await n.json();return console.log("API Response:",c),c}catch(e){throw console.error("Failed to create key:",e),e}},eU=async(e,t,o)=>{try{console.log("Form Values in teamMemberAddCall:",o);let r=a?"".concat(a,"/organization/member_add"):"/organization/member_add",n=await fetch(r,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({organization_id:t,member:o})});if(!n.ok){let e=await n.text();throw l(e),console.error("Error response from the server:",e),Error(e)}let c=await n.json();return console.log("API Response:",c),c}catch(e){throw console.error("Failed to create organization member:",e),e}},ez=async(e,t,o)=>{try{console.log("Form Values in organizationMemberDeleteCall:",o);let r=a?"".concat(a,"/organization/member_delete"):"/organization/member_delete",n=await fetch(r,{method:"DELETE",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({organization_id:t,user_id:o})});if(!n.ok){let e=await n.text();throw l(e),console.error("Error response from the server:",e),Error("Network response was not ok")}let c=await n.json();return console.log("API Response:",c),c}catch(e){throw console.error("Failed to delete organization member:",e),e}},eV=async(e,t,o)=>{try{console.log("Form Values in organizationMemberUpdateCall:",o);let r=a?"".concat(a,"/organization/member_update"):"/organization/member_update",n=await fetch(r,{method:"PATCH",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({organization_id:t,...o})});if(!n.ok){let e=await n.text();throw l(e),console.error("Error response from the server:",e),Error("Network response was not ok")}let c=await n.json();return console.log("API Response:",c),c}catch(e){throw console.error("Failed to update organization member:",e),e}},eL=async(e,t,o)=>{try{console.log("Form Values in userUpdateUserCall:",t);let r=a?"".concat(a,"/user/update"):"/user/update",n={...t};null!==o&&(n.user_role=o),n=JSON.stringify(n);let c=await fetch(r,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:n});if(!c.ok){let e=await c.text();throw l(e),console.error("Error response from the server:",e),Error("Network response was not ok")}let s=await c.json();return console.log("API Response:",s),s}catch(e){throw console.error("Failed to create key:",e),e}},eM=async(e,t)=>{try{let o=a?"".concat(a,"/health/services?service=").concat(t):"/health/services?service=".concat(t);console.log("Checking Slack Budget Alerts service health");let n=await fetch(o,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!n.ok){let e=await n.text();throw l(e),Error(e)}let c=await n.json();return r.ZP.success("Test request to ".concat(t," made - check logs/alerts on ").concat(t," to verify")),c}catch(e){throw console.error("Failed to perform health check:",e),e}},eD=async e=>{try{let t=a?"".concat(a,"/budget/list"):"/budget/list",o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error("Network response was not ok")}return await o.json()}catch(e){throw console.error("Failed to get callbacks:",e),e}},eZ=async(e,t,o)=>{try{let t=a?"".concat(a,"/get/config/callbacks"):"/get/config/callbacks",o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error("Network response was not ok")}return await o.json()}catch(e){throw console.error("Failed to get callbacks:",e),e}},eH=async e=>{try{let t=a?"".concat(a,"/config/list?config_type=general_settings"):"/config/list?config_type=general_settings",o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error("Network response was not ok")}return await o.json()}catch(e){throw console.error("Failed to get callbacks:",e),e}},eq=async e=>{try{let t=a?"".concat(a,"/config/pass_through_endpoint"):"/config/pass_through_endpoint",o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error("Network response was not ok")}return await o.json()}catch(e){throw console.error("Failed to get callbacks:",e),e}},eX=async(e,t)=>{try{let o=a?"".concat(a,"/config/field/info?field_name=").concat(t):"/config/field/info?field_name=".concat(t),r=await fetch(o,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!r.ok)throw await r.text(),Error("Network response was not ok");return await r.json()}catch(e){throw console.error("Failed to set callbacks:",e),e}},eY=async(e,t)=>{try{let o=a?"".concat(a,"/config/pass_through_endpoint"):"/config/pass_through_endpoint",r=await fetch(o,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({...t})});if(!r.ok){let e=await r.text();throw l(e),Error("Network response was not ok")}return await r.json()}catch(e){throw console.error("Failed to set callbacks:",e),e}},e$=async(e,t,o)=>{try{let n=a?"".concat(a,"/config/field/update"):"/config/field/update",c=await fetch(n,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({field_name:t,field_value:o,config_type:"general_settings"})});if(!c.ok){let e=await c.text();throw l(e),Error("Network response was not ok")}let s=await c.json();return r.ZP.success("Successfully updated value!"),s}catch(e){throw console.error("Failed to set callbacks:",e),e}},eK=async(e,t)=>{try{let o=a?"".concat(a,"/config/field/delete"):"/config/field/delete",n=await fetch(o,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({field_name:t,config_type:"general_settings"})});if(!n.ok){let e=await n.text();throw l(e),Error("Network response was not ok")}let c=await n.json();return r.ZP.success("Field reset on proxy"),c}catch(e){throw console.error("Failed to get callbacks:",e),e}},eW=async(e,t)=>{try{let o=a?"".concat(a,"/config/pass_through_endpoint?endpoint_id=").concat(t):"/config/pass_through_endpoint".concat(t),r=await fetch(o,{method:"DELETE",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!r.ok){let e=await r.text();throw l(e),Error("Network response was not ok")}return await r.json()}catch(e){throw console.error("Failed to get callbacks:",e),e}},eQ=async(e,t)=>{try{let o=a?"".concat(a,"/config/update"):"/config/update",r=await fetch(o,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({...t})});if(!r.ok){let e=await r.text();throw l(e),Error("Network response was not ok")}return await r.json()}catch(e){throw console.error("Failed to set callbacks:",e),e}},e0=async e=>{try{let t=a?"".concat(a,"/health"):"/health",o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error("Network response was not ok")}return await o.json()}catch(e){throw console.error("Failed to call /health:",e),e}},e1=async(e,t)=>{try{let o=a?"".concat(a,"/health?model=").concat(encodeURIComponent(t)):"/health?model=".concat(encodeURIComponent(t)),r=await fetch(o,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!r.ok){let e=await r.text();throw Error(e||"Network response was not ok")}return await r.json()}catch(e){throw console.error("Failed to call /health for model ".concat(t,":"),e),e}},e2=async e=>{try{let t=a?"".concat(a,"/cache/ping"):"/cache/ping",o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error(e)}return await o.json()}catch(e){throw console.error("Failed to call /cache/ping:",e),e}},e3=async e=>{try{let t=a?"".concat(a,"/health/latest"):"/health/latest",o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error(e)}return await o.json()}catch(e){throw console.error("Failed to call /health/latest:",e),e}},e4=async e=>{try{console.log("Getting proxy UI settings"),console.log("proxyBaseUrl in getProxyUISettings:",a);let t=a?"".concat(a,"/sso/get/ui_settings"):"/sso/get/ui_settings",o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok)throw await o.text(),Error("Network response was not ok");return await o.json()}catch(e){throw console.error("Failed to get callbacks:",e),e}},e5=async e=>{try{let t=a?"".concat(a,"/v2/guardrails/list"):"/v2/guardrails/list",o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error("Network response was not ok")}return await o.json()}catch(e){throw console.error("Failed to get guardrails list:",e),e}},e6=async(e,t)=>{try{let o=a?"".concat(a,"/guardrails"):"/guardrails",r=await fetch(o,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({guardrail:t})});if(!r.ok){let e=await r.text();throw l(e),Error(e)}let n=await r.json();return console.log("Create guardrail response:",n),n}catch(e){throw console.error("Failed to create guardrail:",e),e}},e8=async(e,t,o)=>{try{let r=a?"".concat(a,"/spend/logs/ui/").concat(t,"?start_date=").concat(encodeURIComponent(o)):"/spend/logs/ui/".concat(t,"?start_date=").concat(encodeURIComponent(o));console.log("Fetching log details from:",r);let n=await fetch(r,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!n.ok){let e=await n.text();throw l(e),Error("Network response was not ok")}let c=await n.json();return console.log("Fetched log details:",c),c}catch(e){throw console.error("Failed to fetch log details:",e),e}},e9=async e=>{try{let t=a?"".concat(a,"/get/internal_user_settings"):"/get/internal_user_settings";console.log("Fetching SSO settings from:",t);let o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error("Network response was not ok")}let r=await o.json();return console.log("Fetched SSO settings:",r),r}catch(e){throw console.error("Failed to fetch SSO settings:",e),e}},e7=async(e,t)=>{try{let o=a?"".concat(a,"/update/internal_user_settings"):"/update/internal_user_settings";console.log("Updating internal user settings:",t);let n=await fetch(o,{method:"PATCH",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify(t)});if(!n.ok){let e=await n.text();throw l(e),Error("Network response was not ok")}let c=await n.json();return console.log("Updated internal user settings:",c),r.ZP.success("Internal user settings updated successfully"),c}catch(e){throw console.error("Failed to update internal user settings:",e),e}},te=async e=>{try{let t=a?"".concat(a,"/v1/mcp/server"):"/v1/mcp/server";console.log("Fetching MCP servers from:",t);let o=await fetch(t,{method:s.GET,headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error("Network response was not ok")}let r=await o.json();return console.log("Fetched MCP servers:",r),r}catch(e){throw console.error("Failed to fetch MCP servers:",e),e}},tt=async(e,t)=>{try{console.log("Form Values in createMCPServer:",t);let o=a?"".concat(a,"/v1/mcp/server"):"/v1/mcp/server",r=await fetch(o,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({...t})});if(!r.ok){let e=await r.text();throw l(e),console.error("Error response from the server:",e),Error("Network response was not ok")}let n=await r.json();return console.log("API Response:",n),n}catch(e){throw console.error("Failed to create key:",e),e}},to=async(e,t)=>{try{let o=a?"".concat(a,"/v1/mcp/server"):"/v1/mcp/server",r=await fetch(o,{method:"PUT",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify(t)});if(!r.ok){let e=await r.text();throw l(e),Error("Network response was not ok")}return await r.json()}catch(e){throw console.error("Failed to update MCP server:",e),e}},tr=async(e,t)=>{try{let o=(a?"".concat(a):"")+"/v1/mcp/server/".concat(t);console.log("in deleteMCPServer:",t);let r=await fetch(o,{method:s.DELETE,headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!r.ok){let e=await r.text();throw l(e),Error("Network response was not ok")}}catch(e){throw console.error("Failed to delete key:",e),e}},ta=async(e,t)=>{try{let o=a?"".concat(a,"/mcp-rest/tools/list?server_id=").concat(t):"/mcp-rest/tools/list?server_id=".concat(t);console.log("Fetching MCP tools from:",o);let r=await fetch(o,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!r.ok){let e=await r.text();throw l(e),Error("Network response was not ok")}let n=await r.json();return console.log("Fetched MCP tools:",n),n}catch(e){throw console.error("Failed to fetch MCP tools:",e),e}},tn=async(e,t,o)=>{try{let r=a?"".concat(a,"/mcp-rest/tools/call"):"/mcp-rest/tools/call";console.log("Calling MCP tool:",t,"with arguments:",o);let n=await fetch(r,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({name:t,arguments:o})});if(!n.ok){let e=await n.text();throw l(e),Error("Network response was not ok")}let c=await n.json();return console.log("MCP tool call response:",c),c}catch(e){throw console.error("Failed to call MCP tool:",e),e}},tc=async(e,t)=>{try{let o=a?"".concat(a,"/tag/new"):"/tag/new",r=await fetch(o,{method:"POST",headers:{"Content-Type":"application/json",Authorization:"Bearer ".concat(e)},body:JSON.stringify(t)});if(!r.ok){let e=await r.text();await l(e);return}return await r.json()}catch(e){throw console.error("Error creating tag:",e),e}},ts=async(e,t)=>{try{let o=a?"".concat(a,"/tag/update"):"/tag/update",r=await fetch(o,{method:"POST",headers:{"Content-Type":"application/json",Authorization:"Bearer ".concat(e)},body:JSON.stringify(t)});if(!r.ok){let e=await r.text();await l(e);return}return await r.json()}catch(e){throw console.error("Error updating tag:",e),e}},ti=async(e,t)=>{try{let o=a?"".concat(a,"/tag/info"):"/tag/info",r=await fetch(o,{method:"POST",headers:{"Content-Type":"application/json",Authorization:"Bearer ".concat(e)},body:JSON.stringify({names:t})});if(!r.ok){let e=await r.text();return await l(e),{}}return await r.json()}catch(e){throw console.error("Error getting tag info:",e),e}},tl=async e=>{try{let t=a?"".concat(a,"/tag/list"):"/tag/list",o=await fetch(t,{method:"GET",headers:{Authorization:"Bearer ".concat(e)}});if(!o.ok){let e=await o.text();return await l(e),{}}return await o.json()}catch(e){throw console.error("Error listing tags:",e),e}},td=async(e,t)=>{try{let o=a?"".concat(a,"/tag/delete"):"/tag/delete",r=await fetch(o,{method:"POST",headers:{"Content-Type":"application/json",Authorization:"Bearer ".concat(e)},body:JSON.stringify({name:t})});if(!r.ok){let e=await r.text();await l(e);return}return await r.json()}catch(e){throw console.error("Error deleting tag:",e),e}},th=async e=>{try{let t=a?"".concat(a,"/get/default_team_settings"):"/get/default_team_settings";console.log("Fetching default team settings from:",t);let o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error("Network response was not ok")}let r=await o.json();return console.log("Fetched default team settings:",r),r}catch(e){throw console.error("Failed to fetch default team settings:",e),e}},tp=async(e,t)=>{try{let o=a?"".concat(a,"/update/default_team_settings"):"/update/default_team_settings";console.log("Updating default team settings:",t);let n=await fetch(o,{method:"PATCH",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify(t)});if(!n.ok){let e=await n.text();throw l(e),Error("Network response was not ok")}let c=await n.json();return console.log("Updated default team settings:",c),r.ZP.success("Default team settings updated successfully"),c}catch(e){throw console.error("Failed to update default team settings:",e),e}},tw=async(e,t)=>{try{let o=a?"".concat(a,"/team/permissions_list?team_id=").concat(t):"/team/permissions_list?team_id=".concat(t),r=await fetch(o,{method:"GET",headers:{"Content-Type":"application/json",Authorization:"Bearer ".concat(e)}});if(!r.ok){let e=await r.text();throw l(e),Error("Network response was not ok")}let n=await r.json();return console.log("Team permissions response:",n),n}catch(e){throw console.error("Failed to get team permissions:",e),e}},tu=async(e,t,o)=>{try{let r=a?"".concat(a,"/team/permissions_update"):"/team/permissions_update",n=await fetch(r,{method:"POST",headers:{"Content-Type":"application/json",Authorization:"Bearer ".concat(e)},body:JSON.stringify({team_id:t,team_member_permissions:o})});if(!n.ok){let e=await n.text();throw l(e),Error("Network response was not ok")}let c=await n.json();return console.log("Team permissions response:",c),c}catch(e){throw console.error("Failed to update team permissions:",e),e}},tg=async(e,t)=>{try{let o=a?"".concat(a,"/spend/logs/session/ui?session_id=").concat(encodeURIComponent(t)):"/spend/logs/session/ui?session_id=".concat(encodeURIComponent(t)),r=await fetch(o,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!r.ok){let e=await r.text();throw l(e),Error("Network response was not ok")}return await r.json()}catch(e){throw console.error("Failed to fetch session logs:",e),e}},tf=async(e,t)=>{try{let o=a?"".concat(a,"/vector_store/new"):"/vector_store/new",r=await fetch(o,{method:"POST",headers:{"Content-Type":"application/json",Authorization:"Bearer ".concat(e)},body:JSON.stringify(t)});if(!r.ok){let e=await r.json();throw Error(e.detail||"Failed to create vector store")}return await r.json()}catch(e){throw console.error("Error creating vector store:",e),e}},ty=async function(e){arguments.length>1&&void 0!==arguments[1]&&arguments[1],arguments.length>2&&void 0!==arguments[2]&&arguments[2];try{let t=a?"".concat(a,"/vector_store/list"):"/vector_store/list",o=await fetch(t,{method:"GET",headers:{"Content-Type":"application/json",Authorization:"Bearer ".concat(e)}});if(!o.ok){let e=await o.json();throw Error(e.detail||"Failed to list vector stores")}return await o.json()}catch(e){throw console.error("Error listing vector stores:",e),e}},tm=async(e,t)=>{try{let o=a?"".concat(a,"/vector_store/delete"):"/vector_store/delete",r=await fetch(o,{method:"POST",headers:{"Content-Type":"application/json",Authorization:"Bearer ".concat(e)},body:JSON.stringify({vector_store_id:t})});if(!r.ok){let e=await r.json();throw Error(e.detail||"Failed to delete vector store")}return await r.json()}catch(e){throw console.error("Error deleting vector store:",e),e}},tk=async e=>{try{let t=a?"".concat(a,"/email/event_settings"):"/email/event_settings",o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error("Failed to get email event settings")}let r=await o.json();return console.log("Email event settings response:",r),r}catch(e){throw console.error("Failed to get email event settings:",e),e}},t_=async(e,t)=>{try{let o=a?"".concat(a,"/email/event_settings"):"/email/event_settings",r=await fetch(o,{method:"PATCH",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify(t)});if(!r.ok){let e=await r.text();throw l(e),Error("Failed to update email event settings")}let n=await r.json();return console.log("Update email event settings response:",n),n}catch(e){throw console.error("Failed to update email event settings:",e),e}},tT=async e=>{try{let t=a?"".concat(a,"/email/event_settings/reset"):"/email/event_settings/reset",o=await fetch(t,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error("Failed to reset email event settings")}let r=await o.json();return console.log("Reset email event settings response:",r),r}catch(e){throw console.error("Failed to reset email event settings:",e),e}},tj=async(e,t)=>{try{let o=a?"".concat(a,"/guardrails/").concat(t):"/guardrails/".concat(t),r=await fetch(o,{method:"DELETE",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!r.ok){let e=await r.text();throw l(e),Error(e)}let n=await r.json();return console.log("Delete guardrail response:",n),n}catch(e){throw console.error("Failed to delete guardrail:",e),e}},tE=async e=>{try{let t=a?"".concat(a,"/guardrails/ui/add_guardrail_settings"):"/guardrails/ui/add_guardrail_settings",o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error("Failed to get guardrail UI settings")}let r=await o.json();return console.log("Guardrail UI settings response:",r),r}catch(e){throw console.error("Failed to get guardrail UI settings:",e),e}},tC=async e=>{try{let t=a?"".concat(a,"/guardrails/ui/provider_specific_params"):"/guardrails/ui/provider_specific_params",o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error("Failed to get guardrail provider specific parameters")}let r=await o.json();return console.log("Guardrail provider specific params response:",r),r}catch(e){throw console.error("Failed to get guardrail provider specific parameters:",e),e}},tS=async(e,t)=>{try{let o=a?"".concat(a,"/guardrails/").concat(t,"/info"):"/guardrails/".concat(t,"/info"),r=await fetch(o,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!r.ok){let e=await r.text();throw l(e),Error("Failed to get guardrail info")}let n=await r.json();return console.log("Guardrail info response:",n),n}catch(e){throw console.error("Failed to get guardrail info:",e),e}},tN=async(e,t,o)=>{try{let r=a?"".concat(a,"/guardrails/").concat(t):"/guardrails/".concat(t),n=await fetch(r,{method:"PATCH",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify(o)});if(!n.ok){let e=await n.text();throw l(e),Error("Failed to update guardrail")}let c=await n.json();return console.log("Update guardrail response:",c),c}catch(e){throw console.error("Failed to update guardrail:",e),e}},tv=async e=>{try{let t=a?"".concat(a,"/get/sso_settings"):"/get/sso_settings";console.log("Fetching SSO configuration from:",t);let o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error("Network response was not ok")}let r=await o.json();return console.log("Fetched SSO configuration:",r),r}catch(e){throw console.error("Failed to fetch SSO configuration:",e),e}},tF=async(e,t)=>{try{let o=a?"".concat(a,"/update/sso_settings"):"/update/sso_settings";console.log("Updating SSO configuration:",t);let r=await fetch(o,{method:"PATCH",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify(t)});if(!r.ok){let e=await r.text();throw l(e),Error("Network response was not ok")}let n=await r.json();return console.log("Updated SSO configuration:",n),n}catch(e){throw console.error("Failed to update SSO configuration:",e),e}},tb=async(e,t,o,r,n)=>{try{let t=a?"".concat(a,"/audit"):"/audit",o=new URLSearchParams;r&&o.append("page",r.toString()),n&&o.append("page_size",n.toString());let c=o.toString();c&&(t+="?".concat(c));let s=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!s.ok){let e=await s.text();throw l(e),Error("Network response was not ok")}return await s.json()}catch(e){throw console.error("Failed to fetch audit logs:",e),e}},tx=async e=>{try{let t=a?"".concat(a,"/user/available_users"):"/user/available_users",o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e)}});if(!o.ok){if(404===o.status)return null;let e=await o.text();throw l(e),Error("Network response was not ok")}return await o.json()}catch(e){throw console.error("Failed to fetch remaining users:",e),e}},tO=async(e,t,o)=>{try{let n=a?"".concat(a,"/config/pass_through_endpoint/").concat(encodeURIComponent(t)):"/config/pass_through_endpoint/".concat(encodeURIComponent(t)),c=await fetch(n,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify(o)});if(!c.ok){let e=await c.text();throw l(e),Error("Network response was not ok")}let s=await c.json();return r.ZP.success("Pass through endpoint updated successfully"),s}catch(e){throw console.error("Failed to update pass through endpoint:",e),e}},tB=async(e,t)=>{try{let o=a?"".concat(a,"/config/callback/delete"):"/config/callback/delete",r=await fetch(o,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({callback_name:t})});if(!r.ok){let e=await r.text();throw l(e),Error("Network response was not ok")}return await r.json()}catch(e){throw console.error("Failed to delete specific callback:",e),e}}}}]); \ No newline at end of file +"use strict";(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[250],{19250:function(e,t,o){o.d(t,{$D:function(){return eJ},$I:function(){return Q},$o:function(){return tF},AZ:function(){return X},Au:function(){return e_},BL:function(){return eZ},Br:function(){return b},Bw:function(){return tx},E9:function(){return eX},EB:function(){return th},EG:function(){return eW},EY:function(){return e0},Eb:function(){return N},FC:function(){return eh},Gh:function(){return eG},H1:function(){return I},H2:function(){return a},Hx:function(){return ej},I1:function(){return S},It:function(){return B},J$:function(){return ec},JO:function(){return O},K8:function(){return h},K_:function(){return eQ},Ko:function(){return tN},LY:function(){return eV},Lp:function(){return eR},MO:function(){return p},Mx:function(){return tf},N3:function(){return eb},N8:function(){return er},NL:function(){return e9},NV:function(){return m},Nc:function(){return eP},Nz:function(){return e1},O3:function(){return eD},OD:function(){return eC},OU:function(){return eu},Of:function(){return F},Og:function(){return y},Ou:function(){return ty},Ov:function(){return C},Oz:function(){return tB},PC:function(){return e6},PT:function(){return K},PY:function(){return tv},Pj:function(){return tt},Pv:function(){return tm},Qg:function(){return ex},RQ:function(){return j},Rg:function(){return et},Sb:function(){return ez},So:function(){return ea},TF:function(){return tu},Tj:function(){return e2},Tx:function(){return tO},U8:function(){return te},UM:function(){return tl},VA:function(){return A},Vt:function(){return eY},W_:function(){return M},X:function(){return es},XB:function(){return tg},XO:function(){return T},Xd:function(){return eS},Xm:function(){return x},YU:function(){return eH},Yi:function(){return tj},Yo:function(){return U},Z9:function(){return V},Zr:function(){return k},a6:function(){return P},aC:function(){return tw},ao:function(){return eK},b1:function(){return ew},cq:function(){return R},cu:function(){return eI},e2:function(){return eT},eH:function(){return W},eW:function(){return tC},eZ:function(){return eB},fE:function(){return td},fP:function(){return eo},fk:function(){return tS},g:function(){return e4},gX:function(){return eO},gl:function(){return to},h3:function(){return ed},hT:function(){return ev},hy:function(){return f},ix:function(){return Y},j2:function(){return ei},jA:function(){return e$},jE:function(){return eM},jr:function(){return tT},kK:function(){return g},kn:function(){return $},lP:function(){return w},lU:function(){return ta},lg:function(){return eN},mC:function(){return ti},mR:function(){return en},mY:function(){return tc},m_:function(){return D},mp:function(){return eq},n$:function(){return em},n9:function(){return ts},nJ:function(){return tb},nd:function(){return e7},o6:function(){return ee},oC:function(){return eF},ol:function(){return L},pf:function(){return eL},pu:function(){return tk},qI:function(){return _},qW:function(){return t_},qd:function(){return tE},qk:function(){return e8},qm:function(){return u},r1:function(){return tp},r6:function(){return G},rs:function(){return v},s0:function(){return Z},sN:function(){return eA},t$:function(){return J},t0:function(){return eE},t3:function(){return e5},tB:function(){return tn},tN:function(){return ep},u5:function(){return el},v9:function(){return ek},vh:function(){return eU},wX:function(){return E},wd:function(){return eg},xA:function(){return ey},xO:function(){return e3},xX:function(){return z},xZ:function(){return tr},zX:function(){return c},zg:function(){return ef}});var r=o(41021);let a=null;console.log=function(){};let n=function(e){let t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:null,o=window.location.origin,r=t||o;console.log("proxyBaseUrl:",a),console.log("serverRootPath:",e),e.length>0&&!r.endsWith(e)&&"/"!=e&&(r+=e,a=r),console.log("Updated proxyBaseUrl:",a)},c=()=>a||window.location.origin,s={GET:"GET",DELETE:"DELETE"},i=0,l=async e=>{let t=Date.now();t-i>6e4?(e.includes("Authentication Error - Expired Key")&&(r.ZP.info("UI Session Expired. Logging out."),i=t,document.cookie="token=; expires=Thu, 01 Jan 1970 00:00:00 UTC; path=/;",window.location.href=window.location.pathname),i=t):console.log("Error suppressed to prevent spam:",e)},d="Authorization";function h(){let e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:"Authorization";console.log("setGlobalLitellmHeaderName: ".concat(e)),d=e}let p=async()=>{console.log("Getting UI config");let e=await fetch("/.well-known/litellm-ui-config"),t=await e.json();return console.log("jsonData in getUiConfig:",t),n(t.server_root_path,t.proxy_base_url),t},w=async()=>{let e=a?"".concat(a,"/openapi.json"):"/openapi.json",t=await fetch(e);return await t.json()},u=async e=>{try{let t=a?"".concat(a,"/get/litellm_model_cost_map"):"/get/litellm_model_cost_map",o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}}),r=await o.json();return console.log("received litellm model cost data: ".concat(r)),r}catch(e){throw console.error("Failed to get model cost map:",e),e}},g=async(e,t)=>{try{let o=a?"".concat(a,"/model/new"):"/model/new",n=await fetch(o,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({...t})});if(!n.ok){let e=await n.text()||"Network response was not ok";throw r.ZP.error(e),Error(e)}let c=await n.json();return console.log("API Response:",c),r.ZP.destroy(),r.ZP.success("Model ".concat(t.model_name," created successfully"),2),c}catch(e){throw console.error("Failed to create key:",e),e}},f=async e=>{try{let t=a?"".concat(a,"/model/settings"):"/model/settings",o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error("Network response was not ok")}return await o.json()}catch(e){console.error("Failed to get model settings:",e)}},y=async(e,t)=>{console.log("model_id in model delete call: ".concat(t));try{let o=a?"".concat(a,"/model/delete"):"/model/delete",r=await fetch(o,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({id:t})});if(!r.ok){let e=await r.text();throw l(e),console.error("Error response from the server:",e),Error("Network response was not ok")}let n=await r.json();return console.log("API Response:",n),n}catch(e){throw console.error("Failed to create key:",e),e}},m=async(e,t)=>{if(console.log("budget_id in budget delete call: ".concat(t)),null!=e)try{let o=a?"".concat(a,"/budget/delete"):"/budget/delete",r=await fetch(o,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({id:t})});if(!r.ok){let e=await r.text();throw l(e),console.error("Error response from the server:",e),Error("Network response was not ok")}let n=await r.json();return console.log("API Response:",n),n}catch(e){throw console.error("Failed to create key:",e),e}},k=async(e,t)=>{try{console.log("Form Values in budgetCreateCall:",t),console.log("Form Values after check:",t);let o=a?"".concat(a,"/budget/new"):"/budget/new",r=await fetch(o,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({...t})});if(!r.ok){let e=await r.text();throw l(e),console.error("Error response from the server:",e),Error("Network response was not ok")}let n=await r.json();return console.log("API Response:",n),n}catch(e){throw console.error("Failed to create key:",e),e}},_=async(e,t)=>{try{console.log("Form Values in budgetUpdateCall:",t),console.log("Form Values after check:",t);let o=a?"".concat(a,"/budget/update"):"/budget/update",r=await fetch(o,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({...t})});if(!r.ok){let e=await r.text();throw l(e),console.error("Error response from the server:",e),Error("Network response was not ok")}let n=await r.json();return console.log("API Response:",n),n}catch(e){throw console.error("Failed to create key:",e),e}},T=async(e,t)=>{try{let o=a?"".concat(a,"/invitation/new"):"/invitation/new",r=await fetch(o,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({user_id:t})});if(!r.ok){let e=await r.text();throw l(e),console.error("Error response from the server:",e),Error("Network response was not ok")}let n=await r.json();return console.log("API Response:",n),n}catch(e){throw console.error("Failed to create key:",e),e}},j=async e=>{try{let t=a?"".concat(a,"/alerting/settings"):"/alerting/settings",o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error("Network response was not ok")}return await o.json()}catch(e){throw console.error("Failed to get callbacks:",e),e}},E=async(e,t,o)=>{try{if(console.log("Form Values in keyCreateCall:",o),o.description&&(o.metadata||(o.metadata={}),o.metadata.description=o.description,delete o.description,o.metadata=JSON.stringify(o.metadata)),o.metadata){console.log("formValues.metadata:",o.metadata);try{o.metadata=JSON.parse(o.metadata)}catch(e){throw Error("Failed to parse metadata: "+e)}}console.log("Form Values after check:",o);let r=a?"".concat(a,"/key/generate"):"/key/generate",n=await fetch(r,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({user_id:t,...o})});if(!n.ok){let e=await n.text();throw l(e),console.error("Error response from the server:",e),Error(e)}let c=await n.json();return console.log("API Response:",c),c}catch(e){throw console.error("Failed to create key:",e),e}},C=async(e,t,o)=>{try{if(console.log("Form Values in keyCreateCall:",o),o.description&&(o.metadata||(o.metadata={}),o.metadata.description=o.description,delete o.description,o.metadata=JSON.stringify(o.metadata)),o.auto_create_key=!1,o.metadata){console.log("formValues.metadata:",o.metadata);try{o.metadata=JSON.parse(o.metadata)}catch(e){throw Error("Failed to parse metadata: "+e)}}console.log("Form Values after check:",o);let r=a?"".concat(a,"/user/new"):"/user/new",n=await fetch(r,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({user_id:t,...o})});if(!n.ok){let e=await n.text();throw l(e),console.error("Error response from the server:",e),Error(e)}let c=await n.json();return console.log("API Response:",c),c}catch(e){throw console.error("Failed to create key:",e),e}},S=async(e,t)=>{try{let o=a?"".concat(a,"/key/delete"):"/key/delete";console.log("in keyDeleteCall:",t);let r=await fetch(o,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({keys:[t]})});if(!r.ok){let e=await r.text();throw l(e),Error("Network response was not ok")}let n=await r.json();return console.log(n),n}catch(e){throw console.error("Failed to create key:",e),e}},N=async(e,t)=>{try{let o=a?"".concat(a,"/user/delete"):"/user/delete";console.log("in userDeleteCall:",t);let r=await fetch(o,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({user_ids:t})});if(!r.ok){let e=await r.text();throw l(e),Error("Network response was not ok")}let n=await r.json();return console.log(n),n}catch(e){throw console.error("Failed to delete user(s):",e),e}},v=async(e,t)=>{try{let o=a?"".concat(a,"/team/delete"):"/team/delete";console.log("in teamDeleteCall:",t);let r=await fetch(o,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({team_ids:[t]})});if(!r.ok){let e=await r.text();throw l(e),Error("Network response was not ok")}let n=await r.json();return console.log(n),n}catch(e){throw console.error("Failed to delete key:",e),e}},F=async function(e){let t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:null,o=arguments.length>2&&void 0!==arguments[2]?arguments[2]:null,r=arguments.length>3&&void 0!==arguments[3]?arguments[3]:null,n=arguments.length>4&&void 0!==arguments[4]?arguments[4]:null,c=arguments.length>5&&void 0!==arguments[5]?arguments[5]:null,s=arguments.length>6&&void 0!==arguments[6]?arguments[6]:null,i=arguments.length>7&&void 0!==arguments[7]?arguments[7]:null,h=arguments.length>8&&void 0!==arguments[8]?arguments[8]:null,p=arguments.length>9&&void 0!==arguments[9]?arguments[9]:null;try{let w=a?"".concat(a,"/user/list"):"/user/list";console.log("in userListCall");let u=new URLSearchParams;if(t&&t.length>0){let e=t.join(",");u.append("user_ids",e)}o&&u.append("page",o.toString()),r&&u.append("page_size",r.toString()),n&&u.append("user_email",n),c&&u.append("role",c),s&&u.append("team",s),i&&u.append("sso_user_ids",i),h&&u.append("sort_by",h),p&&u.append("sort_order",p);let g=u.toString();g&&(w+="?".concat(g));let f=await fetch(w,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!f.ok){let e=await f.text();throw l(e),Error("Network response was not ok")}let y=await f.json();return console.log("/user/list API Response:",y),y}catch(e){throw console.error("Failed to create key:",e),e}},b=async function(e,t,o){let r=arguments.length>3&&void 0!==arguments[3]&&arguments[3],n=arguments.length>4?arguments[4]:void 0,c=arguments.length>5?arguments[5]:void 0,s=arguments.length>6&&void 0!==arguments[6]&&arguments[6];console.log("userInfoCall: ".concat(t,", ").concat(o,", ").concat(r,", ").concat(n,", ").concat(c,", ").concat(s));try{let i;if(r){i=a?"".concat(a,"/user/list"):"/user/list";let e=new URLSearchParams;null!=n&&e.append("page",n.toString()),null!=c&&e.append("page_size",c.toString()),i+="?".concat(e.toString())}else i=a?"".concat(a,"/user/info"):"/user/info",("Admin"!==o&&"Admin Viewer"!==o||s)&&t&&(i+="?user_id=".concat(t));console.log("Requesting user data from:",i);let h=await fetch(i,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!h.ok){let e=await h.text();throw l(e),Error("Network response was not ok")}let p=await h.json();return console.log("API Response:",p),p}catch(e){throw console.error("Failed to fetch user data:",e),e}},x=async(e,t)=>{try{let o=a?"".concat(a,"/team/info"):"/team/info";t&&(o="".concat(o,"?team_id=").concat(t)),console.log("in teamInfoCall");let r=await fetch(o,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!r.ok){let e=await r.text();throw l(e),Error("Network response was not ok")}let n=await r.json();return console.log("API Response:",n),n}catch(e){throw console.error("Failed to create key:",e),e}},O=async function(e,t){let o=arguments.length>2&&void 0!==arguments[2]?arguments[2]:null,r=arguments.length>3&&void 0!==arguments[3]?arguments[3]:null,n=arguments.length>4&&void 0!==arguments[4]?arguments[4]:null;arguments.length>5&&void 0!==arguments[5]&&arguments[5],arguments.length>6&&void 0!==arguments[6]&&arguments[6],arguments.length>7&&void 0!==arguments[7]&&arguments[7],arguments.length>8&&void 0!==arguments[8]&&arguments[8];try{let c=a?"".concat(a,"/v2/team/list"):"/v2/team/list";console.log("in teamInfoCall");let s=new URLSearchParams;o&&s.append("user_id",o.toString()),t&&s.append("organization_id",t.toString()),r&&s.append("team_id",r.toString()),n&&s.append("team_alias",n.toString());let i=s.toString();i&&(c+="?".concat(i));let h=await fetch(c,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!h.ok){let e=await h.text();throw l(e),Error("Network response was not ok")}let p=await h.json();return console.log("/v2/team/list API Response:",p),p}catch(e){throw console.error("Failed to create key:",e),e}},B=async function(e,t){let o=arguments.length>2&&void 0!==arguments[2]?arguments[2]:null,r=arguments.length>3&&void 0!==arguments[3]?arguments[3]:null,n=arguments.length>4&&void 0!==arguments[4]?arguments[4]:null;try{let c=a?"".concat(a,"/team/list"):"/team/list";console.log("in teamInfoCall");let s=new URLSearchParams;o&&s.append("user_id",o.toString()),t&&s.append("organization_id",t.toString()),r&&s.append("team_id",r.toString()),n&&s.append("team_alias",n.toString());let i=s.toString();i&&(c+="?".concat(i));let h=await fetch(c,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!h.ok){let e=await h.text();throw l(e),Error("Network response was not ok")}let p=await h.json();return console.log("/team/list API Response:",p),p}catch(e){throw console.error("Failed to create key:",e),e}},P=async e=>{try{let t=a?"".concat(a,"/team/available"):"/team/available";console.log("in availableTeamListCall");let o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error("Network response was not ok")}let r=await o.json();return console.log("/team/available_teams API Response:",r),r}catch(e){throw e}},G=async e=>{try{let t=a?"".concat(a,"/organization/list"):"/organization/list",o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error("Network response was not ok")}return await o.json()}catch(e){throw console.error("Failed to create key:",e),e}},J=async(e,t)=>{try{let o=a?"".concat(a,"/organization/info"):"/organization/info";t&&(o="".concat(o,"?organization_id=").concat(t)),console.log("in teamInfoCall");let r=await fetch(o,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!r.ok){let e=await r.text();throw l(e),Error("Network response was not ok")}let n=await r.json();return console.log("API Response:",n),n}catch(e){throw console.error("Failed to create key:",e),e}},I=async(e,t)=>{try{if(console.log("Form Values in organizationCreateCall:",t),t.metadata){console.log("formValues.metadata:",t.metadata);try{t.metadata=JSON.parse(t.metadata)}catch(e){throw console.error("Failed to parse metadata:",e),Error("Failed to parse metadata: "+e)}}let o=a?"".concat(a,"/organization/new"):"/organization/new",r=await fetch(o,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({...t})});if(!r.ok){let e=await r.text();throw l(e),console.error("Error response from the server:",e),Error("Network response was not ok")}let n=await r.json();return console.log("API Response:",n),n}catch(e){throw console.error("Failed to create key:",e),e}},A=async(e,t)=>{try{console.log("Form Values in organizationUpdateCall:",t);let o=a?"".concat(a,"/organization/update"):"/organization/update",r=await fetch(o,{method:"PATCH",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({...t})});if(!r.ok){let e=await r.text();throw l(e),console.error("Error response from the server:",e),Error("Network response was not ok")}let n=await r.json();return console.log("Update Team Response:",n),n}catch(e){throw console.error("Failed to create key:",e),e}},R=async(e,t)=>{try{let o=a?"".concat(a,"/organization/delete"):"/organization/delete",r=await fetch(o,{method:"DELETE",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({organization_ids:[t]})});if(!r.ok){let e=await r.text();throw l(e),Error("Error deleting organization: ".concat(e))}return await r.json()}catch(e){throw console.error("Failed to delete organization:",e),e}},U=async(e,t)=>{try{let o=a?"".concat(a,"/utils/transform_request"):"/utils/transform_request",r=await fetch(o,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify(t)});if(!r.ok){let e=await r.text();throw l(e),Error("Network response was not ok")}return await r.json()}catch(e){throw console.error("Failed to create key:",e),e}},z=async function(e,t,o){let r=arguments.length>3&&void 0!==arguments[3]?arguments[3]:1;try{let n=a?"".concat(a,"/user/daily/activity"):"/user/daily/activity",c=new URLSearchParams;c.append("start_date",t.toISOString()),c.append("end_date",o.toISOString()),c.append("page_size","1000"),c.append("page",r.toString());let s=c.toString();s&&(n+="?".concat(s));let i=await fetch(n,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!i.ok){let e=await i.text();throw l(e),Error("Network response was not ok")}return await i.json()}catch(e){throw console.error("Failed to create key:",e),e}},V=async function(e,t,o){let r=arguments.length>3&&void 0!==arguments[3]?arguments[3]:1,n=arguments.length>4&&void 0!==arguments[4]?arguments[4]:null;try{let c=a?"".concat(a,"/tag/daily/activity"):"/tag/daily/activity",s=new URLSearchParams;s.append("start_date",t.toISOString()),s.append("end_date",o.toISOString()),s.append("page_size","1000"),s.append("page",r.toString()),n&&s.append("tags",n.join(","));let i=s.toString();i&&(c+="?".concat(i));let h=await fetch(c,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!h.ok){let e=await h.text();throw l(e),Error("Network response was not ok")}return await h.json()}catch(e){throw console.error("Failed to create key:",e),e}},L=async function(e,t,o){let r=arguments.length>3&&void 0!==arguments[3]?arguments[3]:1,n=arguments.length>4&&void 0!==arguments[4]?arguments[4]:null;try{let c=a?"".concat(a,"/team/daily/activity"):"/team/daily/activity",s=new URLSearchParams;s.append("start_date",t.toISOString()),s.append("end_date",o.toISOString()),s.append("page_size","1000"),s.append("page",r.toString()),n&&s.append("team_ids",n.join(",")),s.append("exclude_team_ids","litellm-dashboard");let i=s.toString();i&&(c+="?".concat(i));let h=await fetch(c,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!h.ok){let e=await h.text();throw l(e),Error("Network response was not ok")}return await h.json()}catch(e){throw console.error("Failed to create key:",e),e}},M=async e=>{try{let t=a?"".concat(a,"/onboarding/get_token"):"/onboarding/get_token";t+="?invite_link=".concat(e);let o=await fetch(t,{method:"GET",headers:{"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error("Network response was not ok")}return await o.json()}catch(e){throw console.error("Failed to create key:",e),e}},D=async(e,t,o,r)=>{let n=a?"".concat(a,"/onboarding/claim_token"):"/onboarding/claim_token";try{let a=await fetch(n,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({invitation_link:t,user_id:o,password:r})});if(!a.ok){let e=await a.text();throw l(e),Error("Network response was not ok")}let c=await a.json();return console.log(c),c}catch(e){throw console.error("Failed to delete key:",e),e}},Z=async(e,t,o)=>{try{let r=a?"".concat(a,"/key/").concat(t,"/regenerate"):"/key/".concat(t,"/regenerate"),n=await fetch(r,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify(o)});if(!n.ok){let e=await n.text();throw l(e),Error("Network response was not ok")}let c=await n.json();return console.log("Regenerate key Response:",c),c}catch(e){throw console.error("Failed to regenerate key:",e),e}},H=!1,q=null,X=async(e,t,o)=>{try{console.log("modelInfoCall:",e,t,o);let n=a?"".concat(a,"/v2/model/info"):"/v2/model/info",c=new URLSearchParams;c.append("include_team_models","true"),c.toString()&&(n+="?".concat(c.toString()));let s=await fetch(n,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!s.ok){let e=await s.text();throw e+="error shown=".concat(H),H||(e.includes("No model list passed")&&(e="No Models Exist. Click Add Model to get started."),r.ZP.info(e,10),H=!0,q&&clearTimeout(q),q=setTimeout(()=>{H=!1},1e4)),Error("Network response was not ok")}let i=await s.json();return console.log("modelInfoCall:",i),i}catch(e){throw console.error("Failed to create key:",e),e}},Y=async(e,t)=>{try{let o=a?"".concat(a,"/v1/model/info"):"/v1/model/info";o+="?litellm_model_id=".concat(t);let r=await fetch(o,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!r.ok)throw await r.text(),Error("Network response was not ok");let n=await r.json();return console.log("modelInfoV1Call:",n),n}catch(e){throw console.error("Failed to create key:",e),e}},$=async e=>{try{let t=a?"".concat(a,"/model_group/info"):"/model_group/info",o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok)throw await o.text(),Error("Network response was not ok");let r=await o.json();return console.log("modelHubCall:",r),r}catch(e){throw console.error("Failed to create key:",e),e}},K=async e=>{try{let t=a?"".concat(a,"/get/allowed_ips"):"/get/allowed_ips",o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw Error("Network response was not ok: ".concat(e))}let r=await o.json();return console.log("getAllowedIPs:",r),r.data}catch(e){throw console.error("Failed to get allowed IPs:",e),e}},W=async(e,t)=>{try{let o=a?"".concat(a,"/add/allowed_ip"):"/add/allowed_ip",r=await fetch(o,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({ip:t})});if(!r.ok){let e=await r.text();throw Error("Network response was not ok: ".concat(e))}let n=await r.json();return console.log("addAllowedIP:",n),n}catch(e){throw console.error("Failed to add allowed IP:",e),e}},Q=async(e,t)=>{try{let o=a?"".concat(a,"/delete/allowed_ip"):"/delete/allowed_ip",r=await fetch(o,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({ip:t})});if(!r.ok){let e=await r.text();throw Error("Network response was not ok: ".concat(e))}let n=await r.json();return console.log("deleteAllowedIP:",n),n}catch(e){throw console.error("Failed to delete allowed IP:",e),e}},ee=async(e,t,o,r,n,c,s,i)=>{try{let t=a?"".concat(a,"/model/metrics"):"/model/metrics";r&&(t="".concat(t,"?_selected_model_group=").concat(r,"&startTime=").concat(n,"&endTime=").concat(c,"&api_key=").concat(s,"&customer=").concat(i));let o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error("Network response was not ok")}return await o.json()}catch(e){throw console.error("Failed to create key:",e),e}},et=async(e,t,o,r)=>{try{let n=a?"".concat(a,"/model/streaming_metrics"):"/model/streaming_metrics";t&&(n="".concat(n,"?_selected_model_group=").concat(t,"&startTime=").concat(o,"&endTime=").concat(r));let c=await fetch(n,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!c.ok){let e=await c.text();throw l(e),Error("Network response was not ok")}return await c.json()}catch(e){throw console.error("Failed to create key:",e),e}},eo=async(e,t,o,r,n,c,s,i)=>{try{let t=a?"".concat(a,"/model/metrics/slow_responses"):"/model/metrics/slow_responses";r&&(t="".concat(t,"?_selected_model_group=").concat(r,"&startTime=").concat(n,"&endTime=").concat(c,"&api_key=").concat(s,"&customer=").concat(i));let o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error("Network response was not ok")}return await o.json()}catch(e){throw console.error("Failed to create key:",e),e}},er=async(e,t,o,r,n,c,s,i)=>{try{let t=a?"".concat(a,"/model/metrics/exceptions"):"/model/metrics/exceptions";r&&(t="".concat(t,"?_selected_model_group=").concat(r,"&startTime=").concat(n,"&endTime=").concat(c,"&api_key=").concat(s,"&customer=").concat(i));let o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error("Network response was not ok")}return await o.json()}catch(e){throw console.error("Failed to create key:",e),e}},ea=async function(e,t,o){let r=arguments.length>3&&void 0!==arguments[3]&&arguments[3],n=arguments.length>4&&void 0!==arguments[4]?arguments[4]:null,c=(arguments.length>5&&void 0!==arguments[5]&&arguments[5],arguments.length>6&&void 0!==arguments[6]&&arguments[6]);console.log("in /models calls, globalLitellmHeaderName",d);try{let t=a?"".concat(a,"/models"):"/models",o=new URLSearchParams;o.append("include_model_access_groups","True"),!0===r&&o.append("return_wildcard_routes","True"),!0===c&&o.append("only_model_access_groups","True"),n&&o.append("team_id",n.toString()),o.toString()&&(t+="?".concat(o.toString()));let s=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!s.ok){let e=await s.text();throw l(e),Error("Network response was not ok")}return await s.json()}catch(e){throw console.error("Failed to create key:",e),e}},en=async e=>{try{let t=a?"".concat(a,"/global/spend/teams"):"/global/spend/teams";console.log("in teamSpendLogsCall:",t);let o=await fetch("".concat(t),{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error("Network response was not ok")}let r=await o.json();return console.log(r),r}catch(e){throw console.error("Failed to create key:",e),e}},ec=async(e,t,o,r)=>{try{let n=a?"".concat(a,"/global/spend/tags"):"/global/spend/tags";t&&o&&(n="".concat(n,"?start_date=").concat(t,"&end_date=").concat(o)),r&&(n+="".concat(n,"&tags=").concat(r.join(","))),console.log("in tagsSpendLogsCall:",n);let c=await fetch("".concat(n),{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!c.ok)throw await c.text(),Error("Network response was not ok");let s=await c.json();return console.log(s),s}catch(e){throw console.error("Failed to create key:",e),e}},es=async e=>{try{let t=a?"".concat(a,"/global/spend/all_tag_names"):"/global/spend/all_tag_names";console.log("in global/spend/all_tag_names call",t);let o=await fetch("".concat(t),{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok)throw await o.text(),Error("Network response was not ok");let r=await o.json();return console.log(r),r}catch(e){throw console.error("Failed to create key:",e),e}},ei=async e=>{try{let t=a?"".concat(a,"/global/all_end_users"):"/global/all_end_users";console.log("in global/all_end_users call",t);let o=await fetch("".concat(t),{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok)throw await o.text(),Error("Network response was not ok");let r=await o.json();return console.log(r),r}catch(e){throw console.error("Failed to create key:",e),e}},el=async(e,t)=>{try{let o=a?"".concat(a,"/user/filter/ui"):"/user/filter/ui";t.get("user_email")&&(o+="?user_email=".concat(t.get("user_email"))),t.get("user_id")&&(o+="?user_id=".concat(t.get("user_id")));let r=await fetch(o,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!r.ok){let e=await r.text();throw l(e),Error("Network response was not ok")}return await r.json()}catch(e){throw console.error("Failed to create key:",e),e}},ed=async(e,t,o,r,n,c,s,i,h,p,w)=>{try{let u=a?"".concat(a,"/spend/logs/ui"):"/spend/logs/ui",g=new URLSearchParams;t&&g.append("api_key",t),o&&g.append("team_id",o),r&&g.append("request_id",r),n&&g.append("start_date",n),c&&g.append("end_date",c),s&&g.append("page",s.toString()),i&&g.append("page_size",i.toString()),h&&g.append("user_id",h),p&&g.append("status_filter",p),w&&g.append("model",w);let f=g.toString();f&&(u+="?".concat(f));let y=await fetch(u,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!y.ok){let e=await y.text();throw l(e),Error("Network response was not ok")}let m=await y.json();return console.log("Spend Logs Response:",m),m}catch(e){throw console.error("Failed to fetch spend logs:",e),e}},eh=async e=>{try{let t=a?"".concat(a,"/global/spend/logs"):"/global/spend/logs",o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error("Network response was not ok")}let r=await o.json();return console.log(r),r}catch(e){throw console.error("Failed to create key:",e),e}},ep=async e=>{try{let t=a?"".concat(a,"/global/spend/keys?limit=5"):"/global/spend/keys?limit=5",o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error("Network response was not ok")}let r=await o.json();return console.log(r),r}catch(e){throw console.error("Failed to create key:",e),e}},ew=async(e,t,o,r)=>{try{let n=a?"".concat(a,"/global/spend/end_users"):"/global/spend/end_users",c="";c=t?JSON.stringify({api_key:t,startTime:o,endTime:r}):JSON.stringify({startTime:o,endTime:r});let s={method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:c},i=await fetch(n,s);if(!i.ok){let e=await i.text();throw l(e),Error("Network response was not ok")}let h=await i.json();return console.log(h),h}catch(e){throw console.error("Failed to create key:",e),e}},eu=async(e,t,o,r)=>{try{let n=a?"".concat(a,"/global/spend/provider"):"/global/spend/provider";o&&r&&(n+="?start_date=".concat(o,"&end_date=").concat(r)),t&&(n+="&api_key=".concat(t));let c={method:"GET",headers:{[d]:"Bearer ".concat(e)}},s=await fetch(n,c);if(!s.ok){let e=await s.text();throw l(e),Error("Network response was not ok")}let i=await s.json();return console.log(i),i}catch(e){throw console.error("Failed to fetch spend data:",e),e}},eg=async(e,t,o)=>{try{let r=a?"".concat(a,"/global/activity"):"/global/activity";t&&o&&(r+="?start_date=".concat(t,"&end_date=").concat(o));let n={method:"GET",headers:{[d]:"Bearer ".concat(e)}},c=await fetch(r,n);if(!c.ok)throw await c.text(),Error("Network response was not ok");let s=await c.json();return console.log(s),s}catch(e){throw console.error("Failed to fetch spend data:",e),e}},ef=async(e,t,o)=>{try{let r=a?"".concat(a,"/global/activity/cache_hits"):"/global/activity/cache_hits";t&&o&&(r+="?start_date=".concat(t,"&end_date=").concat(o));let n={method:"GET",headers:{[d]:"Bearer ".concat(e)}},c=await fetch(r,n);if(!c.ok)throw await c.text(),Error("Network response was not ok");let s=await c.json();return console.log(s),s}catch(e){throw console.error("Failed to fetch spend data:",e),e}},ey=async(e,t,o)=>{try{let r=a?"".concat(a,"/global/activity/model"):"/global/activity/model";t&&o&&(r+="?start_date=".concat(t,"&end_date=").concat(o));let n={method:"GET",headers:{[d]:"Bearer ".concat(e)}},c=await fetch(r,n);if(!c.ok)throw await c.text(),Error("Network response was not ok");let s=await c.json();return console.log(s),s}catch(e){throw console.error("Failed to fetch spend data:",e),e}},em=async(e,t,o,r)=>{try{let n=a?"".concat(a,"/global/activity/exceptions"):"/global/activity/exceptions";t&&o&&(n+="?start_date=".concat(t,"&end_date=").concat(o)),r&&(n+="&model_group=".concat(r));let c={method:"GET",headers:{[d]:"Bearer ".concat(e)}},s=await fetch(n,c);if(!s.ok)throw await s.text(),Error("Network response was not ok");let i=await s.json();return console.log(i),i}catch(e){throw console.error("Failed to fetch spend data:",e),e}},ek=async(e,t,o,r)=>{try{let n=a?"".concat(a,"/global/activity/exceptions/deployment"):"/global/activity/exceptions/deployment";t&&o&&(n+="?start_date=".concat(t,"&end_date=").concat(o)),r&&(n+="&model_group=".concat(r));let c={method:"GET",headers:{[d]:"Bearer ".concat(e)}},s=await fetch(n,c);if(!s.ok)throw await s.text(),Error("Network response was not ok");let i=await s.json();return console.log(i),i}catch(e){throw console.error("Failed to fetch spend data:",e),e}},e_=async e=>{try{let t=a?"".concat(a,"/global/spend/models?limit=5"):"/global/spend/models?limit=5",o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error("Network response was not ok")}let r=await o.json();return console.log(r),r}catch(e){throw console.error("Failed to create key:",e),e}},eT=async(e,t)=>{try{let o=a?"".concat(a,"/v2/key/info"):"/v2/key/info",r=await fetch(o,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({keys:t})});if(!r.ok){let e=await r.text();if(e.includes("Invalid proxy server token passed"))throw Error("Invalid proxy server token passed");throw l(e),Error("Network response was not ok")}let n=await r.json();return console.log(n),n}catch(e){throw console.error("Failed to create key:",e),e}},ej=async(e,t,o)=>{try{console.log("Sending model connection test request:",JSON.stringify(t));let n=a?"".concat(a,"/health/test_connection"):"/health/test_connection",c=await fetch(n,{method:"POST",headers:{"Content-Type":"application/json",[d]:"Bearer ".concat(e)},body:JSON.stringify({litellm_params:t,mode:o})}),s=c.headers.get("content-type");if(!s||!s.includes("application/json")){let e=await c.text();throw console.error("Received non-JSON response:",e),Error("Received non-JSON response (".concat(c.status,": ").concat(c.statusText,"). Check network tab for details."))}let i=await c.json();if(!c.ok||"error"===i.status){if("error"===i.status);else{var r;return{status:"error",message:(null===(r=i.error)||void 0===r?void 0:r.message)||"Connection test failed: ".concat(c.status," ").concat(c.statusText)}}}return i}catch(e){throw console.error("Model connection test error:",e),e}},eE=async(e,t)=>{try{console.log("entering keyInfoV1Call");let o=a?"".concat(a,"/key/info"):"/key/info";o="".concat(o,"?key=").concat(t);let n=await fetch(o,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(console.log("response",n),!n.ok){let e=await n.text();l(e),r.ZP.error("Failed to fetch key info - "+e)}let c=await n.json();return console.log("data",c),c}catch(e){throw console.error("Failed to fetch key info:",e),e}},eC=async function(e,t,o,r,n,c,s,i){let h=arguments.length>8&&void 0!==arguments[8]?arguments[8]:null,p=arguments.length>9&&void 0!==arguments[9]?arguments[9]:null;try{let w=a?"".concat(a,"/key/list"):"/key/list";console.log("in keyListCall");let u=new URLSearchParams;o&&u.append("team_id",o.toString()),t&&u.append("organization_id",t.toString()),r&&u.append("key_alias",r),c&&u.append("key_hash",c),n&&u.append("user_id",n.toString()),s&&u.append("page",s.toString()),i&&u.append("size",i.toString()),h&&u.append("sort_by",h),p&&u.append("sort_order",p),u.append("return_full_object","true"),u.append("include_team_keys","true");let g=u.toString();g&&(w+="?".concat(g));let f=await fetch(w,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!f.ok){let e=await f.text();throw l(e),Error("Network response was not ok")}let y=await f.json();return console.log("/team/list API Response:",y),y}catch(e){throw console.error("Failed to create key:",e),e}},eS=async(e,t)=>{try{let o=a?"".concat(a,"/user/get_users?role=").concat(t):"/user/get_users?role=".concat(t);console.log("in userGetAllUsersCall:",o);let r=await fetch(o,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!r.ok){let e=await r.text();throw l(e),Error("Network response was not ok")}let n=await r.json();return console.log(n),n}catch(e){throw console.error("Failed to get requested models:",e),e}},eN=async e=>{try{let t=a?"".concat(a,"/user/available_roles"):"/user/available_roles",o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok)throw await o.text(),Error("Network response was not ok");let r=await o.json();return console.log("response from user/available_role",r),r}catch(e){throw e}},ev=async(e,t)=>{try{if(console.log("Form Values in teamCreateCall:",t),t.metadata){console.log("formValues.metadata:",t.metadata);try{t.metadata=JSON.parse(t.metadata)}catch(e){throw Error("Failed to parse metadata: "+e)}}let o=a?"".concat(a,"/team/new"):"/team/new",r=await fetch(o,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({...t})});if(!r.ok){let e=await r.text();throw l(e),console.error("Error response from the server:",e),Error("Network response was not ok")}let n=await r.json();return console.log("API Response:",n),n}catch(e){throw console.error("Failed to create key:",e),e}},eF=async(e,t)=>{try{if(console.log("Form Values in credentialCreateCall:",t),t.metadata){console.log("formValues.metadata:",t.metadata);try{t.metadata=JSON.parse(t.metadata)}catch(e){throw Error("Failed to parse metadata: "+e)}}let o=a?"".concat(a,"/credentials"):"/credentials",r=await fetch(o,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({...t})});if(!r.ok){let e=await r.text();throw l(e),console.error("Error response from the server:",e),Error("Network response was not ok")}let n=await r.json();return console.log("API Response:",n),n}catch(e){throw console.error("Failed to create key:",e),e}},eb=async e=>{try{let t=a?"".concat(a,"/credentials"):"/credentials";console.log("in credentialListCall");let o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error("Network response was not ok")}let r=await o.json();return console.log("/credentials API Response:",r),r}catch(e){throw console.error("Failed to create key:",e),e}},ex=async(e,t,o)=>{try{let r=a?"".concat(a,"/credentials"):"/credentials";t?r+="/by_name/".concat(t):o&&(r+="/by_model/".concat(o)),console.log("in credentialListCall");let n=await fetch(r,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!n.ok){let e=await n.text();throw l(e),Error("Network response was not ok")}let c=await n.json();return console.log("/credentials API Response:",c),c}catch(e){throw console.error("Failed to create key:",e),e}},eO=async(e,t)=>{try{let o=a?"".concat(a,"/credentials/").concat(t):"/credentials/".concat(t);console.log("in credentialDeleteCall:",t);let r=await fetch(o,{method:"DELETE",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!r.ok){let e=await r.text();throw l(e),Error("Network response was not ok")}let n=await r.json();return console.log(n),n}catch(e){throw console.error("Failed to delete key:",e),e}},eB=async(e,t,o)=>{try{if(console.log("Form Values in credentialUpdateCall:",o),o.metadata){console.log("formValues.metadata:",o.metadata);try{o.metadata=JSON.parse(o.metadata)}catch(e){throw Error("Failed to parse metadata: "+e)}}let r=a?"".concat(a,"/credentials/").concat(t):"/credentials/".concat(t),n=await fetch(r,{method:"PATCH",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({...o})});if(!n.ok){let e=await n.text();throw l(e),console.error("Error response from the server:",e),Error("Network response was not ok")}let c=await n.json();return console.log("API Response:",c),c}catch(e){throw console.error("Failed to create key:",e),e}},eP=async(e,t)=>{try{if(console.log("Form Values in keyUpdateCall:",t),t.model_tpm_limit){console.log("formValues.model_tpm_limit:",t.model_tpm_limit);try{t.model_tpm_limit=JSON.parse(t.model_tpm_limit)}catch(e){throw Error("Failed to parse model_tpm_limit: "+e)}}if(t.model_rpm_limit){console.log("formValues.model_rpm_limit:",t.model_rpm_limit);try{t.model_rpm_limit=JSON.parse(t.model_rpm_limit)}catch(e){throw Error("Failed to parse model_rpm_limit: "+e)}}let o=a?"".concat(a,"/key/update"):"/key/update",r=await fetch(o,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({...t})});if(!r.ok){let e=await r.text();throw l(e),console.error("Error response from the server:",e),Error("Network response was not ok")}let n=await r.json();return console.log("Update key Response:",n),n}catch(e){throw console.error("Failed to create key:",e),e}},eG=async(e,t)=>{try{console.log("Form Values in teamUpateCall:",t);let o=a?"".concat(a,"/team/update"):"/team/update",n=await fetch(o,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({...t})});if(!n.ok){let e=await n.text();throw l(e),console.error("Error response from the server:",e),r.ZP.error("Failed to update team settings: "+e),Error(e)}let c=await n.json();return console.log("Update Team Response:",c),c}catch(e){throw console.error("Failed to update team:",e),e}},eJ=async(e,t,o)=>{try{console.log("Form Values in modelUpateCall:",t);let r=a?"".concat(a,"/model/").concat(o,"/update"):"/model/".concat(o,"/update"),n=await fetch(r,{method:"PATCH",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({...t})});if(!n.ok){let e=await n.text();throw l(e),console.error("Error update from the server:",e),Error("Network response was not ok")}let c=await n.json();return console.log("Update model Response:",c),c}catch(e){throw console.error("Failed to update model:",e),e}},eI=async(e,t,o)=>{try{console.log("Form Values in teamMemberAddCall:",o);let n=a?"".concat(a,"/team/member_add"):"/team/member_add",c=await fetch(n,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({team_id:t,member:o})});if(!c.ok){var r;let e=await c.text(),t={};try{t=JSON.parse(e)}catch(t){console.warn("Failed to parse error body as JSON:",e)}let o=(null==t?void 0:null===(r=t.detail)||void 0===r?void 0:r.error)||"Failed to add team member",a=Error(o);throw a.raw=t,a}let s=await c.json();return console.log("API Response:",s),s}catch(e){throw console.error("Failed to create key:",e),e}},eA=async(e,t,o)=>{try{console.log("Form Values in teamMemberUpdateCall:",o);let n=a?"".concat(a,"/team/member_update"):"/team/member_update",c=await fetch(n,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({team_id:t,role:o.role,user_id:o.user_id})});if(!c.ok){var r;let e=await c.text(),t={};try{t=JSON.parse(e)}catch(t){console.warn("Failed to parse error body as JSON:",e)}let o=(null==t?void 0:null===(r=t.detail)||void 0===r?void 0:r.error)||"Failed to add team member",a=Error(o);throw a.raw=t,a}let s=await c.json();return console.log("API Response:",s),s}catch(e){throw console.error("Failed to update team member:",e),e}},eR=async(e,t,o)=>{try{console.log("Form Values in teamMemberAddCall:",o);let r=a?"".concat(a,"/team/member_delete"):"/team/member_delete",n=await fetch(r,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({team_id:t,...void 0!==o.user_email&&{user_email:o.user_email},...void 0!==o.user_id&&{user_id:o.user_id}})});if(!n.ok){let e=await n.text();throw l(e),console.error("Error response from the server:",e),Error("Network response was not ok")}let c=await n.json();return console.log("API Response:",c),c}catch(e){throw console.error("Failed to create key:",e),e}},eU=async(e,t,o)=>{try{console.log("Form Values in teamMemberAddCall:",o);let r=a?"".concat(a,"/organization/member_add"):"/organization/member_add",n=await fetch(r,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({organization_id:t,member:o})});if(!n.ok){let e=await n.text();throw l(e),console.error("Error response from the server:",e),Error(e)}let c=await n.json();return console.log("API Response:",c),c}catch(e){throw console.error("Failed to create organization member:",e),e}},ez=async(e,t,o)=>{try{console.log("Form Values in organizationMemberDeleteCall:",o);let r=a?"".concat(a,"/organization/member_delete"):"/organization/member_delete",n=await fetch(r,{method:"DELETE",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({organization_id:t,user_id:o})});if(!n.ok){let e=await n.text();throw l(e),console.error("Error response from the server:",e),Error("Network response was not ok")}let c=await n.json();return console.log("API Response:",c),c}catch(e){throw console.error("Failed to delete organization member:",e),e}},eV=async(e,t,o)=>{try{console.log("Form Values in organizationMemberUpdateCall:",o);let r=a?"".concat(a,"/organization/member_update"):"/organization/member_update",n=await fetch(r,{method:"PATCH",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({organization_id:t,...o})});if(!n.ok){let e=await n.text();throw l(e),console.error("Error response from the server:",e),Error("Network response was not ok")}let c=await n.json();return console.log("API Response:",c),c}catch(e){throw console.error("Failed to update organization member:",e),e}},eL=async(e,t,o)=>{try{console.log("Form Values in userUpdateUserCall:",t);let r=a?"".concat(a,"/user/update"):"/user/update",n={...t};null!==o&&(n.user_role=o),n=JSON.stringify(n);let c=await fetch(r,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:n});if(!c.ok){let e=await c.text();throw l(e),console.error("Error response from the server:",e),Error("Network response was not ok")}let s=await c.json();return console.log("API Response:",s),s}catch(e){throw console.error("Failed to create key:",e),e}},eM=async(e,t)=>{try{let o=a?"".concat(a,"/health/services?service=").concat(t):"/health/services?service=".concat(t);console.log("Checking Slack Budget Alerts service health");let n=await fetch(o,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!n.ok){let e=await n.text();throw l(e),Error(e)}let c=await n.json();return r.ZP.success("Test request to ".concat(t," made - check logs/alerts on ").concat(t," to verify")),c}catch(e){throw console.error("Failed to perform health check:",e),e}},eD=async e=>{try{let t=a?"".concat(a,"/budget/list"):"/budget/list",o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error("Network response was not ok")}return await o.json()}catch(e){throw console.error("Failed to get callbacks:",e),e}},eZ=async(e,t,o)=>{try{let t=a?"".concat(a,"/get/config/callbacks"):"/get/config/callbacks",o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error("Network response was not ok")}return await o.json()}catch(e){throw console.error("Failed to get callbacks:",e),e}},eH=async e=>{try{let t=a?"".concat(a,"/config/list?config_type=general_settings"):"/config/list?config_type=general_settings",o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error("Network response was not ok")}return await o.json()}catch(e){throw console.error("Failed to get callbacks:",e),e}},eq=async e=>{try{let t=a?"".concat(a,"/config/pass_through_endpoint"):"/config/pass_through_endpoint",o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error("Network response was not ok")}return await o.json()}catch(e){throw console.error("Failed to get callbacks:",e),e}},eX=async(e,t)=>{try{let o=a?"".concat(a,"/config/field/info?field_name=").concat(t):"/config/field/info?field_name=".concat(t),r=await fetch(o,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!r.ok)throw await r.text(),Error("Network response was not ok");return await r.json()}catch(e){throw console.error("Failed to set callbacks:",e),e}},eY=async(e,t)=>{try{let o=a?"".concat(a,"/config/pass_through_endpoint"):"/config/pass_through_endpoint",r=await fetch(o,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({...t})});if(!r.ok){let e=await r.text();throw l(e),Error("Network response was not ok")}return await r.json()}catch(e){throw console.error("Failed to set callbacks:",e),e}},e$=async(e,t,o)=>{try{let n=a?"".concat(a,"/config/field/update"):"/config/field/update",c=await fetch(n,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({field_name:t,field_value:o,config_type:"general_settings"})});if(!c.ok){let e=await c.text();throw l(e),Error("Network response was not ok")}let s=await c.json();return r.ZP.success("Successfully updated value!"),s}catch(e){throw console.error("Failed to set callbacks:",e),e}},eK=async(e,t)=>{try{let o=a?"".concat(a,"/config/field/delete"):"/config/field/delete",n=await fetch(o,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({field_name:t,config_type:"general_settings"})});if(!n.ok){let e=await n.text();throw l(e),Error("Network response was not ok")}let c=await n.json();return r.ZP.success("Field reset on proxy"),c}catch(e){throw console.error("Failed to get callbacks:",e),e}},eW=async(e,t)=>{try{let o=a?"".concat(a,"/config/pass_through_endpoint?endpoint_id=").concat(t):"/config/pass_through_endpoint".concat(t),r=await fetch(o,{method:"DELETE",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!r.ok){let e=await r.text();throw l(e),Error("Network response was not ok")}return await r.json()}catch(e){throw console.error("Failed to get callbacks:",e),e}},eQ=async(e,t)=>{try{let o=a?"".concat(a,"/config/update"):"/config/update",r=await fetch(o,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({...t})});if(!r.ok){let e=await r.text();throw l(e),Error("Network response was not ok")}return await r.json()}catch(e){throw console.error("Failed to set callbacks:",e),e}},e0=async e=>{try{let t=a?"".concat(a,"/health"):"/health",o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error("Network response was not ok")}return await o.json()}catch(e){throw console.error("Failed to call /health:",e),e}},e1=async(e,t)=>{try{let o=a?"".concat(a,"/health?model=").concat(encodeURIComponent(t)):"/health?model=".concat(encodeURIComponent(t)),r=await fetch(o,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!r.ok){let e=await r.text();throw Error(e||"Network response was not ok")}return await r.json()}catch(e){throw console.error("Failed to call /health for model ".concat(t,":"),e),e}},e2=async e=>{try{let t=a?"".concat(a,"/cache/ping"):"/cache/ping",o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error(e)}return await o.json()}catch(e){throw console.error("Failed to call /cache/ping:",e),e}},e3=async e=>{try{let t=a?"".concat(a,"/health/latest"):"/health/latest",o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error(e)}return await o.json()}catch(e){throw console.error("Failed to call /health/latest:",e),e}},e4=async e=>{try{console.log("Getting proxy UI settings"),console.log("proxyBaseUrl in getProxyUISettings:",a);let t=a?"".concat(a,"/sso/get/ui_settings"):"/sso/get/ui_settings",o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok)throw await o.text(),Error("Network response was not ok");return await o.json()}catch(e){throw console.error("Failed to get callbacks:",e),e}},e5=async e=>{try{let t=a?"".concat(a,"/v2/guardrails/list"):"/v2/guardrails/list",o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error("Network response was not ok")}return await o.json()}catch(e){throw console.error("Failed to get guardrails list:",e),e}},e6=async(e,t)=>{try{let o=a?"".concat(a,"/guardrails"):"/guardrails",r=await fetch(o,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({guardrail:t})});if(!r.ok){let e=await r.text();throw l(e),Error(e)}let n=await r.json();return console.log("Create guardrail response:",n),n}catch(e){throw console.error("Failed to create guardrail:",e),e}},e8=async(e,t,o)=>{try{let r=a?"".concat(a,"/spend/logs/ui/").concat(t,"?start_date=").concat(encodeURIComponent(o)):"/spend/logs/ui/".concat(t,"?start_date=").concat(encodeURIComponent(o));console.log("Fetching log details from:",r);let n=await fetch(r,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!n.ok){let e=await n.text();throw l(e),Error("Network response was not ok")}let c=await n.json();return console.log("Fetched log details:",c),c}catch(e){throw console.error("Failed to fetch log details:",e),e}},e9=async e=>{try{let t=a?"".concat(a,"/get/internal_user_settings"):"/get/internal_user_settings";console.log("Fetching SSO settings from:",t);let o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error("Network response was not ok")}let r=await o.json();return console.log("Fetched SSO settings:",r),r}catch(e){throw console.error("Failed to fetch SSO settings:",e),e}},e7=async(e,t)=>{try{let o=a?"".concat(a,"/update/internal_user_settings"):"/update/internal_user_settings";console.log("Updating internal user settings:",t);let n=await fetch(o,{method:"PATCH",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify(t)});if(!n.ok){let e=await n.text();throw l(e),Error("Network response was not ok")}let c=await n.json();return console.log("Updated internal user settings:",c),r.ZP.success("Internal user settings updated successfully"),c}catch(e){throw console.error("Failed to update internal user settings:",e),e}},te=async e=>{try{let t=a?"".concat(a,"/v1/mcp/server"):"/v1/mcp/server";console.log("Fetching MCP servers from:",t);let o=await fetch(t,{method:s.GET,headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error("Network response was not ok")}let r=await o.json();return console.log("Fetched MCP servers:",r),r}catch(e){throw console.error("Failed to fetch MCP servers:",e),e}},tt=async(e,t)=>{try{console.log("Form Values in createMCPServer:",t);let o=a?"".concat(a,"/v1/mcp/server"):"/v1/mcp/server",r=await fetch(o,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({...t})});if(!r.ok){let e=await r.text();throw l(e),console.error("Error response from the server:",e),Error("Network response was not ok")}let n=await r.json();return console.log("API Response:",n),n}catch(e){throw console.error("Failed to create key:",e),e}},to=async(e,t)=>{try{let o=a?"".concat(a,"/v1/mcp/server"):"/v1/mcp/server",r=await fetch(o,{method:"PUT",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify(t)});if(!r.ok){let e=await r.text();throw l(e),Error("Network response was not ok")}return await r.json()}catch(e){throw console.error("Failed to update MCP server:",e),e}},tr=async(e,t)=>{try{let o=(a?"".concat(a):"")+"/v1/mcp/server/".concat(t);console.log("in deleteMCPServer:",t);let r=await fetch(o,{method:s.DELETE,headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!r.ok){let e=await r.text();throw l(e),Error("Network response was not ok")}}catch(e){throw console.error("Failed to delete key:",e),e}},ta=async(e,t)=>{try{let o=a?"".concat(a,"/mcp-rest/tools/list?server_id=").concat(t):"/mcp-rest/tools/list?server_id=".concat(t);console.log("Fetching MCP tools from:",o);let r=await fetch(o,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!r.ok){let e=await r.text();throw l(e),Error("Network response was not ok")}let n=await r.json();return console.log("Fetched MCP tools:",n),n}catch(e){throw console.error("Failed to fetch MCP tools:",e),e}},tn=async(e,t,o)=>{try{let r=a?"".concat(a,"/mcp-rest/tools/call"):"/mcp-rest/tools/call";console.log("Calling MCP tool:",t,"with arguments:",o);let n=await fetch(r,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({name:t,arguments:o})});if(!n.ok){let e=await n.text();throw l(e),Error("Network response was not ok")}let c=await n.json();return console.log("MCP tool call response:",c),c}catch(e){throw console.error("Failed to call MCP tool:",e),e}},tc=async(e,t)=>{try{let o=a?"".concat(a,"/tag/new"):"/tag/new",r=await fetch(o,{method:"POST",headers:{"Content-Type":"application/json",Authorization:"Bearer ".concat(e)},body:JSON.stringify(t)});if(!r.ok){let e=await r.text();await l(e);return}return await r.json()}catch(e){throw console.error("Error creating tag:",e),e}},ts=async(e,t)=>{try{let o=a?"".concat(a,"/tag/update"):"/tag/update",r=await fetch(o,{method:"POST",headers:{"Content-Type":"application/json",Authorization:"Bearer ".concat(e)},body:JSON.stringify(t)});if(!r.ok){let e=await r.text();await l(e);return}return await r.json()}catch(e){throw console.error("Error updating tag:",e),e}},ti=async(e,t)=>{try{let o=a?"".concat(a,"/tag/info"):"/tag/info",r=await fetch(o,{method:"POST",headers:{"Content-Type":"application/json",Authorization:"Bearer ".concat(e)},body:JSON.stringify({names:t})});if(!r.ok){let e=await r.text();return await l(e),{}}return await r.json()}catch(e){throw console.error("Error getting tag info:",e),e}},tl=async e=>{try{let t=a?"".concat(a,"/tag/list"):"/tag/list",o=await fetch(t,{method:"GET",headers:{Authorization:"Bearer ".concat(e)}});if(!o.ok){let e=await o.text();return await l(e),{}}return await o.json()}catch(e){throw console.error("Error listing tags:",e),e}},td=async(e,t)=>{try{let o=a?"".concat(a,"/tag/delete"):"/tag/delete",r=await fetch(o,{method:"POST",headers:{"Content-Type":"application/json",Authorization:"Bearer ".concat(e)},body:JSON.stringify({name:t})});if(!r.ok){let e=await r.text();await l(e);return}return await r.json()}catch(e){throw console.error("Error deleting tag:",e),e}},th=async e=>{try{let t=a?"".concat(a,"/get/default_team_settings"):"/get/default_team_settings";console.log("Fetching default team settings from:",t);let o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error("Network response was not ok")}let r=await o.json();return console.log("Fetched default team settings:",r),r}catch(e){throw console.error("Failed to fetch default team settings:",e),e}},tp=async(e,t)=>{try{let o=a?"".concat(a,"/update/default_team_settings"):"/update/default_team_settings";console.log("Updating default team settings:",t);let n=await fetch(o,{method:"PATCH",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify(t)});if(!n.ok){let e=await n.text();throw l(e),Error("Network response was not ok")}let c=await n.json();return console.log("Updated default team settings:",c),r.ZP.success("Default team settings updated successfully"),c}catch(e){throw console.error("Failed to update default team settings:",e),e}},tw=async(e,t)=>{try{let o=a?"".concat(a,"/team/permissions_list?team_id=").concat(t):"/team/permissions_list?team_id=".concat(t),r=await fetch(o,{method:"GET",headers:{"Content-Type":"application/json",Authorization:"Bearer ".concat(e)}});if(!r.ok){let e=await r.text();throw l(e),Error("Network response was not ok")}let n=await r.json();return console.log("Team permissions response:",n),n}catch(e){throw console.error("Failed to get team permissions:",e),e}},tu=async(e,t,o)=>{try{let r=a?"".concat(a,"/team/permissions_update"):"/team/permissions_update",n=await fetch(r,{method:"POST",headers:{"Content-Type":"application/json",Authorization:"Bearer ".concat(e)},body:JSON.stringify({team_id:t,team_member_permissions:o})});if(!n.ok){let e=await n.text();throw l(e),Error("Network response was not ok")}let c=await n.json();return console.log("Team permissions response:",c),c}catch(e){throw console.error("Failed to update team permissions:",e),e}},tg=async(e,t)=>{try{let o=a?"".concat(a,"/spend/logs/session/ui?session_id=").concat(encodeURIComponent(t)):"/spend/logs/session/ui?session_id=".concat(encodeURIComponent(t)),r=await fetch(o,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!r.ok){let e=await r.text();throw l(e),Error("Network response was not ok")}return await r.json()}catch(e){throw console.error("Failed to fetch session logs:",e),e}},tf=async(e,t)=>{try{let o=a?"".concat(a,"/vector_store/new"):"/vector_store/new",r=await fetch(o,{method:"POST",headers:{"Content-Type":"application/json",Authorization:"Bearer ".concat(e)},body:JSON.stringify(t)});if(!r.ok){let e=await r.json();throw Error(e.detail||"Failed to create vector store")}return await r.json()}catch(e){throw console.error("Error creating vector store:",e),e}},ty=async function(e){arguments.length>1&&void 0!==arguments[1]&&arguments[1],arguments.length>2&&void 0!==arguments[2]&&arguments[2];try{let t=a?"".concat(a,"/vector_store/list"):"/vector_store/list",o=await fetch(t,{method:"GET",headers:{"Content-Type":"application/json",Authorization:"Bearer ".concat(e)}});if(!o.ok){let e=await o.json();throw Error(e.detail||"Failed to list vector stores")}return await o.json()}catch(e){throw console.error("Error listing vector stores:",e),e}},tm=async(e,t)=>{try{let o=a?"".concat(a,"/vector_store/delete"):"/vector_store/delete",r=await fetch(o,{method:"POST",headers:{"Content-Type":"application/json",Authorization:"Bearer ".concat(e)},body:JSON.stringify({vector_store_id:t})});if(!r.ok){let e=await r.json();throw Error(e.detail||"Failed to delete vector store")}return await r.json()}catch(e){throw console.error("Error deleting vector store:",e),e}},tk=async e=>{try{let t=a?"".concat(a,"/email/event_settings"):"/email/event_settings",o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error("Failed to get email event settings")}let r=await o.json();return console.log("Email event settings response:",r),r}catch(e){throw console.error("Failed to get email event settings:",e),e}},t_=async(e,t)=>{try{let o=a?"".concat(a,"/email/event_settings"):"/email/event_settings",r=await fetch(o,{method:"PATCH",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify(t)});if(!r.ok){let e=await r.text();throw l(e),Error("Failed to update email event settings")}let n=await r.json();return console.log("Update email event settings response:",n),n}catch(e){throw console.error("Failed to update email event settings:",e),e}},tT=async e=>{try{let t=a?"".concat(a,"/email/event_settings/reset"):"/email/event_settings/reset",o=await fetch(t,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error("Failed to reset email event settings")}let r=await o.json();return console.log("Reset email event settings response:",r),r}catch(e){throw console.error("Failed to reset email event settings:",e),e}},tj=async(e,t)=>{try{let o=a?"".concat(a,"/guardrails/").concat(t):"/guardrails/".concat(t),r=await fetch(o,{method:"DELETE",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!r.ok){let e=await r.text();throw l(e),Error(e)}let n=await r.json();return console.log("Delete guardrail response:",n),n}catch(e){throw console.error("Failed to delete guardrail:",e),e}},tE=async e=>{try{let t=a?"".concat(a,"/guardrails/ui/add_guardrail_settings"):"/guardrails/ui/add_guardrail_settings",o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error("Failed to get guardrail UI settings")}let r=await o.json();return console.log("Guardrail UI settings response:",r),r}catch(e){throw console.error("Failed to get guardrail UI settings:",e),e}},tC=async e=>{try{let t=a?"".concat(a,"/guardrails/ui/provider_specific_params"):"/guardrails/ui/provider_specific_params",o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error("Failed to get guardrail provider specific parameters")}let r=await o.json();return console.log("Guardrail provider specific params response:",r),r}catch(e){throw console.error("Failed to get guardrail provider specific parameters:",e),e}},tS=async(e,t)=>{try{let o=a?"".concat(a,"/guardrails/").concat(t,"/info"):"/guardrails/".concat(t,"/info"),r=await fetch(o,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!r.ok){let e=await r.text();throw l(e),Error("Failed to get guardrail info")}let n=await r.json();return console.log("Guardrail info response:",n),n}catch(e){throw console.error("Failed to get guardrail info:",e),e}},tN=async(e,t,o)=>{try{let r=a?"".concat(a,"/guardrails/").concat(t):"/guardrails/".concat(t),n=await fetch(r,{method:"PATCH",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify(o)});if(!n.ok){let e=await n.text();throw l(e),Error("Failed to update guardrail")}let c=await n.json();return console.log("Update guardrail response:",c),c}catch(e){throw console.error("Failed to update guardrail:",e),e}},tv=async e=>{try{let t=a?"".concat(a,"/get/sso_settings"):"/get/sso_settings";console.log("Fetching SSO configuration from:",t);let o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!o.ok){let e=await o.text();throw l(e),Error("Network response was not ok")}let r=await o.json();return console.log("Fetched SSO configuration:",r),r}catch(e){throw console.error("Failed to fetch SSO configuration:",e),e}},tF=async(e,t)=>{try{let o=a?"".concat(a,"/update/sso_settings"):"/update/sso_settings";console.log("Updating SSO configuration:",t);let r=await fetch(o,{method:"PATCH",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify(t)});if(!r.ok){let e=await r.text();throw l(e),Error("Network response was not ok")}let n=await r.json();return console.log("Updated SSO configuration:",n),n}catch(e){throw console.error("Failed to update SSO configuration:",e),e}},tb=async(e,t,o,r,n)=>{try{let t=a?"".concat(a,"/audit"):"/audit",o=new URLSearchParams;r&&o.append("page",r.toString()),n&&o.append("page_size",n.toString());let c=o.toString();c&&(t+="?".concat(c));let s=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"}});if(!s.ok){let e=await s.text();throw l(e),Error("Network response was not ok")}return await s.json()}catch(e){throw console.error("Failed to fetch audit logs:",e),e}},tx=async e=>{try{let t=a?"".concat(a,"/user/available_users"):"/user/available_users",o=await fetch(t,{method:"GET",headers:{[d]:"Bearer ".concat(e)}});if(!o.ok){if(404===o.status)return null;let e=await o.text();throw l(e),Error("Network response was not ok")}return await o.json()}catch(e){throw console.error("Failed to fetch remaining users:",e),e}},tO=async(e,t,o)=>{try{let n=a?"".concat(a,"/config/pass_through_endpoint/").concat(encodeURIComponent(t)):"/config/pass_through_endpoint/".concat(encodeURIComponent(t)),c=await fetch(n,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify(o)});if(!c.ok){let e=await c.text();throw l(e),Error("Network response was not ok")}let s=await c.json();return r.ZP.success("Pass through endpoint updated successfully"),s}catch(e){throw console.error("Failed to update pass through endpoint:",e),e}},tB=async(e,t)=>{try{let o=a?"".concat(a,"/config/callback/delete"):"/config/callback/delete",r=await fetch(o,{method:"POST",headers:{[d]:"Bearer ".concat(e),"Content-Type":"application/json"},body:JSON.stringify({callback_name:t})});if(!r.ok){let e=await r.text();throw l(e),Error("Network response was not ok")}return await r.json()}catch(e){throw console.error("Failed to delete specific callback:",e),e}}}}]); \ No newline at end of file diff --git a/litellm/proxy/_experimental/out/_next/static/chunks/webpack-a426aae3231a8df1.js b/litellm/proxy/_experimental/out/_next/static/chunks/webpack-a426aae3231a8df1.js index c82df116cd8..21702908136 100644 --- a/litellm/proxy/_experimental/out/_next/static/chunks/webpack-a426aae3231a8df1.js +++ b/litellm/proxy/_experimental/out/_next/static/chunks/webpack-a426aae3231a8df1.js @@ -1 +1 @@ -!function(){"use strict";var e,t,n,r,o,u,i,c,f,a={},l={};function d(e){var t=l[e];if(void 0!==t)return t.exports;var n=l[e]={id:e,loaded:!1,exports:{}},r=!0;try{a[e].call(n.exports,n,n.exports,d),r=!1}finally{r&&delete l[e]}return n.loaded=!0,n.exports}d.m=a,e=[],d.O=function(t,n,r,o){if(n){o=o||0;for(var u=e.length;u>0&&e[u-1][2]>o;u--)e[u]=e[u-1];e[u]=[n,r,o];return}for(var i=1/0,u=0;u=o&&Object.keys(d.O).every(function(e){return d.O[e](n[f])})?n.splice(f--,1):(c=!1,o0&&e[u-1][2]>o;u--)e[u]=e[u-1];e[u]=[n,r,o];return}for(var i=1/0,u=0;u=o&&Object.keys(d.O).every(function(e){return d.O[e](n[f])})?n.splice(f--,1):(c=!1,o - - - - - - - - - - - - - + + + + + + + + + + + + + + diff --git a/litellm/proxy/_experimental/out/assets/logos/cerebras.svg b/litellm/proxy/_experimental/out/assets/logos/cerebras.svg index 426f6430c23..1ff347220c5 100644 --- a/litellm/proxy/_experimental/out/assets/logos/cerebras.svg +++ b/litellm/proxy/_experimental/out/assets/logos/cerebras.svg @@ -1,89 +1,89 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/litellm/proxy/_experimental/out/assets/logos/deepseek.svg b/litellm/proxy/_experimental/out/assets/logos/deepseek.svg index c4754047da2..61760f13190 100644 --- a/litellm/proxy/_experimental/out/assets/logos/deepseek.svg +++ b/litellm/proxy/_experimental/out/assets/logos/deepseek.svg @@ -1,25 +1,25 @@ - - - - - - + + + + + + diff --git a/litellm/proxy/_experimental/out/assets/logos/perplexity-ai.svg b/litellm/proxy/_experimental/out/assets/logos/perplexity-ai.svg index e828b6dfbf1..e3a32be9809 100644 --- a/litellm/proxy/_experimental/out/assets/logos/perplexity-ai.svg +++ b/litellm/proxy/_experimental/out/assets/logos/perplexity-ai.svg @@ -1,16 +1,16 @@ - - - - - - - - - - + + + + + + + + + + \ No newline at end of file diff --git a/litellm/proxy/_experimental/out/index.html b/litellm/proxy/_experimental/out/index.html index 986f93bb911..75df302f7a2 100644 --- a/litellm/proxy/_experimental/out/index.html +++ b/litellm/proxy/_experimental/out/index.html @@ -1 +1 @@ -LiteLLM Dashboard \ No newline at end of file +LiteLLM Dashboard \ No newline at end of file diff --git a/litellm/proxy/_experimental/out/index.txt b/litellm/proxy/_experimental/out/index.txt index ec26580f8a7..a252918fcd5 100644 --- a/litellm/proxy/_experimental/out/index.txt +++ b/litellm/proxy/_experimental/out/index.txt @@ -2,6 +2,6 @@ 3:I[21718,["665","static/chunks/3014691f-b7b79b78e27792f3.js","990","static/chunks/13b76428-ebdf3012af0e4489.js","402","static/chunks/402-239a4ed70dc393da.js","313","static/chunks/313-cf4a28394ee560d6.js","899","static/chunks/899-0459bccb48a6666d.js","539","static/chunks/539-fadc69b2ec7728b1.js","250","static/chunks/250-f07ccab57fe599d4.js","699","static/chunks/699-2e0b76ba9cd1d301.js","931","static/chunks/app/page-2905d62702267b5c.js"],"default",1] 4:I[4707,[],""] 5:I[36423,[],""] -0:["1Uk4UFrWV9mxVjloiprrd",[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],["",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/31b7f215e119031e.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/c1c21001170a99e0.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_b0dd8a","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]] +0:["1Uk4UFrWV9mxVjloiprrd",[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],["",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/_next/static/css/31b7f215e119031e.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/_next/static/css/c1c21001170a99e0.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_b0dd8a","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]] 6:[["$","meta","0",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","1",{"charSet":"utf-8"}],["$","title","2",{"children":"LiteLLM Dashboard"}],["$","meta","3",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","4",{"rel":"icon","href":"/favicon.ico","type":"image/x-icon","sizes":"16x16"}],["$","link","5",{"rel":"icon","href":"./favicon.ico"}],["$","meta","6",{"name":"next-size-adjust"}]] 1:null diff --git a/litellm/proxy/_experimental/out/model_hub.txt b/litellm/proxy/_experimental/out/model_hub.txt index cfe276eec46..a0357d122d5 100644 --- a/litellm/proxy/_experimental/out/model_hub.txt +++ b/litellm/proxy/_experimental/out/model_hub.txt @@ -2,6 +2,6 @@ 3:I[52829,["402","static/chunks/402-239a4ed70dc393da.js","313","static/chunks/313-cf4a28394ee560d6.js","250","static/chunks/250-f07ccab57fe599d4.js","699","static/chunks/699-2e0b76ba9cd1d301.js","418","static/chunks/app/model_hub/page-ce40c5a05f3174ca.js"],"default",1] 4:I[4707,[],""] 5:I[36423,[],""] -0:["1Uk4UFrWV9mxVjloiprrd",[[["",{"children":["model_hub",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],["",{"children":["model_hub",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[null,["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children","model_hub","children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined"}]],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/31b7f215e119031e.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/c1c21001170a99e0.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_b0dd8a","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]] +0:["1Uk4UFrWV9mxVjloiprrd",[[["",{"children":["model_hub",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],["",{"children":["model_hub",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[null,["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children","model_hub","children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined"}]],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/_next/static/css/31b7f215e119031e.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/_next/static/css/c1c21001170a99e0.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_b0dd8a","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]] 6:[["$","meta","0",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","1",{"charSet":"utf-8"}],["$","title","2",{"children":"LiteLLM Dashboard"}],["$","meta","3",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","4",{"rel":"icon","href":"/favicon.ico","type":"image/x-icon","sizes":"16x16"}],["$","link","5",{"rel":"icon","href":"./favicon.ico"}],["$","meta","6",{"name":"next-size-adjust"}]] 1:null diff --git a/litellm/proxy/_experimental/out/onboarding.html b/litellm/proxy/_experimental/out/onboarding.html deleted file mode 100644 index e63b2ae7291..00000000000 --- a/litellm/proxy/_experimental/out/onboarding.html +++ /dev/null @@ -1 +0,0 @@ -LiteLLM Dashboard \ No newline at end of file diff --git a/litellm/proxy/_experimental/out/onboarding.txt b/litellm/proxy/_experimental/out/onboarding.txt index 5f908c80021..d191f23e4c7 100644 --- a/litellm/proxy/_experimental/out/onboarding.txt +++ b/litellm/proxy/_experimental/out/onboarding.txt @@ -2,6 +2,6 @@ 3:I[12011,["665","static/chunks/3014691f-b7b79b78e27792f3.js","402","static/chunks/402-239a4ed70dc393da.js","899","static/chunks/899-0459bccb48a6666d.js","250","static/chunks/250-f07ccab57fe599d4.js","461","static/chunks/app/onboarding/page-e05c770288debeda.js"],"default",1] 4:I[4707,[],""] 5:I[36423,[],""] -0:["1Uk4UFrWV9mxVjloiprrd",[[["",{"children":["onboarding",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],["",{"children":["onboarding",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[null,["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children","onboarding","children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined"}]],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/31b7f215e119031e.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/c1c21001170a99e0.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_b0dd8a","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]] +0:["1Uk4UFrWV9mxVjloiprrd",[[["",{"children":["onboarding",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],["",{"children":["onboarding",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[null,["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children","onboarding","children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined"}]],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/_next/static/css/31b7f215e119031e.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/_next/static/css/c1c21001170a99e0.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_b0dd8a","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]] 6:[["$","meta","0",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","1",{"charSet":"utf-8"}],["$","title","2",{"children":"LiteLLM Dashboard"}],["$","meta","3",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","4",{"rel":"icon","href":"/favicon.ico","type":"image/x-icon","sizes":"16x16"}],["$","link","5",{"rel":"icon","href":"./favicon.ico"}],["$","meta","6",{"name":"next-size-adjust"}]] 1:null diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index f9c4fe15e18..40243ae668b 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -1,119 +1,4 @@ model_list: - - model_name: codex-mini + - model_name: gemini-2.5-pro litellm_params: - model: codex-mini-latest - api_key: os.environ/OPENAI_API_KEY - - model_name: bedrock/* - litellm_params: - model: bedrock/* - - model_name: eu.anthropic.claude-3-5-sonnet-20240620-v1:0 - litellm_params: - model: eu.anthropic.claude-3-5-sonnet-20240620-v1:0 - - model_name: "gpt-4o-mini-openai" - litellm_params: - model: gpt-4o-mini - api_key: os.environ/OPENAI_API_KEY - model_info: - access_groups: ["beta-models"] # 👈 Model Access Group - - model_name: azure_ai/Phi-3-medium - litellm_params: - model: azure_ai/Phi-3-medium - api_key: os.environ/AZURE_AI_PHI_3_MEDIUM_API_KEY - api_base: os.environ/AZURE_AI_PHI_3_MEDIUM_API_BASE - - model_name: "bedrock-nova" - litellm_params: - model: us.amazon.nova-pro-v1:0 - - model_name: openrouter_model - litellm_params: - model: openrouter/openrouter_model - api_key: os.environ/OPENROUTER_API_KEY - api_base: http://0.0.0.0:8090 - - model_name: dall-e-3-azure - litellm_params: - model: azure/dall-e-3-test - api_version: "2023-12-01-preview" - api_base: os.environ/AZURE_SWEDEN_API_BASE - api_key: os.environ/AZURE_SWEDEN_API_KEY - model_info: - input_cost_per_pixel: 10 - - model_name: "claude-3-7-sonnet" - litellm_params: - model: databricks/databricks-claude-3-7-sonnet - api_key: os.environ/DATABRICKS_API_KEY - api_base: os.environ/DATABRICKS_API_BASE - - model_name: "gpt-4.1" - litellm_params: - model: azure/gpt-4.1 - api_key: os.environ/AZURE_API_KEY_REALTIME - api_base: https://krris-m2f9a9i7-eastus2.openai.azure.com/ - - model_name: "xai/*" - litellm_params: - model: xai/* - api_key: os.environ/XAI_API_KEY - - model_name: "text-embedding-ada-002" - litellm_params: - model: text-embedding-ada-002 - api_key: os.environ/OPENAI_API_KEY - - model_name: gemini/* - litellm_params: - model: gemini/* - - model_name: llama-qwen - litellm_params: - model: ollama/qwen2:0.5b - model_info: - input_cost_per_token: 0.75 - output_cost_per_token: 3 - - model_name: gpt-image-1 - litellm_params: - model: gpt-image-1 - api_key: os.environ/OPENAI_API_KEY - # drop_params: true - - model_name: "gpt-4o-batch" - litellm_params: - model: azure/gpt-4o-mini - api_base: os.environ/AZURE_API_BASE - api_key: os.environ/AZURE_API_KEY - model_info: - id: my-general-azure-deployment - mode: batch - - model_name: "gpt-4o-batch" - litellm_params: - model: azure/gpt-4o-mini - api_base: https://krris-m2f9a9i7-eastus2.openai.azure.com - api_key: 04d22fb7e9ad4d9c8afe7c6abf97a6fc - model_info: - id: my-unique-azure-deployment - mode: batch - - model_name: fake-openai-endpoint - litellm_params: - model: openai/fake - api_key: fake-key - api_base: https://exampleopenaiendpoint-production.up.railway.app/ - - model_name: "anthropic-claude-vertex" - litellm_params: - model: vertex_ai/claude-3-5-sonnet@20240620 - vertex_project: internal-litellm-local-dev - - model_name: "openai-custom/*" - litellm_params: - model: "openai/*" - api_key: os.environ/OPENAI_API_KEY_TEST - - model_name: "anthropic-claude" - litellm_params: - model: "anthropic/claude-3-5-sonnet-latest" - api_key: os.environ/ANTHROPIC_API_KEY - -general_settings: - store_model_in_db: true - store_prompts_in_spend_logs: true - token_rate_limit_type: "output" - # master_key: os.environ/PROXY_MASTER_KEY - -litellm_settings: - # cache: true - # cache_params: - # type: redis - # ttl: 600 - # password: os.environ/REDIS_PASSWORD - # supported_call_types: ["acompletion", "completion"] - callbacks: ["prometheus", "langfuse"] - + model: gemini/gemini-2.5-pro diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 7d358cad030..111ef89f7df 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1115,6 +1115,7 @@ class NewTeamRequest(TeamBase): team_member_budget: Optional[float] = ( None # allow user to set a budget for all team members ) + team_member_key_duration: Optional[str] = None # e.g. "1d", "1w", "1m" model_config = ConfigDict(protected_namespaces=()) @@ -1157,6 +1158,7 @@ class UpdateTeamRequest(LiteLLMPydanticObjectBase): guardrails: Optional[List[str]] = None object_permission: Optional[LiteLLM_ObjectPermissionBase] = None team_member_budget: Optional[float] = None + team_member_key_duration: Optional[str] = None class ResetTeamBudgetRequest(LiteLLMPydanticObjectBase): @@ -2792,6 +2794,7 @@ LiteLLM_ManagementEndpoint_MetadataFields = [ LiteLLM_ManagementEndpoint_MetadataFields_Premium = [ "guardrails", "tags", + "team_member_key_duration", ] diff --git a/litellm/proxy/auth/handle_jwt.py b/litellm/proxy/auth/handle_jwt.py index fa91785e6b7..046f66173d2 100644 --- a/litellm/proxy/auth/handle_jwt.py +++ b/litellm/proxy/auth/handle_jwt.py @@ -164,7 +164,9 @@ class JWTHandler: self.litellm_jwtauth.team_ids_jwt_field is not None and token.get(self.litellm_jwtauth.team_ids_jwt_field) is not None ): + return token[self.litellm_jwtauth.team_ids_jwt_field] + return [] def get_end_user_id( diff --git a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py index fb8e9236f48..a16f7f1a0b2 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py +++ b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py @@ -79,9 +79,9 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): if messages: for message in messages: - message_text_content: Optional[ - List[str] - ] = self.get_content_for_message(message=message) + message_text_content: Optional[List[str]] = ( + self.get_content_for_message(message=message) + ) if message_text_content is None: continue for text_content in message_text_content: @@ -241,7 +241,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): self, response: BedrockGuardrailResponse ) -> bool: """ - By default always raise an exception when a guardrail intervention is detected. + Only raise exception for "BLOCKED" actions, not for "ANONYMIZED" actions. If `self.mask_request_content` or `self.mask_response_content` is set to `True`, then use the output from the guardrail to mask the request or response content. """ @@ -250,11 +250,68 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): if self.mask_request_content or self.mask_response_content: return False - # if intervention, return True - if response.get("action") == "GUARDRAIL_INTERVENED": - return True - # if no intervention, return False + if response.get("action") != "GUARDRAIL_INTERVENED": + return False + + # Check assessments to determine if any actions were BLOCKED (vs ANONYMIZED) + assessments = response.get("assessments", []) + if not assessments: + return False + + for assessment in assessments: + # Check topic policy + topic_policy = assessment.get("topicPolicy") + if topic_policy: + topics = topic_policy.get("topics", []) + for topic in topics: + if topic.get("action") == "BLOCKED": + return True + + # Check content policy + content_policy = assessment.get("contentPolicy") + if content_policy: + filters = content_policy.get("filters", []) + for filter_item in filters: + if filter_item.get("action") == "BLOCKED": + return True + + # Check word policy + word_policy = assessment.get("wordPolicy") + if word_policy: + custom_words = word_policy.get("customWords", []) + for custom_word in custom_words: + if custom_word.get("action") == "BLOCKED": + return True + managed_words = word_policy.get("managedWordLists", []) + for managed_word in managed_words: + if managed_word.get("action") == "BLOCKED": + return True + + # Check sensitive information policy + sensitive_info_policy = assessment.get("sensitiveInformationPolicy") + if sensitive_info_policy: + pii_entities = sensitive_info_policy.get("piiEntities", []) + if pii_entities: + for pii_entity in pii_entities: + if pii_entity.get("action") == "BLOCKED": + return True + regexes = sensitive_info_policy.get("regexes", []) + if regexes: + for regex in regexes: + if regex.get("action") == "BLOCKED": + return True + + # Check contextual grounding policy + contextual_grounding_policy = assessment.get("contextualGroundingPolicy") + if contextual_grounding_policy: + grounding_filters = contextual_grounding_policy.get("filters", []) + for grounding_filter in grounding_filters: + if grounding_filter.get("action") == "BLOCKED": + return True + + # If we got here, intervention occurred but no BLOCKED actions found + # This means all actions were ANONYMIZED or NONE, so don't raise exception return False @log_guardrail_information @@ -300,11 +357,11 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): ######################################################### ########## 2. Update the messages with the guardrail response ########## ######################################################### - data[ - "messages" - ] = self._update_messages_with_updated_bedrock_guardrail_response( - messages=new_messages, - bedrock_guardrail_response=bedrock_guardrail_response, + data["messages"] = ( + self._update_messages_with_updated_bedrock_guardrail_response( + messages=new_messages, + bedrock_guardrail_response=bedrock_guardrail_response, + ) ) ######################################################### @@ -354,11 +411,11 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): ######################################################### ########## 2. Update the messages with the guardrail response ########## ######################################################### - data[ - "messages" - ] = self._update_messages_with_updated_bedrock_guardrail_response( - messages=new_messages, - bedrock_guardrail_response=bedrock_guardrail_response, + data["messages"] = ( + self._update_messages_with_updated_bedrock_guardrail_response( + messages=new_messages, + bedrock_guardrail_response=bedrock_guardrail_response, + ) ) ######################################################### @@ -408,11 +465,11 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): ######################################################### ########## 2. Update the messages with the guardrail response ########## ######################################################### - data[ - "messages" - ] = self._update_messages_with_updated_bedrock_guardrail_response( - messages=new_messages, - bedrock_guardrail_response=bedrock_guardrail_response, + data["messages"] = ( + self._update_messages_with_updated_bedrock_guardrail_response( + messages=new_messages, + bedrock_guardrail_response=bedrock_guardrail_response, + ) ) ######################################################### @@ -440,21 +497,29 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): Returns: List of messages with content masked according to guardrail response """ - # Skip processing if masking is not enabled - if not (self.mask_request_content or self.mask_response_content): - return messages - # Get masked texts from guardrail response masked_texts = self._extract_masked_texts_from_response( bedrock_guardrail_response ) - if not masked_texts: - return messages - # Apply masking to messages using index tracking - return self._apply_masking_to_messages( - messages=messages, masked_texts=masked_texts - ) + # If guardrail provided masked output, use it regardless of masking flags + # because the guardrail has already determined this content needs anonymization + if masked_texts: + verbose_proxy_logger.debug( + "Bedrock guardrail provided masked output, applying to messages" + ) + return self._apply_masking_to_messages( + messages=messages, masked_texts=masked_texts + ) + + # If masking is enabled but no masked texts available, still try to apply + # (this maintains backward compatibility for edge cases) + if self.mask_request_content or self.mask_response_content: + verbose_proxy_logger.debug( + "Masking enabled but no masked output from guardrail, returning original messages" + ) + + return messages async def async_post_call_streaming_iterator_hook( self, diff --git a/litellm/proxy/litellm.log b/litellm/proxy/litellm.log new file mode 100644 index 00000000000..4f592f5cc0b --- /dev/null +++ b/litellm/proxy/litellm.log @@ -0,0 +1,357 @@ +18:10:09 - LiteLLM Router:INFO: router.py:660 - Routing strategy: simple-shuffle +18:10:11 - LiteLLM Proxy:INFO: utils.py:1317 - All necessary views exist! +18:10:11 - LiteLLM Router:WARNING: router.py:4862 - Error upserting deployment: vertex_project, and vertex_location must be set in litellm_params for pass-through endpoints., ignoring and continuing with other deployments. +18:10:11 - LiteLLM Router:WARNING: router.py:4862 - Error upserting deployment: vertex_project, and vertex_location must be set in litellm_params for pass-through endpoints., ignoring and continuing with other deployments. +18:10:23 - LiteLLM Proxy:INFO: ui_sso.py:129 - Redirecting to SSO login for http://localhost:4000/sso/callback +18:10:27 - LiteLLM Proxy:INFO: ui_sso.py:495 - Starting SSO callback +18:10:27 - LiteLLM Proxy:INFO: ui_sso.py:550 - Redirecting to http://localhost:4000/sso/callback +18:10:28 - LiteLLM Proxy:INFO: ui_sso.py:581 - SSO callback result: id='krrishd' email='krrishdholakia@gmail.com' first_name=None last_name=None display_name='a3f1c107-04dc-4c93-ae60-7f32eb4b05ce' picture=None provider=None team_ids=[] +18:10:28 - LiteLLM Proxy:INFO: ui_sso.py:671 - user_defined_values for creating ui key: {'models': [], 'user_id': 'krrishd', 'user_email': 'krrishdholakia@gmail.com', 'max_budget': None, 'user_role': 'proxy_admin', 'budget_duration': None} +18:10:28 - LiteLLM Proxy:INFO: utils.py:1856 - Data Inserted into Keys Table +18:10:28 - LiteLLM Proxy:INFO: ui_sso.py:761 - user_id: krrishd; jwt_token: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ1c2VyX2lkIjoia3JyaXNoZCIsImtleSI6InNrLTVvOXVVc0ZaaTVBRFBiWERoanhCZlEiLCJ1c2VyX2VtYWlsIjoia3JyaXNoZGhvbGFraWFAZ21haWwuY29tIiwidXNlcl9yb2xlIjoicHJveHlfYWRtaW4iLCJsb2dpbl9tZXRob2QiOiJzc28iLCJwcmVtaXVtX3VzZXIiOnRydWUsImF1dGhfaGVhZGVyX25hbWUiOiJBdXRob3JpemF0aW9uIiwiZGlzYWJsZWRfbm9uX2FkbWluX3BlcnNvbmFsX2tleV9jcmVhdGlvbiI6ZmFsc2UsInNlcnZlcl9yb290X3BhdGgiOiIvIn0.OiZdFjZ2wiMhFbMCwu2cZYXh7oV5BB8Vta-Ysk5JBQU +18:10:28 - LiteLLM Proxy:INFO: ui_sso.py:764 - Redirecting to http://localhost:4000/ui/?login=success +18:10:30 - LiteLLM Proxy:ERROR: key_management_endpoints.py:2275 - Error in list_keys: Server disconnected without sending a response. +Traceback (most recent call last): + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_transports/default.py", line 101, in map_httpcore_exceptions + yield + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_transports/default.py", line 394, in handle_async_request + resp = await self._pool.handle_async_request(req) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_async/connection_pool.py", line 216, in handle_async_request + raise exc from None + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_async/connection_pool.py", line 196, in handle_async_request + response = await connection.handle_async_request( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_async/connection.py", line 101, in handle_async_request + return await self._connection.handle_async_request(request) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_async/http11.py", line 143, in handle_async_request + raise exc + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_async/http11.py", line 113, in handle_async_request + ) = await self._receive_response_headers(**kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_async/http11.py", line 186, in _receive_response_headers + event = await self._receive_event(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_async/http11.py", line 238, in _receive_event + raise RemoteProtocolError(msg) +httpcore.RemoteProtocolError: Server disconnected without sending a response. + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/Users/krrishdholakia/Documents/litellm/litellm/proxy/management_endpoints/key_management_endpoints.py", line 2255, in list_keys + response = await _list_key_helper( + ^^^^^^^^^^^^^^^^^^^^^^^ + File "/Users/krrishdholakia/Documents/litellm/litellm/proxy/management_endpoints/key_management_endpoints.py", line 2434, in _list_key_helper + total_count = await prisma_client.db.litellm_verificationtoken.count( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/prisma/actions.py", line 10157, in count + resp = await self._client._execute( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/prisma/_base_client.py", line 543, in _execute + return await self._engine.query(builder.build(), tx_id=self._tx_id) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/prisma/engine/_query.py", line 402, in query + return await self.request( + ^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/prisma/engine/_http.py", line 217, in request + response = await self.session.request(method, url, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/prisma/_async_http.py", line 26, in request + return Response(await self.session.request(method, url, **kwargs)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_client.py", line 1540, in request + return await self.send(request, auth=auth, follow_redirects=follow_redirects) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_client.py", line 1629, in send + response = await self._send_handling_auth( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_client.py", line 1657, in _send_handling_auth + response = await self._send_handling_redirects( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_client.py", line 1694, in _send_handling_redirects + response = await self._send_single_request(request) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_client.py", line 1730, in _send_single_request + response = await transport.handle_async_request(request) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_transports/default.py", line 393, in handle_async_request + with map_httpcore_exceptions(): + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/contextlib.py", line 155, in __exit__ + self.gen.throw(typ, value, traceback) + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_transports/default.py", line 118, in map_httpcore_exceptions + raise mapped_exc(message) from exc +httpx.RemoteProtocolError: Server disconnected without sending a response. +18:10:30 - LiteLLM Proxy:ERROR: proxy_server.py:2730 - litellm.proxy_server.py::add_deployment() - Error getting new models from DB - All connection attempts failed +Traceback (most recent call last): + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_transports/default.py", line 101, in map_httpcore_exceptions + yield + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_transports/default.py", line 394, in handle_async_request + resp = await self._pool.handle_async_request(req) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_async/connection_pool.py", line 216, in handle_async_request + raise exc from None + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_async/connection_pool.py", line 196, in handle_async_request + response = await connection.handle_async_request( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_async/connection.py", line 99, in handle_async_request + raise exc + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_async/connection.py", line 76, in handle_async_request + stream = await self._connect(request) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_async/connection.py", line 122, in _connect + stream = await self._network_backend.connect_tcp(**kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_backends/auto.py", line 30, in connect_tcp + return await self._backend.connect_tcp( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_backends/anyio.py", line 112, in connect_tcp + with map_exceptions(exc_map): + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/contextlib.py", line 155, in __exit__ + self.gen.throw(typ, value, traceback) + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_exceptions.py", line 14, in map_exceptions + raise to_exc(exc) from exc +httpcore.ConnectError: All connection attempts failed + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/Users/krrishdholakia/Documents/litellm/litellm/proxy/proxy_server.py", line 2728, in _get_models_from_db + new_models = await prisma_client.db.litellm_proxymodeltable.find_many() + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/prisma/actions.py", line 2540, in find_many + resp = await self._client._execute( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/prisma/_base_client.py", line 543, in _execute + return await self._engine.query(builder.build(), tx_id=self._tx_id) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/prisma/engine/_query.py", line 402, in query + return await self.request( + ^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/prisma/engine/_http.py", line 217, in request + response = await self.session.request(method, url, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/prisma/_async_http.py", line 26, in request + return Response(await self.session.request(method, url, **kwargs)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_client.py", line 1540, in request + return await self.send(request, auth=auth, follow_redirects=follow_redirects) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_client.py", line 1629, in send + response = await self._send_handling_auth( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_client.py", line 1657, in _send_handling_auth + response = await self._send_handling_redirects( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_client.py", line 1694, in _send_handling_redirects + response = await self._send_single_request(request) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_client.py", line 1730, in _send_single_request + response = await transport.handle_async_request(request) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_transports/default.py", line 393, in handle_async_request + with map_httpcore_exceptions(): + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/contextlib.py", line 155, in __exit__ + self.gen.throw(typ, value, traceback) + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_transports/default.py", line 118, in map_httpcore_exceptions + raise mapped_exc(message) from exc +httpx.ConnectError: All connection attempts failed +18:10:30 - LiteLLM Proxy:ERROR: utils.py:1404 - LiteLLM Prisma Client Exception get_generic_data: All connection attempts failed +18:10:30 - LiteLLM Proxy:ERROR: utils.py:1404 - LiteLLM Prisma Client Exception get_generic_data: All connection attempts failed +18:10:30 - LiteLLM Proxy:ERROR: proxy_server.py:2778 - litellm.proxy.proxy_server.py::ProxyConfig:add_deployment - All connection attempts failed +Traceback (most recent call last): + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_transports/default.py", line 101, in map_httpcore_exceptions + yield + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_transports/default.py", line 394, in handle_async_request + resp = await self._pool.handle_async_request(req) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_async/connection_pool.py", line 216, in handle_async_request + raise exc from None + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_async/connection_pool.py", line 196, in handle_async_request + response = await connection.handle_async_request( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_async/connection.py", line 99, in handle_async_request + raise exc + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_async/connection.py", line 76, in handle_async_request + stream = await self._connect(request) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_async/connection.py", line 122, in _connect + stream = await self._network_backend.connect_tcp(**kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_backends/auto.py", line 30, in connect_tcp + return await self._backend.connect_tcp( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_backends/anyio.py", line 112, in connect_tcp + with map_exceptions(exc_map): + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/contextlib.py", line 155, in __exit__ + self.gen.throw(typ, value, traceback) + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_exceptions.py", line 14, in map_exceptions + raise to_exc(exc) from exc +httpcore.ConnectError: All connection attempts failed + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/Users/krrishdholakia/Documents/litellm/litellm/proxy/proxy_server.py", line 2760, in add_deployment + await self._update_llm_router( + File "/Users/krrishdholakia/Documents/litellm/litellm/proxy/proxy_server.py", line 2418, in _update_llm_router + config_data = await proxy_config.get_config() + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Users/krrishdholakia/Documents/litellm/litellm/proxy/proxy_server.py", line 1584, in get_config + config = await self._update_config_from_db( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Users/krrishdholakia/Documents/litellm/litellm/proxy/proxy_server.py", line 2706, in _update_config_from_db + responses = await asyncio.gather(*_tasks) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Users/krrishdholakia/Documents/litellm/litellm/proxy/db/log_db_metrics.py", line 99, in wrapper + raise e + File "/Users/krrishdholakia/Documents/litellm/litellm/proxy/db/log_db_metrics.py", line 42, in wrapper + result = await func(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/backoff/_async.py", line 151, in retry + ret = await target(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Users/krrishdholakia/Documents/litellm/litellm/proxy/utils.py", line 1418, in get_generic_data + raise e + File "/Users/krrishdholakia/Documents/litellm/litellm/proxy/utils.py", line 1392, in get_generic_data + response = await self.db.litellm_config.find_first( # type: ignore + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/prisma/actions.py", line 11822, in find_first + resp = await self._client._execute( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/prisma/_base_client.py", line 543, in _execute + return await self._engine.query(builder.build(), tx_id=self._tx_id) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/prisma/engine/_query.py", line 402, in query + return await self.request( + ^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/prisma/engine/_http.py", line 217, in request + response = await self.session.request(method, url, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/prisma/_async_http.py", line 26, in request + return Response(await self.session.request(method, url, **kwargs)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_client.py", line 1540, in request + return await self.send(request, auth=auth, follow_redirects=follow_redirects) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_client.py", line 1629, in send + response = await self._send_handling_auth( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_client.py", line 1657, in _send_handling_auth + response = await self._send_handling_redirects( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_client.py", line 1694, in _send_handling_redirects + response = await self._send_single_request(request) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_client.py", line 1730, in _send_single_request + response = await transport.handle_async_request(request) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_transports/default.py", line 393, in handle_async_request + with map_httpcore_exceptions(): + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/contextlib.py", line 155, in __exit__ + self.gen.throw(typ, value, traceback) + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_transports/default.py", line 118, in map_httpcore_exceptions + raise mapped_exc(message) from exc +httpx.ConnectError: All connection attempts failed +18:10:30 - LiteLLM Proxy:ERROR: utils.py:1404 - LiteLLM Prisma Client Exception get_generic_data: All connection attempts failed +18:10:30 - LiteLLM Proxy:ERROR: utils.py:1404 - LiteLLM Prisma Client Exception get_generic_data: All connection attempts failed +18:10:30 - LiteLLM Proxy:INFO: proxy_server.py:490 - Shutting down LiteLLM Proxy Server +18:11:47 - LiteLLM Router:INFO: router.py:660 - Routing strategy: simple-shuffle +18:11:49 - LiteLLM Proxy:INFO: utils.py:1317 - All necessary views exist! +18:11:50 - LiteLLM Router:WARNING: router.py:4862 - Error upserting deployment: vertex_project, and vertex_location must be set in litellm_params for pass-through endpoints., ignoring and continuing with other deployments. +18:11:50 - LiteLLM Router:WARNING: router.py:4862 - Error upserting deployment: vertex_project, and vertex_location must be set in litellm_params for pass-through endpoints., ignoring and continuing with other deployments. +18:12:00 - LiteLLM Proxy:ERROR: proxy_server.py:2925 - litellm.proxy_server.py::get_credentials() - Error getting credentials from DB - Server disconnected without sending a response. +Traceback (most recent call last): + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_transports/default.py", line 101, in map_httpcore_exceptions + yield + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_transports/default.py", line 394, in handle_async_request + resp = await self._pool.handle_async_request(req) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_async/connection_pool.py", line 216, in handle_async_request + raise exc from None + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_async/connection_pool.py", line 196, in handle_async_request + response = await connection.handle_async_request( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_async/connection.py", line 101, in handle_async_request + return await self._connection.handle_async_request(request) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_async/http11.py", line 143, in handle_async_request + raise exc + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_async/http11.py", line 113, in handle_async_request + ) = await self._receive_response_headers(**kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_async/http11.py", line 186, in _receive_response_headers + event = await self._receive_event(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_async/http11.py", line 238, in _receive_event + raise RemoteProtocolError(msg) +httpcore.RemoteProtocolError: Server disconnected without sending a response. + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/Users/krrishdholakia/Documents/litellm/litellm/proxy/proxy_server.py", line 2916, in get_credentials + credentials = await prisma_client.db.litellm_credentialstable.find_many() + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/prisma/actions.py", line 1502, in find_many + resp = await self._client._execute( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/prisma/_base_client.py", line 543, in _execute + return await self._engine.query(builder.build(), tx_id=self._tx_id) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/prisma/engine/_query.py", line 402, in query + return await self.request( + ^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/prisma/engine/_http.py", line 217, in request + response = await self.session.request(method, url, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/prisma/_async_http.py", line 26, in request + return Response(await self.session.request(method, url, **kwargs)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_client.py", line 1540, in request + return await self.send(request, auth=auth, follow_redirects=follow_redirects) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_client.py", line 1629, in send + response = await self._send_handling_auth( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_client.py", line 1657, in _send_handling_auth + response = await self._send_handling_redirects( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_client.py", line 1694, in _send_handling_redirects + response = await self._send_single_request(request) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_client.py", line 1730, in _send_single_request + response = await transport.handle_async_request(request) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_transports/default.py", line 393, in handle_async_request + with map_httpcore_exceptions(): + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/contextlib.py", line 155, in __exit__ + self.gen.throw(typ, value, traceback) + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_transports/default.py", line 118, in map_httpcore_exceptions + raise mapped_exc(message) from exc +httpx.RemoteProtocolError: Server disconnected without sending a response. +18:12:01 - LiteLLM Proxy:INFO: proxy_server.py:490 - Shutting down LiteLLM Proxy Server +18:12:14 - LiteLLM Router:INFO: router.py:660 - Routing strategy: simple-shuffle +18:12:16 - LiteLLM Proxy:INFO: utils.py:1317 - All necessary views exist! +18:12:16 - LiteLLM Router:WARNING: router.py:4862 - Error upserting deployment: vertex_project, and vertex_location must be set in litellm_params for pass-through endpoints., ignoring and continuing with other deployments. +18:12:16 - LiteLLM Router:WARNING: router.py:4862 - Error upserting deployment: vertex_project, and vertex_location must be set in litellm_params for pass-through endpoints., ignoring and continuing with other deployments. +18:12:21 - LiteLLM Proxy:INFO: ui_sso.py:129 - Redirecting to SSO login for http://localhost:4000/sso/callback +18:12:26 - LiteLLM Proxy:INFO: ui_sso.py:495 - Starting SSO callback +18:12:26 - LiteLLM Proxy:INFO: ui_sso.py:550 - Redirecting to http://localhost:4000/sso/callback +18:12:26 - LiteLLM Proxy:INFO: ui_sso.py:581 - SSO callback result: id='krrishd' email='krrishdholakia@gmail.com' first_name=None last_name=None display_name='a3f1c107-04dc-4c93-ae60-7f32eb4b05ce' picture=None provider=None team_ids=[] +18:12:27 - LiteLLM Proxy:INFO: ui_sso.py:672 - user_defined_values for creating ui key: {'models': [], 'user_id': 'krrishd', 'user_email': 'krrishdholakia@gmail.com', 'max_budget': None, 'user_role': 'proxy_admin', 'budget_duration': None} +18:12:27 - LiteLLM Proxy:INFO: utils.py:1856 - Data Inserted into Keys Table +18:12:27 - LiteLLM Proxy:INFO: ui_sso.py:762 - user_id: krrishd; jwt_token: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ1c2VyX2lkIjoia3JyaXNoZCIsImtleSI6InNrLUQzMEFpdW9lckU3YlMyakFXWVFLd1EiLCJ1c2VyX2VtYWlsIjoia3JyaXNoZGhvbGFraWFAZ21haWwuY29tIiwidXNlcl9yb2xlIjoicHJveHlfYWRtaW4iLCJsb2dpbl9tZXRob2QiOiJzc28iLCJwcmVtaXVtX3VzZXIiOnRydWUsImF1dGhfaGVhZGVyX25hbWUiOiJBdXRob3JpemF0aW9uIiwiZGlzYWJsZWRfbm9uX2FkbWluX3BlcnNvbmFsX2tleV9jcmVhdGlvbiI6ZmFsc2UsInNlcnZlcl9yb290X3BhdGgiOiIvIn0.EzYP86hw12J4WHLe6ZZz4YgVNGPnxM_PHqLjINH2_-U +18:12:27 - LiteLLM Proxy:INFO: ui_sso.py:765 - Redirecting to http://localhost:4000/ui/?login=success +18:12:31 - LiteLLM Proxy:INFO: proxy_server.py:490 - Shutting down LiteLLM Proxy Server +18:15:07 - LiteLLM Router:INFO: router.py:660 - Routing strategy: simple-shuffle +18:15:09 - LiteLLM Proxy:INFO: utils.py:1317 - All necessary views exist! +18:15:09 - LiteLLM Router:WARNING: router.py:4862 - Error upserting deployment: vertex_project, and vertex_location must be set in litellm_params for pass-through endpoints., ignoring and continuing with other deployments. +18:15:09 - LiteLLM Router:WARNING: router.py:4862 - Error upserting deployment: vertex_project, and vertex_location must be set in litellm_params for pass-through endpoints., ignoring and continuing with other deployments. +18:15:17 - LiteLLM Proxy:INFO: utils.py:1916 - Data Inserted into Config Table +18:15:28 - LiteLLM Proxy:INFO: ui_sso.py:129 - Redirecting to SSO login for http://localhost:4000/sso/callback +18:15:32 - LiteLLM Proxy:INFO: ui_sso.py:495 - Starting SSO callback +18:15:32 - LiteLLM Proxy:INFO: ui_sso.py:550 - Redirecting to http://localhost:4000/sso/callback +18:15:32 - LiteLLM Proxy:INFO: ui_sso.py:581 - SSO callback result: id='krrishd' email='krrishdholakia@gmail.com' first_name=None last_name=None display_name='a3f1c107-04dc-4c93-ae60-7f32eb4b05ce' picture=None provider=None team_ids=[] +18:15:37 - LiteLLM Proxy:INFO: proxy_server.py:490 - Shutting down LiteLLM Proxy Server diff --git a/litellm/proxy/management_endpoints/callback_management_endpoints.py b/litellm/proxy/management_endpoints/callback_management_endpoints.py new file mode 100644 index 00000000000..3f611715e41 --- /dev/null +++ b/litellm/proxy/management_endpoints/callback_management_endpoints.py @@ -0,0 +1,27 @@ +""" +Endpoints for managing callbacks +""" +from fastapi import APIRouter, Depends + +from litellm.litellm_core_utils.logging_callback_manager import CallbacksByType +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + +router = APIRouter() + + +@router.get( + "/callbacks/list", + tags=["Logging Callbacks"], + dependencies=[Depends(user_api_key_auth)], + response_model=CallbacksByType, +) +async def list_callbacks(): + """ + View List of Active Logging Callbacks + """ + from litellm import logging_callback_manager + + # Get callbacks organized by type using the callback manager utility + callbacks_by_type = logging_callback_manager.get_callbacks_by_type() + + return callbacks_by_type \ No newline at end of file diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index fdc8e73dad3..0b1c7f523c0 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -512,6 +512,20 @@ async def generate_key_fn( # noqa: PLR0915 }, ) + # APPLY ENTERPRISE KEY MANAGEMENT PARAMS + try: + from litellm_enterprise.proxy.management_endpoints.key_management_endpoints import ( + apply_enterprise_key_management_params, + ) + + data = apply_enterprise_key_management_params(data, team_table) + except Exception as e: + verbose_proxy_logger.info( + "litellm.proxy.proxy_server.generate_key_fn(): Enterprise key management params not applied - {}".format( + str(e) + ) + ) + # TODO: @ishaan-jaff: Migrate all budget tracking to use LiteLLM_BudgetTable _budget_id = data.budget_id if prisma_client is not None and data.soft_budget is not None: @@ -536,7 +550,7 @@ async def generate_key_fn( # noqa: PLR0915 # ADD METADATA FIELDS # Set Management Endpoint Metadata Fields for field in LiteLLM_ManagementEndpoint_MetadataFields_Premium: - if getattr(data, field) is not None: + if getattr(data, field, None) is not None: _set_object_metadata_field( object_data=data, field_name=field, @@ -589,9 +603,9 @@ async def generate_key_fn( # noqa: PLR0915 request_type="key", **data_json, table_name="key" ) - response[ - "soft_budget" - ] = data.soft_budget # include the user-input soft budget in the response + response["soft_budget"] = ( + data.soft_budget + ) # include the user-input soft budget in the response response = GenerateKeyResponse(**response) @@ -667,9 +681,9 @@ async def _set_object_permission( data=data_json["object_permission"], ) ) - data_json[ - "object_permission_id" - ] = created_object_permission.object_permission_id + data_json["object_permission_id"] = ( + created_object_permission.object_permission_id + ) # delete the object_permission from the data_json data_json.pop("object_permission") @@ -1652,10 +1666,10 @@ async def delete_verification_tokens( try: if prisma_client: tokens = [_hash_token_if_needed(token=key) for key in tokens] - _keys_being_deleted: List[ - LiteLLM_VerificationToken - ] = await prisma_client.db.litellm_verificationtoken.find_many( - where={"token": {"in": tokens}} + _keys_being_deleted: List[LiteLLM_VerificationToken] = ( + await prisma_client.db.litellm_verificationtoken.find_many( + where={"token": {"in": tokens}} + ) ) if len(_keys_being_deleted) == 0: @@ -1763,9 +1777,9 @@ async def _rotate_master_key( from litellm.proxy.proxy_server import proxy_config try: - models: Optional[ - List - ] = await prisma_client.db.litellm_proxymodeltable.find_many() + models: Optional[List] = ( + await prisma_client.db.litellm_proxymodeltable.find_many() + ) except Exception: models = None # 2. process model table @@ -2057,11 +2071,11 @@ async def validate_key_list_check( param="user_id", code=status.HTTP_403_FORBIDDEN, ) - complete_user_info_db_obj: Optional[ - BaseModel - ] = await prisma_client.db.litellm_usertable.find_unique( - where={"user_id": user_api_key_dict.user_id}, - include={"organization_memberships": True}, + complete_user_info_db_obj: Optional[BaseModel] = ( + await prisma_client.db.litellm_usertable.find_unique( + where={"user_id": user_api_key_dict.user_id}, + include={"organization_memberships": True}, + ) ) if complete_user_info_db_obj is None: @@ -2147,10 +2161,10 @@ async def get_admin_team_ids( if complete_user_info is None: return [] # Get all teams that user is an admin of - teams: Optional[ - List[BaseModel] - ] = await prisma_client.db.litellm_teamtable.find_many( - where={"team_id": {"in": complete_user_info.teams}} + teams: Optional[List[BaseModel]] = ( + await prisma_client.db.litellm_teamtable.find_many( + where={"team_id": {"in": complete_user_info.teams}} + ) ) if teams is None: return [] @@ -2403,12 +2417,14 @@ async def _list_key_helper( where=where, # type: ignore skip=skip, # type: ignore take=size, # type: ignore - order=order_by - if order_by - else [ - {"created_at": "desc"}, - {"token": "desc"}, # fallback sort - ], + order=( + order_by + if order_by + else [ + {"created_at": "desc"}, + {"token": "desc"}, # fallback sort + ] + ), include={"object_permission": True}, ) diff --git a/litellm/proxy/management_endpoints/scim/scim_v2.py b/litellm/proxy/management_endpoints/scim/scim_v2.py index b88413f180e..1ef4cdc974d 100644 --- a/litellm/proxy/management_endpoints/scim/scim_v2.py +++ b/litellm/proxy/management_endpoints/scim/scim_v2.py @@ -18,6 +18,7 @@ from fastapi import ( Response, ) +import litellm from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.proxy._types import ( @@ -361,6 +362,18 @@ async def create_user( # Create user in database user_id = user.userName or str(uuid.uuid4()) metadata = _build_scim_metadata(user_data["given_name"], user_data["family_name"]) + + default_role: Optional[ + Literal[ + LitellmUserRoles.PROXY_ADMIN, + LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, + LitellmUserRoles.INTERNAL_USER, + LitellmUserRoles.INTERNAL_USER_VIEW_ONLY, + ] + ] = LitellmUserRoles.INTERNAL_USER_VIEW_ONLY + if litellm.default_internal_user_params: + default_role = litellm.default_internal_user_params.get("user_role") + new_user_request = NewUserRequest( user_id=user_id, user_email=user_data["user_email"], @@ -368,6 +381,7 @@ async def create_user( teams=user_data["teams"], metadata=metadata, auto_create_key=False, + user_role=default_role, ) # Check if user with email already exists and update if found diff --git a/litellm/proxy/management_endpoints/sso_helper_utils.py b/litellm/proxy/management_endpoints/sso_helper_utils.py index 45906b2fce0..7b296a6646f 100644 --- a/litellm/proxy/management_endpoints/sso_helper_utils.py +++ b/litellm/proxy/management_endpoints/sso_helper_utils.py @@ -1,9 +1,14 @@ +from typing import Dict, Union + from litellm.proxy._types import LitellmUserRoles -def check_is_admin_only_access(ui_access_mode: str) -> bool: +def check_is_admin_only_access(ui_access_mode: Union[str, Dict]) -> bool: """Checks ui access mode is admin_only""" - return ui_access_mode == "admin_only" + if isinstance(ui_access_mode, str): + return ui_access_mode == "admin_only" + else: + return False def has_admin_ui_access(user_role: str) -> bool: diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 31d2877ea69..3d4ae4e945f 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -262,6 +262,7 @@ async def new_team( # noqa: PLR0915 - guardrails: Optional[List[str]] - Guardrails for the team. [Docs](https://docs.litellm.ai/docs/proxy/guardrails) - object_permission: Optional[LiteLLM_ObjectPermissionBase] - team-specific object permission. Example - {"vector_stores": ["vector_store_1", "vector_store_2"]}. IF null or {} then no object permission. - team_member_budget: Optional[float] - The maximum budget allocated to an individual team member. + - team_member_key_duration: Optional[str] - The duration for a team member's key. e.g. "1d", "1w", "1mo" Returns: - team_id: (str) Unique team id - used for tracking spend across multiple keys for same team id. @@ -688,6 +689,7 @@ async def update_team( - guardrails: Optional[List[str]] - Guardrails for the team. [Docs](https://docs.litellm.ai/docs/proxy/guardrails) - object_permission: Optional[LiteLLM_ObjectPermissionBase] - team-specific object permission. Example - {"vector_stores": ["vector_store_1", "vector_store_2"]}. IF null or {} then no object permission. - team_member_budget: Optional[float] - The maximum budget allocated to an individual team member. + - team_member_key_duration: Optional[str] - The duration for a team member's key. e.g. "1d", "1w", "1mo" Example - update team TPM Limit ``` diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index 29a078ec06c..77fdb64b0e9 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -145,7 +145,11 @@ async def google_login(request: Request): # noqa: PLR0915 return HTMLResponse(content=html_form, status_code=200) -def generic_response_convertor(response, jwt_handler: JWTHandler): +def generic_response_convertor( + response, + jwt_handler: JWTHandler, + sso_jwt_handler: Optional[JWTHandler] = None, +): generic_user_id_attribute_name = os.getenv( "GENERIC_USER_ID_ATTRIBUTE", "preferred_username" ) @@ -171,6 +175,13 @@ def generic_response_convertor(response, jwt_handler: JWTHandler): f" generic_user_id_attribute_name: {generic_user_id_attribute_name}\n generic_user_email_attribute_name: {generic_user_email_attribute_name}" ) + all_teams = [] + if sso_jwt_handler is not None: + team_ids = sso_jwt_handler.get_team_ids_from_jwt(cast(dict, response)) + all_teams.extend(team_ids) + + team_ids = jwt_handler.get_team_ids_from_jwt(cast(dict, response)) + all_teams.extend(team_ids) return CustomOpenID( id=response.get(generic_user_id_attribute_name), display_name=response.get(generic_user_display_name_attribute_name), @@ -178,20 +189,24 @@ def generic_response_convertor(response, jwt_handler: JWTHandler): first_name=response.get(generic_user_first_name_attribute_name), last_name=response.get(generic_user_last_name_attribute_name), provider=response.get(generic_provider_attribute_name), - team_ids=jwt_handler.get_team_ids_from_jwt(cast(dict, response)), + team_ids=all_teams, ) async def get_generic_sso_response( request: Request, jwt_handler: JWTHandler, + sso_jwt_handler: Optional[ + JWTHandler + ], # sso specific jwt handler - used for restricted sso group access control generic_client_id: str, redirect_url: str, -) -> Union[OpenID, dict]: +) -> Tuple[Union[OpenID, dict], Optional[dict]]: # return received response # make generic sso provider from fastapi_sso.sso.base import DiscoveryDocument from fastapi_sso.sso.generic import create_provider + received_response: Optional[dict] = None generic_client_secret = os.getenv("GENERIC_CLIENT_SECRET", None) generic_scope = os.getenv("GENERIC_SCOPE", "openid email profile").split(" ") generic_authorization_endpoint = os.getenv("GENERIC_AUTHORIZATION_ENDPOINT", None) @@ -242,9 +257,12 @@ async def get_generic_sso_response( ) def response_convertor(response, client): + nonlocal received_response # return for user debugging + received_response = response return generic_response_convertor( response=response, jwt_handler=jwt_handler, + sso_jwt_handler=sso_jwt_handler, ) SSOProvider = create_provider( @@ -284,7 +302,7 @@ async def get_generic_sso_response( ) raise e verbose_proxy_logger.debug("generic result: %s", result) - return result or {} + return result or {}, received_response async def create_team_member_add_task(team_id, user_info): @@ -480,6 +498,8 @@ async def check_and_update_if_proxy_admin_id( async def auth_callback(request: Request): # noqa: PLR0915 """Verify login""" verbose_proxy_logger.info("Starting SSO callback") + from litellm.proxy._types import LiteLLM_JWTAuth + from litellm.proxy.auth.handle_jwt import JWTHandler from litellm.proxy.management_endpoints.key_management_endpoints import ( generate_key_helper_fn, ) @@ -490,7 +510,6 @@ async def auth_callback(request: Request): # noqa: PLR0915 premium_user, prisma_client, proxy_logging_obj, - ui_access_mode, user_api_key_cache, user_custom_sso, ) @@ -502,9 +521,25 @@ async def auth_callback(request: Request): # noqa: PLR0915 status_code=500, detail=CommonProxyErrors.db_not_connected_error.value ) + sso_jwt_handler: Optional[JWTHandler] = None + ui_access_mode = general_settings.get("ui_access_mode", None) + if ui_access_mode is not None and isinstance(ui_access_mode, dict): + sso_jwt_handler = JWTHandler() + sso_jwt_handler.update_environment( + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + litellm_jwtauth=LiteLLM_JWTAuth( + team_ids_jwt_field=general_settings.get("ui_access_mode", {}).get( + "sso_group_jwt_field", None + ), + ), + leeway=0, + ) + microsoft_client_id = os.getenv("MICROSOFT_CLIENT_ID", None) google_client_id = os.getenv("GOOGLE_CLIENT_ID", None) generic_client_id = os.getenv("GENERIC_CLIENT_ID", None) + received_response: Optional[dict] = None # get url from request if master_key is None: raise ProxyException( @@ -532,11 +567,12 @@ async def auth_callback(request: Request): # noqa: PLR0915 redirect_url=redirect_url, ) elif generic_client_id is not None: - result = await get_generic_sso_response( + result, received_response = await get_generic_sso_response( request=request, jwt_handler=jwt_handler, generic_client_id=generic_client_id, redirect_url=redirect_url, + sso_jwt_handler=sso_jwt_handler, ) if result is None: @@ -547,6 +583,7 @@ async def auth_callback(request: Request): # noqa: PLR0915 # User is Authe'd in - generate key for the UI to access Proxy verbose_proxy_logger.info(f"SSO callback result: {result}") + user_email: Optional[str] = getattr(result, "email", None) user_id: Optional[str] = getattr(result, "id", None) if result is not None else None @@ -612,6 +649,13 @@ async def auth_callback(request: Request): # noqa: PLR0915 budget_duration=internal_user_budget_duration, ) + # (IF SET) Verify user is in restricted SSO group + SSOAuthenticationHandler.verify_user_in_restricted_sso_group( + general_settings=general_settings, + result=result, + received_response=received_response, + ) + user_info = await get_user_info_from_db( result=result, prisma_client=prisma_client, @@ -1055,6 +1099,44 @@ class SSOAuthenticationHandler: sso_teams = getattr(result, "team_ids", []) await add_missing_team_member(user_info=user_info, sso_teams=sso_teams) + @staticmethod + def verify_user_in_restricted_sso_group( + general_settings: Dict, + result: Optional[Union[CustomOpenID, OpenID, dict]], + received_response: Optional[dict], + ) -> Literal[True]: + """ + when ui_access_mode.type == "restricted_sso_group": + + - result.team_ids should contain the restricted_sso_group + - if not, raise a ProxyException + - if so, return True + - if result.team_ids is None, return False + - if result.team_ids is an empty list, return False + - if result.team_ids is a list, return True if the restricted_sso_group is in the list, otherwise return False + """ + + ui_access_mode = cast( + Optional[Union[Dict, str]], general_settings.get("ui_access_mode") + ) + + if ui_access_mode is None: + return True + if isinstance(ui_access_mode, str): + return True + team_ids = getattr(result, "team_ids", []) + + if ui_access_mode.get("type") == "restricted_sso_group": + restricted_sso_group = ui_access_mode.get("restricted_sso_group") + if restricted_sso_group not in team_ids: + raise ProxyException( + message=f"User is not in the restricted SSO group: {restricted_sso_group}. User groups: {team_ids}. Received SSO response: {received_response}", + type=ProxyErrorTypes.auth_error, + param="restricted_sso_group", + code=status.HTTP_403_FORBIDDEN, + ) + return True + @staticmethod async def create_litellm_team_from_sso_group( litellm_team_id: str, @@ -1551,7 +1633,29 @@ async def debug_sso_callback(request: Request): from fastapi.responses import HTMLResponse - from litellm.proxy.proxy_server import jwt_handler + from litellm.proxy._types import LiteLLM_JWTAuth + from litellm.proxy.auth.handle_jwt import JWTHandler + from litellm.proxy.proxy_server import ( + general_settings, + jwt_handler, + prisma_client, + user_api_key_cache, + ) + + sso_jwt_handler: Optional[JWTHandler] = None + ui_access_mode = general_settings.get("ui_access_mode", None) + if ui_access_mode is not None and isinstance(ui_access_mode, dict): + sso_jwt_handler = JWTHandler() + sso_jwt_handler.update_environment( + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + litellm_jwtauth=LiteLLM_JWTAuth( + team_ids_jwt_field=general_settings.get("ui_access_mode", {}).get( + "sso_group_jwt_field", None + ), + ), + leeway=0, + ) microsoft_client_id = os.getenv("MICROSOFT_CLIENT_ID", None) google_client_id = os.getenv("GOOGLE_CLIENT_ID", None) @@ -1580,11 +1684,12 @@ async def debug_sso_callback(request: Request): ) elif generic_client_id is not None: - result = await get_generic_sso_response( + result, _ = await get_generic_sso_response( request=request, jwt_handler=jwt_handler, generic_client_id=generic_client_id, redirect_url=redirect_url, + sso_jwt_handler=sso_jwt_handler, ) # If result is None, return a basic error message diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py index eac0e42ca4a..5ef1a98ef8d 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py @@ -272,26 +272,11 @@ class VertexPassthroughLoggingHandler: return None if len(parsed_chunks) == 0: return None - litellm_custom_stream_wrapper = litellm.CustomStreamWrapper( - completion_stream=vertex_iterator, - model=model, - logging_obj=litellm_logging_obj, - custom_llm_provider="vertex_ai", - ) all_openai_chunks = [] for parsed_chunk in parsed_chunks: - try: - litellm_chunk = litellm_custom_stream_wrapper.chunk_creator( - chunk=parsed_chunk - ) - except Exception as e: - verbose_proxy_logger.error( - "Error creating litellm chunk from vertex passthrough endpoint: %s", - str(e), - ) + if parsed_chunk is None: continue - if litellm_chunk is not None: - all_openai_chunks.append(litellm_chunk) + all_openai_chunks.append(parsed_chunk) complete_streaming_response = litellm.stream_chunk_builder( chunks=all_openai_chunks diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index fe0ede1726e..fda5e941484 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -442,8 +442,18 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils): _parsed_body: Optional[dict] = None, litellm_call_id: Optional[str] = None, ) -> dict: + """ + Filter out litellm params from the request body + """ + from litellm.types.utils import all_litellm_params + _parsed_body = _parsed_body or {} - _litellm_metadata: Optional[dict] = _parsed_body.pop("litellm_metadata", None) + + litellm_params_in_body = {} + for k in all_litellm_params: + if k in _parsed_body: + litellm_params_in_body[k] = _parsed_body.pop(k, None) + _metadata = dict( StandardLoggingUserAPIKeyMetadata( user_api_key_hash=user_api_key_dict.api_key, @@ -457,9 +467,15 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils): user_api_key_request_route=user_api_key_dict.request_route, ) ) + _metadata["user_api_key"] = user_api_key_dict.api_key - if _litellm_metadata: - _metadata.update(_litellm_metadata) + + litellm_metadata = litellm_params_in_body.pop("litellm_metadata", None) + metadata = litellm_params_in_body.pop("metadata", None) + if litellm_metadata: + _metadata.update(litellm_metadata) + if metadata: + _metadata.update(metadata) _metadata = _update_metadata_with_tags_in_header( request=request, @@ -468,12 +484,13 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils): kwargs = { "litellm_params": { + **litellm_params_in_body, "metadata": _metadata, "proxy_server_request": { - "url": str(request.url), - "method": request.method, - "body": copy.copy(_parsed_body), # use copy instead of deepcopy - } + "url": str(request.url), + "method": request.method, + "body": copy.copy(_parsed_body), # use copy instead of deepcopy + }, }, "call_type": "pass_through_endpoint", "litellm_call_id": litellm_call_id, @@ -629,6 +646,7 @@ async def pass_through_request( # noqa: PLR0915 request=request, logging_obj=logging_obj, ) + # done for supporting 'parallel_request_limiter.py' with pass-through endpoints logging_obj.update_environment_variables( model="unknown", @@ -962,7 +980,7 @@ class InitPassThroughEndpointHelpers: app.add_api_route( path=path, - endpoint=create_pass_through_route( + endpoint=create_pass_through_route( # type: ignore path, target, custom_headers, @@ -996,7 +1014,7 @@ class InitPassThroughEndpointHelpers: app.add_api_route( path=wildcard_path, - endpoint=create_pass_through_route( + endpoint=create_pass_through_route( # type: ignore path, target, custom_headers, @@ -1024,6 +1042,7 @@ async def initialize_pass_through_endpoints( None """ import uuid + verbose_proxy_logger.debug("initializing pass through endpoints") from litellm.proxy._types import CommonProxyErrors, LiteLLMRoutes from litellm.proxy.proxy_server import app, premium_user @@ -1031,11 +1050,11 @@ async def initialize_pass_through_endpoints( for endpoint in pass_through_endpoints: if isinstance(endpoint, PassThroughGenericEndpoint): endpoint = endpoint.model_dump() - + # Auto-generate ID for backwards compatibility if not present if endpoint.get("id") is None: endpoint["id"] = str(uuid.uuid4()) - + _target = endpoint.get("target", None) _path: Optional[str] = endpoint.get("path", None) if _path is None: @@ -1062,7 +1081,9 @@ async def initialize_pass_through_endpoints( continue # Add exact path route - verbose_proxy_logger.debug("Initializing pass through endpoint: %s (ID: %s)", _path, endpoint.get("id")) + verbose_proxy_logger.debug( + "Initializing pass through endpoint: %s (ID: %s)", _path, endpoint.get("id") + ) InitPassThroughEndpointHelpers.add_exact_path_route( app=app, path=_path, @@ -1087,7 +1108,9 @@ async def initialize_pass_through_endpoints( cost_per_request=endpoint.get("cost_per_request", None), ) - verbose_proxy_logger.debug("Added new pass through endpoint: %s (ID: %s)", _path, endpoint.get("id")) + verbose_proxy_logger.debug( + "Added new pass through endpoint: %s (ID: %s)", _path, endpoint.get("id") + ) async def _get_pass_through_endpoints_from_db( @@ -1117,12 +1140,10 @@ async def _get_pass_through_endpoints_from_db( returned_endpoints.append(endpoint) else: # Find specific endpoint by ID - found_endpoint = _find_endpoint_by_id( - pass_through_endpoint_data, endpoint_id - ) + found_endpoint = _find_endpoint_by_id(pass_through_endpoint_data, endpoint_id) if found_endpoint is not None: returned_endpoints.append(found_endpoint) - + return returned_endpoints @@ -1182,22 +1203,22 @@ async def update_pass_through_endpoints( ) # Find the endpoint to update - found_endpoint = _find_endpoint_by_id( - pass_through_endpoint_data, endpoint_id - ) - + found_endpoint = _find_endpoint_by_id(pass_through_endpoint_data, endpoint_id) + if found_endpoint is None: raise HTTPException( status_code=404, - detail={ - "error": f"Endpoint with ID '{endpoint_id}' not found" - }, + detail={"error": f"Endpoint with ID '{endpoint_id}' not found"}, ) # Find the index for updating the list endpoint_index = None for idx, endpoint in enumerate(pass_through_endpoint_data): - _endpoint = PassThroughGenericEndpoint(**endpoint) if isinstance(endpoint, dict) else endpoint + _endpoint = ( + PassThroughGenericEndpoint(**endpoint) + if isinstance(endpoint, dict) + else endpoint + ) if _endpoint.id == endpoint_id: endpoint_index = idx break @@ -1212,20 +1233,20 @@ async def update_pass_through_endpoints( # Get the update data as dict, excluding None values for partial updates update_data = data.model_dump(exclude_none=True) - + # Start with existing endpoint data endpoint_dict = found_endpoint.model_dump() - + # Update with new data (only non-None values) endpoint_dict.update(update_data) - + # Preserve existing ID if not provided in update and endpoint has ID if "id" not in update_data and found_endpoint.id is not None: endpoint_dict["id"] = found_endpoint.id - + # Create updated endpoint object updated_endpoint = PassThroughGenericEndpoint(**endpoint_dict) - + # Update the list pass_through_endpoint_data[endpoint_index] = endpoint_dict @@ -1239,7 +1260,9 @@ async def update_pass_through_endpoints( data=updated_data, user_api_key_dict=user_api_key_dict ) - return PassThroughEndpointResponse(endpoints=[updated_endpoint] if updated_endpoint else []) + return PassThroughEndpointResponse( + endpoints=[updated_endpoint] if updated_endpoint else [] + ) @router.post( @@ -1335,10 +1358,8 @@ async def delete_pass_through_endpoints( ) # Find the endpoint to delete - found_endpoint = _find_endpoint_by_id( - pass_through_endpoint_data, endpoint_id - ) - + found_endpoint = _find_endpoint_by_id(pass_through_endpoint_data, endpoint_id) + if found_endpoint is None: raise HTTPException( status_code=400, @@ -1348,11 +1369,15 @@ async def delete_pass_through_endpoints( ) }, ) - + # Find the index for deleting from the list endpoint_index = None for idx, endpoint in enumerate(pass_through_endpoint_data): - _endpoint = PassThroughGenericEndpoint(**endpoint) if isinstance(endpoint, dict) else endpoint + _endpoint = ( + PassThroughGenericEndpoint(**endpoint) + if isinstance(endpoint, dict) + else endpoint + ) if _endpoint.id == endpoint_id: endpoint_index = idx break @@ -1364,7 +1389,7 @@ async def delete_pass_through_endpoints( "error": f"Could not find index for endpoint with ID '{endpoint_id}'" }, ) - + # Remove the endpoint pass_through_endpoint_data.pop(endpoint_index) response_obj = found_endpoint @@ -1388,11 +1413,11 @@ def _find_endpoint_by_id( ) -> Optional[PassThroughGenericEndpoint]: """ Find an endpoint by ID. - + Args: endpoints_data: List of endpoint data (dicts or PassThroughGenericEndpoint objects) endpoint_id: ID to search for - + Returns: Found endpoint or None if not found """ @@ -1406,7 +1431,7 @@ def _find_endpoint_by_id( # Only compare IDs to IDs if _endpoint is not None and _endpoint.id == endpoint_id: return _endpoint - + return None diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 4e7aa675f08..4550c1760a8 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -15,4 +15,7 @@ mcp_servers: general_settings: store_model_in_db: true - store_prompts_in_spend_logs: true \ No newline at end of file + store_prompts_in_spend_logs: true + +litellm_settings: + callbacks: ["langfuse", "datadog"] \ No newline at end of file diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 375833b7de5..0a8abdd19ec 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -231,6 +231,9 @@ from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request from litellm.proxy.management_endpoints.budget_management_endpoints import ( router as budget_management_router, ) +from litellm.proxy.management_endpoints.callback_management_endpoints import ( + router as callback_management_endpoints_router, +) from litellm.proxy.management_endpoints.common_utils import _user_has_admin_view from litellm.proxy.management_endpoints.customer_endpoints import ( router as customer_router, @@ -902,7 +905,7 @@ health_check_results: Dict[str, Union[int, List[Dict[str, Any]]]] = {} queue: List = [] litellm_proxy_budget_name = "litellm-proxy-budget" litellm_proxy_admin_name = LITELLM_PROXY_ADMIN_NAME -ui_access_mode: Literal["admin", "all"] = "all" +ui_access_mode: Union[Literal["admin", "all"], Dict] = "all" proxy_budget_rescheduler_min_time = PROXY_BUDGET_RESCHEDULER_MIN_TIME proxy_budget_rescheduler_max_time = PROXY_BUDGET_RESCHEDULER_MAX_TIME proxy_batch_write_at = PROXY_BATCH_WRITE_AT @@ -1432,11 +1435,13 @@ class ProxyConfig: - Do not write restricted params like 'api_key' to the database - if api_key is passed, save that to the local environment or connected secret manage (maybe expose `litellm.save_secret()`) """ + if prisma_client is not None and ( general_settings.get("store_model_in_db", False) is True or store_model_in_db ): # if using - db for config - models are in ModelTable + new_config.pop("model_list", None) await prisma_client.insert_data(data=new_config, table_name="config") else: @@ -2622,6 +2627,10 @@ class ProxyConfig: pass_through_endpoints=general_settings["pass_through_endpoints"] ) + ## UI ACCESS MODE ## + if "ui_access_mode" in _general_settings: + general_settings["ui_access_mode"] = _general_settings["ui_access_mode"] + def _update_config_fields( self, current_config: dict, @@ -8595,6 +8604,7 @@ app.include_router(spend_management_router) app.include_router(caching_router) app.include_router(analytics_router) app.include_router(guardrails_router) +app.include_router(callback_management_endpoints_router) app.include_router(debugging_endpoints_router) app.include_router(ui_crud_endpoints_router) app.include_router(openai_files_router) diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index 970526afad6..ce623aa7f0c 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -81,7 +81,10 @@ async def route_request( team_id = get_team_id_from_data(data) router_model_names = llm_router.model_names if llm_router is not None else [] if "api_key" in data or "api_base" in data: - return getattr(llm_router, f"{route_type}")(**data) + if llm_router is not None: + return getattr(llm_router, f"{route_type}")(**data) + else: + return getattr(litellm, f"{route_type}")(**data) elif "user_config" in data: router_config = data.pop("user_config") diff --git a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py index 25991862fa1..884afd02bc4 100644 --- a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py +++ b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py @@ -7,7 +7,10 @@ import litellm from litellm._logging import verbose_proxy_logger from litellm.proxy._types import * from litellm.proxy.auth.user_api_key_auth import user_api_key_auth -from litellm.types.proxy.management_endpoints.ui_sso import DefaultTeamSSOParams, SSOConfig +from litellm.types.proxy.management_endpoints.ui_sso import ( + DefaultTeamSSOParams, + SSOConfig, +) router = APIRouter() @@ -18,26 +21,29 @@ class IPAddress(BaseModel): class SettingsResponse(BaseModel): """Base response model for settings with values and schema information""" - + values: Dict[str, Any] """The current configuration values""" - + field_schema: Dict[str, Any] """Schema information including descriptions and property types for UI display""" class SSOSettingsResponse(SettingsResponse): """Response model for SSO settings""" + pass class InternalUserSettingsResponse(SettingsResponse): """Response model for internal user settings""" + pass class DefaultTeamSettingsResponse(SettingsResponse): """Response model for default team settings""" + pass @@ -166,7 +172,10 @@ async def _get_settings_with_schema( # Add descriptions to the response result = { "values": settings_dict, - "field_schema": {"description": schema.get("description", ""), "properties": {}}, + "field_schema": { + "description": schema.get("description", ""), + "properties": {}, + }, } # Add property descriptions @@ -322,20 +331,21 @@ async def get_sso_settings(): Returns a structured object with values and descriptions for UI display. """ import os + from litellm.proxy.proxy_server import proxy_config - + # Load existing config to get both environment variables and general settings config = await proxy_config.get_config() general_settings = config.get("general_settings", {}) or {} environment_variables = config.get("environment_variables", {}) or {} - + # Get user_email from general_settings proxy_admin_email = general_settings.get("proxy_admin_email", None) - + # Helper function to get env var value (first from config, then from environment) def get_env_value(env_var_name: str): return environment_variables.get(env_var_name) or os.getenv(env_var_name) - + # Get current environment variables for SSO sso_config = SSOConfig( google_client_id=get_env_value("GOOGLE_CLIENT_ID"), @@ -351,27 +361,31 @@ async def get_sso_settings(): proxy_base_url=get_env_value("PROXY_BASE_URL"), user_email=proxy_admin_email, # Get from config instead of environment ) - + # Get the schema for UI display from pydantic import TypeAdapter + schema = TypeAdapter(SSOConfig).json_schema(by_alias=True) - + # Convert to dict for response sso_dict = sso_config.model_dump() - + # Add descriptions to the response result = { "values": sso_dict, - "field_schema": {"description": schema.get("description", ""), "properties": {}}, + "field_schema": { + "description": schema.get("description", ""), + "properties": {}, + }, } - + # Add property descriptions for field_name, field_info in schema["properties"].items(): result["field_schema"]["properties"][field_name] = { "description": field_info.get("description", ""), "type": field_info.get("type", "string"), } - + return result @@ -384,51 +398,56 @@ async def update_sso_settings(sso_config: SSOConfig): """ Update SSO configuration by saving to both environment variables and config file. """ - from litellm.proxy.proxy_server import proxy_config import os - + + from litellm.proxy.proxy_server import proxy_config + # Update environment variables env_var_mapping = { - 'google_client_id': 'GOOGLE_CLIENT_ID', - 'google_client_secret': 'GOOGLE_CLIENT_SECRET', - 'microsoft_client_id': 'MICROSOFT_CLIENT_ID', - 'microsoft_client_secret': 'MICROSOFT_CLIENT_SECRET', - 'microsoft_tenant': 'MICROSOFT_TENANT', - 'generic_client_id': 'GENERIC_CLIENT_ID', - 'generic_client_secret': 'GENERIC_CLIENT_SECRET', - 'generic_authorization_endpoint': 'GENERIC_AUTHORIZATION_ENDPOINT', - 'generic_token_endpoint': 'GENERIC_TOKEN_ENDPOINT', - 'generic_userinfo_endpoint': 'GENERIC_USERINFO_ENDPOINT', - 'proxy_base_url': 'PROXY_BASE_URL', + "google_client_id": "GOOGLE_CLIENT_ID", + "google_client_secret": "GOOGLE_CLIENT_SECRET", + "microsoft_client_id": "MICROSOFT_CLIENT_ID", + "microsoft_client_secret": "MICROSOFT_CLIENT_SECRET", + "microsoft_tenant": "MICROSOFT_TENANT", + "generic_client_id": "GENERIC_CLIENT_ID", + "generic_client_secret": "GENERIC_CLIENT_SECRET", + "generic_authorization_endpoint": "GENERIC_AUTHORIZATION_ENDPOINT", + "generic_token_endpoint": "GENERIC_TOKEN_ENDPOINT", + "generic_userinfo_endpoint": "GENERIC_USERINFO_ENDPOINT", + "proxy_base_url": "PROXY_BASE_URL", } - + # Load existing config config = await proxy_config.get_config() - + # Update config with new environment variables if "environment_variables" not in config: config["environment_variables"] = {} - + # Update general_settings for user_email (admin email) if "general_settings" not in config: config["general_settings"] = {} - + # Update environment variables in config and in memory sso_data = sso_config.model_dump(exclude_none=True) for field_name, value in sso_data.items(): - if field_name == 'user_email' and value is not None: + + if field_name == "user_email" and value is not None: # Store user_email in general_settings instead of environment variables config["general_settings"]["proxy_admin_email"] = value + elif field_name == "ui_access_mode" and value is not None: + + config["general_settings"]["ui_access_mode"] = value elif field_name in env_var_mapping and value is not None: env_var_name = env_var_mapping[field_name] # Update in config config["environment_variables"][env_var_name] = value # Update in runtime environment os.environ[env_var_name] = value - + # Save the updated config await proxy_config.save_config(new_config=config) - + return { "message": "SSO settings updated successfully", "status": "success", diff --git a/litellm/router.py b/litellm/router.py index cd4b94d4965..acb3fb6f6ea 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -4108,20 +4108,26 @@ class Router: original_exception=exception ) - _time_to_cooldown = kwargs.get("litellm_params", {}).get( - "cooldown_time", self.cooldown_time - ) - + # Determine cooldown time with priority: deployment config > response header > router default + deployment_cooldown = kwargs.get("litellm_params", {}).get("cooldown_time", None) + + header_cooldown = None if exception_headers is not None: - _time_to_cooldown = ( - litellm.utils._get_retry_after_from_exception_header( - response_headers=exception_headers - ) + header_cooldown = litellm.utils._get_retry_after_from_exception_header( + response_headers=exception_headers ) - - if _time_to_cooldown is None or _time_to_cooldown < 0: - # if the response headers did not read it -> set to default cooldown time - _time_to_cooldown = self.cooldown_time + ############################################## + # Logic to determine cooldown time + # 1. Check if a cooldown time is set in the deployment config + # 2. Check if a cooldown time is set in the response header + # 3. If no cooldown time is set, use the router default cooldown time + ############################################## + if deployment_cooldown is not None and deployment_cooldown >= 0: + _time_to_cooldown = deployment_cooldown + elif header_cooldown is not None and header_cooldown >= 0: + _time_to_cooldown = header_cooldown + else: + _time_to_cooldown = self.cooldown_time if isinstance(_model_info, dict): deployment_id = _model_info.get("id", None) diff --git a/litellm/types/llms/vertex_ai.py b/litellm/types/llms/vertex_ai.py index ded64cee6c3..1c3eb49fb0f 100644 --- a/litellm/types/llms/vertex_ai.py +++ b/litellm/types/llms/vertex_ai.py @@ -89,7 +89,7 @@ class SystemInstructions(TypedDict): class Schema(TypedDict, total=False): type: Literal["STRING", "INTEGER", "BOOLEAN", "NUMBER", "ARRAY", "OBJECT"] - format: str + format: Literal["enum", "date-time"] title: str description: str nullable: bool diff --git a/litellm/types/proxy/management_endpoints/ui_sso.py b/litellm/types/proxy/management_endpoints/ui_sso.py index f6838b61703..3cdb5cb6398 100644 --- a/litellm/types/proxy/management_endpoints/ui_sso.py +++ b/litellm/types/proxy/management_endpoints/ui_sso.py @@ -1,4 +1,4 @@ -from typing import List, Literal, Optional, TypedDict +from typing import List, Literal, Optional, TypedDict, Union from pydantic import Field @@ -31,6 +31,14 @@ class MicrosoftServicePrincipalTeam(TypedDict, total=False): principalId: Optional[str] +class AccessControl_UI_AccessMode(LiteLLMPydanticObjectBase): + """Model for Controlling UI Access Mode via SSO Groups""" + + type: Literal["restricted_sso_group"] + restricted_sso_group: str + sso_group_jwt_field: str + + class SSOConfig(LiteLLMPydanticObjectBase): """ Configuration for SSO environment variables and settings @@ -45,7 +53,7 @@ class SSOConfig(LiteLLMPydanticObjectBase): default=None, description="Google OAuth Client Secret for SSO authentication", ) - + # Microsoft SSO microsoft_client_id: Optional[str] = Field( default=None, @@ -59,7 +67,7 @@ class SSOConfig(LiteLLMPydanticObjectBase): default=None, description="Microsoft Azure Tenant ID for SSO authentication", ) - + # Generic/Okta SSO generic_client_id: Optional[str] = Field( default=None, @@ -81,7 +89,7 @@ class SSOConfig(LiteLLMPydanticObjectBase): default=None, description="User info endpoint URL for generic OAuth provider", ) - + # Common settings proxy_base_url: Optional[str] = Field( default=None, @@ -92,6 +100,12 @@ class SSOConfig(LiteLLMPydanticObjectBase): description="Email of the proxy admin user", ) + # Access Mode + ui_access_mode: Optional[Union[AccessControl_UI_AccessMode, str]] = Field( + default=None, + description="Access mode for the UI", + ) + class DefaultTeamSSOParams(LiteLLMPydanticObjectBase): """ diff --git a/litellm/types/utils.py b/litellm/types/utils.py index fd8ecd9e14a..68d861cbc11 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -177,7 +177,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): search_context_cost_per_query: Optional[ SearchContextCostPerQuery ] # Cost for using web search tool - + citation_cost_per_token: Optional[float] # Cost per citation token for Perplexity litellm_provider: Required[str] mode: Required[ Literal[ @@ -947,24 +947,6 @@ class Usage(CompletionUsage): elif isinstance(completion_tokens_details, CompletionTokensDetails): _completion_tokens_details = completion_tokens_details - ## DEEPSEEK MAPPING ## - if "prompt_cache_hit_tokens" in params and isinstance( - params["prompt_cache_hit_tokens"], int - ): - if prompt_tokens_details is None: - prompt_tokens_details = PromptTokensDetailsWrapper( - cached_tokens=params["prompt_cache_hit_tokens"] - ) - - ## ANTHROPIC MAPPING ## - if "cache_read_input_tokens" in params and isinstance( - params["cache_read_input_tokens"], int - ): - if prompt_tokens_details is None: - prompt_tokens_details = PromptTokensDetailsWrapper( - cached_tokens=params["cache_read_input_tokens"] - ) - # handle prompt_tokens_details _prompt_tokens_details: Optional[PromptTokensDetailsWrapper] = None if prompt_tokens_details: @@ -975,6 +957,28 @@ class Usage(CompletionUsage): elif isinstance(prompt_tokens_details, PromptTokensDetails): _prompt_tokens_details = prompt_tokens_details + ## DEEPSEEK MAPPING ## + if "prompt_cache_hit_tokens" in params and isinstance( + params["prompt_cache_hit_tokens"], int + ): + if _prompt_tokens_details is None: + _prompt_tokens_details = PromptTokensDetailsWrapper( + cached_tokens=params["prompt_cache_hit_tokens"] + ) + else: + _prompt_tokens_details.cached_tokens = params["prompt_cache_hit_tokens"] + + ## ANTHROPIC MAPPING ## + if "cache_read_input_tokens" in params and isinstance( + params["cache_read_input_tokens"], int + ): + if _prompt_tokens_details is None: + _prompt_tokens_details = PromptTokensDetailsWrapper( + cached_tokens=params["cache_read_input_tokens"] + ) + else: + _prompt_tokens_details.cached_tokens = params["cache_read_input_tokens"] + super().__init__( prompt_tokens=prompt_tokens or 0, completion_tokens=completion_tokens or 0, @@ -2489,3 +2493,9 @@ class DynamicPromptManagementParamLiteral(str, Enum): @classmethod def list_all_params(cls): return [param.value for param in cls] + + +class CallbacksByType(TypedDict): + success: List[str] + failure: List[str] + success_and_failure: List[str] diff --git a/litellm/types/vector_stores.py b/litellm/types/vector_stores.py index cd8280ac960..3d7190d7f9b 100644 --- a/litellm/types/vector_stores.py +++ b/litellm/types/vector_stores.py @@ -85,3 +85,83 @@ class VectorStoreSearchResponse(TypedDict, total=False): ] # Always "vector_store.search_results.page" search_query: Optional[str] data: Optional[List[VectorStoreSearchResult]] + +class VectorStoreSearchOptionalRequestParams(TypedDict, total=False): + """TypedDict for Optional parameters supported by the vector store search API.""" + filters: Optional[Dict] + max_num_results: Optional[int] + ranking_options: Optional[Dict] + rewrite_query: Optional[bool] + +class VectorStoreSearchRequest(VectorStoreSearchOptionalRequestParams, total=False): + """Request body for searching a vector store""" + query: Union[str, List[str]] + + +# Vector Store Creation Types +class VectorStoreExpirationPolicy(TypedDict, total=False): + """The expiration policy for a vector store""" + anchor: Literal["last_active_at"] # Anchor timestamp after which the expiration policy applies + days: int # Number of days after anchor time that the vector store will expire + + +class VectorStoreAutoChunkingStrategy(TypedDict, total=False): + """Auto chunking strategy configuration""" + type: Literal["auto"] # Always "auto" + + +class VectorStoreStaticChunkingStrategyConfig(TypedDict, total=False): + """Static chunking strategy configuration""" + max_chunk_size_tokens: int # Maximum number of tokens per chunk + chunk_overlap_tokens: int # Number of tokens to overlap between chunks + + +class VectorStoreStaticChunkingStrategy(TypedDict, total=False): + """Static chunking strategy""" + type: Literal["static"] # Always "static" + static: VectorStoreStaticChunkingStrategyConfig + + +class VectorStoreChunkingStrategy(TypedDict, total=False): + """Union type for chunking strategies""" + # This can be either auto or static + type: Literal["auto", "static"] + static: Optional[VectorStoreStaticChunkingStrategyConfig] + + +class VectorStoreFileCounts(TypedDict, total=False): + """File counts for a vector store""" + in_progress: int + completed: int + failed: int + cancelled: int + total: int + + +class VectorStoreCreateOptionalRequestParams(TypedDict, total=False): + """TypedDict for Optional parameters supported by the vector store create API.""" + name: Optional[str] # Name of the vector store + file_ids: Optional[List[str]] # List of File IDs that the vector store should use + expires_after: Optional[VectorStoreExpirationPolicy] # Expiration policy for the vector store + chunking_strategy: Optional[VectorStoreChunkingStrategy] # Chunking strategy for the files + metadata: Optional[Dict[str, str]] # Set of key-value pairs for metadata + + +class VectorStoreCreateRequest(VectorStoreCreateOptionalRequestParams, total=False): + """Request body for creating a vector store""" + pass # All fields are optional for vector store creation + + +class VectorStoreCreateResponse(TypedDict, total=False): + """Response after creating a vector store""" + id: str # ID of the vector store + object: Literal["vector_store"] # Always "vector_store" + created_at: int # Unix timestamp of when the vector store was created + name: Optional[str] # Name of the vector store + bytes: int # Size of the vector store in bytes + file_counts: VectorStoreFileCounts # File counts for the vector store + status: Literal["expired", "in_progress", "completed"] # Status of the vector store + expires_after: Optional[VectorStoreExpirationPolicy] # Expiration policy + expires_at: Optional[int] # Unix timestamp of when the vector store expires + last_active_at: Optional[int] # Unix timestamp of when the vector store was last active + metadata: Optional[Dict[str, str]] # Metadata associated with the vector store \ No newline at end of file diff --git a/litellm/utils.py b/litellm/utils.py index e3b14a16763..05328c3750b 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -78,7 +78,9 @@ from litellm.constants import ( ) from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.custom_logger import CustomLogger -from litellm.integrations.vector_stores.base_vector_store import BaseVectorStore +from litellm.integrations.vector_store_integrations.base_vector_store import ( + BaseVectorStore, +) from litellm.litellm_core_utils.core_helpers import ( map_finish_reason, process_response_headers, @@ -242,6 +244,7 @@ from litellm.llms.base_llm.image_variations.transformation import ( from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig +from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig from ._logging import _is_debugging_on, verbose_logger from .caching.caching import ( @@ -4650,6 +4653,7 @@ def _get_model_info_helper( # noqa: PLR0915 output_cost_per_second=_model_info.get("output_cost_per_second", None), output_cost_per_image=_model_info.get("output_cost_per_image", None), output_vector_size=_model_info.get("output_vector_size", None), + citation_cost_per_token=_model_info.get("citation_cost_per_token", None), litellm_provider=_model_info.get( "litellm_provider", custom_llm_provider ), @@ -6901,13 +6905,33 @@ class ProviderConfigManager: def get_provider_vector_store_config( provider: LlmProviders, ) -> Optional[CustomLogger]: - from litellm.integrations.vector_stores.bedrock_vector_store import ( + from litellm.integrations.vector_store_integrations.bedrock_vector_store import ( BedrockVectorStore, ) if LlmProviders.BEDROCK == provider: return BedrockVectorStore.get_initialized_custom_logger() return None + + + @staticmethod + def get_provider_vector_stores_config( + provider: LlmProviders, + ) -> Optional[BaseVectorStoreConfig]: + """ + v2 vector store config, use this for new vector store integrations + """ + if litellm.LlmProviders.OPENAI == provider: + from litellm.llms.openai.vector_stores.transformation import ( + OpenAIVectorStoreConfig, + ) + return OpenAIVectorStoreConfig() + elif litellm.LlmProviders.AZURE == provider: + from litellm.llms.azure.vector_stores.transformation import ( + AzureOpenAIVectorStoreConfig, + ) + return AzureOpenAIVectorStoreConfig() + return None @staticmethod def get_provider_image_generation_config( diff --git a/litellm/vector_stores/__init__.py b/litellm/vector_stores/__init__.py new file mode 100644 index 00000000000..6bcc6540328 --- /dev/null +++ b/litellm/vector_stores/__init__.py @@ -0,0 +1,4 @@ +from .main import acreate, asearch, create, search +from .vector_store_registry import VectorStoreRegistry + +__all__ = ["search", "asearch", "create", "acreate", "VectorStoreRegistry"] diff --git a/litellm/vector_stores/main.py b/litellm/vector_stores/main.py new file mode 100644 index 00000000000..80d6341146a --- /dev/null +++ b/litellm/vector_stores/main.py @@ -0,0 +1,434 @@ +""" +LiteLLM SDK Functions for Creating and Searching Vector Stores +""" +import asyncio +import contextvars +from functools import partial +from typing import Any, Coroutine, Dict, List, Optional, Union + +import httpx + +import litellm +from litellm.constants import request_timeout +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler +from litellm.types.router import GenericLiteLLMParams +from litellm.types.vector_stores import ( + VectorStoreCreateOptionalRequestParams, + VectorStoreCreateResponse, + VectorStoreFileCounts, + VectorStoreResultContent, + VectorStoreSearchOptionalRequestParams, + VectorStoreSearchResponse, + VectorStoreSearchResult, +) +from litellm.utils import ProviderConfigManager, client +from litellm.vector_stores.utils import VectorStoreRequestUtils + +####### ENVIRONMENT VARIABLES ################### +# Initialize any necessary instances or variables here +base_llm_http_handler = BaseLLMHTTPHandler() +################################################# + + +def mock_vector_store_search_response( + mock_results: Optional[List[VectorStoreSearchResult]] = None, +): + """Mock response for vector store search""" + if mock_results is None: + mock_results = [ + VectorStoreSearchResult( + score=0.95, + content=[ + VectorStoreResultContent( + text="This is a sample search result from the vector store.", + type="text" + ) + ] + ) + ] + + return VectorStoreSearchResponse( + object="vector_store.search_results.page", + search_query="sample query", + data=mock_results, + ) + + +def mock_vector_store_create_response( + mock_response: Optional[VectorStoreCreateResponse] = None, +): + """Mock response for vector store create""" + if mock_response is None: + mock_response = VectorStoreCreateResponse( + id="vs_mock123", + object="vector_store", + created_at=1699061776, + name="Mock Vector Store", + bytes=0, + file_counts=VectorStoreFileCounts( + in_progress=0, + completed=0, + failed=0, + cancelled=0, + total=0, + ), + status="completed", + expires_after=None, + expires_at=None, + last_active_at=None, + metadata=None, + ) + + return mock_response + + +@client +async def acreate( + name: Optional[str] = None, + file_ids: Optional[List[str]] = None, + expires_after: Optional[Dict] = None, + chunking_strategy: Optional[Dict] = None, + metadata: Optional[Dict[str, str]] = None, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Optional[Dict[str, Any]] = None, + extra_query: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, + # LiteLLM specific params, + custom_llm_provider: Optional[str] = None, + **kwargs, +) -> VectorStoreCreateResponse: + """ + Async: Create a vector store. + """ + local_vars = locals() + try: + loop = asyncio.get_event_loop() + kwargs["acreate"] = True + + # get custom llm provider so we can use this for mapping exceptions + if custom_llm_provider is None: + custom_llm_provider = "openai" # Default to OpenAI for vector stores + + func = partial( + create, + name=name, + file_ids=file_ids, + expires_after=expires_after, + chunking_strategy=chunking_strategy, + metadata=metadata, + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + timeout=timeout, + custom_llm_provider=custom_llm_provider, + **kwargs, + ) + + ctx = contextvars.copy_context() + func_with_context = partial(ctx.run, func) + init_response = await loop.run_in_executor(None, func_with_context) + + if asyncio.iscoroutine(init_response): + response = await init_response + else: + response = init_response + + return response + except Exception as e: + raise litellm.exception_type( + model=None, + custom_llm_provider=custom_llm_provider, + original_exception=e, + completion_kwargs=local_vars, + extra_kwargs=kwargs, + ) + + +@client +def create( + name: Optional[str] = None, + file_ids: Optional[List[str]] = None, + expires_after: Optional[Dict] = None, + chunking_strategy: Optional[Dict] = None, + metadata: Optional[Dict[str, str]] = None, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Optional[Dict[str, Any]] = None, + extra_query: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, + # LiteLLM specific params, + custom_llm_provider: Optional[str] = None, + **kwargs, +) -> Union[VectorStoreCreateResponse, Coroutine[Any, Any, VectorStoreCreateResponse]]: + """ + Create a vector store. + + Args: + name: The name of the vector store. + file_ids: A list of File IDs that the vector store should use. + expires_after: The expiration policy for the vector store. + chunking_strategy: The chunking strategy used to chunk the file(s). + metadata: Set of 16 key-value pairs that can be attached to an object. + + Returns: + VectorStoreCreateResponse containing the created vector store details. + """ + local_vars = locals() + try: + litellm_logging_obj: LiteLLMLoggingObj = kwargs.get("litellm_logging_obj") # type: ignore + litellm_call_id: Optional[str] = kwargs.get("litellm_call_id", None) + _is_async = kwargs.pop("acreate", False) is True + + # get llm provider logic + litellm_params = GenericLiteLLMParams(**kwargs) + + ## MOCK RESPONSE LOGIC + if litellm_params.mock_response and isinstance( + litellm_params.mock_response, dict + ): + return mock_vector_store_create_response( + mock_response=VectorStoreCreateResponse(**litellm_params.mock_response) + ) + + # Default to OpenAI for vector stores + if custom_llm_provider is None: + custom_llm_provider = "openai" + + # get provider config - using vector store custom logger for now + vector_store_provider_config = ProviderConfigManager.get_provider_vector_stores_config( + provider=litellm.LlmProviders(custom_llm_provider), + ) + + if vector_store_provider_config is None: + raise ValueError( + f"Vector store create is not supported for {custom_llm_provider}" + ) + + local_vars.update(kwargs) + + # Get VectorStoreCreateOptionalRequestParams with only valid parameters + vector_store_create_optional_params: VectorStoreCreateOptionalRequestParams = ( + VectorStoreRequestUtils.get_requested_vector_store_create_optional_param( + local_vars + ) + ) + + # Pre Call logging + litellm_logging_obj.update_environment_variables( + model=None, + optional_params={ + "name": name, + **vector_store_create_optional_params, + }, + litellm_params={ + "litellm_call_id": litellm_call_id, + }, + custom_llm_provider=custom_llm_provider, + ) + + response = base_llm_http_handler.vector_store_create_handler( + vector_store_create_optional_params=vector_store_create_optional_params, + vector_store_provider_config=vector_store_provider_config, + custom_llm_provider=custom_llm_provider, + litellm_params=litellm_params, + logging_obj=litellm_logging_obj, + extra_headers=extra_headers, + extra_body=extra_body, + timeout=timeout or request_timeout, + _is_async=_is_async, + client=kwargs.get("client"), + ) + + return response + except Exception as e: + raise litellm.exception_type( + model=None, + custom_llm_provider=custom_llm_provider, + original_exception=e, + completion_kwargs=local_vars, + extra_kwargs=kwargs, + ) + + +@client +async def asearch( + vector_store_id: str, + query: Union[str, List[str]], + filters: Optional[Dict] = None, + max_num_results: Optional[int] = None, + ranking_options: Optional[Dict] = None, + rewrite_query: Optional[bool] = None, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Optional[Dict[str, Any]] = None, + extra_query: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, + # LiteLLM specific params, + custom_llm_provider: Optional[str] = None, + **kwargs, +) -> VectorStoreSearchResponse: + """ + Async: Search a vector store for relevant chunks based on a query and file attributes filter. + """ + local_vars = locals() + try: + loop = asyncio.get_event_loop() + kwargs["asearch"] = True + + # get custom llm provider so we can use this for mapping exceptions + if custom_llm_provider is None: + custom_llm_provider = "openai" # Default to OpenAI for vector stores + + func = partial( + search, + vector_store_id=vector_store_id, + query=query, + filters=filters, + max_num_results=max_num_results, + ranking_options=ranking_options, + rewrite_query=rewrite_query, + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + timeout=timeout, + custom_llm_provider=custom_llm_provider, + **kwargs, + ) + + ctx = contextvars.copy_context() + func_with_context = partial(ctx.run, func) + init_response = await loop.run_in_executor(None, func_with_context) + + if asyncio.iscoroutine(init_response): + response = await init_response + else: + response = init_response + + return response + except Exception as e: + raise litellm.exception_type( + model=None, + custom_llm_provider=custom_llm_provider, + original_exception=e, + completion_kwargs=local_vars, + extra_kwargs=kwargs, + ) + + +@client +def search( + vector_store_id: str, + query: Union[str, List[str]], + filters: Optional[Dict] = None, + max_num_results: Optional[int] = None, + ranking_options: Optional[Dict] = None, + rewrite_query: Optional[bool] = None, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Optional[Dict[str, Any]] = None, + extra_query: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, + # LiteLLM specific params, + custom_llm_provider: Optional[str] = None, + **kwargs, +) -> Union[VectorStoreSearchResponse, Coroutine[Any, Any, VectorStoreSearchResponse]]: + """ + Search a vector store for relevant chunks based on a query and file attributes filter. + + Args: + vector_store_id: The ID of the vector store to search. + query: A query string or array for the search. + filters: Optional filter to apply based on file attributes. + max_num_results: Maximum number of results to return (1-50, default 10). + ranking_options: Optional ranking options for search. + rewrite_query: Whether to rewrite the natural language query for vector search. + + Returns: + VectorStoreSearchResponse containing the search results. + """ + local_vars = locals() + try: + litellm_logging_obj: LiteLLMLoggingObj = kwargs.get("litellm_logging_obj") # type: ignore + litellm_call_id: Optional[str] = kwargs.get("litellm_call_id", None) + _is_async = kwargs.pop("asearch", False) is True + + # get llm provider logic + litellm_params = GenericLiteLLMParams(**kwargs) + + ## MOCK RESPONSE LOGIC + if litellm_params.mock_response and isinstance( + litellm_params.mock_response, (str, list) + ): + mock_results = None + if isinstance(litellm_params.mock_response, list): + mock_results = litellm_params.mock_response + return mock_vector_store_search_response(mock_results=mock_results) + + # Default to OpenAI for vector stores + if custom_llm_provider is None: + custom_llm_provider = "openai" + + # get provider config - using vector store custom logger for now + vector_store_provider_config = ProviderConfigManager.get_provider_vector_stores_config( + provider=litellm.LlmProviders(custom_llm_provider), + ) + + if vector_store_provider_config is None: + raise ValueError( + f"Vector store search is not supported for {custom_llm_provider}" + ) + + local_vars.update(kwargs) + + # Get VectorStoreSearchOptionalRequestParams with only valid parameters + vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams = ( + VectorStoreRequestUtils.get_requested_vector_store_search_optional_param( + local_vars + ) + ) + + # Pre Call logging + litellm_logging_obj.update_environment_variables( + model=None, + optional_params={ + "vector_store_id": vector_store_id, + "query": query, + **vector_store_search_optional_params, + }, + litellm_params={ + "litellm_call_id": litellm_call_id, + "vector_store_id": vector_store_id, + }, + custom_llm_provider=custom_llm_provider, + ) + + response = base_llm_http_handler.vector_store_search_handler( + vector_store_id=vector_store_id, + query=query, + vector_store_search_optional_params=vector_store_search_optional_params, + vector_store_provider_config=vector_store_provider_config, + custom_llm_provider=custom_llm_provider, + litellm_params=litellm_params, + logging_obj=litellm_logging_obj, + extra_headers=extra_headers, + extra_body=extra_body, + timeout=timeout or request_timeout, + _is_async=_is_async, + client=kwargs.get("client"), + ) + + return response + except Exception as e: + raise litellm.exception_type( + model=None, + custom_llm_provider=custom_llm_provider, + original_exception=e, + completion_kwargs=local_vars, + extra_kwargs=kwargs, + ) \ No newline at end of file diff --git a/litellm/vector_stores/utils.py b/litellm/vector_stores/utils.py new file mode 100644 index 00000000000..b7eb7790add --- /dev/null +++ b/litellm/vector_stores/utils.py @@ -0,0 +1,51 @@ +from typing import Any, Dict, cast, get_type_hints + +from litellm.types.vector_stores import ( + VectorStoreCreateOptionalRequestParams, + VectorStoreSearchOptionalRequestParams, +) + + +class VectorStoreRequestUtils: + """Helper utils for constructing Vector Store search requests""" + + @staticmethod + def get_requested_vector_store_search_optional_param( + params: Dict[str, Any], + ) -> VectorStoreSearchOptionalRequestParams: + """ + Filter parameters to only include those defined in VectorStoreSearchOptionalRequestParams. + + Args: + params: Dictionary of parameters to filter + + Returns: + VectorStoreSearchOptionalRequestParams instance with only the valid parameters + """ + valid_keys = get_type_hints(VectorStoreSearchOptionalRequestParams).keys() + filtered_params = { + k: v for k, v in params.items() if k in valid_keys and v is not None + } + + return cast(VectorStoreSearchOptionalRequestParams, filtered_params) + + @staticmethod + def get_requested_vector_store_create_optional_param( + params: Dict[str, Any], + ) -> VectorStoreCreateOptionalRequestParams: + """ + Filter parameters to only include those defined in VectorStoreCreateOptionalRequestParams. + + Args: + params: Dictionary of parameters to filter + + Returns: + VectorStoreCreateOptionalRequestParams instance with only the valid parameters + """ + valid_keys = get_type_hints(VectorStoreCreateOptionalRequestParams).keys() + filtered_params = { + k: v for k, v in params.items() if k in valid_keys and v is not None + } + + return cast(VectorStoreCreateOptionalRequestParams, filtered_params) + diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 9d72f852de1..8d72806c32b 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -2156,6 +2156,66 @@ "supports_tool_choice": true, "supports_native_streaming": true }, + "azure/o3-pro": { + "max_tokens": 100000, + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "input_cost_per_token": 2e-05, + "output_cost_per_token": 8e-05, + "input_cost_per_token_batches": 1e-05, + "output_cost_per_token_batches": 4e-05, + "litellm_provider": "azure", + "mode": "chat", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": false, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, + "azure/o3-pro-2025-06-10": { + "max_tokens": 100000, + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "input_cost_per_token": 2e-05, + "output_cost_per_token": 8e-05, + "input_cost_per_token_batches": 1e-05, + "output_cost_per_token_batches": 4e-05, + "litellm_provider": "azure", + "mode": "chat", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": false, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, "azure/o3": { "max_tokens": 100000, "max_input_tokens": 200000, @@ -3986,7 +4046,8 @@ "litellm_provider": "mistral", "mode": "chat", "supports_assistant_prefill": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_response_schema": true }, "mistral/mistral-small": { "max_tokens": 8191, @@ -3998,7 +4059,8 @@ "supports_function_calling": true, "mode": "chat", "supports_assistant_prefill": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_response_schema": true }, "mistral/mistral-small-latest": { "max_tokens": 8191, @@ -4010,7 +4072,8 @@ "supports_function_calling": true, "mode": "chat", "supports_assistant_prefill": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_response_schema": true }, "mistral/mistral-medium": { "max_tokens": 8191, @@ -4021,7 +4084,8 @@ "litellm_provider": "mistral", "mode": "chat", "supports_assistant_prefill": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_response_schema": true }, "mistral/mistral-medium-latest": { "max_tokens": 8191, @@ -4033,7 +4097,8 @@ "mode": "chat", "supports_function_calling": true, "supports_assistant_prefill": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_response_schema": true }, "mistral/mistral-medium-2505": { "max_tokens": 8191, @@ -4045,7 +4110,8 @@ "mode": "chat", "supports_function_calling": true, "supports_assistant_prefill": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_response_schema": true }, "mistral/mistral-medium-2312": { "max_tokens": 8191, @@ -4056,7 +4122,8 @@ "litellm_provider": "mistral", "mode": "chat", "supports_assistant_prefill": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_response_schema": true }, "mistral/mistral-large-latest": { "max_tokens": 128000, @@ -4068,7 +4135,8 @@ "mode": "chat", "supports_function_calling": true, "supports_assistant_prefill": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_response_schema": true }, "mistral/mistral-large-2411": { "max_tokens": 128000, @@ -4080,7 +4148,8 @@ "mode": "chat", "supports_function_calling": true, "supports_assistant_prefill": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_response_schema": true }, "mistral/mistral-large-2402": { "max_tokens": 8191, @@ -4092,7 +4161,8 @@ "mode": "chat", "supports_function_calling": true, "supports_assistant_prefill": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_response_schema": true }, "mistral/mistral-large-2407": { "max_tokens": 128000, @@ -4104,7 +4174,8 @@ "mode": "chat", "supports_function_calling": true, "supports_assistant_prefill": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_response_schema": true }, "mistral/pixtral-large-latest": { "max_tokens": 128000, @@ -4117,7 +4188,8 @@ "supports_function_calling": true, "supports_assistant_prefill": true, "supports_vision": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_response_schema": true }, "mistral/pixtral-large-2411": { "max_tokens": 128000, @@ -4130,7 +4202,8 @@ "supports_function_calling": true, "supports_assistant_prefill": true, "supports_vision": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_response_schema": true }, "mistral/pixtral-12b-2409": { "max_tokens": 128000, @@ -4143,7 +4216,8 @@ "supports_function_calling": true, "supports_assistant_prefill": true, "supports_vision": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_response_schema": true }, "mistral/open-mistral-7b": { "max_tokens": 8191, @@ -4154,7 +4228,8 @@ "litellm_provider": "mistral", "mode": "chat", "supports_assistant_prefill": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_response_schema": true }, "mistral/open-mixtral-8x7b": { "max_tokens": 8191, @@ -4166,7 +4241,8 @@ "mode": "chat", "supports_function_calling": true, "supports_assistant_prefill": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_response_schema": true }, "mistral/open-mixtral-8x22b": { "max_tokens": 8191, @@ -4178,7 +4254,8 @@ "mode": "chat", "supports_function_calling": true, "supports_assistant_prefill": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_response_schema": true }, "mistral/codestral-latest": { "max_tokens": 8191, @@ -4189,7 +4266,8 @@ "litellm_provider": "mistral", "mode": "chat", "supports_assistant_prefill": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_response_schema": true }, "mistral/codestral-2405": { "max_tokens": 8191, @@ -4200,7 +4278,8 @@ "litellm_provider": "mistral", "mode": "chat", "supports_assistant_prefill": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_response_schema": true }, "mistral/open-mistral-nemo": { "max_tokens": 128000, @@ -4212,7 +4291,8 @@ "mode": "chat", "source": "https://mistral.ai/technology/", "supports_assistant_prefill": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_response_schema": true }, "mistral/open-mistral-nemo-2407": { "max_tokens": 128000, @@ -4224,7 +4304,8 @@ "mode": "chat", "source": "https://mistral.ai/technology/", "supports_assistant_prefill": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_response_schema": true }, "mistral/open-codestral-mamba": { "max_tokens": 256000, @@ -4261,7 +4342,8 @@ "source": "https://mistral.ai/news/devstral", "supports_function_calling": true, "supports_assistant_prefill": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_response_schema": true }, "mistral/magistral-medium-latest": { "max_tokens": 40000, @@ -4275,7 +4357,8 @@ "supports_function_calling": true, "supports_assistant_prefill": true, "supports_tool_choice": true, - "supports_reasoning": true + "supports_reasoning": true, + "supports_response_schema": true }, "mistral/magistral-medium-2506": { "max_tokens": 40000, @@ -4289,7 +4372,8 @@ "supports_function_calling": true, "supports_assistant_prefill": true, "supports_tool_choice": true, - "supports_reasoning": true + "supports_reasoning": true, + "supports_response_schema": true }, "mistral/magistral-small-latest": { "max_tokens": 40000, @@ -4303,7 +4387,8 @@ "supports_function_calling": true, "supports_assistant_prefill": true, "supports_tool_choice": true, - "supports_reasoning": true + "supports_reasoning": true, + "supports_response_schema": true }, "mistral/magistral-small-2506": { "max_tokens": 40000, @@ -4317,7 +4402,8 @@ "supports_function_calling": true, "supports_assistant_prefill": true, "supports_tool_choice": true, - "supports_reasoning": true + "supports_reasoning": true, + "supports_response_schema": true }, "mistral/mistral-embed": { "max_tokens": 8192, @@ -9995,7 +10081,15 @@ "mode": "chat", "supports_tool_choice": true }, - "mistralai/mistral-small-3.1-24b-instruct": { + "openrouter/mistralai/mistral-small-3.1-24b-instruct": { + "max_tokens": 32000, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "supports_tool_choice": true + }, + "openrouter/mistralai/mistral-small-3.2-24b-instruct": { "max_tokens": 32000, "input_cost_per_token": 1e-07, "output_cost_per_token": 3e-07, @@ -13647,13 +13741,14 @@ "input_cost_per_token": 2e-06, "output_cost_per_token": 8e-06, "output_cost_per_reasoning_token": 3e-06, - "litellm_provider": "perplexity", - "mode": "chat", + "citation_cost_per_token": 2e-06, "search_context_cost_per_query": { "search_context_size_low": 0.005, "search_context_size_medium": 0.005, "search_context_size_high": 0.005 }, + "litellm_provider": "perplexity", + "mode": "chat", "supports_reasoning": true, "supports_web_search": true }, diff --git a/poetry.lock b/poetry.lock index 509c19d4283..d179e90d80e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1833,14 +1833,14 @@ openai = ["openai (>=0.27.8)"] [[package]] name = "litellm-enterprise" -version = "0.1.8" +version = "0.1.9" description = "Package for LiteLLM Enterprise features" optional = true python-versions = "!=2.7.*,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,!=3.7.*,>=3.8" groups = ["main"] markers = "extra == \"proxy\"" files = [ - {file = "litellm_enterprise-0.1.8.tar.gz", hash = "sha256:59b8c8a18f51c1e5fe2e341d66ba268c2814c8035825ae929530438524dc08ca"}, + {file = "litellm_enterprise-0.1.9.tar.gz", hash = "sha256:2bdf629cf8bd36805bad70acb609bfa0c00eaf72d3b42f6e17c54c5b50758c4a"}, ] [[package]] @@ -4991,4 +4991,4 @@ utils = ["numpydoc"] [metadata] lock-version = "2.1" python-versions = ">=3.8.1,<4.0, !=3.9.7" -content-hash = "1d0d0c73c55208694ca58a8a8d12ee0e95e265bb16aefd72d79ca8b5e49a8bca" +content-hash = "c3d56e337720ab9c5ab2ec87794f16d1ba831ee86edeb236ae7429f3a7717677" diff --git a/pyproject.toml b/pyproject.toml index af13459b70f..5ecd47df50f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "1.73.0" +version = "1.73.2" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT" @@ -58,7 +58,7 @@ redisvl = {version = "^0.4.1", optional = true, markers = "python_version >= '3. mcp = {version = "1.9.3", optional = true, python = ">=3.10"} litellm-proxy-extras = {version = "0.2.5", optional = true} rich = {version = "13.7.1", optional = true} -litellm-enterprise = {version = "0.1.8", optional = true} +litellm-enterprise = {version = "0.1.9", optional = true} diskcache = {version = "^5.6.1", optional = true} [tool.poetry.extras] @@ -141,7 +141,7 @@ requires = ["poetry-core", "wheel"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "1.73.0" +version = "1.73.2" version_files = [ "pyproject.toml:^version" ] diff --git a/requirements.txt b/requirements.txt index c5b18f1b0d8..905a41fd4a6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -56,4 +56,4 @@ websockets==13.1.0 # for realtime API ######################## # LITELLM ENTERPRISE DEPENDENCIES ######################## -litellm-enterprise==0.1.8 +litellm-enterprise==0.1.9 diff --git a/test_script.py b/test_script.py index 0dce4b62563..d0fec5035b3 100644 --- a/test_script.py +++ b/test_script.py @@ -1,24 +1,20 @@ import os import json -gemini_model_cost_map = json.load(open("model_prices_and_context_window.json")) +mistral_model_cost_map = json.load(open("model_prices_and_context_window.json")) -for model, model_info in gemini_model_cost_map.items(): +for model, model_info in mistral_model_cost_map.items(): if ( - ( - model_info.get("litellm_provider") == "gemini" - or model_info.get("litellm_provider") == "vertex_ai-language-models" - ) + (model_info.get("litellm_provider") == "mistral") and model_info.get("mode") == "chat" - and ("gemini-2.5" in model and "tts" not in model) - and model_info.get("supports_pdf_input") is None + and ("codestral-mamba" not in model) ): """ - Update all gemini chat models to support pdf input + Update all mistral models to supports_response_schema """ - model_info["supports_pdf_input"] = True - print(f"Updated {model} to support pdf input") + model_info["supports_response_schema"] = True + print(f"Updated {model} to support response schema") json.dump( - gemini_model_cost_map, open("model_prices_and_context_window.json", "w"), indent=4 + mistral_model_cost_map, open("model_prices_and_context_window.json", "w"), indent=4 ) diff --git a/tests/guardrails_tests/test_bedrock_guardrails.py b/tests/guardrails_tests/test_bedrock_guardrails.py index 998c27a93db..246338fd1d3 100644 --- a/tests/guardrails_tests/test_bedrock_guardrails.py +++ b/tests/guardrails_tests/test_bedrock_guardrails.py @@ -10,14 +10,13 @@ from litellm.caching import DualCache from unittest.mock import MagicMock, AsyncMock, patch @pytest.mark.asyncio -async def test_bedrock_guardrails(): +async def test_bedrock_guardrails_pii_masking(): # Create proper mock objects mock_user_api_key_dict = UserAPIKeyAuth() guardrail = BedrockGuardrail( guardrailIdentifier="wf0hkdb5x07f", guardrailVersion="DRAFT", - mask_request_content=True, ) request_data = { @@ -35,7 +34,7 @@ async def test_bedrock_guardrails(): user_api_key_dict=mock_user_api_key_dict, call_type="completion" ) - print(response) + print("response after moderation hook", response) if response: # Only assert if response is not None assert response["messages"][0]["content"] == "Hello, my phone number is {PHONE}" @@ -45,14 +44,13 @@ async def test_bedrock_guardrails(): @pytest.mark.asyncio -async def test_bedrock_guardrails_content_list(): +async def test_bedrock_guardrails_pii_masking_content_list(): # Create proper mock objects mock_user_api_key_dict = UserAPIKeyAuth() guardrail = BedrockGuardrail( guardrailIdentifier="wf0hkdb5x07f", guardrailVersion="DRAFT", - mask_request_content=True, ) request_data = { @@ -105,7 +103,7 @@ async def test_bedrock_guardrails_with_streaming(): ) guardrail = BedrockGuardrail( - guardrailIdentifier="wf0hkdb5x07f", + guardrailIdentifier="ff6ujrregl1q", guardrailVersion="DRAFT", supported_event_hooks=[GuardrailEventHooks.post_call], guardrail_name="bedrock-post-guard", @@ -118,7 +116,7 @@ async def test_bedrock_guardrails_with_streaming(): "messages": [ { "role": "user", - "content": "My name is ishaan@gmail.com" + "content": "Hi I like coffee" } ], "stream": True, @@ -154,7 +152,7 @@ async def test_bedrock_guardrails_with_streaming_no_violation(): ) guardrail = BedrockGuardrail( - guardrailIdentifier="wf0hkdb5x07f", + guardrailIdentifier="ff6ujrregl1q", guardrailVersion="DRAFT", supported_event_hooks=[GuardrailEventHooks.post_call], guardrail_name="bedrock-post-guard", @@ -334,4 +332,241 @@ async def test_bedrock_guardrail_aws_param_persistence(): assert kwargs["aws_secret_access_key"] == "test-secret-key" assert kwargs["aws_region_name"] == "us-east-1" +@pytest.mark.asyncio +async def test_bedrock_guardrail_blocked_vs_anonymized_actions(): + """Test that BLOCKED actions raise exceptions but ANONYMIZED actions do not""" + from unittest.mock import MagicMock + from litellm.proxy.guardrails.guardrail_hooks.bedrock_guardrails import BedrockGuardrail + from litellm.types.proxy.guardrails.guardrail_hooks.bedrock_guardrails import BedrockGuardrailResponse + + guardrail = BedrockGuardrail( + guardrailIdentifier="test-guardrail", + guardrailVersion="DRAFT" + ) + + # Test 1: ANONYMIZED action should NOT raise exception + anonymized_response: BedrockGuardrailResponse = { + "action": "GUARDRAIL_INTERVENED", + "outputs": [{ + "text": "Hello, my phone number is {PHONE}" + }], + "assessments": [{ + "sensitiveInformationPolicy": { + "piiEntities": [{ + "type": "PHONE", + "match": "+1 412 555 1212", + "action": "ANONYMIZED" + }] + } + }] + } + + should_raise = guardrail._should_raise_guardrail_blocked_exception(anonymized_response) + assert should_raise is False, "ANONYMIZED actions should not raise exceptions" + + # Test 2: BLOCKED action should raise exception + blocked_response: BedrockGuardrailResponse = { + "action": "GUARDRAIL_INTERVENED", + "outputs": [{ + "text": "I can't provide that information." + }], + "assessments": [{ + "topicPolicy": { + "topics": [{ + "name": "Sensitive Topic", + "type": "DENY", + "action": "BLOCKED" + }] + } + }] + } + + should_raise = guardrail._should_raise_guardrail_blocked_exception(blocked_response) + assert should_raise is True, "BLOCKED actions should raise exceptions" + + # Test 3: Mixed actions - should raise if ANY action is BLOCKED + mixed_response: BedrockGuardrailResponse = { + "action": "GUARDRAIL_INTERVENED", + "outputs": [{ + "text": "I can't provide that information." + }], + "assessments": [{ + "sensitiveInformationPolicy": { + "piiEntities": [{ + "type": "PHONE", + "match": "+1 412 555 1212", + "action": "ANONYMIZED" + }] + }, + "topicPolicy": { + "topics": [{ + "name": "Blocked Topic", + "type": "DENY", + "action": "BLOCKED" + }] + } + }] + } + + should_raise = guardrail._should_raise_guardrail_blocked_exception(mixed_response) + assert should_raise is True, "Mixed actions with any BLOCKED should raise exceptions" + + # Test 4: NONE action should not raise exception + none_response: BedrockGuardrailResponse = { + "action": "NONE", + "outputs": [], + "assessments": [] + } + + should_raise = guardrail._should_raise_guardrail_blocked_exception(none_response) + assert should_raise is False, "NONE actions should not raise exceptions" + + # Test 5: Test other policy types with BLOCKED actions + content_blocked_response: BedrockGuardrailResponse = { + "action": "GUARDRAIL_INTERVENED", + "outputs": [{ + "text": "I can't provide that information." + }], + "assessments": [{ + "contentPolicy": { + "filters": [{ + "type": "VIOLENCE", + "confidence": "HIGH", + "action": "BLOCKED" + }] + } + }] + } + + should_raise = guardrail._should_raise_guardrail_blocked_exception(content_blocked_response) + assert should_raise is True, "Content policy BLOCKED actions should raise exceptions" + + +@pytest.mark.asyncio +async def test_bedrock_guardrail_masking_with_anonymized_response(): + """Test that masking works correctly when guardrail returns ANONYMIZED actions""" + from unittest.mock import AsyncMock, MagicMock, patch + from litellm.proxy._types import UserAPIKeyAuth + from litellm.caching import DualCache + + # Create proper mock objects + mock_user_api_key_dict = UserAPIKeyAuth() + + guardrail = BedrockGuardrail( + guardrailIdentifier="test-guardrail", + guardrailVersion="DRAFT", + mask_request_content=True, + ) + + # Mock the Bedrock API response with ANONYMIZED action + mock_bedrock_response = MagicMock() + mock_bedrock_response.status_code = 200 + mock_bedrock_response.json.return_value = { + "action": "GUARDRAIL_INTERVENED", + "outputs": [{ + "text": "Hello, my phone number is {PHONE}" + }], + "assessments": [{ + "sensitiveInformationPolicy": { + "piiEntities": [{ + "type": "PHONE", + "match": "+1 412 555 1212", + "action": "ANONYMIZED" + }] + } + }] + } + + request_data = { + "model": "gpt-4o", + "messages": [ + {"role": "user", "content": "Hello, my phone number is +1 412 555 1212"}, + ], + } + + # Patch the async_handler.post method + with patch.object(guardrail.async_handler, 'post', new_callable=AsyncMock) as mock_post: + mock_post.return_value = mock_bedrock_response + + # This should NOT raise an exception since action is ANONYMIZED + try: + response = await guardrail.async_moderation_hook( + data=request_data, + user_api_key_dict=mock_user_api_key_dict, + call_type="completion" + ) + # Should succeed and return data with masked content + assert response is not None + assert response["messages"][0]["content"] == "Hello, my phone number is {PHONE}" + except Exception as e: + pytest.fail(f"Should not raise exception for ANONYMIZED actions, but got: {e}") + + +@pytest.mark.asyncio +async def test_bedrock_guardrail_uses_masked_output_without_masking_flags(): + """Test that masked output from guardrails is used even when masking flags are not enabled""" + from unittest.mock import AsyncMock, MagicMock, patch + from litellm.proxy._types import UserAPIKeyAuth + + # Create proper mock objects + mock_user_api_key_dict = UserAPIKeyAuth() + + # Create guardrail WITHOUT masking flags enabled + guardrail = BedrockGuardrail( + guardrailIdentifier="test-guardrail", + guardrailVersion="DRAFT", + # Note: No mask_request_content=True or mask_response_content=True + ) + + # Mock the Bedrock API response with ANONYMIZED action and masked output + mock_bedrock_response = MagicMock() + mock_bedrock_response.status_code = 200 + mock_bedrock_response.json.return_value = { + "action": "GUARDRAIL_INTERVENED", + "outputs": [{ + "text": "Hello, my phone number is {PHONE} and email is {EMAIL}" + }], + "assessments": [{ + "sensitiveInformationPolicy": { + "piiEntities": [ + { + "type": "PHONE", + "match": "+1 412 555 1212", + "action": "ANONYMIZED" + }, + { + "type": "EMAIL", + "match": "user@example.com", + "action": "ANONYMIZED" + } + ] + } + }] + } + + request_data = { + "model": "gpt-4o", + "messages": [ + {"role": "user", "content": "Hello, my phone number is +1 412 555 1212 and email is user@example.com"}, + ], + } + + # Patch the async_handler.post method + with patch.object(guardrail.async_handler, 'post', new_callable=AsyncMock) as mock_post: + mock_post.return_value = mock_bedrock_response + + # This should use the masked output even without masking flags + response = await guardrail.async_moderation_hook( + data=request_data, + user_api_key_dict=mock_user_api_key_dict, + call_type="completion" + ) + + # Should use the masked content from guardrail output + assert response is not None + assert response["messages"][0]["content"] == "Hello, my phone number is {PHONE} and email is {EMAIL}" + print("✅ Masked output was applied even without masking flags enabled") + + + \ No newline at end of file diff --git a/tests/llm_translation/test_azure_openai.py b/tests/llm_translation/test_azure_openai.py index 039eb434f54..a27d0dd165d 100644 --- a/tests/llm_translation/test_azure_openai.py +++ b/tests/llm_translation/test_azure_openai.py @@ -594,6 +594,7 @@ async def test_azure_embedding_max_retries_0( def test_azure_safety_result(): """Bubble up safety result from Azure OpenAI""" from litellm import completion + litellm._turn_on_debug() response = completion( @@ -602,4 +603,30 @@ def test_azure_safety_result(): ) print(f"response: {response}") assert response.choices[0].message.content is not None - assert response.choices[0].provider_specific_fields is not None \ No newline at end of file + assert response.choices[0].provider_specific_fields is not None + + +def test_azure_openai_responses_bridge(): + from litellm import completion + import litellm + + litellm._turn_on_debug() + + with patch.object(litellm, "responses") as mock_responses: + try: + response = completion( + model="azure/responses/test-azure-computer-use-preview", + messages=[{"role": "user", "content": "Hello world"}], + api_base=os.getenv("AZURE_COMPUTER_USE_API_BASE"), + api_version="2025-04-01-preview", + api_key=os.getenv("AZURE_COMPUTER_USE_API_KEY"), + ) + except Exception as e: + print(e) + + mock_responses.assert_called_once() + assert ( + mock_responses.call_args.kwargs["model"] + == "test-azure-computer-use-preview" + ) + assert mock_responses.call_args.kwargs["custom_llm_provider"] == "azure" diff --git a/tests/logging_callback_tests/test_bedrock_knowledgebase_hook.py b/tests/logging_callback_tests/test_bedrock_knowledgebase_hook.py index 532a5c11c52..4c75d485277 100644 --- a/tests/logging_callback_tests/test_bedrock_knowledgebase_hook.py +++ b/tests/logging_callback_tests/test_bedrock_knowledgebase_hook.py @@ -19,7 +19,7 @@ import pytest import litellm from litellm import completion from litellm._logging import verbose_logger -from litellm.integrations.vector_stores.bedrock_vector_store import BedrockVectorStore +from litellm.integrations.vector_store_integrations.bedrock_vector_store import BedrockVectorStore from litellm.llms.custom_httpx.http_handler import HTTPHandler, AsyncHTTPHandler from litellm.integrations.custom_logger import CustomLogger from litellm.types.utils import StandardLoggingPayload, StandardLoggingVectorStoreRequest diff --git a/tests/logging_callback_tests/test_unit_tests_init_callbacks.py b/tests/logging_callback_tests/test_unit_tests_init_callbacks.py index 74165ccdb7d..aff622201fb 100644 --- a/tests/logging_callback_tests/test_unit_tests_init_callbacks.py +++ b/tests/logging_callback_tests/test_unit_tests_init_callbacks.py @@ -16,48 +16,10 @@ import asyncio import logging from litellm._logging import verbose_logger from prometheus_client import REGISTRY, CollectorRegistry - -from litellm.integrations.lago import LagoLogger -from litellm.integrations.deepeval import DeepEvalLogger -from litellm.integrations.openmeter import OpenMeterLogger -from litellm.integrations.braintrust_logging import BraintrustLogger -from litellm.integrations.galileo import GalileoObserve -from litellm.integrations.langsmith import LangsmithLogger -from litellm.integrations.literal_ai import LiteralAILogger -from litellm.integrations.prometheus import PrometheusLogger -from litellm.integrations.datadog.datadog import DataDogLogger -from litellm.integrations.datadog.datadog_llm_obs import DataDogLLMObsLogger -from litellm.integrations.gcs_bucket.gcs_bucket import GCSBucketLogger -from litellm.integrations.gcs_pubsub.pub_sub import GcsPubSubLogger -from litellm.integrations.opik.opik import OpikLogger -from litellm.integrations.opentelemetry import OpenTelemetry -from litellm.integrations.mlflow import MlflowLogger -from litellm.integrations.argilla import ArgillaLogger -from litellm.integrations.deepeval.deepeval import DeepEvalLogger -from litellm.integrations.s3_v2 import S3Logger -from litellm.integrations.langfuse.langfuse_otel import LangfuseOtelLogger -from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook -from litellm.integrations.vector_stores.bedrock_vector_store import BedrockVectorStore -from litellm.integrations.langfuse.langfuse_prompt_management import ( - LangfusePromptManagement, -) -from litellm.integrations.azure_storage.azure_storage import AzureBlobStorageLogger -from litellm.integrations.agentops import AgentOps -from litellm.integrations.humanloop import HumanloopLogger -from litellm.proxy.hooks.dynamic_rate_limiter import _PROXY_DynamicRateLimitHandler -from litellm_enterprise.enterprise_callbacks.generic_api_callback import ( - GenericAPILogger, -) -from litellm_enterprise.enterprise_callbacks.send_emails.resend_email import ( - ResendEmailLogger, -) -from litellm_enterprise.enterprise_callbacks.send_emails.smtp_email import ( - SMTPEmailLogger, -) -from litellm_enterprise.enterprise_callbacks.pagerduty.pagerduty import ( - PagerDutyAlerting, -) from unittest.mock import patch +from litellm.litellm_core_utils.custom_logger_registry import ( + CustomLoggerRegistry, +) # clear prometheus collectors / registry collectors = list(REGISTRY._collector_to_names.keys()) @@ -65,43 +27,7 @@ for collector in collectors: REGISTRY.unregister(collector) ###################################### -callback_class_str_to_classType = { - "lago": LagoLogger, - "openmeter": OpenMeterLogger, - "braintrust": BraintrustLogger, - "galileo": GalileoObserve, - "langsmith": LangsmithLogger, - "literalai": LiteralAILogger, - "prometheus": PrometheusLogger, - "datadog": DataDogLogger, - "datadog_llm_observability": DataDogLLMObsLogger, - "gcs_bucket": GCSBucketLogger, - "opik": OpikLogger, - "argilla": ArgillaLogger, - "opentelemetry": OpenTelemetry, - "azure_storage": AzureBlobStorageLogger, - "humanloop": HumanloopLogger, - # OTEL compatible loggers - "logfire": OpenTelemetry, - "arize": OpenTelemetry, - "langfuse_otel": OpenTelemetry, - "arize_phoenix": OpenTelemetry, - "langtrace": OpenTelemetry, - "mlflow": MlflowLogger, - "langfuse": LangfusePromptManagement, - "otel": OpenTelemetry, - "pagerduty": PagerDutyAlerting, - "gcs_pubsub": GcsPubSubLogger, - "anthropic_cache_control_hook": AnthropicCacheControlHook, - "agentops": AgentOps, - "bedrock_vector_store": BedrockVectorStore, - "generic_api": GenericAPILogger, - "resend_email": ResendEmailLogger, - "smtp_email": SMTPEmailLogger, - "deepeval": DeepEvalLogger, - "s3_v2": S3Logger, - "langfuse_otel": OpenTelemetry, -} + expected_env_vars = { "LAGO_API_KEY": "api_key", @@ -215,7 +141,7 @@ async def use_callback_in_llm_call( await asyncio.sleep(0.5) - expected_class = callback_class_str_to_classType[callback] + expected_class = CustomLoggerRegistry.CALLBACK_CLASS_STR_TO_CLASS_TYPE[callback] if used_in == "callbacks": assert isinstance(litellm._async_success_callback[0], expected_class) diff --git a/tests/router_unit_tests/test_router_helper_utils.py b/tests/router_unit_tests/test_router_helper_utils.py index 8a27f2147ce..a3c3a1201ca 100644 --- a/tests/router_unit_tests/test_router_helper_utils.py +++ b/tests/router_unit_tests/test_router_helper_utils.py @@ -446,6 +446,40 @@ async def test_deployment_callback_on_failure(model_list): ) +def test_deployment_callback_respects_cooldown_time(model_list): + """Ensure per-model cooldown_time is honored even when exception headers are present.""" + import httpx + import time + from unittest.mock import patch + + router = Router(model_list=model_list) + + class FakeException(Exception): + def __init__(self): + self.status_code = 429 + self.headers = httpx.Headers({"x-test": "1"}) + + kwargs = { + "exception": FakeException(), + "litellm_params": { + "metadata": {"model_group": "gpt-3.5-turbo"}, + "model_info": {"id": 100}, + "cooldown_time": 0, + }, + } + + with patch("litellm.router._set_cooldown_deployments") as mock_set: + router.deployment_callback_on_failure( + kwargs=kwargs, + completion_response=None, + start_time=time.time(), + end_time=time.time(), + ) + + mock_set.assert_called_once() + assert mock_set.call_args.kwargs["time_to_cooldown"] == 0 + + def test_log_retry(model_list): """Test if the '_log_retry' function is working correctly""" import time diff --git a/tests/test_litellm/enterprise/enterprise_callbacks/test_callback_controls.py b/tests/test_litellm/enterprise/enterprise_callbacks/test_callback_controls.py new file mode 100644 index 00000000000..a06911170c0 --- /dev/null +++ b/tests/test_litellm/enterprise/enterprise_callbacks/test_callback_controls.py @@ -0,0 +1,176 @@ +import unittest.mock as mock +from unittest.mock import MagicMock, patch + +import pytest + +from enterprise.litellm_enterprise.enterprise_callbacks.callback_controls import ( + EnterpriseCallbackControls, +) +from litellm.constants import X_LITELLM_DISABLE_CALLBACKS +from litellm.integrations.custom_logger import CustomLogger +from litellm.integrations.datadog.datadog import DataDogLogger +from litellm.integrations.langfuse.langfuse_prompt_management import ( + LangfusePromptManagement, +) +from litellm.integrations.s3_v2 import S3Logger + + +class TestEnterpriseCallbackControls: + + @pytest.fixture + def mock_premium_user(self): + """Fixture to mock premium user check as True""" + with patch.object(EnterpriseCallbackControls, '_premium_user_check', return_value=True): + yield + + @pytest.fixture + def mock_non_premium_user(self): + """Fixture to mock premium user check as False""" + with patch.object(EnterpriseCallbackControls, '_premium_user_check', return_value=False): + yield + + @pytest.fixture + def mock_request_headers(self): + """Fixture to mock get_proxy_server_request_headers""" + with patch('enterprise.litellm_enterprise.enterprise_callbacks.callback_controls.get_proxy_server_request_headers') as mock_headers: + yield mock_headers + + def test_callback_disabled_langfuse_string(self, mock_premium_user, mock_request_headers): + """Test that 'langfuse' string callback is disabled when specified in headers""" + mock_request_headers.return_value = {X_LITELLM_DISABLE_CALLBACKS: "langfuse"} + litellm_params = {"proxy_server_request": {"url": "test"}} + + result = EnterpriseCallbackControls.is_callback_disabled_via_headers("langfuse", litellm_params) + assert result is True + + def test_callback_disabled_langfuse_customlogger(self, mock_premium_user, mock_request_headers): + """Test that LangfusePromptManagement CustomLogger instance is disabled when 'langfuse' specified in headers""" + mock_request_headers.return_value = {X_LITELLM_DISABLE_CALLBACKS: "langfuse"} + litellm_params = {"proxy_server_request": {"url": "test"}} + + langfuse_logger = LangfusePromptManagement() + result = EnterpriseCallbackControls.is_callback_disabled_via_headers(langfuse_logger, litellm_params) + assert result is True + + def test_callback_disabled_s3_v2_string(self, mock_premium_user, mock_request_headers): + """Test that 's3_v2' string callback is disabled when specified in headers""" + mock_request_headers.return_value = {X_LITELLM_DISABLE_CALLBACKS: "s3_v2"} + litellm_params = {"proxy_server_request": {"url": "test"}} + + result = EnterpriseCallbackControls.is_callback_disabled_via_headers("s3_v2", litellm_params) + assert result is True + + def test_callback_disabled_s3_v2_customlogger(self, mock_premium_user, mock_request_headers): + """Test that S3Logger CustomLogger instance is disabled when 's3_v2' specified in headers""" + mock_request_headers.return_value = {X_LITELLM_DISABLE_CALLBACKS: "s3_v2"} + litellm_params = {"proxy_server_request": {"url": "test"}} + + # Mock S3Logger to avoid async initialization issues + with patch('litellm.integrations.s3_v2.S3Logger.__init__', return_value=None): + s3_logger = S3Logger() + result = EnterpriseCallbackControls.is_callback_disabled_via_headers(s3_logger, litellm_params) + assert result is True + + def test_callback_disabled_datadog_string(self, mock_premium_user, mock_request_headers): + """Test that 'datadog' string callback is disabled when specified in headers""" + mock_request_headers.return_value = {X_LITELLM_DISABLE_CALLBACKS: "datadog"} + litellm_params = {"proxy_server_request": {"url": "test"}} + + result = EnterpriseCallbackControls.is_callback_disabled_via_headers("datadog", litellm_params) + assert result is True + + def test_callback_disabled_datadog_customlogger(self, mock_premium_user, mock_request_headers): + """Test that DataDogLogger CustomLogger instance is disabled when 'datadog' specified in headers""" + mock_request_headers.return_value = {X_LITELLM_DISABLE_CALLBACKS: "datadog"} + litellm_params = {"proxy_server_request": {"url": "test"}} + + # Mock DataDogLogger to avoid async initialization issues + with patch('litellm.integrations.datadog.datadog.DataDogLogger.__init__', return_value=None): + datadog_logger = DataDogLogger() + result = EnterpriseCallbackControls.is_callback_disabled_via_headers(datadog_logger, litellm_params) + assert result is True + + def test_multiple_callbacks_disabled(self, mock_premium_user, mock_request_headers): + """Test that multiple callbacks can be disabled with comma-separated list""" + mock_request_headers.return_value = {X_LITELLM_DISABLE_CALLBACKS: "langfuse,datadog,s3_v2"} + litellm_params = {"proxy_server_request": {"url": "test"}} + + # Test each callback is disabled + assert EnterpriseCallbackControls.is_callback_disabled_via_headers("langfuse", litellm_params) is True + assert EnterpriseCallbackControls.is_callback_disabled_via_headers("datadog", litellm_params) is True + assert EnterpriseCallbackControls.is_callback_disabled_via_headers("s3_v2", litellm_params) is True + + # Test non-disabled callback is not disabled + assert EnterpriseCallbackControls.is_callback_disabled_via_headers("prometheus", litellm_params) is False + + def test_callback_not_disabled_when_not_in_list(self, mock_premium_user, mock_request_headers): + """Test that callbacks not in the disabled list are not disabled""" + mock_request_headers.return_value = {X_LITELLM_DISABLE_CALLBACKS: "langfuse"} + litellm_params = {"proxy_server_request": {"url": "test"}} + + result = EnterpriseCallbackControls.is_callback_disabled_via_headers("datadog", litellm_params) + assert result is False + + def test_callback_not_disabled_when_no_header(self, mock_premium_user, mock_request_headers): + """Test that callbacks are not disabled when the header is not present""" + mock_request_headers.return_value = {} + litellm_params = {"proxy_server_request": {"url": "test"}} + + result = EnterpriseCallbackControls.is_callback_disabled_via_headers("langfuse", litellm_params) + assert result is False + + def test_callback_not_disabled_when_header_none(self, mock_premium_user, mock_request_headers): + """Test that callbacks are not disabled when the header value is None""" + mock_request_headers.return_value = {X_LITELLM_DISABLE_CALLBACKS: None} + litellm_params = {"proxy_server_request": {"url": "test"}} + + result = EnterpriseCallbackControls.is_callback_disabled_via_headers("langfuse", litellm_params) + assert result is False + + def test_non_premium_user_cannot_disable_callbacks(self, mock_non_premium_user, mock_request_headers): + """Test that non-premium users cannot disable callbacks even with the header""" + mock_request_headers.return_value = {X_LITELLM_DISABLE_CALLBACKS: "langfuse"} + litellm_params = {"proxy_server_request": {"url": "test"}} + + result = EnterpriseCallbackControls.is_callback_disabled_via_headers("langfuse", litellm_params) + assert result is False + + def test_case_insensitive_callback_matching(self, mock_premium_user, mock_request_headers): + """Test that callback matching is case insensitive""" + mock_request_headers.return_value = {X_LITELLM_DISABLE_CALLBACKS: "LANGFUSE,DataDog"} + litellm_params = {"proxy_server_request": {"url": "test"}} + + # Test lowercase callbacks are disabled + assert EnterpriseCallbackControls.is_callback_disabled_via_headers("langfuse", litellm_params) is True + assert EnterpriseCallbackControls.is_callback_disabled_via_headers("datadog", litellm_params) is True + + def test_whitespace_handling_in_disabled_callbacks(self, mock_premium_user, mock_request_headers): + """Test that whitespace around callback names is handled correctly""" + mock_request_headers.return_value = {X_LITELLM_DISABLE_CALLBACKS: " langfuse , datadog , s3_v2 "} + litellm_params = {"proxy_server_request": {"url": "test"}} + + assert EnterpriseCallbackControls.is_callback_disabled_via_headers("langfuse", litellm_params) is True + assert EnterpriseCallbackControls.is_callback_disabled_via_headers("datadog", litellm_params) is True + assert EnterpriseCallbackControls.is_callback_disabled_via_headers("s3_v2", litellm_params) is True + + def test_custom_logger_not_in_registry(self, mock_premium_user, mock_request_headers): + """Test that CustomLogger not in registry is not disabled""" + mock_request_headers.return_value = {X_LITELLM_DISABLE_CALLBACKS: "unknown_logger"} + litellm_params = {"proxy_server_request": {"url": "test"}} + + # Create a mock CustomLogger that's not in the registry + class UnknownLogger(CustomLogger): + pass + + unknown_logger = UnknownLogger() + result = EnterpriseCallbackControls.is_callback_disabled_via_headers(unknown_logger, litellm_params) + assert result is False + + def test_exception_handling(self, mock_premium_user, mock_request_headers): + """Test that exceptions are handled gracefully and return False""" + # Make get_proxy_server_request_headers raise an exception + mock_request_headers.side_effect = Exception("Test exception") + litellm_params = {"proxy_server_request": {"url": "test"}} + + result = EnterpriseCallbackControls.is_callback_disabled_via_headers("langfuse", litellm_params) + assert result is False diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index 4210d27e8a6..56ea91aef24 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -325,3 +325,50 @@ async def test_bedrock_process_image_async_factory(): image_url=image_url, format=None ) print(f"content_block: {content_block}") + + +def test_unpack_defs_resolves_nested_ref_inside_anyof_items(): + """Ensure unpack_defs correctly resolves $ref inside items within anyOf (Issue #11372).""" + from litellm.litellm_core_utils.prompt_templates.common_utils import unpack_defs + + # Define a minimal schema reproducing the bug scenario + schema = { + "type": "object", + "properties": { + "vatAmounts": { + "anyOf": [ + { # List of VatAmount + "type": "array", + "items": {"$ref": "#/$defs/VatAmount"}, + }, + {"type": "null"}, + ], + "title": "Vat Amounts", + } + }, + "$defs": { + "VatAmount": { + "type": "object", + "properties": { + "vatRate": {"type": "number"}, + "vatAmount": {"type": "number"}, + }, + "required": ["vatRate", "vatAmount"], + "title": "VatAmount", + } + }, + } + + # Perform unpacking + unpack_defs(schema, schema["$defs"]) + + # Extract the items schema after unpacking + items_schema = ( + schema["properties"]["vatAmounts"]["anyOf"][0]["items"] + ) + + # Assertions: items_schema should now be the resolved object, not an empty dict + assert isinstance(items_schema, dict), "Items schema should be a dict after unpacking" + assert items_schema.get("type") == "object" + # Ensure essential properties are present + assert set(items_schema.get("properties", {}).keys()) == {"vatRate", "vatAmount"} diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 9d95749827b..6687fbd0c17 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -178,3 +178,39 @@ def test_get_request_tags(): assert "test-tag" in tags assert "User-Agent: litellm" in tags assert "User-Agent: litellm/0.1.0" in tags + + +def test_response_cost_calculator_with_response_cost_in_hidden_params(logging_obj): + from litellm import Router + from litellm.litellm_core_utils.litellm_logging import Logging + + router = Router( + model_list=[ + { + "model_name": "DeepSeek-R1", + "litellm_params": { + "model": "together_ai/deepseek-ai/DeepSeek-R1", + }, + "model_info": { + "access_groups": ["agent-models"], + "supports_tool_choice": True, + "supports_function_calling": True, + "input_cost_per_token": 100, + "output_cost_per_token": 100, + }, + } + ] + ) + + mock_response = router.completion( + model="DeepSeek-R1", + messages=[{"role": "user", "content": "Hey"}], + mock_response="Hello, world!", + ) + + response_cost = logging_obj._response_cost_calculator( + result=mock_response, + ) + + assert response_cost is not None + assert response_cost > 100 diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py index ac01e495184..7f97d208460 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py @@ -34,3 +34,33 @@ def test_anthropic_experimental_pass_through_messages_handler(): print(f"Error: {e}") mock_completion.assert_called_once() mock_completion.call_args.kwargs["api_key"] == "test-api-key" + + +def test_anthropic_experimental_pass_through_messages_handler_custom_llm_provider(): + """ + Test that litellm.completion is called when a custom LLM provider is given + """ + from litellm.llms.anthropic.experimental_pass_through.messages.handler import ( + anthropic_messages_handler, + ) + + with patch("litellm.completion", return_value="test-response") as mock_completion: + try: + anthropic_messages_handler( + max_tokens=100, + messages=[{"role": "user", "content": "Hello, how are you?"}], + model="my-custom-model", + custom_llm_provider="my-custom-llm", + api_key="test-api-key", + ) + except Exception as e: + print(f"Error: {e}") + + # Assert that litellm.completion was called when using a custom LLM provider + mock_completion.assert_called_once() + + # Verify that the custom provider was passed through + call_kwargs = mock_completion.call_args.kwargs + assert call_kwargs["custom_llm_provider"] == "my-custom-llm" + assert call_kwargs["model"] == "my-custom-llm/my-custom-model" + assert call_kwargs["api_key"] == "test-api-key" diff --git a/tests/test_litellm/llms/azure/response/test_azure_transformation.py b/tests/test_litellm/llms/azure/response/test_azure_transformation.py new file mode 100644 index 00000000000..8edd87a12fc --- /dev/null +++ b/tests/test_litellm/llms/azure/response/test_azure_transformation.py @@ -0,0 +1,97 @@ +import os +import sys +from unittest.mock import patch + +sys.path.insert( + 0, os.path.abspath("../../../../..") +) # Adds the parent directory to the system path + +from litellm.llms.azure.responses.transformation import AzureOpenAIResponsesAPIConfig +from litellm.types.router import GenericLiteLLMParams + + +def test_validate_environment_api_key_within_litellm_params(): + azure_openai_responses_apiconfig = AzureOpenAIResponsesAPIConfig() + litellm_params = GenericLiteLLMParams(api_key="test-api-key") + + result = azure_openai_responses_apiconfig.validate_environment( + headers={}, model="", litellm_params=litellm_params + ) + + expected = {"api-key": "test-api-key"} + + assert result == expected + + +def test_validate_environment_api_key_within_litellm(): + azure_openai_responses_apiconfig = AzureOpenAIResponsesAPIConfig() + + with patch("litellm.api_key", "test-api-key"): + litellm_params = GenericLiteLLMParams() + result = azure_openai_responses_apiconfig.validate_environment( + headers={}, model="", litellm_params=litellm_params + ) + + expected = {"api-key": "test-api-key"} + + assert result == expected + + +def test_validate_environment_azure_key_within_litellm(): + azure_openai_responses_apiconfig = AzureOpenAIResponsesAPIConfig() + + with patch("litellm.azure_key", "test-azure-key"): + litellm_params = GenericLiteLLMParams() + result = azure_openai_responses_apiconfig.validate_environment( + headers={}, model="", litellm_params=litellm_params + ) + + expected = {"api-key": "test-azure-key"} + + assert result == expected + + +def test_validate_environment_azure_openai_api_key_within_secret_str(): + azure_openai_responses_apiconfig = AzureOpenAIResponsesAPIConfig() + + with patch("litellm.api_key", None), \ + patch("litellm.azure_key", None), \ + patch("litellm.llms.azure.common_utils.get_secret_str") as mock_get_secret_str: + # Configure the mock to return "test-api-key" when called with "AZURE_OPENAI_API_KEY" + mock_get_secret_str.side_effect = ( + lambda key: "test-api-key" if key == "AZURE_OPENAI_API_KEY" else None + ) + + litellm_params = GenericLiteLLMParams() + result = azure_openai_responses_apiconfig.validate_environment( + headers={}, model="", litellm_params=litellm_params + ) + expected = {"api-key": "test-api-key"} + + assert result == expected + + +def test_validate_environment_azure_api_key_within_secret_str(): + azure_openai_responses_apiconfig = AzureOpenAIResponsesAPIConfig() + + with patch("litellm.api_key", None), \ + patch("litellm.azure_key", None), \ + patch("litellm.llms.azure.common_utils.get_secret_str") as mock_get_secret_str: + # Configure the mock to return None for "AZURE_OPENAI_API_KEY" and "test-api-key" for "AZURE_API_KEY" + def mock_side_effect(key): + if key == "AZURE_OPENAI_API_KEY": + return None + elif key == "AZURE_API_KEY": + return "test-api-key" + else: + return None + + mock_get_secret_str.side_effect = mock_side_effect + + litellm_params = GenericLiteLLMParams() + result = azure_openai_responses_apiconfig.validate_environment( + headers={}, model="", litellm_params=litellm_params + ) + expected = {"api-key": "test-api-key"} + + assert result == expected diff --git a/tests/test_litellm/llms/azure/test_azure_common_utils.py b/tests/test_litellm/llms/azure/test_azure_common_utils.py index f304ce43923..8c5fd7e5e08 100644 --- a/tests/test_litellm/llms/azure/test_azure_common_utils.py +++ b/tests/test_litellm/llms/azure/test_azure_common_utils.py @@ -11,13 +11,23 @@ sys.path.insert( 0, os.path.abspath("../../../..") ) # Adds the parent directory to the system path import litellm -from litellm.llms.azure.common_utils import BaseAzureLLM +from litellm.llms.azure.common_utils import BaseAzureLLM, get_azure_ad_token +from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import CallTypes # Mock the necessary dependencies @pytest.fixture -def setup_mocks(): +def setup_mocks(monkeypatch): + # Clear Azure environment variables that might interfere with tests + monkeypatch.delenv("AZURE_USERNAME", raising=False) + monkeypatch.delenv("AZURE_PASSWORD", raising=False) + monkeypatch.delenv("AZURE_CLIENT_SECRET", raising=False) + monkeypatch.delenv("AZURE_CLIENT_ID", raising=False) + monkeypatch.delenv("AZURE_TENANT_ID", raising=False) + monkeypatch.delenv("AZURE_SCOPE", raising=False) + monkeypatch.delenv("AZURE_AD_TOKEN", raising=False) + with patch( "litellm.llms.azure.common_utils.get_azure_ad_token_from_entra_id" ) as mock_entra_token, patch( @@ -994,3 +1004,375 @@ def test_scope_always_string_in_initialize_azure_sdk_client(setup_mocks, monkeyp ) print("All scope tests passed - scope is always a string") + + +def test_with_existing_token_provider(setup_mocks): + """Test get_azure_ad_token with an existing token provider.""" + token_provider = lambda: "test-token" + litellm_params = GenericLiteLLMParams(azure_ad_token_provider=token_provider) + + token = get_azure_ad_token(litellm_params) + + assert token == "test-token" + + +def test_with_existing_azure_ad_token(setup_mocks): + """Test get_azure_ad_token with an existing azure ad token.""" + litellm_params = GenericLiteLLMParams(azure_ad_token="test-token") + + token = get_azure_ad_token(litellm_params) + + assert token == "test-token" + + +def test_with_existing_azure_ad_token_from_env(setup_mocks): + """Test get_azure_ad_token with an existing AZURE_AD_TOKEN from env.""" + + # mock get_secret_str("AZURE_AD_TOKEN") to "test-token" + with patch("litellm.llms.azure.common_utils.get_secret_str") as mock_get_secret_str: + # Configure the mock to return "test-token" when called with "AZURE_AD_TOKEN" + mock_get_secret_str.side_effect = ( + lambda key: "test-token" if key == "AZURE_AD_TOKEN" else None + ) + + litellm_params = GenericLiteLLMParams() + + token = get_azure_ad_token(litellm_params) + + assert token == "test-token" + # Verify that get_secret_str was called with "AZURE_AD_TOKEN" + mock_get_secret_str.assert_called_with("AZURE_AD_TOKEN") + + +def test_get_azure_ad_token_with_client_id_and_client_secret(setup_mocks): + """Test get_azure_ad_token with tenant_id, client_id, and client_secret.""" + # Reset mocks to ensure clean state + setup_mocks["entra_token"].reset_mock() + + # Create test parameters with username, password, and client_id + # but no other authentication methods + litellm_params = GenericLiteLLMParams( + tenant_id="test-tenant-id", + client_id="test-client-id", + client_secret="test-client-secret", + azure_scope="test-azure-scope", + ) + + # Call the function + token = get_azure_ad_token(litellm_params) + + # Verify the debug message was logged + setup_mocks["logger"].debug.assert_any_call( + "Using Azure AD Token Provider from Entra ID for Azure Auth" + ) + + # Verify get_azure_ad_token_from_entra_id was called with correct params + setup_mocks["entra_token"].assert_called_once_with( + tenant_id="test-tenant-id", + client_id="test-client-id", + client_secret="test-client-secret", + scope="test-azure-scope", + ) + + # Verify the token is what we expect from our mock + assert token == "mock-entra-token" + + +def test_get_azure_ad_token_with_client_id_and_client_secret_from_env( + setup_mocks, monkeypatch +): + """Test get_azure_ad_token with tenant_id, client_id, and client_secret from env.""" + # Reset mocks to ensure clean state + setup_mocks["entra_token"].reset_mock() + + # Set environment variables + monkeypatch.setenv("AZURE_TENANT_ID", "test-tenant-id") + monkeypatch.setenv("AZURE_CLIENT_ID", "test-client-id") + monkeypatch.setenv("AZURE_CLIENT_SECRET", "test-client-secret") + monkeypatch.setenv("AZURE_SCOPE", "test-azure-scope") + + # Create test parameters with username, password, and client_id + # but no other authentication methods + litellm_params = GenericLiteLLMParams() + + # Call the function + token = get_azure_ad_token(litellm_params) + + # Verify the debug message was logged + setup_mocks["logger"].debug.assert_any_call( + "Using Azure AD Token Provider from Entra ID for Azure Auth" + ) + + # Verify get_azure_ad_token_from_entra_id was called with correct params + setup_mocks["entra_token"].assert_called_once_with( + tenant_id="test-tenant-id", + client_id="test-client-id", + client_secret="test-client-secret", + scope="test-azure-scope", + ) + + # Verify the token is what we expect from our mock + assert token == "mock-entra-token" + + +def test_get_azure_ad_token_with_username_password(setup_mocks): + """Test get_azure_ad_token with username, password, and client_id.""" + # Reset mocks to ensure clean state + setup_mocks["username_password_token"].reset_mock() + + # Create test parameters with username, password, and client_id + # but no other authentication methods + litellm_params = GenericLiteLLMParams( + azure_username="test-username", + azure_password="test-password", + client_id="test-client-id", + azure_scope="test-azure-scope", + # Ensure no other auth methods are available + azure_ad_token_provider=None, + azure_ad_token=None, + tenant_id=None, + client_secret=None, + ) + + # Call the function + token = get_azure_ad_token(litellm_params) + + # Verify the debug message was logged + setup_mocks["logger"].debug.assert_any_call( + "Using Azure Username and Password for Azure Auth" + ) + + # Verify get_azure_ad_token_from_username_password was called with correct params + setup_mocks["username_password_token"].assert_called_once_with( + azure_username="test-username", + azure_password="test-password", + client_id="test-client-id", + scope="test-azure-scope", + ) + + # Verify the token is what we expect from our mock + assert token == "mock-username-password-token" + + +def test_get_azure_ad_token_with_missing_username_password(setup_mocks): + """Test get_azure_ad_token skips username/password auth when credentials are incomplete.""" + # Reset mocks to ensure clean state + setup_mocks["username_password_token"].reset_mock() + + # Test cases with missing credentials + test_cases = [ + # Missing username + GenericLiteLLMParams( + azure_username=None, + azure_password="test-password", + client_id="test-client-id", + ), + # Missing password + GenericLiteLLMParams( + azure_username="test-username", + azure_password=None, + client_id="test-client-id", + ), + # Missing client_id + GenericLiteLLMParams( + azure_username="test-username", + azure_password="test-password", + client_id=None, + ), + ] + + for params in test_cases: + # Call the function + get_azure_ad_token(params) + + # Verify username/password auth was not used + setup_mocks["username_password_token"].assert_not_called() + + # Reset mock for next test case + setup_mocks["username_password_token"].reset_mock() + + +def test_get_azure_ad_token_with_username_password_from_env(setup_mocks, monkeypatch): + """Test get_azure_ad_token with username, password, and client_id from environment variables.""" + # Reset mocks to ensure clean state + setup_mocks["username_password_token"].reset_mock() + + # Set environment variables + monkeypatch.setenv("AZURE_USERNAME", "env-username") + monkeypatch.setenv("AZURE_PASSWORD", "env-password") + monkeypatch.setenv("AZURE_CLIENT_ID", "env-client-id") + monkeypatch.setenv("AZURE_SCOPE", "test-azure-scope") + + # Create test parameters with no explicit credentials + litellm_params = GenericLiteLLMParams( + # Ensure no other auth methods are available + azure_ad_token_provider=None, + azure_ad_token=None, + tenant_id=None, + client_secret=None, + # Don't set username, password, or client_id directly + ) + + # Call the function + token = get_azure_ad_token(litellm_params) + + # Verify the debug message was logged + setup_mocks["logger"].debug.assert_any_call( + "Using Azure Username and Password for Azure Auth" + ) + + # Verify get_azure_ad_token_from_username_password was called with correct params from env + setup_mocks["username_password_token"].assert_called_once_with( + azure_username="env-username", + azure_password="env-password", + client_id="env-client-id", + scope="test-azure-scope", + ) + + # Verify the token is what we expect from our mock + assert token == "mock-username-password-token" + + +def test_get_azure_ad_token_with_oidc_token(setup_mocks, monkeypatch): + """Test get_azure_ad_token with OIDC token.""" + # Reset mocks to ensure clean state + setup_mocks["oidc_token"].reset_mock() + + # Clear environment variables that might interfere with OIDC token logic + monkeypatch.delenv("AZURE_USERNAME", raising=False) + monkeypatch.delenv("AZURE_PASSWORD", raising=False) + monkeypatch.delenv("AZURE_CLIENT_SECRET", raising=False) + + # Create test parameters with OIDC token, client_id, and tenant_id + litellm_params = GenericLiteLLMParams( + azure_ad_token="oidc/test-token", + client_id="test-client-id", + tenant_id="test-tenant-id", + azure_scope="test-azure-scope", + # Ensure no other auth methods are available + azure_ad_token_provider=None, + client_secret=None, + azure_username=None, + azure_password=None, + ) + + # Call the function + token = get_azure_ad_token(litellm_params) + + # Verify the debug message was logged + setup_mocks["logger"].debug.assert_any_call("Using Azure OIDC Token for Azure Auth") + + # Verify get_azure_ad_token_from_oidc was called with correct params + setup_mocks["oidc_token"].assert_called_once_with( + azure_ad_token="oidc/test-token", + azure_client_id="test-client-id", + azure_tenant_id="test-tenant-id", + scope="test-azure-scope", + ) + + # Verify the token is what we expect from our mock + assert token == "mock-oidc-token" + + +def test_get_azure_ad_token_with_token_refresh(setup_mocks, monkeypatch): + """Test get_azure_ad_token with token refresh enabled.""" + # Reset mocks to ensure clean state + monkeypatch.delenv("AZURE_USERNAME", raising=False) + monkeypatch.delenv("AZURE_PASSWORD", raising=False) + monkeypatch.delenv("AZURE_CLIENT_SECRET", raising=False) + + setup_mocks["token_provider"].reset_mock() + + # Enable token refresh + setup_mocks["litellm"].enable_azure_ad_token_refresh = True + + # Create test parameters with no other auth methods available + litellm_params = GenericLiteLLMParams() + + # Call the function + token = get_azure_ad_token(litellm_params) + + # Verify the debug message was logged + setup_mocks["logger"].debug.assert_any_call( + "Using Azure AD token provider based on Service Principal with Secret workflow for Azure Auth" + ) + + # Verify get_azure_ad_token_provider was called + setup_mocks["token_provider"].assert_called_once() + + # Verify the token is what we expect from our mock + assert token == "mock-default-token" + + +def test_get_azure_ad_token_with_token_refresh_error(setup_mocks): + """Test get_azure_ad_token with token refresh enabled but raising an error.""" + # Reset mocks to ensure clean state + setup_mocks["token_provider"].reset_mock() + + # Enable token refresh but make it raise an error + setup_mocks["litellm"].enable_azure_ad_token_refresh = True + setup_mocks["token_provider"].side_effect = ValueError("Token provider error") + + # Create test parameters with no other auth methods available + litellm_params = GenericLiteLLMParams() + + # Call the function + token = get_azure_ad_token(litellm_params) + + # Verify the debug message was logged + setup_mocks["logger"].debug.assert_any_call( + "Using Azure AD token provider based on Service Principal with Secret workflow for Azure Auth" + ) + + # Verify error was logged + setup_mocks["logger"].debug.assert_any_call( + "Azure AD Token Provider could not be used." + ) + + # Verify get_azure_ad_token_provider was called + setup_mocks["token_provider"].assert_called_once() + + # Verify the token is None since the provider raised an error + assert token is None + + +def test_token_provider_returns_non_string(setup_mocks): + """Test that get_azure_ad_token raises TypeError when token provider returns non-string value.""" + # Create a token provider that returns a non-string value + non_string_provider = lambda: 123 # Returns an integer instead of a string + + # Create test parameters with the non-string token provider + litellm_params = GenericLiteLLMParams(azure_ad_token_provider=non_string_provider) + + # Call the function and expect a TypeError + with pytest.raises(TypeError) as excinfo: + get_azure_ad_token(litellm_params) + + # Verify the error message + assert "Azure AD token must be a string" in str(excinfo.value) + + # Verify the error was logged + setup_mocks["logger"].error.assert_any_call( + "Azure AD token provider returned non-string value: " + ) + + +def test_token_provider_raises_exception(setup_mocks): + """Test that get_azure_ad_token raises RuntimeError when token provider raises an exception.""" + # Create a token provider that raises an exception + error_message = "Test provider error" + error_provider = lambda: exec('raise ValueError("' + error_message + '")') + + # Create test parameters with the error-raising token provider + litellm_params = GenericLiteLLMParams(azure_ad_token_provider=error_provider) + + # Call the function and expect a RuntimeError + with pytest.raises(RuntimeError) as excinfo: + get_azure_ad_token(litellm_params) + + # Verify the error message + assert "Failed to get Azure AD token" in str(excinfo.value) + assert error_message in str(excinfo.value) + + # Verify the error was logged + setup_mocks["logger"].error.assert_called() diff --git a/tests/test_litellm/llms/custom_httpx/test_aiohttp_transport.py b/tests/test_litellm/llms/custom_httpx/test_aiohttp_transport.py index 4d1023d873f..f296f076178 100644 --- a/tests/test_litellm/llms/custom_httpx/test_aiohttp_transport.py +++ b/tests/test_litellm/llms/custom_httpx/test_aiohttp_transport.py @@ -176,3 +176,46 @@ async def test_timeout_exception_gets_mapped(): # Should have received the first chunk before the error assert received_chunks == [b"chunk1"] + + +@pytest.mark.asyncio +async def test_handle_async_request_uses_env_proxy(monkeypatch): + """Aiohttp transport should honor HTTP(S)_PROXY env vars""" + proxy_url = "http://proxy.local:3128" + monkeypatch.setenv("HTTP_PROXY", proxy_url) + monkeypatch.setenv("http_proxy", proxy_url) + monkeypatch.setenv("HTTPS_PROXY", proxy_url) + monkeypatch.setenv("https_proxy", proxy_url) + monkeypatch.delenv("DISABLE_AIOHTTP_TRUST_ENV", raising=False) + + captured = {} + + class FakeSession: + def request(self, *args, **kwargs): + captured["proxy"] = kwargs.get("proxy") + + class Resp: + status = 200 + headers = {} + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, tb): + pass + + @property + def content(self): + class C: + async def iter_chunked(self, size): + yield b"" + + return C() + + return Resp() + + transport = LiteLLMAiohttpTransport(client=lambda: FakeSession()) + request = httpx.Request("GET", "http://example.com") + await transport.handle_async_request(request) + + assert captured["proxy"] == proxy_url diff --git a/tests/test_litellm/llms/mistral/test_mistral_chat_transformation.py b/tests/test_litellm/llms/mistral/test_mistral_chat_transformation.py index c46413799fe..be0e73d8506 100644 --- a/tests/test_litellm/llms/mistral/test_mistral_chat_transformation.py +++ b/tests/test_litellm/llms/mistral/test_mistral_chat_transformation.py @@ -95,10 +95,6 @@ class TestMistralReasoningSupport: def test_get_mistral_reasoning_system_prompt(self): """Test that the reasoning system prompt is properly formatted.""" prompt = MistralConfig._get_mistral_reasoning_system_prompt() - - assert "" in prompt - assert "" in prompt - assert "step-by-step" in prompt assert isinstance(prompt, str) assert len(prompt) > 50 # Ensure it's not empty diff --git a/tests/test_litellm/llms/ollama/test_ollama_embedding.py b/tests/test_litellm/llms/ollama/test_ollama_embedding.py index 30248f26b68..a5cbd36ceea 100644 --- a/tests/test_litellm/llms/ollama/test_ollama_embedding.py +++ b/tests/test_litellm/llms/ollama/test_ollama_embedding.py @@ -1,8 +1,9 @@ -import pytest from unittest.mock import AsyncMock, MagicMock, patch -from litellm.types.utils import EmbeddingResponse -from litellm.llms.ollama.completion.handler import ollama_embeddings, ollama_aembeddings +import pytest + +from litellm.llms.ollama.completion.handler import ollama_aembeddings, ollama_embeddings +from litellm.types.utils import EmbeddingResponse @pytest.fixture @@ -26,8 +27,9 @@ def mock_encoding(): def test_ollama_embeddings(mock_response_data, mock_embedding_response, mock_encoding): - with patch("litellm.module_level_client.post") as mock_post, \ - patch("litellm.OllamaConfig.get_config", return_value={"truncate": 512}): + with patch("litellm.module_level_client.post") as mock_post, patch( + "litellm.OllamaConfig.get_config", return_value={"truncate": 512} + ): mock_response = MagicMock() mock_response.json.return_value = mock_response_data @@ -50,11 +52,17 @@ def test_ollama_embeddings(mock_response_data, mock_embedding_response, mock_enc @pytest.mark.asyncio -async def test_ollama_aembeddings(mock_response_data, mock_embedding_response, mock_encoding): - with patch("litellm.module_level_aclient.post", new_callable=AsyncMock) as mock_post, \ - patch("litellm.OllamaConfig.get_config", return_value={"truncate": 512}): - - mock_post.return_value.json.return_value = mock_response_data +async def test_ollama_aembeddings( + mock_response_data, mock_embedding_response, mock_encoding +): + mock_response = AsyncMock() + # Make json() a regular synchronous method, not async + mock_response.json = MagicMock(return_value=mock_response_data) + with patch( + "litellm.module_level_aclient.post", return_value=mock_response + ) as mock_post, patch( + "litellm.OllamaConfig.get_config", return_value={"truncate": 512} + ): response = await ollama_aembeddings( api_base="http://localhost:11434", @@ -78,9 +86,10 @@ def test_prompt_eval_fallback_when_missing(mock_embedding_response, mock_encodin # No "prompt_eval_count" } - with patch("litellm.module_level_client.post") as mock_post, \ - patch("litellm.OllamaConfig.get_config", return_value={}): - + with patch("litellm.module_level_client.post") as mock_post, patch( + "litellm.OllamaConfig.get_config", return_value={} + ): + mock_response = MagicMock() mock_response.json.return_value = response_data mock_post.return_value = mock_response @@ -99,4 +108,4 @@ def test_prompt_eval_fallback_when_missing(mock_embedding_response, mock_encodin assert response.usage.prompt_tokens == 5 assert response.usage.total_tokens == 5 assert response.usage.completion_tokens == 0 - assert response.data[0]['embedding'] == [0.1, 0.2, 0.3] + assert response.data[0]["embedding"] == [0.1, 0.2, 0.3] diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py b/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py index a926c33f53b..db0a69482b2 100644 --- a/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py @@ -153,9 +153,9 @@ class TestOpenAIResponsesAPIConfig: # Test with provided API key headers = {} api_key = "test_api_key" - + litellm_params = GenericLiteLLMParams(api_key=api_key) result = self.config.validate_environment( - headers=headers, model=self.model, api_key=api_key + headers=headers, model=self.model, litellm_params=litellm_params ) assert "Authorization" in result @@ -165,7 +165,10 @@ class TestOpenAIResponsesAPIConfig: headers = {} with patch("litellm.api_key", "litellm_api_key"): - result = self.config.validate_environment(headers=headers, model=self.model) + litellm_params = GenericLiteLLMParams() + result = self.config.validate_environment( + headers=headers, model=self.model, litellm_params=litellm_params + ) assert "Authorization" in result assert result["Authorization"] == "Bearer litellm_api_key" @@ -175,8 +178,9 @@ class TestOpenAIResponsesAPIConfig: with patch("litellm.openai_key", "openai_key"): with patch("litellm.api_key", None): + litellm_params = GenericLiteLLMParams() result = self.config.validate_environment( - headers=headers, model=self.model + headers=headers, model=self.model, litellm_params=litellm_params ) assert "Authorization" in result @@ -193,8 +197,9 @@ class TestOpenAIResponsesAPIConfig: "litellm.llms.openai.responses.transformation.get_secret_str", return_value="env_api_key", ): + litellm_params = GenericLiteLLMParams() result = self.config.validate_environment( - headers=headers, model=self.model + headers=headers, model=self.model, litellm_params=litellm_params ) assert "Authorization" in result diff --git a/tests/test_litellm/llms/perplexity/chat/test_perplexity_chat_transformation.py b/tests/test_litellm/llms/perplexity/chat/test_perplexity_chat_transformation.py new file mode 100644 index 00000000000..6f64f46b4a7 --- /dev/null +++ b/tests/test_litellm/llms/perplexity/chat/test_perplexity_chat_transformation.py @@ -0,0 +1,409 @@ +""" +Test file for Perplexity chat transformation functionality. + +Tests the response transformation to extract citation tokens and search queries +from Perplexity API responses. +""" + +import os +import sys +from unittest.mock import Mock + +import pytest + +# Add the project root to Python path +sys.path.insert(0, os.path.abspath("../../../../..")) + +from litellm import ModelResponse +from litellm.llms.perplexity.chat.transformation import PerplexityChatConfig +from litellm.types.utils import Usage + + +class TestPerplexityChatTransformation: + """Test suite for Perplexity chat transformation functionality.""" + + def test_enhance_usage_with_citation_tokens(self): + """Test extraction of citation tokens from API response.""" + config = PerplexityChatConfig() + + # Create a ModelResponse with basic usage + model_response = ModelResponse() + model_response.usage = Usage( + prompt_tokens=100, + completion_tokens=50, + total_tokens=150 + ) + + # Mock raw response with citations + raw_response_dict = { + "choices": [{"message": {"content": "Test response"}}], + "usage": { + "prompt_tokens": 100, + "completion_tokens": 50, + "total_tokens": 150 + }, + "citations": [ + "This is a citation with some text content", + "Another citation with more text here", + "Third citation with additional information" + ] + } + + # Enhance the usage with Perplexity fields + config._enhance_usage_with_perplexity_fields(model_response, raw_response_dict) + + # Check that citation tokens were added + assert hasattr(model_response.usage, "citation_tokens") + citation_tokens = getattr(model_response.usage, "citation_tokens") + + # Should have extracted citation tokens (estimated based on character count) + assert citation_tokens > 0 + assert isinstance(citation_tokens, int) + + def test_enhance_usage_with_search_queries_from_usage(self): + """Test extraction of search queries from usage field in API response.""" + config = PerplexityChatConfig() + + # Create a ModelResponse with basic usage + model_response = ModelResponse() + model_response.usage = Usage( + prompt_tokens=100, + completion_tokens=50, + total_tokens=150 + ) + + # Mock raw response with search queries in usage + raw_response_dict = { + "choices": [{"message": {"content": "Test response"}}], + "usage": { + "prompt_tokens": 100, + "completion_tokens": 50, + "total_tokens": 150, + "num_search_queries": 3 + } + } + + # Enhance the usage with Perplexity fields + config._enhance_usage_with_perplexity_fields(model_response, raw_response_dict) + + # Check that search queries were added to prompt_tokens_details + assert hasattr(model_response.usage, "prompt_tokens_details") + assert model_response.usage.prompt_tokens_details is not None + assert hasattr(model_response.usage.prompt_tokens_details, "web_search_requests") + + web_search_requests = model_response.usage.prompt_tokens_details.web_search_requests + assert web_search_requests == 3 + + def test_enhance_usage_with_search_queries_from_root(self): + """Test extraction of search queries from root level in API response.""" + config = PerplexityChatConfig() + + # Create a ModelResponse with basic usage + model_response = ModelResponse() + model_response.usage = Usage( + prompt_tokens=100, + completion_tokens=50, + total_tokens=150 + ) + + # Mock raw response with search queries at root level + raw_response_dict = { + "choices": [{"message": {"content": "Test response"}}], + "usage": { + "prompt_tokens": 100, + "completion_tokens": 50, + "total_tokens": 150 + }, + "num_search_queries": 2 + } + + # Enhance the usage with Perplexity fields + config._enhance_usage_with_perplexity_fields(model_response, raw_response_dict) + + # Check that search queries were added to prompt_tokens_details + assert hasattr(model_response.usage, "prompt_tokens_details") + assert model_response.usage.prompt_tokens_details is not None + assert hasattr(model_response.usage.prompt_tokens_details, "web_search_requests") + + web_search_requests = model_response.usage.prompt_tokens_details.web_search_requests + assert web_search_requests == 2 + + def test_enhance_usage_with_both_citations_and_search_queries(self): + """Test extraction of both citation tokens and search queries.""" + config = PerplexityChatConfig() + + # Create a ModelResponse with basic usage + model_response = ModelResponse() + model_response.usage = Usage( + prompt_tokens=100, + completion_tokens=50, + total_tokens=150 + ) + + # Mock raw response with both citations and search queries + raw_response_dict = { + "choices": [{"message": {"content": "Test response"}}], + "usage": { + "prompt_tokens": 100, + "completion_tokens": 50, + "total_tokens": 150, + "num_search_queries": 2 + }, + "citations": [ + "Citation one with some content", + "Citation two with more information" + ] + } + + # Enhance the usage with Perplexity fields + config._enhance_usage_with_perplexity_fields(model_response, raw_response_dict) + + # Check that both fields were added + assert hasattr(model_response.usage, "citation_tokens") + assert hasattr(model_response.usage, "prompt_tokens_details") + assert model_response.usage.prompt_tokens_details is not None + assert hasattr(model_response.usage.prompt_tokens_details, "web_search_requests") + + citation_tokens = getattr(model_response.usage, "citation_tokens") + web_search_requests = model_response.usage.prompt_tokens_details.web_search_requests + + assert citation_tokens > 0 + assert web_search_requests == 2 + + def test_enhance_usage_with_empty_citations(self): + """Test handling of empty citations array.""" + config = PerplexityChatConfig() + + # Create a ModelResponse with basic usage + model_response = ModelResponse() + model_response.usage = Usage( + prompt_tokens=100, + completion_tokens=50, + total_tokens=150 + ) + + # Mock raw response with empty citations + raw_response_dict = { + "choices": [{"message": {"content": "Test response"}}], + "usage": { + "prompt_tokens": 100, + "completion_tokens": 50, + "total_tokens": 150 + }, + "citations": [] + } + + # Enhance the usage with Perplexity fields + config._enhance_usage_with_perplexity_fields(model_response, raw_response_dict) + + # Should not set citation_tokens for empty citations + citation_tokens = getattr(model_response.usage, "citation_tokens", 0) + assert citation_tokens == 0 + + def test_enhance_usage_with_missing_fields(self): + """Test handling when both citations and search queries are missing.""" + config = PerplexityChatConfig() + + # Create a ModelResponse with basic usage + model_response = ModelResponse() + model_response.usage = Usage( + prompt_tokens=100, + completion_tokens=50, + total_tokens=150 + ) + + # Mock raw response without citations or search queries + raw_response_dict = { + "choices": [{"message": {"content": "Test response"}}], + "usage": { + "prompt_tokens": 100, + "completion_tokens": 50, + "total_tokens": 150 + } + } + + # Should not raise an error + config._enhance_usage_with_perplexity_fields(model_response, raw_response_dict) + + # Should not have added custom fields + citation_tokens = getattr(model_response.usage, "citation_tokens", 0) + assert citation_tokens == 0 + + # prompt_tokens_details might be None or have web_search_requests as 0 + if hasattr(model_response.usage, "prompt_tokens_details") and model_response.usage.prompt_tokens_details: + web_search_requests = getattr(model_response.usage.prompt_tokens_details, "web_search_requests", 0) + assert web_search_requests == 0 + + def test_citation_token_estimation(self): + """Test that citation token estimation is reasonable.""" + config = PerplexityChatConfig() + + # Test cases with known character counts + test_cases = [ + # (citation_text, expected_min_tokens, expected_max_tokens) + ("Short", 1, 2), + ("This is a longer citation with multiple words", 10, 15), + ("A very long citation with many words and characters that should result in more tokens", 18, 25), + ] + + for citation_text, min_tokens, max_tokens in test_cases: + model_response = ModelResponse() + model_response.usage = Usage( + prompt_tokens=100, + completion_tokens=50, + total_tokens=150 + ) + + raw_response_dict = { + "usage": {"prompt_tokens": 100, "completion_tokens": 50, "total_tokens": 150}, + "citations": [citation_text] + } + + config._enhance_usage_with_perplexity_fields(model_response, raw_response_dict) + + citation_tokens = getattr(model_response.usage, "citation_tokens") + + # Should be within reasonable range + assert min_tokens <= citation_tokens <= max_tokens, f"Citation '{citation_text}' resulted in {citation_tokens} tokens, expected {min_tokens}-{max_tokens}" + + def test_multiple_citations_aggregation(self): + """Test that multiple citations are aggregated correctly.""" + config = PerplexityChatConfig() + + model_response = ModelResponse() + model_response.usage = Usage( + prompt_tokens=100, + completion_tokens=50, + total_tokens=150 + ) + + raw_response_dict = { + "usage": {"prompt_tokens": 100, "completion_tokens": 50, "total_tokens": 150}, + "citations": [ + "First citation with some text", + "Second citation with different content", + "Third citation with more information" + ] + } + + config._enhance_usage_with_perplexity_fields(model_response, raw_response_dict) + + citation_tokens = getattr(model_response.usage, "citation_tokens") + + # Should have aggregated all citations + total_chars = sum(len(citation) for citation in raw_response_dict["citations"]) + expected_tokens = total_chars // 4 # Our estimation logic + + assert citation_tokens == expected_tokens + + def test_search_queries_priority_usage_over_root(self): + """Test that search queries from usage field take priority over root level.""" + config = PerplexityChatConfig() + + # Create a ModelResponse with basic usage + model_response = ModelResponse() + model_response.usage = Usage( + prompt_tokens=100, + completion_tokens=50, + total_tokens=150 + ) + + # Mock raw response with search queries in both locations + raw_response_dict = { + "choices": [{"message": {"content": "Test response"}}], + "usage": { + "prompt_tokens": 100, + "completion_tokens": 50, + "total_tokens": 150, + "num_search_queries": 5 # This should take priority + }, + "num_search_queries": 3 # This should be ignored + } + + # Enhance the usage with Perplexity fields + config._enhance_usage_with_perplexity_fields(model_response, raw_response_dict) + + # Check that usage field took priority + assert hasattr(model_response.usage, "prompt_tokens_details") + assert model_response.usage.prompt_tokens_details is not None + web_search_requests = model_response.usage.prompt_tokens_details.web_search_requests + + assert web_search_requests == 5 # Should use the usage field value, not root + + def test_no_usage_object_handling(self): + """Test handling when model_response has no usage object.""" + config = PerplexityChatConfig() + + # Create a ModelResponse without usage + model_response = ModelResponse() + + # Mock raw response with Perplexity-specific fields + raw_response_dict = { + "choices": [{"message": {"content": "Test response"}}], + "usage": { + "prompt_tokens": 100, + "completion_tokens": 50, + "total_tokens": 150, + "num_search_queries": 2 + }, + "citations": ["Some citation"] + } + + # Should not raise an error when usage is None + config._enhance_usage_with_perplexity_fields(model_response, raw_response_dict) + + # Usage should be created with the Perplexity fields + assert model_response.usage is not None + assert hasattr(model_response.usage, "citation_tokens") + assert hasattr(model_response.usage, "prompt_tokens_details") + assert model_response.usage.prompt_tokens_details is not None + assert hasattr(model_response.usage.prompt_tokens_details, "web_search_requests") + + citation_tokens = getattr(model_response.usage, "citation_tokens") + web_search_requests = model_response.usage.prompt_tokens_details.web_search_requests + + assert citation_tokens > 0 + assert web_search_requests == 2 + + @pytest.mark.parametrize("search_query_location", ["usage", "root"]) + def test_search_queries_extraction_locations(self, search_query_location): + """Test search queries extraction from different response locations.""" + config = PerplexityChatConfig() + + # Create a ModelResponse with basic usage + model_response = ModelResponse() + model_response.usage = Usage( + prompt_tokens=100, + completion_tokens=50, + total_tokens=150 + ) + + # Create response dict based on parameter + if search_query_location == "usage": + raw_response_dict = { + "usage": { + "prompt_tokens": 100, + "completion_tokens": 50, + "total_tokens": 150, + "num_search_queries": 4 + } + } + else: # root + raw_response_dict = { + "usage": { + "prompt_tokens": 100, + "completion_tokens": 50, + "total_tokens": 150 + }, + "num_search_queries": 4 + } + + # Enhance the usage with Perplexity fields + config._enhance_usage_with_perplexity_fields(model_response, raw_response_dict) + + # Should extract search queries from either location + assert hasattr(model_response.usage, "prompt_tokens_details") + assert model_response.usage.prompt_tokens_details is not None + web_search_requests = model_response.usage.prompt_tokens_details.web_search_requests + + assert web_search_requests == 4 \ No newline at end of file diff --git a/tests/test_litellm/llms/perplexity/test_perplexity.py b/tests/test_litellm/llms/perplexity/test_perplexity.py new file mode 100644 index 00000000000..5c8eead4d6d --- /dev/null +++ b/tests/test_litellm/llms/perplexity/test_perplexity.py @@ -0,0 +1,25 @@ +import os +import sys + +sys.path.insert(0, os.path.abspath("../../..")) + +import pytest + + +class TestPerplexityWebSearch: + """Test suite for Perplexity web search functionality.""" + + @pytest.mark.parametrize( + "model", + ["perplexity/sonar", "perplexity/sonar-pro"] + ) + def test_web_search_options_in_supported_params(self, model): + """ + Test that web_search_options is in the list of supported parameters for Perplexity sonar models + """ + from litellm.llms.perplexity.chat.transformation import PerplexityChatConfig + + config = PerplexityChatConfig() + supported_params = config.get_supported_openai_params(model=model) + + assert "web_search_options" in supported_params, f"web_search_options should be supported for {model}" diff --git a/tests/test_litellm/llms/perplexity/test_perplexity_cost_calculator.py b/tests/test_litellm/llms/perplexity/test_perplexity_cost_calculator.py new file mode 100644 index 00000000000..f9a52100070 --- /dev/null +++ b/tests/test_litellm/llms/perplexity/test_perplexity_cost_calculator.py @@ -0,0 +1,373 @@ +""" +Test file for Perplexity cost calculator functionality. + +Tests the cost calculation for Perplexity models including citation tokens, +search queries, and reasoning tokens. +""" + +import json +import math +import os +import sys +from unittest.mock import Mock, patch + +import pytest + +# Add the project root to Python path +sys.path.insert(0, os.path.abspath("../../../..")) + +import litellm +from litellm.cost_calculator import completion_cost, cost_per_token +from litellm.llms.perplexity.cost_calculator import cost_per_token as perplexity_cost_per_token +from litellm.types.utils import Usage, PromptTokensDetailsWrapper +from litellm.utils import get_model_info + + +class TestPerplexityCostCalculator: + """Test suite for Perplexity cost calculation functionality.""" + + @pytest.fixture(autouse=True) + def setup_model_cost_map(self): + """Set up the model cost map for testing.""" + # Ensure we use local model cost map for consistent testing + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + + # Load the model cost map + try: + with open("model_prices_and_context_window.json", "r") as f: + model_cost_map = json.load(f) + litellm.model_cost = model_cost_map + except FileNotFoundError: + # Fallback to ensure we have the Perplexity model configuration + litellm.model_cost = { + "perplexity/sonar-deep-research": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 8e-06, + "output_cost_per_reasoning_token": 3e-06, + "citation_cost_per_token": 2e-06, + "search_context_cost_per_query": { + "search_context_size_low": 0.005, + "search_context_size_medium": 0.005, + "search_context_size_high": 0.005 + }, + "litellm_provider": "perplexity", + "mode": "chat", + "supports_reasoning": True, + "supports_web_search": True, + } + } + + def test_basic_cost_calculation(self): + """Test basic cost calculation without additional fields.""" + usage = Usage( + prompt_tokens=100, + completion_tokens=50, + total_tokens=150 + ) + + prompt_cost, completion_cost = perplexity_cost_per_token( + model="sonar-deep-research", + usage=usage + ) + + # Expected costs: + # Input: 100 tokens * $2e-6 = $0.0002 + # Output: 50 tokens * $8e-6 = $0.0004 + expected_prompt_cost = 100 * 2e-6 + expected_completion_cost = 50 * 8e-6 + + assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-6) + assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-6) + + def test_citation_tokens_cost_calculation(self): + """Test cost calculation with citation tokens.""" + usage = Usage( + prompt_tokens=100, + completion_tokens=50, + total_tokens=150 + ) + + # Add citation tokens + usage.citation_tokens = 25 + + prompt_cost, completion_cost = perplexity_cost_per_token( + model="sonar-deep-research", + usage=usage + ) + + # Expected costs: + # Input: 100 tokens * $2e-6 = $0.0002 + # Citation: 25 tokens * $2e-6 = $0.00005 + # Total prompt cost: $0.00025 + # Output: 50 tokens * $8e-6 = $0.0004 + expected_prompt_cost = (100 * 2e-6) + (25 * 2e-6) + expected_completion_cost = 50 * 8e-6 + + assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-6) + assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-6) + + def test_search_queries_cost_calculation(self): + """Test cost calculation with search queries.""" + usage = Usage( + prompt_tokens=100, + completion_tokens=50, + total_tokens=150, + prompt_tokens_details=PromptTokensDetailsWrapper(web_search_requests=3) + ) + + prompt_cost, completion_cost = perplexity_cost_per_token( + model="sonar-deep-research", + usage=usage + ) + + # Expected costs: + # Input: 100 tokens * $2e-6 = $0.0002 + # Output: 50 tokens * $8e-6 = $0.0004 + # Search: 3 queries * ($0.005 / 1000) = $0.000015 + # Total completion cost: $0.000415 + expected_prompt_cost = 100 * 2e-6 + expected_completion_cost = (50 * 8e-6) + (3 / 1000 * 0.005) + + assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-6) + assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-6) + + def test_reasoning_tokens_from_direct_attribute(self): + """Test reasoning tokens cost calculation from direct attribute.""" + usage = Usage( + prompt_tokens=100, + completion_tokens=50, + total_tokens=150 + ) + + # Set reasoning tokens directly + usage.reasoning_tokens = 20 + + prompt_cost, completion_cost = perplexity_cost_per_token( + model="sonar-deep-research", + usage=usage + ) + + # Expected costs: + # Input: 100 tokens * $2e-6 = $0.0002 + # Output: 50 tokens * $8e-6 = $0.0004 + # Reasoning: 20 tokens * $3e-6 = $0.00006 + # Total completion cost: $0.00046 + expected_prompt_cost = 100 * 2e-6 + expected_completion_cost = (50 * 8e-6) + (20 * 3e-6) + + assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-6) + assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-6) + + def test_reasoning_tokens_from_completion_tokens_details(self): + """Test reasoning tokens cost calculation from completion_tokens_details.""" + usage = Usage( + prompt_tokens=100, + completion_tokens=50, + total_tokens=150, + reasoning_tokens=20 # This should be stored in completion_tokens_details + ) + + prompt_cost, completion_cost = perplexity_cost_per_token( + model="sonar-deep-research", + usage=usage + ) + + # Expected costs: + # Input: 100 tokens * $2e-6 = $0.0002 + # Output: 50 tokens * $8e-6 = $0.0004 + # Reasoning: 20 tokens * $3e-6 = $0.00006 + # Total completion cost: $0.00046 + expected_prompt_cost = 100 * 2e-6 + expected_completion_cost = (50 * 8e-6) + (20 * 3e-6) + + assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-6) + assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-6) + + def test_comprehensive_cost_calculation(self): + """Test cost calculation with all fields combined.""" + usage = Usage( + prompt_tokens=100, + completion_tokens=50, + total_tokens=150, + reasoning_tokens=15, + prompt_tokens_details=PromptTokensDetailsWrapper(web_search_requests=2) + ) + + # Add custom fields + usage.citation_tokens = 30 + + prompt_cost, completion_cost = perplexity_cost_per_token( + model="sonar-deep-research", + usage=usage + ) + + # Expected costs: + # Input: 100 tokens * $2e-6 = $0.0002 + # Citation: 30 tokens * $2e-6 = $0.00006 + # Total prompt cost: $0.00026 + # Output: 50 tokens * $8e-6 = $0.0004 + # Reasoning: 15 tokens * $3e-6 = $0.000045 + # Search: 2 queries * ($0.005 / 1000) = $0.00001 + # Total completion cost: $0.000455 + expected_prompt_cost = (100 * 2e-6) + (30 * 2e-6) + expected_completion_cost = (50 * 8e-6) + (15 * 3e-6) + (2 / 1000 * 0.005) + + assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-6) + assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-6) + + def test_zero_values_handling(self): + """Test that zero or missing values are handled correctly.""" + usage = Usage( + prompt_tokens=100, + completion_tokens=50, + total_tokens=150, + prompt_tokens_details=PromptTokensDetailsWrapper(web_search_requests=0) + ) + + # These should not raise errors and should not affect cost + usage.citation_tokens = 0 + + prompt_cost, completion_cost = perplexity_cost_per_token( + model="sonar-deep-research", + usage=usage + ) + + # Should be same as basic calculation + expected_prompt_cost = 100 * 2e-6 + expected_completion_cost = 50 * 8e-6 + + assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-6) + assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-6) + + def test_missing_model_info_fields(self): + """Test behavior when model info is missing some fields.""" + usage = Usage( + prompt_tokens=100, + completion_tokens=50, + total_tokens=150, + prompt_tokens_details=PromptTokensDetailsWrapper(web_search_requests=2) + ) + + usage.citation_tokens = 25 + + # Mock get_model_info to return incomplete model info + with patch('litellm.llms.perplexity.cost_calculator.get_model_info') as mock_get_model_info: + mock_get_model_info.return_value = { + "input_cost_per_token": 2e-6, + "output_cost_per_token": 8e-6, + # Missing search_queries_cost_per_query + } + + prompt_cost, completion_cost = perplexity_cost_per_token( + model="sonar-deep-research", + usage=usage + ) + + # Should only calculate basic costs when fields are missing + expected_prompt_cost = 100 * 2e-6 + expected_completion_cost = 50 * 8e-6 + + assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-6) + assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-6) + + def test_integration_with_main_cost_calculator(self): + """Test integration with the main LiteLLM cost calculator.""" + usage = Usage( + prompt_tokens=100, + completion_tokens=50, + total_tokens=150, + reasoning_tokens=10, + prompt_tokens_details=PromptTokensDetailsWrapper(web_search_requests=1) + ) + + usage.citation_tokens = 20 + + # Test main cost calculator + prompt_cost, completion_cost_val = cost_per_token( + model="sonar-deep-research", + custom_llm_provider="perplexity", + usage_object=usage + ) + + # Should match direct call to perplexity cost calculator + expected_prompt, expected_completion = perplexity_cost_per_token( + model="sonar-deep-research", + usage=usage + ) + + assert math.isclose(prompt_cost, expected_prompt, rel_tol=1e-6) + assert math.isclose(completion_cost_val, expected_completion, rel_tol=1e-6) + + def test_integration_with_completion_cost_function(self): + """Test integration with the completion_cost function.""" + from litellm import ModelResponse + + # Create a mock ModelResponse + usage = Usage( + prompt_tokens=100, + completion_tokens=50, + total_tokens=150, + reasoning_tokens=10, + prompt_tokens_details=PromptTokensDetailsWrapper(web_search_requests=1) + ) + usage.citation_tokens = 15 + + response = ModelResponse() + response.usage = usage + response.model = "sonar-deep-research" + + # Test completion_cost function + total_cost = completion_cost(completion_response=response, custom_llm_provider="perplexity") + + # Calculate expected total cost + expected_prompt_cost = (100 * 2e-6) + (15 * 2e-6) # Input + citation + expected_completion_cost = (50 * 8e-6) + (10 * 3e-6) + (1 / 1000 * 0.005) # Output + reasoning + search + expected_total = expected_prompt_cost + expected_completion_cost + + assert math.isclose(total_cost, expected_total, rel_tol=1e-6) + + def test_model_info_access(self): + """Test that model info correctly returns the new cost fields.""" + model_info = get_model_info(model="sonar-deep-research", custom_llm_provider="perplexity") + + # Check that the new fields are accessible + assert "citation_cost_per_token" in model_info + assert model_info["citation_cost_per_token"] == 2e-6 + assert model_info["search_context_cost_per_query"] == { + "search_context_size_low": 0.005, + "search_context_size_medium": 0.005, + "search_context_size_high": 0.005 + } + + @pytest.mark.parametrize("citation_tokens", [0, 10, 25, 100]) + @pytest.mark.parametrize("search_queries", [0, 1, 5, 10]) + @pytest.mark.parametrize("reasoning_tokens", [0, 15, 30]) + def test_cost_calculation_combinations(self, citation_tokens, search_queries, reasoning_tokens): + """Test various combinations of citation tokens, search queries, and reasoning tokens.""" + usage = Usage( + prompt_tokens=100, + completion_tokens=50, + total_tokens=150, + reasoning_tokens=reasoning_tokens, + prompt_tokens_details=PromptTokensDetailsWrapper(web_search_requests=search_queries) + ) + + usage.citation_tokens = citation_tokens + + prompt_cost, completion_cost = perplexity_cost_per_token( + model="sonar-deep-research", + usage=usage + ) + + # Calculate expected costs + expected_prompt_cost = (100 * 2e-6) + (citation_tokens * 2e-6) + expected_completion_cost = (50 * 8e-6) + (reasoning_tokens * 3e-6) + (search_queries / 1000 * 0.005) + + assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-6) + assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-6) + + # Ensure costs are non-negative + assert prompt_cost >= 0 + assert completion_cost >= 0 \ No newline at end of file diff --git a/tests/test_litellm/llms/perplexity/test_perplexity_integration.py b/tests/test_litellm/llms/perplexity/test_perplexity_integration.py new file mode 100644 index 00000000000..ae72b8a9625 --- /dev/null +++ b/tests/test_litellm/llms/perplexity/test_perplexity_integration.py @@ -0,0 +1,319 @@ +""" +Integration tests for Perplexity cost calculation and transformation. + +Tests the end-to-end functionality of Perplexity cost calculation +including integration with the main LiteLLM cost calculator. +""" + +import json +import math +import os +import sys +from unittest.mock import Mock, patch + +import pytest + +# Add the project root to Python path +sys.path.insert(0, os.path.abspath("../../../..")) + +import litellm +from litellm import ModelResponse +from litellm.cost_calculator import completion_cost, cost_per_token +from litellm.llms.perplexity.chat.transformation import PerplexityChatConfig +from litellm.types.utils import Usage, PromptTokensDetailsWrapper +from litellm.utils import get_model_info + + +class TestPerplexityIntegration: + """Integration test suite for Perplexity functionality.""" + + @pytest.fixture(autouse=True) + def setup_model_cost_map(self): + """Set up the model cost map for testing.""" + # Ensure we use local model cost map for consistent testing + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + + # Load the model cost map + try: + with open("model_prices_and_context_window.json", "r") as f: + model_cost_map = json.load(f) + litellm.model_cost = model_cost_map + except FileNotFoundError: + # Fallback to ensure we have the Perplexity model configuration + litellm.model_cost = { + "perplexity/sonar-deep-research": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 8e-06, + "output_cost_per_reasoning_token": 3e-06, + "citation_cost_per_token": 2e-06, + "search_queries_cost_per_query": { + "search_queries_size_low": 0.005, + "search_queries_size_medium": 0.005, + "search_queries_size_high": 0.005 + }, + "litellm_provider": "perplexity", + "mode": "chat", + "supports_reasoning": True, + "supports_web_search": True, + } + } + + def test_end_to_end_cost_calculation_with_transformation(self): + """Test end-to-end cost calculation with response transformation.""" + # Create a Perplexity API response that includes citations and search queries + config = PerplexityChatConfig() + + # Create a ModelResponse with basic usage (before transformation) + model_response = ModelResponse() + model_response.model = "sonar-deep-research" + model_response.usage = Usage( + prompt_tokens=100, + completion_tokens=50, + total_tokens=150, + reasoning_tokens=10 + ) + + # Simulate raw response from Perplexity API + raw_response_dict = { + "choices": [{"message": {"content": "Test response with citations"}}], + "usage": { + "prompt_tokens": 100, + "completion_tokens": 50, + "total_tokens": 150, + "num_search_queries": 2 + }, + "citations": [ + "This is the first citation with important information about the topic", + "Another citation providing additional context for the response" + ] + } + + # Apply transformation to extract Perplexity-specific fields + config._enhance_usage_with_perplexity_fields(model_response, raw_response_dict) + + # Now calculate the cost with the enhanced usage + total_cost = completion_cost(completion_response=model_response, custom_llm_provider="perplexity") + + # Calculate expected cost + citation_chars = sum(len(citation) for citation in raw_response_dict["citations"]) + citation_tokens = citation_chars // 4 + + expected_prompt_cost = (100 * 2e-6) + (citation_tokens * 2e-6) # Input + citation + expected_completion_cost = (50 * 8e-6) + (10 * 3e-6) + (2 / 1000 * 0.005) # Output + reasoning + search + expected_total = expected_prompt_cost + expected_completion_cost + + assert math.isclose(total_cost, expected_total, rel_tol=1e-6) + + def test_cost_calculation_without_custom_fields(self): + """Test that cost calculation works normally when custom fields are absent.""" + # Create a standard response without Perplexity-specific fields + model_response = ModelResponse() + model_response.model = "sonar-deep-research" + model_response.usage = Usage( + prompt_tokens=100, + completion_tokens=50, + total_tokens=150 + ) + + # Calculate cost without custom fields + total_cost = completion_cost(completion_response=model_response, custom_llm_provider="perplexity") + + # Should only include basic input/output costs + expected_cost = (100 * 2e-6) + (50 * 8e-6) + + assert math.isclose(total_cost, expected_cost, rel_tol=1e-6) + + def test_main_cost_calculator_integration(self): + """Test integration with the main LiteLLM cost calculator.""" + # Create usage with all Perplexity fields + usage = Usage( + prompt_tokens=200, + completion_tokens=100, + total_tokens=300, + reasoning_tokens=25, + prompt_tokens_details=PromptTokensDetailsWrapper(web_search_requests=3) + ) + usage.citation_tokens = 40 + + # Test main cost calculator + prompt_cost, completion_cost_val = cost_per_token( + model="sonar-deep-research", + custom_llm_provider="perplexity", + usage_object=usage + ) + + # Calculate expected costs + expected_prompt_cost = (200 * 2e-6) + (40 * 2e-6) # Input + citation + expected_completion_cost = (100 * 8e-6) + (25 * 3e-6) + (3 / 1000 * 0.005) # Output + reasoning + search + + assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-6) + assert math.isclose(completion_cost_val, expected_completion_cost, rel_tol=1e-6) + + def test_model_info_includes_custom_fields(self): + """Test that get_model_info returns the custom Perplexity cost fields.""" + model_info = get_model_info(model="sonar-deep-research", custom_llm_provider="perplexity") + + # Verify custom fields are included + required_fields = [ + "citation_cost_per_token", + "search_context_cost_per_query", + "input_cost_per_token", + "output_cost_per_token", + "output_cost_per_reasoning_token" + ] + + for field in required_fields: + assert field in model_info, f"Missing field: {field}" + assert model_info[field] is not None, f"Null value for field: {field}" + + def test_various_citation_sizes(self): + """Test cost calculation with various citation sizes.""" + config = PerplexityChatConfig() + + test_cases = [ + # (citations, expected_approximate_tokens) + (["Short"], 1), + (["This is a medium-length citation with some content"], 12), + (["Very short", "Another citation", "Third one with more text content"], 15), + ([""], 0), # Empty citation + ] + + for citations, expected_approx_tokens in test_cases: + model_response = ModelResponse() + model_response.model = "sonar-deep-research" + model_response.usage = Usage( + prompt_tokens=100, + completion_tokens=50, + total_tokens=150 + ) + + raw_response_dict = { + "usage": {"prompt_tokens": 100, "completion_tokens": 50, "total_tokens": 150}, + "citations": citations + } + + config._enhance_usage_with_perplexity_fields(model_response, raw_response_dict) + + citation_tokens = getattr(model_response.usage, "citation_tokens", 0) + + # Allow for reasonable variance in token estimation + if expected_approx_tokens == 0: + assert citation_tokens == 0 + else: + assert abs(citation_tokens - expected_approx_tokens) <= 5 + + def test_cost_calculation_with_zero_values(self): + """Test cost calculation handles zero values for custom fields correctly.""" + usage = Usage( + prompt_tokens=100, + completion_tokens=50, + total_tokens=150 + ) + + # Set custom fields to zero + usage.citation_tokens = 0 + usage.prompt_tokens_details = PromptTokensDetailsWrapper(web_search_requests=0) + + # Should not add any extra cost + prompt_cost, completion_cost_val = cost_per_token( + model="sonar-deep-research", + custom_llm_provider="perplexity", + usage_object=usage + ) + + expected_prompt_cost = 100 * 2e-6 + expected_completion_cost = 50 * 8e-6 + + assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-6) + assert math.isclose(completion_cost_val, expected_completion_cost, rel_tol=1e-6) + + def test_high_volume_cost_calculation(self): + """Test cost calculation with high token and query counts.""" + usage = Usage( + prompt_tokens=50000, + completion_tokens=25000, + total_tokens=75000, + reasoning_tokens=10000 + ) + + usage.citation_tokens = 5000 + usage.prompt_tokens_details = PromptTokensDetailsWrapper(web_search_requests=100) + + total_cost = completion_cost( + completion_response=ModelResponse(usage=usage, model="sonar-deep-research"), + custom_llm_provider="perplexity" + ) + + # Calculate expected cost + expected_prompt_cost = (50000 * 2e-6) + (5000 * 2e-6) # $0.11 + expected_completion_cost = (25000 * 8e-6) + (10000 * 3e-6) + (100 / 1000 * 0.005) # $0.23 + expected_total = expected_prompt_cost + expected_completion_cost # $0.34 + + assert math.isclose(total_cost, expected_total, rel_tol=1e-6) + assert total_cost > 0.3 # Sanity check for high-volume scenario + + def test_transformation_preserves_existing_usage_fields(self): + """Test that transformation doesn't overwrite existing standard usage fields.""" + config = PerplexityChatConfig() + + model_response = ModelResponse() + model_response.usage = Usage( + prompt_tokens=100, + completion_tokens=50, + total_tokens=150, + reasoning_tokens=20 + ) + + # Store original values + original_prompt_tokens = model_response.usage.prompt_tokens + original_completion_tokens = model_response.usage.completion_tokens + original_total_tokens = model_response.usage.total_tokens + + raw_response_dict = { + "usage": { + "prompt_tokens": 999, # Different from original + "completion_tokens": 999, # Different from original + "total_tokens": 999, # Different from original + "num_search_queries": 3 + }, + "citations": ["Some citation"] + } + + config._enhance_usage_with_perplexity_fields(model_response, raw_response_dict) + + # Original usage fields should be preserved + assert model_response.usage.prompt_tokens == original_prompt_tokens + assert model_response.usage.completion_tokens == original_completion_tokens + assert model_response.usage.total_tokens == original_total_tokens + + # But custom fields should be added + assert hasattr(model_response.usage, "prompt_tokens_details") + assert hasattr(model_response.usage, "citation_tokens") + assert model_response.usage.prompt_tokens_details.web_search_requests == 3 + + @pytest.mark.parametrize("provider_name", ["perplexity", "PERPLEXITY", "Perplexity"]) + def test_case_insensitive_provider_matching(self, provider_name): + """Test that cost calculation works with different case variations of provider name.""" + usage = Usage( + prompt_tokens=100, + completion_tokens=50, + total_tokens=150 + ) + usage.citation_tokens = 10 + usage.prompt_tokens_details = PromptTokensDetailsWrapper(web_search_requests=1) + + # Should work regardless of case + prompt_cost, completion_cost_val = cost_per_token( + model="sonar-deep-research", + custom_llm_provider=provider_name.lower(), # Normalize to lowercase + usage_object=usage + ) + + # Should calculate costs correctly + expected_prompt_cost = (100 * 2e-6) + (10 * 2e-6) + expected_completion_cost = (50 * 8e-6) + (1 / 1000 * 0.005) + + assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-6) + assert math.isclose(completion_cost_val, expected_completion_cost, rel_tol=1e-6) \ No newline at end of file diff --git a/tests/test_litellm/llms/test_volcengine.py b/tests/test_litellm/llms/test_volcengine.py index cb9bc0f521c..4904124d37e 100644 --- a/tests/test_litellm/llms/test_volcengine.py +++ b/tests/test_litellm/llms/test_volcengine.py @@ -1,5 +1,6 @@ import os import sys +from unittest.mock import MagicMock, patch from pydantic import BaseModel @@ -23,7 +24,9 @@ class TestVolcEngineConfig: ) assert mapped_params == { - "thinking": {"type": "disabled"}, + "extra_body": { + "thinking": {"type": "disabled"}, + } } e2e_mapped_params = get_optional_params( @@ -33,6 +36,48 @@ class TestVolcEngineConfig: drop_params=False, ) - assert "thinking" in e2e_mapped_params and e2e_mapped_params["thinking"] == { + assert "thinking" in e2e_mapped_params["extra_body"] and e2e_mapped_params[ + "extra_body" + ]["thinking"] == { "type": "enabled", } + + def test_e2e_completion(self): + from openai import OpenAI + + from litellm import completion + from litellm.types.utils import ModelResponse + + client = OpenAI(api_key="test_api_key") + + mock_raw_response = MagicMock() + mock_raw_response.headers = { + "x-request-id": "123", + "openai-organization": "org-123", + "x-ratelimit-limit-requests": "100", + "x-ratelimit-remaining-requests": "99", + } + mock_raw_response.parse.return_value = ModelResponse() + + with patch.object( + client.chat.completions.with_raw_response, "create", mock_raw_response + ) as mock_create: + completion( + model="volcengine/doubao-seed-1.6", + messages=[ + { + "role": "system", + "content": "**Tell me your model detail information.**", + } + ], + user="guest", + stream=True, + thinking={"type": "disabled"}, + client=client, + ) + + mock_create.assert_called_once() + print(mock_create.call_args.kwargs) + assert mock_create.call_args.kwargs["extra_body"] == { + "thinking": {"type": "disabled"}, + } diff --git a/tests/test_litellm/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py b/tests/test_litellm/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py new file mode 100644 index 00000000000..ef3404353ae --- /dev/null +++ b/tests/test_litellm/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py @@ -0,0 +1,643 @@ +import os +import sys +from typing import List +from unittest.mock import AsyncMock, MagicMock, patch + +import httpx +import pytest + +sys.path.insert( + 0, os.path.abspath("../../../..") +) # Adds the parent directory to the system path + +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.llms.vertex_ai.common_utils import VertexAIError +from litellm.llms.vertex_ai.context_caching.vertex_ai_context_caching import ( + ContextCachingEndpoints, +) + + +class TestContextCachingEndpoints: + """Test class for ContextCachingEndpoints methods""" + + def setup_method(self): + """Setup for each test method""" + self.context_caching = ContextCachingEndpoints() + self.mock_logging = MagicMock(spec=Logging) + self.mock_client = MagicMock(spec=HTTPHandler) + self.mock_async_client = MagicMock(spec=AsyncHTTPHandler) + + # Sample messages for testing + self.sample_messages = [ + { + "role": "system", + "content": "You are a helpful assistant", + "cache_control": {"type": "ephemeral"}, + }, + {"role": "user", "content": "Hello, how are you?"}, + ] + + # Sample tools for testing + self.sample_tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather information", + "parameters": { + "type": "object", + "properties": {"location": {"type": "string"}}, + }, + }, + } + ] + + self.sample_optional_params = {"tools": self.sample_tools.copy()} + + @patch( + "litellm.llms.vertex_ai.context_caching.vertex_ai_context_caching.separate_cached_messages" + ) + @patch( + "litellm.llms.vertex_ai.context_caching.vertex_ai_context_caching.local_cache_obj" + ) + def test_check_and_create_cache_with_cached_content( + self, mock_cache_obj, mock_separate + ): + """Test check_and_create_cache when cached_content is provided""" + # Setup + cached_content = "cached_content_123" + optional_params = self.sample_optional_params.copy() + + # Execute + result = self.context_caching.check_and_create_cache( + messages=self.sample_messages, + optional_params=optional_params, + api_key="test_key", + api_base=None, + model="gemini-1.5-pro", + client=self.mock_client, + timeout=30.0, + logging_obj=self.mock_logging, + cached_content=cached_content, + ) + + # Assert + messages, returned_params, returned_cache = result + assert messages == self.sample_messages + assert returned_params == optional_params + assert returned_cache == cached_content + + # Verify mocks weren't called since we short-circuited + mock_separate.assert_not_called() + mock_cache_obj.get_cache_key.assert_not_called() + + @patch( + "litellm.llms.vertex_ai.context_caching.vertex_ai_context_caching.separate_cached_messages" + ) + def test_check_and_create_cache_no_cached_messages(self, mock_separate): + """Test check_and_create_cache when no cached messages are found""" + # Setup + mock_separate.return_value = ([], self.sample_messages) # No cached messages + optional_params = self.sample_optional_params.copy() + + # Execute + result = self.context_caching.check_and_create_cache( + messages=self.sample_messages, + optional_params=optional_params, + api_key="test_key", + api_base=None, + model="gemini-1.5-pro", + client=self.mock_client, + timeout=30.0, + logging_obj=self.mock_logging, + ) + + # Assert + messages, returned_params, returned_cache = result + assert messages == self.sample_messages + assert returned_params == optional_params + assert returned_cache is None + + @patch( + "litellm.llms.vertex_ai.context_caching.vertex_ai_context_caching.separate_cached_messages" + ) + @patch( + "litellm.llms.vertex_ai.context_caching.vertex_ai_context_caching.local_cache_obj" + ) + @patch.object(ContextCachingEndpoints, "check_cache") + def test_check_and_create_cache_existing_cache_found( + self, mock_check_cache, mock_cache_obj, mock_separate + ): + """Test check_and_create_cache when existing cache is found""" + # Setup + cached_messages = [self.sample_messages[0]] # System message with cache_control + non_cached_messages = [self.sample_messages[1]] # User message + mock_separate.return_value = (cached_messages, non_cached_messages) + + mock_cache_obj.get_cache_key.return_value = "test_cache_key" + mock_check_cache.return_value = "existing_cache_name" + + optional_params = self.sample_optional_params.copy() + + # Execute + result = self.context_caching.check_and_create_cache( + messages=self.sample_messages, + optional_params=optional_params, + api_key="test_key", + api_base=None, + model="gemini-1.5-pro", + client=self.mock_client, + timeout=30.0, + logging_obj=self.mock_logging, + ) + + # Assert + messages, returned_params, returned_cache = result + assert messages == non_cached_messages + assert returned_params == optional_params + assert returned_cache == "existing_cache_name" + + # Verify cache key was generated with tools + mock_cache_obj.get_cache_key.assert_called_once_with( + messages=cached_messages, tools=self.sample_tools + ) + + @patch( + "litellm.llms.vertex_ai.context_caching.vertex_ai_context_caching.separate_cached_messages" + ) + @patch( + "litellm.llms.vertex_ai.context_caching.vertex_ai_context_caching.local_cache_obj" + ) + @patch( + "litellm.llms.vertex_ai.context_caching.vertex_ai_context_caching.transform_openai_messages_to_gemini_context_caching" + ) + @patch.object(ContextCachingEndpoints, "check_cache") + @patch.object(ContextCachingEndpoints, "_get_token_and_url_context_caching") + def test_check_and_create_cache_create_new_cache( + self, + mock_get_token_url, + mock_check_cache, + mock_transform, + mock_cache_obj, + mock_separate, + ): + """Test check_and_create_cache when creating new cache""" + # Setup + cached_messages = [self.sample_messages[0]] + non_cached_messages = [self.sample_messages[1]] + mock_separate.return_value = (cached_messages, non_cached_messages) + + mock_cache_obj.get_cache_key.return_value = "test_cache_key" + mock_check_cache.return_value = None # No existing cache + mock_get_token_url.return_value = ("token", "https://test-url.com") + + mock_transform.return_value = {"model": "gemini-1.5-pro", "contents": []} + + # Mock successful HTTP response + mock_response = MagicMock() + mock_response.json.return_value = { + "name": "new_cache_name", + "model": "gemini-1.5-pro", + } + self.mock_client.post.return_value = mock_response + + optional_params = self.sample_optional_params.copy() + + # Execute + result = self.context_caching.check_and_create_cache( + messages=self.sample_messages, + optional_params=optional_params, + api_key="test_key", + api_base=None, + model="gemini-1.5-pro", + client=self.mock_client, + timeout=30.0, + logging_obj=self.mock_logging, + ) + + # Assert + messages, returned_params, returned_cache = result + assert messages == non_cached_messages + assert returned_params == optional_params + assert returned_cache == "new_cache_name" + + # Verify HTTP request was made + self.mock_client.post.assert_called_once() + call_args = self.mock_client.post.call_args + assert "tools" in call_args.kwargs["json"] + assert call_args.kwargs["json"]["tools"] == self.sample_tools + + @patch( + "litellm.llms.vertex_ai.context_caching.vertex_ai_context_caching.separate_cached_messages" + ) + @patch( + "litellm.llms.vertex_ai.context_caching.vertex_ai_context_caching.local_cache_obj" + ) + @patch.object(ContextCachingEndpoints, "check_cache") + @patch.object(ContextCachingEndpoints, "_get_token_and_url_context_caching") + def test_check_and_create_cache_http_error( + self, mock_get_token_url, mock_check_cache, mock_cache_obj, mock_separate + ): + """Test check_and_create_cache handles HTTP errors properly""" + # Setup + cached_messages = [self.sample_messages[0]] + non_cached_messages = [self.sample_messages[1]] + mock_separate.return_value = (cached_messages, non_cached_messages) + + mock_cache_obj.get_cache_key.return_value = "test_cache_key" + mock_check_cache.return_value = None + mock_get_token_url.return_value = ("token", "https://test-url.com") + + # Mock HTTP error + mock_response = MagicMock() + mock_response.status_code = 400 + mock_response.text = "Bad Request" + http_error = httpx.HTTPStatusError( + "Error", request=MagicMock(), response=mock_response + ) + self.mock_client.post.side_effect = http_error + + optional_params = self.sample_optional_params.copy() + + # Execute and Assert + with pytest.raises(VertexAIError) as exc_info: + self.context_caching.check_and_create_cache( + messages=self.sample_messages, + optional_params=optional_params, + api_key="test_key", + api_base=None, + model="gemini-1.5-pro", + client=self.mock_client, + timeout=30.0, + logging_obj=self.mock_logging, + ) + + assert exc_info.value.status_code == 400 + assert "Bad Request" in str(exc_info.value.message) + + @pytest.mark.asyncio + @patch( + "litellm.llms.vertex_ai.context_caching.vertex_ai_context_caching.separate_cached_messages" + ) + @patch( + "litellm.llms.vertex_ai.context_caching.vertex_ai_context_caching.local_cache_obj" + ) + async def test_async_check_and_create_cache_with_cached_content( + self, mock_cache_obj, mock_separate + ): + """Test async_check_and_create_cache when cached_content is provided""" + # Setup + cached_content = "cached_content_123" + optional_params = self.sample_optional_params.copy() + + # Execute + result = await self.context_caching.async_check_and_create_cache( + messages=self.sample_messages, + optional_params=optional_params, + api_key="test_key", + api_base=None, + model="gemini-1.5-pro", + client=self.mock_async_client, + timeout=30.0, + logging_obj=self.mock_logging, + cached_content=cached_content, + ) + + # Assert + messages, returned_params, returned_cache = result + assert messages == self.sample_messages + assert returned_params == optional_params + assert returned_cache == cached_content + + @pytest.mark.asyncio + @patch( + "litellm.llms.vertex_ai.context_caching.vertex_ai_context_caching.separate_cached_messages" + ) + async def test_async_check_and_create_cache_no_cached_messages(self, mock_separate): + """Test async_check_and_create_cache when no cached messages are found""" + # Setup + mock_separate.return_value = ([], self.sample_messages) + optional_params = self.sample_optional_params.copy() + + # Execute + result = await self.context_caching.async_check_and_create_cache( + messages=self.sample_messages, + optional_params=optional_params, + api_key="test_key", + api_base=None, + model="gemini-1.5-pro", + client=self.mock_async_client, + timeout=30.0, + logging_obj=self.mock_logging, + ) + + # Assert + messages, returned_params, returned_cache = result + assert messages == self.sample_messages + assert returned_params == optional_params + assert returned_cache is None + + @pytest.mark.asyncio + @patch( + "litellm.llms.vertex_ai.context_caching.vertex_ai_context_caching.separate_cached_messages" + ) + @patch( + "litellm.llms.vertex_ai.context_caching.vertex_ai_context_caching.local_cache_obj" + ) + @patch.object(ContextCachingEndpoints, "async_check_cache") + async def test_async_check_and_create_cache_existing_cache_found( + self, mock_async_check_cache, mock_cache_obj, mock_separate + ): + """Test async_check_and_create_cache when existing cache is found""" + # Setup + cached_messages = [self.sample_messages[0]] + non_cached_messages = [self.sample_messages[1]] + mock_separate.return_value = (cached_messages, non_cached_messages) + + mock_cache_obj.get_cache_key.return_value = "test_cache_key" + mock_async_check_cache.return_value = "existing_cache_name" + + optional_params = self.sample_optional_params.copy() + + # Execute + result = await self.context_caching.async_check_and_create_cache( + messages=self.sample_messages, + optional_params=optional_params, + api_key="test_key", + api_base=None, + model="gemini-1.5-pro", + client=self.mock_async_client, + timeout=30.0, + logging_obj=self.mock_logging, + ) + + # Assert + messages, returned_params, returned_cache = result + assert messages == non_cached_messages + assert returned_params == optional_params + assert returned_cache == "existing_cache_name" + + # Verify cache key was generated with tools + mock_cache_obj.get_cache_key.assert_called_once_with( + messages=cached_messages, tools=self.sample_tools + ) + + @pytest.mark.asyncio + @patch( + "litellm.llms.vertex_ai.context_caching.vertex_ai_context_caching.separate_cached_messages" + ) + @patch( + "litellm.llms.vertex_ai.context_caching.vertex_ai_context_caching.local_cache_obj" + ) + @patch( + "litellm.llms.vertex_ai.context_caching.vertex_ai_context_caching.transform_openai_messages_to_gemini_context_caching" + ) + @patch.object(ContextCachingEndpoints, "async_check_cache") + @patch.object(ContextCachingEndpoints, "_get_token_and_url_context_caching") + @patch( + "litellm.llms.vertex_ai.context_caching.vertex_ai_context_caching.get_async_httpx_client" + ) + async def test_async_check_and_create_cache_create_new_cache( + self, + mock_get_client, + mock_get_token_url, + mock_async_check_cache, + mock_transform, + mock_cache_obj, + mock_separate, + ): + """Test async_check_and_create_cache when creating new cache""" + # Setup + cached_messages = [self.sample_messages[0]] + non_cached_messages = [self.sample_messages[1]] + mock_separate.return_value = (cached_messages, non_cached_messages) + + mock_cache_obj.get_cache_key.return_value = "test_cache_key" + mock_async_check_cache.return_value = None + mock_get_token_url.return_value = ("token", "https://test-url.com") + + mock_transform.return_value = {"model": "gemini-1.5-pro", "contents": []} + + # Mock successful HTTP response + mock_response = MagicMock() + mock_response.json.return_value = { + "name": "new_cache_name", + "model": "gemini-1.5-pro", + } + self.mock_async_client.post = AsyncMock(return_value=mock_response) + + optional_params = self.sample_optional_params.copy() + + # Execute + result = await self.context_caching.async_check_and_create_cache( + messages=self.sample_messages, + optional_params=optional_params, + api_key="test_key", + api_base=None, + model="gemini-1.5-pro", + client=self.mock_async_client, + timeout=30.0, + logging_obj=self.mock_logging, + ) + + # Assert + messages, returned_params, returned_cache = result + assert messages == non_cached_messages + assert returned_params == optional_params + assert returned_cache == "new_cache_name" + + # Verify HTTP request was made + self.mock_async_client.post.assert_called_once() + call_args = self.mock_async_client.post.call_args + assert "tools" in call_args.kwargs["json"] + assert call_args.kwargs["json"]["tools"] == self.sample_tools + + @pytest.mark.asyncio + @patch( + "litellm.llms.vertex_ai.context_caching.vertex_ai_context_caching.separate_cached_messages" + ) + @patch( + "litellm.llms.vertex_ai.context_caching.vertex_ai_context_caching.local_cache_obj" + ) + @patch.object(ContextCachingEndpoints, "async_check_cache") + @patch.object(ContextCachingEndpoints, "_get_token_and_url_context_caching") + @patch( + "litellm.llms.vertex_ai.context_caching.vertex_ai_context_caching.get_async_httpx_client" + ) + async def test_async_check_and_create_cache_timeout_error( + self, + mock_get_client, + mock_get_token_url, + mock_async_check_cache, + mock_cache_obj, + mock_separate, + ): + """Test async_check_and_create_cache handles timeout errors properly""" + # Setup + cached_messages = [self.sample_messages[0]] + non_cached_messages = [self.sample_messages[1]] + mock_separate.return_value = (cached_messages, non_cached_messages) + + mock_cache_obj.get_cache_key.return_value = "test_cache_key" + mock_async_check_cache.return_value = None + mock_get_token_url.return_value = ("token", "https://test-url.com") + + # Mock timeout error + self.mock_async_client.post = AsyncMock( + side_effect=httpx.TimeoutException("Timeout") + ) + + optional_params = self.sample_optional_params.copy() + + # Execute and Assert + with pytest.raises(VertexAIError) as exc_info: + await self.context_caching.async_check_and_create_cache( + messages=self.sample_messages, + optional_params=optional_params, + api_key="test_key", + api_base=None, + model="gemini-1.5-pro", + client=self.mock_async_client, + timeout=30.0, + logging_obj=self.mock_logging, + ) + + assert exc_info.value.status_code == 408 + assert "Timeout error occurred" in str(exc_info.value.message) + + def test_check_and_create_cache_tools_popped_from_optional_params(self): + """Test that tools are properly popped from optional_params when there are cached messages""" + with patch( + "litellm.llms.vertex_ai.context_caching.vertex_ai_context_caching.separate_cached_messages" + ) as mock_separate: + # Mock to return cached messages so tools get popped + cached_messages = [ + self.sample_messages[0] + ] # System message with cache_control + non_cached_messages = [self.sample_messages[1]] # User message + mock_separate.return_value = (cached_messages, non_cached_messages) + + optional_params = self.sample_optional_params.copy() + original_tools = optional_params["tools"].copy() + + # Mock the check_cache to return existing cache so we don't make HTTP calls + with patch.object( + self.context_caching, "check_cache", return_value="existing_cache" + ): + # Execute + result = self.context_caching.check_and_create_cache( + messages=self.sample_messages, + optional_params=optional_params, + api_key="test_key", + api_base=None, + model="gemini-1.5-pro", + client=self.mock_client, + timeout=30.0, + logging_obj=self.mock_logging, + ) + + # Assert tools were popped from optional_params + assert "tools" not in optional_params + + # But original tools should still be available for comparison + assert original_tools == self.sample_tools + + def test_check_and_create_cache_tools_not_popped_when_no_cached_messages(self): + """Test that tools are NOT popped from optional_params when there are no cached messages""" + with patch( + "litellm.llms.vertex_ai.context_caching.vertex_ai_context_caching.separate_cached_messages" + ) as mock_separate: + mock_separate.return_value = ( + [], + self.sample_messages, + ) # No cached messages + + optional_params = self.sample_optional_params.copy() + original_tools = optional_params["tools"].copy() + + # Execute + result = self.context_caching.check_and_create_cache( + messages=self.sample_messages, + optional_params=optional_params, + api_key="test_key", + api_base=None, + model="gemini-1.5-pro", + client=self.mock_client, + timeout=30.0, + logging_obj=self.mock_logging, + ) + + # Assert tools were NOT popped from optional_params (early return) + assert "tools" in optional_params + assert optional_params["tools"] == original_tools + + @pytest.mark.asyncio + async def test_async_check_and_create_cache_tools_not_popped_when_no_cached_messages( + self, + ): + """Test that tools are NOT popped from optional_params in async version when there are no cached messages""" + with patch( + "litellm.llms.vertex_ai.context_caching.vertex_ai_context_caching.separate_cached_messages" + ) as mock_separate: + mock_separate.return_value = ( + [], + self.sample_messages, + ) # No cached messages + + optional_params = self.sample_optional_params.copy() + original_tools = optional_params["tools"].copy() + + # Execute + result = await self.context_caching.async_check_and_create_cache( + messages=self.sample_messages, + optional_params=optional_params, + api_key="test_key", + api_base=None, + model="gemini-1.5-pro", + client=self.mock_async_client, + timeout=30.0, + logging_obj=self.mock_logging, + ) + + # Assert tools were NOT popped from optional_params (early return) + assert "tools" in optional_params + assert optional_params["tools"] == original_tools + + @pytest.mark.asyncio + async def test_async_check_and_create_cache_tools_popped_from_optional_params(self): + """Test that tools are properly popped from optional_params in async version when there are cached messages""" + with patch( + "litellm.llms.vertex_ai.context_caching.vertex_ai_context_caching.separate_cached_messages" + ) as mock_separate: + # Mock to return cached messages so tools get popped + cached_messages = [ + self.sample_messages[0] + ] # System message with cache_control + non_cached_messages = [self.sample_messages[1]] # User message + mock_separate.return_value = (cached_messages, non_cached_messages) + + optional_params = self.sample_optional_params.copy() + original_tools = optional_params["tools"].copy() + + # Mock the async_check_cache to return existing cache so we don't make HTTP calls + with patch.object( + self.context_caching, "async_check_cache", return_value="existing_cache" + ): + # Execute + result = await self.context_caching.async_check_and_create_cache( + messages=self.sample_messages, + optional_params=optional_params, + api_key="test_key", + api_base=None, + model="gemini-1.5-pro", + client=self.mock_async_client, + timeout=30.0, + logging_obj=self.mock_logging, + ) + + # Assert tools were popped from optional_params + assert "tools" not in optional_params + + # But original tools should still be available for comparison + assert original_tools == self.sample_tools diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py index b43e5a113cf..9ebb9004b7f 100644 --- a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py +++ b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py @@ -237,11 +237,7 @@ def test_build_vertex_schema(): }, "recursion_limit": {"type": "integer"}, "configurable": {"type": "object"}, - "run_id": { - "anyOf": [ - {"format": "uuid", "type": "string", "nullable": True} - ] - }, + "run_id": {"anyOf": [{"type": "string", "nullable": True}]}, }, "type": "object", }, @@ -627,3 +623,57 @@ def test_get_vertex_region_global_only_model( assert result == expected_region mock_is_global_only.assert_called_once_with("test-model") + + +def test_vertex_filter_format_uri(): + import json + + from litellm.llms.vertex_ai.common_utils import filter_schema_fields + + parameters = { + "type": "object", + "properties": { + "url": { + "type": "string", + "format": "uri", + "description": "The URL to fetch content from", + }, + "prompt": { + "type": "string", + "description": "The prompt to run on the fetched content", + }, + }, + "required": ["url", "prompt"], + "$schema": "http://json-schema.org/draft-07/schema#", + } + valid_schema_fields = { + "minLength", + "nullable", + "maxItems", + "required", + "default", + "items", + "propertyOrdering", + "maximum", + "properties", + "anyOf", + "description", + "minProperties", + "minimum", + "minItems", + "maxProperties", + "title", + "pattern", + "example", + "format", + "enum", + "maxLength", + "type", + } + + new_parameters = filter_schema_fields( + schema_dict=parameters, + valid_fields=valid_schema_fields, + ) + + assert "uri" not in json.dumps(new_parameters) diff --git a/tests/test_litellm/llms/watsonx/test_watsonx.py b/tests/test_litellm/llms/watsonx/test_watsonx.py new file mode 100644 index 00000000000..13ca6ad8e6c --- /dev/null +++ b/tests/test_litellm/llms/watsonx/test_watsonx.py @@ -0,0 +1,205 @@ +import json +import os +import sys + +sys.path.insert( + 0, os.path.abspath("../..") +) # Adds the parent directory to the system path +import litellm +from litellm import completion +from litellm.llms.custom_httpx.http_handler import HTTPHandler +from unittest.mock import patch, Mock +import pytest +from typing import Optional + + +@pytest.fixture +def watsonx_chat_completion_call(): + def _call( + model="watsonx/my-test-model", + messages=None, + api_key="test_api_key", + space_id: Optional[str] = None, + headers=None, + client=None, + patch_token_call=True, + ): + if messages is None: + messages = [{"role": "user", "content": "Hello, how are you?"}] + if client is None: + client = HTTPHandler() + + if patch_token_call: + mock_response = Mock() + mock_response.json.return_value = { + "access_token": "mock_access_token", + "expires_in": 3600, + } + mock_response.raise_for_status = Mock() # No-op to simulate no exception + + with patch.object(client, "post") as mock_post, patch.object( + litellm.module_level_client, "post", return_value=mock_response + ) as mock_get: + try: + completion( + model=model, + messages=messages, + api_key=api_key, + headers=headers or {}, + client=client, + space_id=space_id, + ) + except Exception as e: + print(e) + + return mock_post, mock_get + else: + with patch.object(client, "post") as mock_post: + try: + completion( + model=model, + messages=messages, + api_key=api_key, + headers=headers or {}, + client=client, + space_id=space_id, + ) + except Exception as e: + print(e) + return mock_post, None + + return _call + + +def test_watsonx_deployment_model_id_not_in_payload( + monkeypatch, watsonx_chat_completion_call +): + """Test that deployment models do not include 'model_id' in the request payload""" + monkeypatch.setenv("WATSONX_PROJECT_ID", "test-project-id") + monkeypatch.setenv("WATSONX_API_BASE", "https://test-api.watsonx.ai") + model = "watsonx/deployment/test-deployment-id" + messages = [{"role": "user", "content": "Test message"}] + + mock_post, _ = watsonx_chat_completion_call(model=model, messages=messages) + + assert mock_post.call_count == 1 + json_data = json.loads(mock_post.call_args.kwargs["data"]) + # Ensure model_id is not in the payload for deployment models + assert "model_id" not in json_data or json_data["model_id"] is None + # Ensure project_id is also not in the payload for deployment models + assert "project_id" not in json_data or json_data["project_id"] is None + + +def test_watsonx_regular_model_includes_model_id( + monkeypatch, watsonx_chat_completion_call +): + """Test that regular models include 'model_id' in the request payload""" + monkeypatch.setenv("WATSONX_PROJECT_ID", "test-project-id") + monkeypatch.setenv("WATSONX_API_BASE", "https://test-api.watsonx.ai") + model = "watsonx/regular-model" + messages = [{"role": "user", "content": "Test message"}] + + mock_post, _ = watsonx_chat_completion_call(model=model, messages=messages) + + assert mock_post.call_count == 1 + json_data = json.loads(mock_post.call_args.kwargs["data"]) + # Ensure model_id is included in the payload for regular models + assert "model_id" in json_data + assert json_data["model_id"] == "regular-model" # Provider prefix is stripped + # Ensure project_id is also included for regular models + assert "project_id" in json_data + + +@pytest.fixture +def watsonx_completion_call(): + def _call( + model="watsonx_text/my-test-model", + prompt="Hello, how are you?", + api_key="test_api_key", + space_id: Optional[str] = None, + headers=None, + client=None, + patch_token_call=True, + ): + if client is None: + client = HTTPHandler() + + if patch_token_call: + mock_response = Mock() + mock_response.json.return_value = { + "access_token": "mock_access_token", + "expires_in": 3600, + } + mock_response.raise_for_status = Mock() + + with patch.object(client, "post") as mock_post, patch.object( + litellm.module_level_client, "post", return_value=mock_response + ) as mock_get: + try: + litellm.text_completion( + model=model, + prompt=prompt, + api_key=api_key, + headers=headers or {}, + client=client, + space_id=space_id, + ) + except Exception as e: + print(e) + + return mock_post, mock_get + else: + with patch.object(client, "post") as mock_post: + try: + litellm.text_completion( + model=model, + prompt=prompt, + api_key=api_key, + headers=headers or {}, + client=client, + space_id=space_id, + ) + except Exception as e: + print(e) + return mock_post, None + + return _call + + +def test_watsonx_completion_deployment_model_id_not_in_payload( + monkeypatch, watsonx_completion_call +): + """Test that deployment models do not include 'model_id' in completion request payload""" + monkeypatch.setenv("WATSONX_PROJECT_ID", "test-project-id") + monkeypatch.setenv("WATSONX_API_BASE", "https://test-api.watsonx.ai") + model = "watsonx_text/deployment/test-deployment-id" + prompt = "Test prompt" + + mock_post, _ = watsonx_completion_call(model=model, prompt=prompt) + + assert mock_post.call_count == 1 + json_data = json.loads(mock_post.call_args.kwargs["data"]) + # Ensure model_id is not in the payload for deployment models + assert "model_id" not in json_data + # Ensure project_id is also not in the payload for deployment models + assert "project_id" not in json_data + + +def test_watsonx_completion_regular_model_includes_model_id( + monkeypatch, watsonx_completion_call +): + """Test that regular models include 'model_id' in completion request payload""" + monkeypatch.setenv("WATSONX_PROJECT_ID", "test-project-id") + monkeypatch.setenv("WATSONX_API_BASE", "https://test-api.watsonx.ai") + model = "watsonx_text/regular-model" + prompt = "Test prompt" + + mock_post, _ = watsonx_completion_call(model=model, prompt=prompt) + + assert mock_post.call_count == 1 + json_data = json.loads(mock_post.call_args.kwargs["data"]) + # Ensure model_id is included in the payload for regular models + assert "model_id" in json_data + assert json_data["model_id"] == "regular-model" # Provider prefix is stripped + # Ensure project_id is also included for regular models + assert "project_id" in json_data diff --git a/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py b/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py index aab700e2a72..f93728b90c3 100644 --- a/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py @@ -3,7 +3,7 @@ from unittest.mock import AsyncMock import pytest from fastapi import HTTPException -from litellm.proxy._types import NewUserRequest, ProxyException +from litellm.proxy._types import LitellmUserRoles, NewUserRequest, ProxyException from litellm.proxy.management_endpoints.scim.scim_v2 import ( UserProvisionerHelpers, _handle_team_membership_changes, @@ -58,6 +58,94 @@ async def test_create_user_existing_user_conflict(mocker): mocked_new_user.assert_not_called() +@pytest.mark.asyncio +async def test_create_user_defaults_to_viewer(mocker, monkeypatch): + """If no role provided, new user should default to viewer""" + + scim_user = SCIMUser( + schemas=["urn:ietf:params:scim:schemas:core:2.0:User"], + userName="new-user", + name=SCIMUserName(familyName="User", givenName="New"), + emails=[SCIMUserEmail(value="new@example.com")], + ) + + mock_prisma_client = mocker.MagicMock() + mock_prisma_client.db = mocker.MagicMock() + mock_prisma_client.db.litellm_usertable = mocker.MagicMock() + mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=None) + mock_prisma_client.db.litellm_usertable.find_first = AsyncMock(return_value=None) + + monkeypatch.setattr( + "litellm.default_internal_user_params", None, raising=False + ) + + mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2._get_prisma_client_or_raise_exception", + AsyncMock(return_value=mock_prisma_client), + ) + + new_user_mock = mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2.new_user", + AsyncMock(return_value=NewUserRequest(user_id="new-user")), + ) + + mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2.ScimTransformations.transform_litellm_user_to_scim_user", + AsyncMock(return_value=scim_user), + ) + + await create_user(user=scim_user) + + called_args = new_user_mock.call_args.kwargs["data"] + assert called_args.user_role == LitellmUserRoles.INTERNAL_USER_VIEW_ONLY + + +@pytest.mark.asyncio +async def test_create_user_uses_default_internal_user_params_role(mocker, monkeypatch): + """If role is set in default_internal_user_params, new user should use that role""" + + scim_user = SCIMUser( + schemas=["urn:ietf:params:scim:schemas:core:2.0:User"], + userName="new-user", + name=SCIMUserName(familyName="User", givenName="New"), + emails=[SCIMUserEmail(value="new@example.com")], + ) + + mock_prisma_client = mocker.MagicMock() + mock_prisma_client.db = mocker.MagicMock() + mock_prisma_client.db.litellm_usertable = mocker.MagicMock() + mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=None) + mock_prisma_client.db.litellm_usertable.find_first = AsyncMock(return_value=None) + + # Set default_internal_user_params with a specific role + default_params = { + "user_role": LitellmUserRoles.PROXY_ADMIN, + } + monkeypatch.setattr( + "litellm.default_internal_user_params", default_params, raising=False + ) + + mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2._get_prisma_client_or_raise_exception", + AsyncMock(return_value=mock_prisma_client), + ) + + new_user_mock = mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2.new_user", + AsyncMock(return_value=NewUserRequest(user_id="new-user")), + ) + + mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2.ScimTransformations.transform_litellm_user_to_scim_user", + AsyncMock(return_value=scim_user), + ) + + await create_user(user=scim_user) + + called_args = new_user_mock.call_args.kwargs["data"] + assert called_args.user_role == LitellmUserRoles.PROXY_ADMIN + + @pytest.mark.asyncio async def test_handle_existing_user_by_email_no_email(mocker): """Should return None when new_user_request has no email""" diff --git a/tests/test_litellm/proxy/management_endpoints/test_callback_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_callback_management_endpoints.py new file mode 100644 index 00000000000..bb622304ef0 --- /dev/null +++ b/tests/test_litellm/proxy/management_endpoints/test_callback_management_endpoints.py @@ -0,0 +1,202 @@ +import json +import os +import sys +from datetime import datetime, timezone +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from fastapi.testclient import TestClient + +sys.path.insert( + 0, os.path.abspath("../../../..") +) # + +from typing import cast + +import litellm +from litellm.integrations.datadog.datadog import DataDogLogger +from litellm.integrations.langfuse.langfuse import LangFuseLogger +from litellm.proxy.management_endpoints.callback_management_endpoints import router +from litellm.proxy.proxy_server import app + + +class TestCallbackManagementEndpoints: + """Test suite for callback management endpoints""" + + @pytest.fixture(autouse=True) + def setup_and_teardown(self): + """Setup and teardown for each test""" + # Reset callbacks before each test + litellm.success_callback = [] + litellm.failure_callback = [] + litellm._async_success_callback = [] + litellm._async_failure_callback = [] + litellm.callbacks = [] + + yield + + # Clean up after each test + litellm.success_callback = [] + litellm.failure_callback = [] + litellm._async_success_callback = [] + litellm._async_failure_callback = [] + litellm.callbacks = [] + + def test_list_callbacks_no_active_callbacks(self): + """Test /callbacks/list endpoint with no active callbacks""" + # Setup test client + client = TestClient(app) + + # Make request to list callbacks endpoint + response = client.get( + "/callbacks/list", + headers={"Authorization": "Bearer sk-1234"} + ) + + # Verify response + assert response.status_code == 200 + + response_data = response.json() + assert "success" in response_data + assert "failure" in response_data + assert "success_and_failure" in response_data + + # All lists should be empty + assert response_data["success"] == [] + assert response_data["failure"] == [] + assert response_data["success_and_failure"] == [] + + @patch.dict(os.environ, { + "LANGFUSE_PUBLIC_KEY": "test_public_key", + "LANGFUSE_SECRET_KEY": "test_secret_key", + "LANGFUSE_HOST": "https://test.langfuse.com" + }) + def test_list_callbacks_with_langfuse_logger(self): + """Test /callbacks/list endpoint with real Langfuse logger initialized""" + # Setup test client + client = TestClient(app) + + # Initialize Langfuse logger and add to callbacks + with patch('litellm.integrations.langfuse.langfuse.Langfuse') as mock_langfuse: + # Mock the Langfuse client initialization + mock_langfuse_client = MagicMock() + mock_langfuse.return_value = mock_langfuse_client + + + # Add string representation to callback lists (this is how the system typically works) + litellm.success_callback.append("langfuse") + litellm._async_success_callback.append("langfuse") + + # Make request to list callbacks endpoint + response = client.get( + "/callbacks/list", + headers={"Authorization": "Bearer sk-1234"} + ) + + # Verify response + assert response.status_code == 200 + + response_data = response.json() + + # Verify langfuse appears in success callbacks + assert "langfuse" in response_data["success"] + assert response_data["failure"] == [] + assert response_data["success_and_failure"] == [] + + # Verify the response structure is correct + assert isinstance(response_data["success"], list) + assert isinstance(response_data["failure"], list) + assert isinstance(response_data["success_and_failure"], list) + + def test_list_callbacks_with_datadog_logger(self): + """Test /callbacks/list endpoint with DataDog logger configuration""" + # Setup test client + client = TestClient(app) + + # Test with datadog callbacks added directly (without initializing the logger to avoid async issues) + # Add string representations to different callback types to test comprehensive categorization + litellm.success_callback.append("datadog") + litellm.failure_callback.append("datadog") + litellm.callbacks.append("datadog") + + # Make request to list callbacks endpoint + response = client.get( + "/callbacks/list", + headers={"Authorization": "Bearer sk-1234"} + ) + + # Verify response + assert response.status_code == 200 + + response_data = response.json() + + # Verify datadog appears in the correct categorization + # Since datadog is in both success and failure, it should appear in success_and_failure + assert "datadog" in response_data["success_and_failure"] + + # The categorization logic should deduplicate properly + assert len([cb for cb in response_data["success"] if cb == "datadog"]) <= 1 + assert len([cb for cb in response_data["failure"] if cb == "datadog"]) <= 1 + assert len([cb for cb in response_data["success_and_failure"] if cb == "datadog"]) <= 1 + + # Verify the response structure is correct + assert isinstance(response_data["success"], list) + assert isinstance(response_data["failure"], list) + assert isinstance(response_data["success_and_failure"], list) + + def test_list_callbacks_mixed_callback_types(self): + """Test /callbacks/list endpoint with mixed callback types (string and logger instances)""" + # Setup test client + client = TestClient(app) + + # Setup mixed callbacks + litellm.success_callback.append("langfuse") + litellm.failure_callback.append("datadog") + litellm.callbacks.append("prometheus") + + # Make request to list callbacks endpoint + response = client.get( + "/callbacks/list", + headers={"Authorization": "Bearer sk-1234"} + ) + + # Verify response + assert response.status_code == 200 + + response_data = response.json() + + # Verify callbacks are properly categorized + assert "prometheus" in response_data["success_and_failure"] # callbacks list items go to success_and_failure + assert "langfuse" in response_data["success"] + assert "datadog" in response_data["failure"] + + # Verify no duplicates + all_callbacks = ( + response_data["success"] + + response_data["failure"] + + response_data["success_and_failure"] + ) + assert len(set(all_callbacks)) == len(all_callbacks) + + + def test_list_callbacks_empty_response_structure(self): + """Test that response always has correct structure even with no callbacks""" + # Setup test client + client = TestClient(app) + + # Make request to list callbacks endpoint + response = client.get( + "/callbacks/list", + headers={"Authorization": "Bearer sk-1234"} + ) + + # Verify response structure + assert response.status_code == 200 + response_data = response.json() + + # Verify all required keys are present + required_keys = ["success", "failure", "success_and_failure"] + for key in required_keys: + assert key in response_data + assert isinstance(response_data[key], list) + diff --git a/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py b/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py index 2ce4cf29380..60199b335a5 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py +++ b/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py @@ -696,11 +696,12 @@ async def test_get_generic_sso_response_with_additional_headers(): "fastapi_sso.sso.generic.create_provider", return_value=mock_sso_class ) as mock_create_provider: # Act - result = await get_generic_sso_response( + result, received_response = await get_generic_sso_response( request=mock_request, jwt_handler=mock_jwt_handler, generic_client_id=generic_client_id, redirect_url=redirect_url, + sso_jwt_handler=None, ) # Assert @@ -756,11 +757,12 @@ async def test_get_generic_sso_response_with_empty_headers(): "fastapi_sso.sso.generic.create_provider", return_value=mock_sso_class ) as mock_create_provider: # Act - result = await get_generic_sso_response( + result, received_response = await get_generic_sso_response( request=mock_request, jwt_handler=mock_jwt_handler, generic_client_id=generic_client_id, redirect_url=redirect_url, + sso_jwt_handler=None, ) # Assert diff --git a/tests/test_litellm/proxy/test_route_llm_request.py b/tests/test_litellm/proxy/test_route_llm_request.py index c99815bcd91..d4a65d8847d 100644 --- a/tests/test_litellm/proxy/test_route_llm_request.py +++ b/tests/test_litellm/proxy/test_route_llm_request.py @@ -102,3 +102,25 @@ async def test_route_request_no_model_required_with_router_settings(): # Reset the mock for the next route llm_router.reset_mock() + + +@pytest.mark.asyncio +async def test_route_request_no_model_required_with_router_settings_and_no_router(): + """Test route types that don't require model parameter with router settings and no router""" + from unittest.mock import patch + + import litellm + from litellm.proxy.route_llm_request import route_request + + data = { + "model": "my-model-id", + "api_key": "my-api-key", + "messages": [{"role": "user", "content": "what llm are you"}], + } + + with patch.object( + litellm, "acompletion", return_value="fake_response" + ) as mock_completion: + response = await route_request(data, None, "gpt-3.5-turbo", "acompletion") + + mock_completion.assert_called_once_with(**data) diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index 600075cc7be..9645bb04f20 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -266,3 +266,62 @@ def test_default_image_cost_calculator(monkeypatch): } cost = default_image_cost_calculator(**args) assert cost == 10485760 + + +def test_cost_calculator_with_cache_creation(): + from litellm import completion_cost + from litellm.types.utils import ( + Choices, + CompletionTokensDetailsWrapper, + Message, + PromptTokensDetailsWrapper, + Usage, + ) + + litellm_model_response = ModelResponse( + id="chatcmpl-cc5638bc-fdfe-48e4-8884-57c8f4fb7c63", + created=1750733889, + model=None, + object="chat.completion", + system_fingerprint=None, + choices=[ + Choices( + finish_reason="stop", + index=0, + message=Message( + content="Hello! How can I help you today?", + role="assistant", + tool_calls=None, + function_call=None, + provider_specific_fields=None, + ), + ) + ], + usage=Usage( + **{ + "total_tokens": 28508, + "prompt_tokens": 28495, + "completion_tokens": 13, + "prompt_tokens_details": {"audio_tokens": None, "cached_tokens": 0}, + "cache_read_input_tokens": 28491, + "completion_tokens_details": { + "audio_tokens": None, + "reasoning_tokens": 0, + "accepted_prediction_tokens": None, + "rejected_prediction_tokens": None, + }, + "cache_creation_input_tokens": 15, + } + ), + ) + model = "claude-sonnet-4@20250514" + + assert litellm_model_response.usage.prompt_tokens_details.cached_tokens == 28491 + + result = completion_cost( + completion_response=litellm_model_response, + model=model, + custom_llm_provider="vertex_ai", + ) + + print(result) diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 6dfbb8b7e5b..3fd1274eb19 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -466,6 +466,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): }, "additionalProperties": False, }, + "citation_cost_per_token": {"type": "number"}, "supported_modalities": { "type": "array", "items": { @@ -2010,6 +2011,8 @@ class TestProxyFunctionCalling: print(f"Could not test {model}: {e}") + + def test_register_model_with_scientific_notation(): """ Test that the register_model function can handle scientific notation in the model name. diff --git a/tests/vector_store_tests/base_vector_store_test.py b/tests/vector_store_tests/base_vector_store_test.py new file mode 100644 index 00000000000..ac1ec704f36 --- /dev/null +++ b/tests/vector_store_tests/base_vector_store_test.py @@ -0,0 +1,255 @@ +import httpx +import json +import pytest +import sys +from typing import Any, Dict, List +from unittest.mock import MagicMock, Mock, patch +import os +import uuid +import time +import base64 + +sys.path.insert( + 0, os.path.abspath("../..") +) # Adds the parent directory to the system path +import litellm +from abc import ABC, abstractmethod +from litellm.integrations.custom_logger import CustomLogger +import json +from litellm.types.utils import StandardLoggingPayload + +class BaseVectorStoreTest(ABC): + """ + Abstract base test class that enforces a common test across all test classes. + """ + @abstractmethod + def get_base_request_args(self) -> dict: + """Must return the base request args""" + pass + + @abstractmethod + def get_base_create_vector_store_args(self) -> dict: + """Must return the base create vector store args""" + pass + + @pytest.mark.parametrize("sync_mode", [True, False]) + @pytest.mark.asyncio + async def test_basic_search_vector_store(self, sync_mode): + litellm._turn_on_debug() + litellm.set_verbose = True + base_request_args = self.get_base_request_args() + try: + if sync_mode: + response = litellm.vector_stores.search( + query="Basic ping", + **base_request_args + ) + else: + response = await litellm.vector_stores.asearch( + query="Basic ping", + **base_request_args + ) + except litellm.InternalServerError: + pytest.skip("Skipping test due to litellm.InternalServerError") + + print("litellm response=", json.dumps(response, indent=4, default=str)) + + # Validate response structure + self._validate_vector_store_response(response) + + @pytest.mark.parametrize("sync_mode", [True, False]) + @pytest.mark.asyncio + async def test_basic_create_vector_store(self, sync_mode): + litellm._turn_on_debug() + litellm.set_verbose = True + base_request_args = self.get_base_create_vector_store_args() + + # Extract custom_llm_provider from base args if present + create_args = base_request_args + try: + if sync_mode: + response = litellm.vector_stores.create( + name="Test Vector Store", + **create_args + ) + else: + response = await litellm.vector_stores.acreate( + name="Test Vector Store", + **create_args + ) + except litellm.InternalServerError: + pytest.skip("Skipping test due to litellm.InternalServerError") + except Exception as e: + # If this is an authentication or permission error, skip the test + if "authentication" in str(e).lower() or "permission" in str(e).lower() or "unauthorized" in str(e).lower(): + pytest.skip(f"Skipping test due to authentication/permission error: {e}") + raise + + print("litellm create response=", json.dumps(response, indent=4, default=str)) + + # Validate response structure + self._validate_vector_store_create_response(response) + + def _validate_vector_store_response(self, response): + """Validate the structure and content of a vector store search response""" + + # Check that response is a dictionary + assert isinstance(response, dict), f"Response should be a dict, got {type(response)}" + + # Check required top-level fields + required_fields = ['object', 'search_query', 'data'] + for field in required_fields: + assert field in response, f"Missing required field '{field}' in response" + + # Validate object field + assert response['object'] == 'vector_store.search_results.page', \ + f"Expected object to be 'vector_store.search_results.page', got '{response['object']}'" + + # Validate search_query field + assert isinstance(response['search_query'], list), \ + f"search_query should be a list, got {type(response['search_query'])}" + assert len(response['search_query']) > 0, "search_query should not be empty" + assert all(isinstance(query, str) for query in response['search_query']), \ + "All items in search_query should be strings" + + # Validate data field + assert isinstance(response['data'], list), \ + f"data should be a list, got {type(response['data'])}" + + # Validate each result in data + for i, result in enumerate(response['data']): + self._validate_search_result(result, i) + + print(f"✅ Response validation passed: Found {len(response['data'])} search results") + + def _validate_vector_store_create_response(self, response): + """Validate the structure and content of a vector store create response""" + + # Check that response is a dictionary + assert isinstance(response, dict), f"Response should be a dict, got {type(response)}" + + # Check required top-level fields for create response + required_fields = ['id', 'object', 'created_at'] + for field in required_fields: + assert field in response, f"Missing required field '{field}' in create response" + + # Validate object field + assert response['object'] == 'vector_store', \ + f"Expected object to be 'vector_store', got '{response['object']}'" + + # Validate id field + assert isinstance(response['id'], str), \ + f"id should be a string, got {type(response['id'])}" + assert len(response['id']) > 0, "id should not be empty" + assert response['id'].startswith('vs_'), \ + f"id should start with 'vs_', got '{response['id']}'" + + # Validate created_at field + assert isinstance(response['created_at'], int), \ + f"created_at should be an integer, got {type(response['created_at'])}" + assert response['created_at'] > 0, "created_at should be a positive timestamp" + + # Validate optional fields if present + if 'name' in response: + assert isinstance(response['name'], str), \ + f"name should be a string, got {type(response['name'])}" + + if 'bytes' in response: + assert isinstance(response['bytes'], int), \ + f"bytes should be an integer, got {type(response['bytes'])}" + assert response['bytes'] >= 0, "bytes should be non-negative" + + if 'file_counts' in response: + self._validate_file_counts(response['file_counts']) + + if 'status' in response: + valid_statuses = ['expired', 'in_progress', 'completed'] + assert response['status'] in valid_statuses, \ + f"status should be one of {valid_statuses}, got '{response['status']}'" + + if 'expires_at' in response and response['expires_at'] is not None: + assert isinstance(response['expires_at'], int), \ + f"expires_at should be an integer, got {type(response['expires_at'])}" + + if 'last_active_at' in response and response['last_active_at'] is not None: + assert isinstance(response['last_active_at'], int), \ + f"last_active_at should be an integer, got {type(response['last_active_at'])}" + + if 'metadata' in response and response['metadata'] is not None: + assert isinstance(response['metadata'], dict), \ + f"metadata should be a dict, got {type(response['metadata'])}" + + print(f"✅ Create response validation passed: Vector store '{response['id']}' created successfully") + + def _validate_file_counts(self, file_counts): + """Validate file_counts structure""" + assert isinstance(file_counts, dict), \ + f"file_counts should be a dict, got {type(file_counts)}" + + required_count_fields = ['in_progress', 'completed', 'failed', 'cancelled', 'total'] + for field in required_count_fields: + assert field in file_counts, f"Missing required field '{field}' in file_counts" + assert isinstance(file_counts[field], int), \ + f"{field} should be an integer, got {type(file_counts[field])}" + assert file_counts[field] >= 0, f"{field} should be non-negative" + + # Validate that total equals sum of other counts + calculated_total = ( + file_counts['in_progress'] + + file_counts['completed'] + + file_counts['failed'] + + file_counts['cancelled'] + ) + assert file_counts['total'] == calculated_total, \ + f"total should equal sum of other counts ({calculated_total}), got {file_counts['total']}" + + def _validate_search_result(self, result, index): + """Validate an individual search result""" + + # Check that result is a dictionary + assert isinstance(result, dict), f"Result {index} should be a dict, got {type(result)}" + + # Check required fields in each result + required_result_fields = ['file_id', 'filename', 'score', 'attributes', 'content'] + for field in required_result_fields: + assert field in result, f"Missing required field '{field}' in result {index}" + + # Validate file_id + assert isinstance(result['file_id'], str), \ + f"file_id should be a string, got {type(result['file_id'])} in result {index}" + assert len(result['file_id']) > 0, f"file_id should not be empty in result {index}" + + # Validate filename + assert isinstance(result['filename'], str), \ + f"filename should be a string, got {type(result['filename'])} in result {index}" + assert len(result['filename']) > 0, f"filename should not be empty in result {index}" + + # Validate score + assert isinstance(result['score'], (int, float)), \ + f"score should be a number, got {type(result['score'])} in result {index}" + assert 0.0 <= result['score'] <= 1.0, \ + f"score should be between 0.0 and 1.0, got {result['score']} in result {index}" + + # Validate attributes + assert isinstance(result['attributes'], dict), \ + f"attributes should be a dict, got {type(result['attributes'])} in result {index}" + + # Validate content + assert isinstance(result['content'], list), \ + f"content should be a list, got {type(result['content'])} in result {index}" + assert len(result['content']) > 0, f"content should not be empty in result {index}" + + # Validate each content item + for j, content_item in enumerate(result['content']): + assert isinstance(content_item, dict), \ + f"Content item {j} in result {index} should be a dict, got {type(content_item)}" + assert 'type' in content_item, \ + f"Content item {j} in result {index} missing 'type' field" + assert 'text' in content_item, \ + f"Content item {j} in result {index} missing 'text' field" + assert isinstance(content_item['text'], str), \ + f"Content text should be a string in item {j} of result {index}" + assert len(content_item['text']) > 0, \ + f"Content text should not be empty in item {j} of result {index}" + + print(f"✅ Result {index} validation passed: {result['filename']} (score: {result['score']:.4f})") diff --git a/tests/vector_store_tests/conftest.py b/tests/vector_store_tests/conftest.py new file mode 100644 index 00000000000..b3561d8a626 --- /dev/null +++ b/tests/vector_store_tests/conftest.py @@ -0,0 +1,63 @@ +# conftest.py + +import importlib +import os +import sys + +import pytest + +sys.path.insert( + 0, os.path.abspath("../..") +) # Adds the parent directory to the system path +import litellm + + +@pytest.fixture(scope="function", autouse=True) +def setup_and_teardown(): + """ + This fixture reloads litellm before every function. To speed up testing by removing callbacks being chained. + """ + curr_dir = os.getcwd() # Get the current working directory + sys.path.insert( + 0, os.path.abspath("../..") + ) # Adds the project directory to the system path + + import litellm + from litellm import Router + + importlib.reload(litellm) + + try: + if hasattr(litellm, "proxy") and hasattr(litellm.proxy, "proxy_server"): + import litellm.proxy.proxy_server + + importlib.reload(litellm.proxy.proxy_server) + except Exception as e: + print(f"Error reloading litellm.proxy.proxy_server: {e}") + + import asyncio + + loop = asyncio.get_event_loop_policy().new_event_loop() + asyncio.set_event_loop(loop) + print(litellm) + # from litellm import Router, completion, aembedding, acompletion, embedding + yield + + # Teardown code (executes after the yield point) + loop.close() # Close the loop created earlier + asyncio.set_event_loop(None) # Remove the reference to the loop + + +def pytest_collection_modifyitems(config, items): + # Separate tests in 'test_amazing_proxy_custom_logger.py' and other tests + custom_logger_tests = [ + item for item in items if "custom_logger" in item.parent.name + ] + other_tests = [item for item in items if "custom_logger" not in item.parent.name] + + # Sort tests based on their names + custom_logger_tests.sort(key=lambda x: x.name) + other_tests.sort(key=lambda x: x.name) + + # Reorder the items list + items[:] = custom_logger_tests + other_tests diff --git a/tests/vector_store_tests/test_azure_vector_store.py b/tests/vector_store_tests/test_azure_vector_store.py new file mode 100644 index 00000000000..783417cb742 --- /dev/null +++ b/tests/vector_store_tests/test_azure_vector_store.py @@ -0,0 +1,25 @@ +from base_vector_store_test import BaseVectorStoreTest +import os +import pytest + +class TestAzureOpenAIVectorStore(BaseVectorStoreTest): + def get_base_request_args(self) -> dict: + """Must return the base request args""" + return {} + + @pytest.mark.parametrize("sync_mode", [True, False]) + @pytest.mark.asyncio + async def test_basic_search_vector_store(self, sync_mode): + pass + + + def get_base_create_vector_store_args(self) -> dict: + """ + This is a real vector store on Azure + """ + return { + "custom_llm_provider": "azure", + "api_base": os.getenv("AZURE_RESPONSES_OPENAI_ENDPOINT"), + "api_key": os.getenv("AZURE_RESPONSES_OPENAI_API_KEY"), + "api_version": "2025-04-01-preview", + } \ No newline at end of file diff --git a/tests/vector_store_tests/test_openai_vector_store.py b/tests/vector_store_tests/test_openai_vector_store.py new file mode 100644 index 00000000000..3e27be2f64a --- /dev/null +++ b/tests/vector_store_tests/test_openai_vector_store.py @@ -0,0 +1,20 @@ +from base_vector_store_test import BaseVectorStoreTest + +class TestOpenAIVectorStore(BaseVectorStoreTest): + def get_base_request_args(self) -> dict: + """ + This is a real vector store on OpenAI + """ + return { + "vector_store_id": "vs_685b14b1a1b88191bc27e04f1917fddd", + "custom_llm_provider": "openai", + } + + + def get_base_create_vector_store_args(self) -> dict: + """ + This is a real vector store on OpenAI + """ + return { + "custom_llm_provider": "openai", + } \ No newline at end of file diff --git a/ui/litellm-dashboard/src/components/SSOModals.tsx b/ui/litellm-dashboard/src/components/SSOModals.tsx index 0779edb89d2..763242aeb63 100644 --- a/ui/litellm-dashboard/src/components/SSOModals.tsx +++ b/ui/litellm-dashboard/src/components/SSOModals.tsx @@ -132,7 +132,7 @@ const SSOModals: React.FC = ({ } } - // Set form values with existing data + // Set form values with existing data (excluding UI access control fields) const formValues = { sso_provider: selectedProvider, proxy_base_url: ssoData.values.proxy_base_url, diff --git a/ui/litellm-dashboard/src/components/UIAccessControlForm.tsx b/ui/litellm-dashboard/src/components/UIAccessControlForm.tsx new file mode 100644 index 00000000000..def9bf6bb91 --- /dev/null +++ b/ui/litellm-dashboard/src/components/UIAccessControlForm.tsx @@ -0,0 +1,148 @@ +import React, { useEffect, useState } from "react"; +import { Form, Button as Button2, Select, message } from "antd"; +import { Text, TextInput } from "@tremor/react"; +import { getSSOSettings, updateSSOSettings } from "./networking"; + +interface UIAccessControlFormProps { + accessToken: string | null; + onSuccess: () => void; +} + +// Separate UI Access Control Form Component +const UIAccessControlForm: React.FC = ({ accessToken, onSuccess }) => { + const [form] = Form.useForm(); + const [loading, setLoading] = useState(false); + + // Load existing UI access control settings + useEffect(() => { + const loadUIAccessSettings = async () => { + if (accessToken) { + try { + const ssoData = await getSSOSettings(accessToken); + if (ssoData && ssoData.values) { + // Handle nested ui_access_mode structure + const uiAccessMode = ssoData.values.ui_access_mode; + let formValues = {}; + + if (uiAccessMode && typeof uiAccessMode === 'object') { + formValues = { + ui_access_mode_type: uiAccessMode.type, + restricted_sso_group: uiAccessMode.restricted_sso_group, + sso_group_jwt_field: uiAccessMode.sso_group_jwt_field, + }; + } else if (typeof uiAccessMode === 'string') { + // Handle legacy flat structure + formValues = { + ui_access_mode_type: uiAccessMode, + restricted_sso_group: ssoData.values.restricted_sso_group, + sso_group_jwt_field: ssoData.values.team_ids_jwt_field || ssoData.values.sso_group_jwt_field, + }; + } + + form.setFieldsValue(formValues); + } + } catch (error) { + console.error("Failed to load UI access settings:", error); + } + } + }; + + loadUIAccessSettings(); + }, [accessToken, form]); + + const handleUIAccessSubmit = async (formValues: Record) => { + if (!accessToken) { + message.error("No access token available"); + return; + } + + setLoading(true); + try { + // Transform form data to match API expected structure + const apiPayload = { + ui_access_mode: { + type: formValues.ui_access_mode_type, + restricted_sso_group: formValues.restricted_sso_group, + sso_group_jwt_field: formValues.sso_group_jwt_field, + } + }; + + await updateSSOSettings(accessToken, apiPayload); + onSuccess(); + } catch (error) { + console.error("Failed to save UI access settings:", error); + message.error("Failed to save UI access settings"); + } finally { + setLoading(false); + } + }; + + return ( +
+
+ + Configure who can access the UI interface and how group information is extracted from JWT tokens. + +
+ +
+ + + + + prevValues.ui_access_mode_type !== currentValues.ui_access_mode_type} + > + {({ getFieldValue }) => { + const uiAccessModeType = getFieldValue('ui_access_mode_type'); + return uiAccessModeType === 'restricted_sso_group' ? ( + + + + ) : null; + }} + + + + + + +
+ + Update UI Access Control + +
+
+
+ ); +}; + +export default UIAccessControlForm; \ No newline at end of file diff --git a/ui/litellm-dashboard/src/components/admins.tsx b/ui/litellm-dashboard/src/components/admins.tsx index 6b3ddc094d5..a876aa9d519 100644 --- a/ui/litellm-dashboard/src/components/admins.tsx +++ b/ui/litellm-dashboard/src/components/admins.tsx @@ -44,6 +44,7 @@ import { InvitationLink } from "./onboarding_link"; import SSOModals from "./SSOModals"; import { ssoProviderConfigs } from './SSOModals'; import SCIMConfig from "./SCIM"; +import UIAccessControlForm from "./UIAccessControlForm"; interface AdminPanelProps { searchParams: any; @@ -97,6 +98,7 @@ const AdminPanel: React.FC = ({ const [isAllowedIPModalVisible, setIsAllowedIPModalVisible] = useState(false); const [isAddIPModalVisible, setIsAddIPModalVisible] = useState(false); const [isDeleteIPModalVisible, setIsDeleteIPModalVisible] = useState(false); + const [isUIAccessControlModalVisible, setIsUIAccessControlModalVisible] = useState(false); const [allowedIPs, setAllowedIPs] = useState([]); const [ipToDelete, setIPToDelete] = useState(null); const [ssoConfigured, setSsoConfigured] = useState(false); @@ -532,6 +534,14 @@ const AdminPanel: React.FC = ({ } }; + const handleUIAccessControlOk = () => { + setIsUIAccessControlModalVisible(false); + }; + + const handleUIAccessControlCancel = () => { + setIsUIAccessControlModalVisible(false); + }; + console.log(`admins: ${admins?.length}`); return (
@@ -563,6 +573,14 @@ const AdminPanel: React.FC = ({ Allowed IPs
+
+ +
@@ -654,6 +672,24 @@ const AdminPanel: React.FC = ({ >

Are you sure you want to delete the IP address: {ipToDelete}?

+ + {/* UI Access Control Modal */} + + { + handleUIAccessControlOk(); + message.success("UI Access Control settings updated successfully"); + }} + /> + If you need to login without sso, you can access{" "} diff --git a/ui/litellm-dashboard/src/components/mcp_tools/columns.tsx b/ui/litellm-dashboard/src/components/mcp_tools/columns.tsx index 72c326da565..92ebd23ebb9 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/columns.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/columns.tsx @@ -1,7 +1,34 @@ import React from "react"; import { ColumnDef } from "@tanstack/react-table"; import { MCPTool, InputSchema } from "./types"; -import { Button } from "@tremor/react" +import { Button, Callout, Icon } from "@tremor/react" + +const AuthBanner = ({needsAuth, authValue}: {needsAuth: boolean, authValue?: string | null}) => { + if(!needsAuth || (needsAuth && authValue)) { + return ( + + This tool does not require authentication or has authentication added. + + ) + } + + if (needsAuth && !authValue) { + return ( + + Please provide authentication details if this tool call requires auth. + + ); + } + return null; +} export const columns: ColumnDef[] = [ { @@ -78,6 +105,8 @@ export const columns: ColumnDef[] = [ // Tool Panel component to display when a tool is selected export function ToolTestPanel({ tool, + needsAuth, + authValue, onSubmit, isLoading, result, @@ -85,6 +114,8 @@ export function ToolTestPanel({ onClose }: { tool: MCPTool; + needsAuth: boolean; + authValue?: string | null; onSubmit: (args: Record) => void; isLoading: boolean; result: any | null; @@ -131,6 +162,12 @@ export function ToolTestPanel({

{tool.description}

Provider: {tool.mcp_info.server_name}

+
+ +
+ + + ); +}; + +// Wrapper to handle the type mismatch between MCPTool and DataTable's expected type +function DataTableWrapper({ + columns, + data, + isLoading, +}: { + columns: any; + data: MCPTool[]; + isLoading: boolean; +}) { + // Create a dummy renderSubComponent and getRowCanExpand function + const renderSubComponent = () =>
; + const getRowCanExpand = () => false; + + return ( + + ); +} + +const MCPToolsViewer = ({ + serverId, + accessToken, + auth_type, + userRole, + userID, +}: MCPToolsViewerProps) => { + const [searchTerm, setSearchTerm] = useState(""); + const [mcpAuthValue, setMcpAuthValue] = useState(""); + const [selectedTool, setSelectedTool] = useState(null); + const [toolResult, setToolResult] = useState( + null + ); + const [toolError, setToolError] = useState(null); + + // Query to fetch MCP tools + const { data: mcpTools, isLoading: isLoadingTools } = useQuery({ + queryKey: ["mcpTools"], + queryFn: () => { + if (!accessToken) throw new Error("Access Token required"); + return listMCPTools(accessToken, serverId); + }, + enabled: !!accessToken, + }); + + // Mutation for calling a tool + const { mutate: executeTool, isPending: isCallingTool } = useMutation({ + mutationFn: (args: { tool: MCPTool; arguments: Record, authValue: string }) => { + if (!accessToken) throw new Error("Access Token required"); + return callMCPTool(accessToken, args.tool.name, args.arguments, args.authValue); + }, + onSuccess: (data) => { + setToolResult(data); + setToolError(null); + }, + onError: (error: Error) => { + setToolError(error); + setToolResult(null); + }, + }); + + // Add onToolSelect handler to each tool + const toolsData = React.useMemo(() => { + if (!mcpTools) return []; + + return mcpTools.map((tool: MCPTool) => ({ + ...tool, + onToolSelect: (tool: MCPTool) => { + setSelectedTool(tool); + setToolResult(null); + setToolError(null); + }, + })); + }, [mcpTools]); + + // Filter tools based on search term + const filteredTools = React.useMemo(() => { + return toolsData.filter((tool: MCPTool) => { + const searchLower = searchTerm.toLowerCase(); + return ( + tool.name.toLowerCase().includes(searchLower) || + (tool.description != null && + tool.description.toLowerCase().includes(searchLower)) || + tool.mcp_info.server_name.toLowerCase().includes(searchLower) + ); + }); + }, [toolsData, searchTerm]); + + // Handle tool call submission + const handleToolSubmit = (args: Record) => { + if (!selectedTool) return; + + executeTool({ + tool: selectedTool, + arguments: args, + authValue: mcpAuthValue + }); + }; + + if (!accessToken || !userRole || !userID) { + return ( +
+ Missing required authentication parameters. +
+ ); + } + + return ( +
+
+

MCP Tools

+
+ + {mcpServerHasAuth(auth_type) && ( + { + setMcpAuthValue(value); + }} + /> + )} + +
+
+
+
+ setSearchTerm(e.target.value)} + /> + + + +
+
+ {filteredTools.length} tool{filteredTools.length !== 1 ? "s" : ""}{" "} + available +
+
+
+ + +
+ + {/* Tool Test Panel - Show when a tool is selected */} + {selectedTool && ( +
+ setSelectedTool(null)} + /> +
+ )} +
+ ); +}; -// TODO: Move Tools viewer from index file to this file export default MCPToolsViewer; diff --git a/ui/litellm-dashboard/src/components/mcp_tools/types.tsx b/ui/litellm-dashboard/src/components/mcp_tools/types.tsx index de85778701d..40ea23007ab 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/types.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/types.tsx @@ -1,3 +1,11 @@ +// Default no auth value +export const AUTH_TYPE = { + NONE: "none", + API_KEY: "api_key", + BEARER_TOKEN: "bearer_token", + BASIC: "basic", +}; + export const TRANSPORT = { SSE: "sse", HTTP: "http", @@ -14,12 +22,16 @@ export const handleTransport = (transport?: string | null): string => { export const handleAuth = (authType?: string | null): string => { if (authType === null || authType === undefined) { - return "none"; + return AUTH_TYPE.NONE; } return authType; }; +export const mcpServerHasAuth = (authType?: string | null): boolean => { + return handleAuth(authType) !== AUTH_TYPE.NONE; +} + // Define the structure for tool input schema properties export interface InputSchemaProperty { type: string; @@ -89,6 +101,7 @@ export interface InputSchemaProperty { export interface MCPToolsViewerProps { serverId: string; accessToken: string | null; + auth_type?: string | null; userRole: string | null; userID: string | null; } diff --git a/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.tsx b/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.tsx index aac92e98c12..834a1c834f8 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.tsx @@ -261,6 +261,7 @@ const HealthCheckComponent: React.FC = ({ [modelName]: { status: 'unhealthy', lastCheck: currentTime, + lastSuccess: prev[modelName]?.lastSuccess || 'None', loading: false, error: errorMessage, fullError: rawError @@ -319,6 +320,7 @@ const HealthCheckComponent: React.FC = ({ [modelName]: { status: 'unhealthy', lastCheck: currentTime, + lastSuccess: prev[modelName]?.lastSuccess || 'None', loading: false, error: errorMessage, fullError: rawError @@ -365,6 +367,7 @@ const HealthCheckComponent: React.FC = ({ [modelName]: { status: 'unhealthy', lastCheck: currentTime, + lastSuccess: prev[modelName]?.lastSuccess || 'None', loading: false, error: errorMessage, fullError: rawError @@ -393,6 +396,7 @@ const HealthCheckComponent: React.FC = ({ [modelName]: { status: 'unhealthy', lastCheck: currentTime, + lastSuccess: prev[modelName]?.lastSuccess || 'None', loading: false, error: errorMessage, fullError: rawError @@ -562,6 +566,7 @@ const HealthCheckComponent: React.FC = ({ litellm_model_name: model.litellm_model_name, health_status: healthStatus.status, last_check: healthStatus.lastCheck, + last_success: healthStatus.lastSuccess || 'None', health_loading: healthStatus.loading, health_error: healthStatus.error, health_full_error: healthStatus.fullError, diff --git a/ui/litellm-dashboard/src/components/model_dashboard/health_check_columns.tsx b/ui/litellm-dashboard/src/components/model_dashboard/health_check_columns.tsx index 14cb200858a..2e70105a241 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard/health_check_columns.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard/health_check_columns.tsx @@ -15,6 +15,7 @@ interface HealthCheckData { litellm_model_name?: string; health_status: string; last_check: string; + last_success: string; health_loading: boolean; health_error?: string; health_full_error?: string; @@ -235,6 +236,47 @@ export const healthCheckColumns = ( ); }, }, + { + header: "Last Success", + accessorKey: "last_success", + enableSorting: true, + sortingFn: (rowA, rowB, columnId) => { + const lastSuccessA = rowA.getValue("last_success") as string || 'Never succeeded'; + const lastSuccessB = rowB.getValue("last_success") as string || 'Never succeeded'; + + // Handle special cases + if (lastSuccessA === 'Never succeeded' && lastSuccessB === 'Never succeeded') return 0; + if (lastSuccessA === 'Never succeeded') return 1; // Never succeeded goes to bottom + if (lastSuccessB === 'Never succeeded') return -1; + if (lastSuccessA === 'None' && lastSuccessB === 'None') return 0; + if (lastSuccessA === 'None') return 1; // None goes to bottom + if (lastSuccessB === 'None') return -1; + + // Parse dates for comparison + const dateA = new Date(lastSuccessA); + const dateB = new Date(lastSuccessB); + + // If dates are invalid, treat as never succeeded + if (isNaN(dateA.getTime()) && isNaN(dateB.getTime())) return 0; + if (isNaN(dateA.getTime())) return 1; + if (isNaN(dateB.getTime())) return -1; + + // Sort by date (most recent first) + return dateB.getTime() - dateA.getTime(); + }, + cell: ({ row }) => { + const model = row.original; + const modelName = model.model_name; + const healthStatus = modelHealthStatuses[modelName]; + const lastSuccess = healthStatus?.lastSuccess || 'None'; + + return ( + + {lastSuccess} + + ); + }, + }, { header: "Actions", id: "actions", diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index d51d83b7bf6..2ec8642458d 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -136,7 +136,8 @@ const handleError = async (errorData: string) => { }; // Global variable for the header name -let globalLitellmHeaderName: string = "Authorization"; +let globalLitellmHeaderName: string = "Authorization"; +const MCP_AUTH_HEADER: string = "x-mcp-auth"; // Function to set the global header name export function setGlobalLitellmHeaderName( @@ -4882,10 +4883,12 @@ export const listMCPTools = async (accessToken: string, serverId: string) => { } }; + export const callMCPTool = async ( accessToken: string, toolName: string, - toolArguments: Record + toolArguments: Record, + authValue: string, ) => { try { // Construct base URL @@ -4904,6 +4907,7 @@ export const callMCPTool = async ( method: "POST", headers: { [globalLitellmHeaderName]: `Bearer ${accessToken}`, + [MCP_AUTH_HEADER]: authValue, "Content-Type": "application/json", }, body: JSON.stringify({ diff --git a/ui/litellm-dashboard/src/components/new_usage.tsx b/ui/litellm-dashboard/src/components/new_usage.tsx index 0040937245e..1bd395a2540 100644 --- a/ui/litellm-dashboard/src/components/new_usage.tsx +++ b/ui/litellm-dashboard/src/components/new_usage.tsx @@ -224,23 +224,31 @@ const NewUsagePage: React.FC = ({ } // If only one page, just set the data - if (firstPageData.metadata.total_pages === 1) { + if (firstPageData.metadata.total_pages <= 1) { setUserSpendData(firstPageData); return; } // Fetch all pages const allResults = [...firstPageData.results]; + const aggregatedMetadata = { ...firstPageData.metadata }; for (let page = 2; page <= firstPageData.metadata.total_pages; page++) { const pageData = await userDailyActivityCall(accessToken, startTime, endTime, page); allResults.push(...pageData.results); + if (pageData.metadata) { + aggregatedMetadata.total_spend += pageData.metadata.total_spend || 0; + aggregatedMetadata.total_api_requests += pageData.metadata.total_api_requests || 0; + aggregatedMetadata.total_successful_requests += pageData.metadata.total_successful_requests || 0; + aggregatedMetadata.total_failed_requests += pageData.metadata.total_failed_requests || 0; + aggregatedMetadata.total_tokens += pageData.metadata.total_tokens || 0; + } } // Combine all results with the first page's metadata setUserSpendData({ results: allResults, - metadata: firstPageData.metadata + metadata: aggregatedMetadata }); } catch (error) { console.error("Error fetching user spend data:", error); diff --git a/ui/litellm-dashboard/src/components/team/team_info.tsx b/ui/litellm-dashboard/src/components/team/team_info.tsx index 0d79f56262d..439e7f4a6d0 100644 --- a/ui/litellm-dashboard/src/components/team/team_info.tsx +++ b/ui/litellm-dashboard/src/components/team/team_info.tsx @@ -260,6 +260,10 @@ const TeamInfoView: React.FC = ({ updateData.team_member_budget = Number(values.team_member_budget); } + if (values.team_member_key_duration !== undefined) { + updateData.team_member_key_duration = values.team_member_key_duration; + } + // Handle object_permission updates if (values.vector_stores !== undefined || values.mcp_servers !== undefined) { updateData.object_permission = { @@ -453,6 +457,15 @@ const TeamInfoView: React.FC = ({ + + + + +