From 7ec0c096d503176f63367d0da9b4869924b77893 Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Tue, 13 Jan 2026 18:31:19 -0800 Subject: [PATCH] v0 of this --- ARCHITECTURE.md | 629 ++++++++++++++++++ ...odel_prices_and_context_window_backup.json | 545 ++++++++------- 2 files changed, 919 insertions(+), 255 deletions(-) create mode 100644 ARCHITECTURE.md diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 00000000000..f0cce63b5dd --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,629 @@ +# LiteLLM Architecture + +## 1. System Overview + +LiteLLM is a unified interface for 100+ LLM providers. The system consists of two main components: +the **Core Library** for direct LLM interactions and the **Proxy Server** (LLM Gateway) for +production deployments with authentication, rate limiting, and observability. + +```mermaid +graph TD + subgraph Client Application + A[User/Application] --> B["litellm.completion()
litellm/main.py"] + end + + subgraph "LiteLLM Core (litellm/)" + B --> C["get_llm_provider()
litellm/utils.py"] + C --> D["Provider Handler
litellm/llms/{provider}/"] + D --> E["BaseConfig.transform_request()
litellm/llms/base_llm/"] + E --> F["HTTPHandler
litellm/llms/custom_httpx/http_handler.py"] + end + + subgraph LLM Providers + F --> G[OpenAI API] + F --> H[Anthropic API] + F --> I[Azure API] + F --> J[Bedrock API] + F --> K[100+ Provider APIs] + end +``` + +**Core Library (`litellm/`):** +- **Purpose:** Provides a unified `completion()` interface that translates OpenAI-format requests + to provider-specific formats and normalizes responses back to OpenAI format. +- **Mechanism:** Uses transformation classes to convert inputs/outputs, handles streaming, + function calling, and error mapping across all providers. +- **Use Case:** Direct integration into Python applications for LLM calls. + +**Proxy Server (`litellm/proxy/`):** +- **Purpose:** Production-ready LLM Gateway with authentication, rate limiting, load balancing, + spend tracking, and admin UI. +- **Mechanism:** FastAPI server that wraps the core library with enterprise features. +- **Use Case:** Centralized LLM access for organizations with multiple teams and applications. + +## 2. Request Flow + +Every request flows through a standard chain of handlers. The transformation layer converts +inputs to provider-specific formats only after routing decisions are made. + +```mermaid +sequenceDiagram + participant Client + participant ProxyServer as proxy/proxy_server.py
chat_completion() + participant Auth as proxy/auth/
user_api_key_auth.py + participant PreCall as proxy/litellm_pre_call_utils.py
add_litellm_data_to_request() + participant Router as router.py
Router.acompletion() + participant Main as main.py
completion() + participant Transform as llms/base_llm/chat/
transformation.py + participant HTTP as llms/custom_httpx/
http_handler.py + participant Provider as LLM Provider API + + Client->>ProxyServer: POST /v1/chat/completions + ProxyServer->>Auth: user_api_key_auth() + Auth-->>ProxyServer: UserAPIKeyAuth + ProxyServer->>PreCall: add_litellm_data_to_request() + PreCall-->>ProxyServer: Enhanced Request Data + ProxyServer->>Router: route_request() -> acompletion() + Router->>Main: litellm.acompletion() + Main->>Transform: BaseConfig.transform_request() + Transform->>HTTP: AsyncHTTPHandler.post() + HTTP->>Provider: Provider-specific HTTP Request + Provider-->>HTTP: Provider Response + HTTP-->>Transform: Raw Response + Transform-->>Main: BaseConfig.transform_response() + Main-->>Router: ModelResponse + Router-->>ProxyServer: ModelResponse + ProxyServer-->>Client: OpenAI-format JSON Response +``` + +### Request Processing Stages + +1. **Authentication (`proxy/auth/user_api_key_auth.py`):** Validates API keys, JWT tokens, or OAuth2 credentials. + Extracts user, team, and organization context for downstream processing. + +2. **Pre-call Processing (`proxy/litellm_pre_call_utils.py`):** Adds metadata, applies guardrails via + `proxy/hooks/`, and prepares request data. + +3. **Routing (`proxy/route_llm_request.py` -> `router.py`):** The `route_request()` function selects + the appropriate model deployment based on load balancing strategy, cooldowns, and rate limits. + +4. **Provider Resolution (`litellm/utils.py`):** The `get_llm_provider()` function determines which + provider handler to use based on the model name. + +5. **Transformation (`llms/base_llm/chat/transformation.py`):** The provider's `BaseConfig` subclass + converts OpenAI-format requests to provider-specific formats. + +6. **HTTP Request (`llms/custom_httpx/http_handler.py`):** `AsyncHTTPHandler` or `HTTPHandler` makes + the actual HTTP request to the LLM provider. + +7. **Response Processing (`llms/{provider}/chat/transformation.py`):** Provider's `transform_response()` + normalizes the response back to OpenAI format. + +8. **Post-call Hooks (`integrations/custom_logger.py`):** Logs to observability platforms, updates + spend tracking via callbacks registered in `litellm.callbacks`. + +## 3. Core Library Architecture + +### Main Entry Points + +The core library exposes several main functions in `litellm/main.py`: + +| Function | File Location | Purpose | +|----------|---------------|---------| +| `completion()` | `litellm/main.py:992` | Chat completions (sync) | +| `acompletion()` | `litellm/main.py:369` | Chat completions (async) | +| `embedding()` | `litellm/main.py:4352` | Text embeddings | +| `text_completion()` | `litellm/main.py:5445` | Legacy text completions | +| `image_generation()` | `litellm/main.py` | Image generation | +| `transcription()` | `litellm/main.py` | Audio transcription | +| `speech()` | `litellm/main.py` | Text-to-speech | + +### Provider Resolution + +When `completion()` is called, the provider is determined by `get_llm_provider()` in `litellm/utils.py`. +The function parses the model string (e.g., `anthropic/claude-3-opus`) and returns: +- `model` - The model name without provider prefix +- `custom_llm_provider` - The provider identifier (e.g., "anthropic", "openai", "bedrock") +- `api_key` - Resolved API key +- `api_base` - Provider endpoint URL + +### Provider Implementation Pattern + +Each provider follows a consistent implementation pattern: + +```mermaid +graph TD + subgraph "Provider Implementation (litellm/llms/anthropic/)" + A["BaseConfig
llms/base_llm/chat/transformation.py"] --> B["AnthropicConfig
llms/anthropic/chat/transformation.py"] + B --> C["transform_request()"] + B --> D["transform_response()"] + B --> E["get_supported_openai_params()"] + end + + subgraph "Provider Directory Structure" + F["litellm/llms/anthropic/"] + F --> G["chat/transformation.py
(AnthropicConfig)"] + F --> H["chat/handler.py
(AnthropicChatCompletion)"] + F --> I["common_utils.py
(AnthropicModelInfo)"] + end +``` + +**Key Files per Provider (`litellm/llms/{provider}/`):** +- `chat/transformation.py` - Request/response transformation inheriting from `BaseConfig` +- `chat/handler.py` - HTTP request handling and streaming logic +- `common_utils.py` - Shared utilities, model info, and constants + +### Transformation Layer + +The transformation layer (`litellm/llms/base_llm/`) provides base classes for all API types: + +| Base Class | File | Purpose | +|------------|------|---------| +| `BaseConfig` | `llms/base_llm/chat/transformation.py` | Chat completions transformation | +| `BaseEmbeddingConfig` | `llms/base_llm/embedding/transformation.py` | Embedding transformation | +| `BaseImageGenerationConfig` | `llms/base_llm/image_generation/transformation.py` | Image generation transformation | +| `BaseAudioTranscriptionConfig` | `llms/base_llm/audio_transcription/transformation.py` | Audio transcription transformation | +| `BaseBatchesConfig` | `llms/base_llm/batches/transformation.py` | Batch API transformation | + +Each provider implements these base classes to handle format conversion. Example from +`litellm/llms/anthropic/chat/transformation.py`: + +```python +class AnthropicConfig(BaseConfig): + def transform_request( + self, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: + # Convert OpenAI format to Anthropic format + return {"model": model, "messages": transformed_messages, ...} + + def transform_response( + self, + model: str, + raw_response: httpx.Response, + model_response: ModelResponse, + logging_obj: LiteLLMLoggingObj, + ... + ) -> ModelResponse: + # Convert Anthropic response to OpenAI format + return ModelResponse(choices=[...], usage=Usage(...)) +``` + +## 4. Router System + +The Router (`litellm/router.py`) manages multiple model deployments with load balancing, +fallbacks, and health monitoring. + +```mermaid +graph TD + subgraph "Router Configuration (router.py)" + A["Model Group: gpt-4
Router.model_list"] --> B["Deployment 1: Azure
litellm_params.model=azure/gpt-4"] + A --> C["Deployment 2: OpenAI
litellm_params.model=gpt-4"] + A --> D["Deployment 3: Bedrock
litellm_params.model=bedrock/anthropic.claude-3"] + end + + subgraph "Routing Decision (router_strategy/)" + E["Router.acompletion()"] --> F{"routing_strategy
router.py:200"} + F -->|"simple-shuffle"| G["simple_shuffle.py"] + F -->|"least-busy"| H["least_busy.py"] + F -->|"lowest-latency"| I["lowest_latency.py"] + F -->|"lowest-cost"| J["lowest_cost.py"] + F -->|"lowest-tpm-rpm"| K["lowest_tpm_rpm.py"] + end + + subgraph "Health Management (router_utils/)" + L["cooldown_cache.py
CooldownCache"] --> M["Failed Deployments"] + N["cooldown_handlers.py"] --> O["_set_cooldown_deployments()"] + end +``` + +### Routing Strategies (`litellm/router_strategy/`) + +| Strategy | File | Description | +|----------|------|-------------| +| `simple-shuffle` | `simple_shuffle.py` | Random selection across healthy deployments | +| `least-busy` | `least_busy.py` | Routes to deployment with lowest active requests | +| `lowest-latency` | `lowest_latency.py` | Routes to historically fastest deployment | +| `lowest-cost` | `lowest_cost.py` | Routes to cheapest available deployment | +| `lowest-tpm-rpm` | `lowest_tpm_rpm.py` | Routes based on token/request capacity | +| `tag-based` | `tag_based_routing.py` | Routes based on request metadata tags | + +### Fallback and Retry Logic (`litellm/router_utils/`) + +| File | Purpose | +|------|---------| +| `fallback_event_handlers.py` | `run_async_fallback()`, `get_fallback_model_group()` | +| `cooldown_handlers.py` | `_set_cooldown_deployments()`, `_async_get_cooldown_deployments()` | +| `cooldown_cache.py` | `CooldownCache` class for tracking failed deployments | +| `handle_error.py` | `send_llm_exception_alert()`, exception handling | +| `get_retry_from_policy.py` | `get_num_retries_from_retry_policy()` | + +## 5. Proxy Server Architecture + +The Proxy Server (`litellm/proxy/proxy_server.py`) is a FastAPI application that wraps the +core library with enterprise features. + +```mermaid +graph TD + subgraph "Proxy Server (proxy/proxy_server.py)" + A["FastAPI app"] --> B["chat_completion()
Line 5149"] + A --> C["embeddings()
proxy_server.py"] + A --> D["image_generation()
proxy_server.py"] + end + + subgraph "Authentication (proxy/auth/)" + E["user_api_key_auth.py
user_api_key_auth()"] --> F["auth_checks.py"] + G["handle_jwt.py
JWTHandler"] --> F + H["oauth2_check.py"] --> F + end + + subgraph "Request Processing" + I["common_request_processing.py
ProxyBaseLLMRequestProcessing"] --> J["route_llm_request.py
route_request()"] + J --> K["router.py
Router.acompletion()"] + end + + subgraph "Data Layer (proxy/db/)" + L["prisma_client.py
PrismaClient"] --> M["PostgreSQL/SQLite"] + N["caching/redis_cache.py"] --> O["Redis"] + end +``` + +### Endpoint Categories + +**OpenAI-compatible Endpoints (defined in `proxy/proxy_server.py`):** + +| Endpoint | Function | Line | +|----------|----------|------| +| `POST /v1/chat/completions` | `chat_completion()` | ~5149 | +| `POST /v1/completions` | `completion()` | proxy_server.py | +| `POST /v1/embeddings` | `embeddings()` | proxy_server.py | +| `POST /v1/images/generations` | `image_generation()` | image_endpoints/ | +| `POST /v1/audio/transcriptions` | `audio_transcriptions()` | proxy_server.py | +| `POST /v1/audio/speech` | `audio_speech()` | proxy_server.py | + +**Management Endpoints (`proxy/management_endpoints/`):** + +| File | Endpoints | +|------|-----------| +| `key_management_endpoints.py` | `/key/generate`, `/key/delete`, `/key/info` | +| `team_endpoints.py` | `/team/new`, `/team/update`, `/team/delete` | +| `internal_user_endpoints.py` | `/user/new`, `/user/update`, `/user/delete` | +| `model_management_endpoints.py` | `/model/new`, `/model/delete`, `/model/info` | +| `budget_management_endpoints.py` | `/budget/new`, `/budget/info` | +| `organization_endpoints.py` | `/organization/new`, `/organization/update` | + +**Pass-through Endpoints (`proxy/pass_through_endpoints/`):** + +| File | Purpose | +|------|---------| +| `llm_passthrough_endpoints.py` | Provider-specific API forwarding | +| `pass_through_endpoints.py` | Custom pass-through route initialization | + +### Authentication System (`proxy/auth/`) + +| File | Purpose | +|------|---------| +| `user_api_key_auth.py` | Main `user_api_key_auth()` dependency for FastAPI routes | +| `auth_checks.py` | `get_team_object()`, permission and budget validation | +| `handle_jwt.py` | `JWTHandler` class for JWT token processing | +| `oauth2_check.py` | OAuth2 flow handling | +| `model_checks.py` | `get_key_models()`, `get_team_models()` for access validation | +| `route_checks.py` | Endpoint permission checks | + +### Database Schema (`proxy/schema.prisma`) + +The proxy uses Prisma ORM with the following key entities: + +| Table | Purpose | +|-------|---------| +| `LiteLLM_UserTable` | User accounts and settings | +| `LiteLLM_TeamTable` | Team definitions and membership | +| `LiteLLM_OrganizationTable` | Organization hierarchy | +| `LiteLLM_VerificationToken` | API keys (hashed) | +| `LiteLLM_SpendLogs` | Usage and spend tracking | +| `LiteLLM_ModelTable` | Model configurations | +| `LiteLLM_BudgetTable` | Budget definitions | + +## 6. Caching System + +LiteLLM provides multiple caching backends (`litellm/caching/`): + +```mermaid +graph TD + subgraph "Cache Backends (litellm/caching/)" + A["in_memory_cache.py
InMemoryCache"] --> B["Local LRU Cache"] + C["redis_cache.py
RedisCache"] --> D["Redis Server"] + E["redis_cluster_cache.py
RedisClusterCache"] --> F["Redis Cluster"] + G["s3_cache.py
S3Cache"] --> H["S3 Bucket"] + I["disk_cache.py
DiskCache"] --> J["Local Filesystem"] + end + + subgraph "Cache Strategy" + K["dual_cache.py
DualCache"] --> L["In-Memory + Redis"] + M["redis_semantic_cache.py
RedisSemanticCache"] --> N["Vector Similarity"] + end +``` + +| Cache Type | File | Use Case | +|------------|------|----------| +| `InMemoryCache` | `in_memory_cache.py` | Single-instance deployments | +| `RedisCache` | `redis_cache.py` | Multi-instance with shared state | +| `RedisClusterCache` | `redis_cluster_cache.py` | Redis Cluster deployments | +| `DualCache` | `dual_cache.py` | Fast local + persistent remote | +| `S3Cache` | `s3_cache.py` | Long-term response storage | +| `RedisSemanticCache` | `redis_semantic_cache.py` | Similar query deduplication | +| `DiskCache` | `disk_cache.py` | Local filesystem caching | + +## 7. Integrations and Observability + +### Callback System (`litellm/integrations/`) + +LiteLLM supports 30+ observability integrations through a callback system: + +```mermaid +graph LR + subgraph "LiteLLM (litellm/main.py)" + A["completion()"] --> B["litellm.callbacks
List[CustomLogger]"] + end + + subgraph "Observability (integrations/)" + B --> C["langfuse/
langfuse.py"] + B --> D["datadog/
datadog.py"] + B --> E["prometheus.py"] + B --> F["opentelemetry.py"] + B --> G["weights_biases.py"] + B --> H["mlflow.py"] + end + + subgraph "Alerting (integrations/)" + B --> I["SlackAlerting/
slack_alerting.py"] + B --> J["email_alerting.py"] + end +``` + +**Key Integration Files (`litellm/integrations/`):** + +| Category | Files | +|----------|-------| +| **Tracing** | `langfuse/langfuse.py`, `datadog/datadog.py`, `opentelemetry.py`, `arize/arize.py` | +| **Metrics** | `prometheus.py`, `cloudzero/cloudzero.py`, `openmeter.py` | +| **Logging** | `s3.py`, `gcs_bucket/gcs_bucket.py`, `dynamodb.py` | +| **Alerting** | `SlackAlerting/slack_alerting.py`, `email_alerting.py` | + +### Custom Callbacks + +Implement `CustomLogger` (from `integrations/custom_logger.py`) for custom integrations: + +```python +from litellm.integrations.custom_logger import CustomLogger + +class MyCallback(CustomLogger): + def log_success_event(self, kwargs, response_obj, start_time, end_time): + # Log successful completion + pass + + def log_failure_event(self, kwargs, response_obj, start_time, end_time): + # Log failed completion + pass + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + # Async version for non-blocking logging + pass +``` + +Register callbacks via `litellm.callbacks.append(MyCallback())` or in proxy config YAML. + +## 8. Guardrails System + +The guardrails system (`litellm/proxy/guardrails/`) provides content filtering and safety checks: + +```mermaid +graph TD + subgraph "Pre-call Guardrails (proxy/hooks/)" + A["prompt_injection_detection.py"] --> B["Content Filtering"] + B --> C["guardrails/guardrail_hooks/"] + end + + subgraph "Guardrail Providers (guardrails/guardrail_hooks/)" + D["lakera_ai.py"] + E["bedrock_guardrails.py"] + F["azure/prompt_shield.py"] + G["presidio.py"] + H["custom_guardrail.py"] + end + + subgraph "Initialization" + I["init_guardrails.py
init_guardrails_v2()"] --> J["guardrail_registry.py"] + end +``` + +**Guardrail Providers (`proxy/guardrails/guardrail_hooks/`):** + +| Provider | File | +|----------|------| +| Lakera AI | `lakera_ai.py`, `lakera_ai_v2.py` | +| Bedrock Guardrails | `bedrock_guardrails.py` | +| Azure Content Safety | `azure/prompt_shield.py`, `azure/text_moderation.py` | +| OpenAI Moderation | `openai/moderations.py` | +| Presidio (PII) | `presidio.py` | +| Aporia AI | `aporia_ai/aporia_ai.py` | +| Custom | `custom_guardrail.py` | + +**Key Files:** +- `init_guardrails.py` - `init_guardrails_v2()` initializes guardrails from config +- `guardrail_registry.py` - Registry of available guardrail implementations +- `guardrail_helpers.py` - Shared helper functions + +## 9. Type System + +LiteLLM uses Pydantic models for type safety (`litellm/types/`): + +| File | Purpose | +|------|---------| +| `types/utils.py` | Core response types (`ModelResponse`, `Usage`, `EmbeddingResponse`) | +| `types/router.py` | Router configuration (`Deployment`, `LiteLLM_Params`, `RetryPolicy`) | +| `types/llms/openai.py` | OpenAI-specific types (`ChatCompletionRequest`, `AllMessageValues`) | +| `types/llms/anthropic.py` | Anthropic-specific types | +| `types/integrations/*.py` | Integration configuration types | +| `types/guardrails.py` | Guardrail configuration types | + +### Key Types (`litellm/types/utils.py`) + +```python +# Core response type - returned by all completion calls +class ModelResponse(BaseModel): + id: str + choices: List[Choices] + created: int + model: str + usage: Usage + +# Usage tracking +class Usage(BaseModel): + prompt_tokens: int + completion_tokens: int + total_tokens: int +``` + +### Router Types (`litellm/types/router.py`) + +```python +# Model deployment configuration +class Deployment(BaseModel): + model_name: str # User-facing model name + litellm_params: LiteLLM_Params # Provider-specific params + model_info: Optional[ModelInfo] # Pricing, context window info + +class LiteLLM_Params(BaseModel): + model: str # Provider model string (e.g., "azure/gpt-4") + api_key: Optional[str] + api_base: Optional[str] + # ... additional provider params +``` + +## 10. Directory Structure Reference + +``` +litellm/ +├── main.py # completion(), acompletion(), embedding() - core entry points +├── router.py # Router class - load balancing, fallbacks, health checks +├── utils.py # get_llm_provider(), helper functions, response types +├── exceptions.py # LiteLLM exception classes +├── cost_calculator.py # completion_cost(), token counting +├── _logging.py # Logging configuration +│ +├── llms/ # Provider implementations (100+ providers) +│ ├── base_llm/ # Base classes all providers inherit from +│ │ ├── chat/transformation.py # BaseConfig class +│ │ ├── embedding/transformation.py # BaseEmbeddingConfig +│ │ └── ... +│ ├── openai/ +│ │ ├── chat/transformation.py # OpenAIConfig +│ │ ├── chat/handler.py # OpenAIChatCompletion +│ │ └── openai.py +│ ├── anthropic/ +│ │ ├── chat/transformation.py # AnthropicConfig +│ │ └── chat/handler.py # AnthropicChatCompletion +│ ├── azure/ # Azure OpenAI +│ ├── bedrock/ # AWS Bedrock +│ ├── vertex_ai/ # Google Vertex AI +│ └── custom_httpx/ +│ └── http_handler.py # HTTPHandler, AsyncHTTPHandler +│ +├── proxy/ # Proxy server (LLM Gateway) +│ ├── proxy_server.py # FastAPI app, chat_completion(), embeddings() +│ ├── route_llm_request.py # route_request() - routes to router +│ ├── common_request_processing.py # ProxyBaseLLMRequestProcessing +│ ├── litellm_pre_call_utils.py # add_litellm_data_to_request() +│ ├── auth/ +│ │ ├── user_api_key_auth.py # user_api_key_auth() dependency +│ │ ├── auth_checks.py # Permission validation +│ │ └── handle_jwt.py # JWTHandler +│ ├── management_endpoints/ +│ │ ├── key_management_endpoints.py +│ │ ├── team_endpoints.py +│ │ └── model_management_endpoints.py +│ ├── guardrails/ +│ │ ├── init_guardrails.py +│ │ └── guardrail_hooks/ # Provider implementations +│ ├── hooks/ # Pre/post call hooks +│ ├── db/ +│ │ └── prisma_client.py # PrismaClient +│ └── schema.prisma # Database schema +│ +├── router_utils/ # Router helper modules +│ ├── cooldown_handlers.py # Deployment cooldown logic +│ ├── fallback_event_handlers.py # Fallback handling +│ └── handle_error.py # Error handling utilities +│ +├── router_strategy/ # Load balancing strategies +│ ├── simple_shuffle.py +│ ├── lowest_latency.py +│ ├── lowest_cost.py +│ └── tag_based_routing.py +│ +├── caching/ # Cache implementations +│ ├── redis_cache.py +│ ├── in_memory_cache.py +│ ├── dual_cache.py +│ └── caching_handler.py +│ +├── integrations/ # Observability callbacks +│ ├── custom_logger.py # CustomLogger base class +│ ├── langfuse/langfuse.py +│ ├── datadog/datadog.py +│ ├── prometheus.py +│ └── SlackAlerting/slack_alerting.py +│ +├── types/ # Pydantic type definitions +│ ├── utils.py # ModelResponse, Usage, etc. +│ ├── router.py # Deployment, LiteLLM_Params +│ └── llms/ # Provider-specific types +│ +└── litellm_core_utils/ # Internal utilities + ├── litellm_logging.py # Logging class + ├── streaming_handler.py # Stream processing + └── exception_mapping_utils.py # Exception mapping +``` + +## 11. Contributing Guidelines + +### Adding a New Provider + +1. Create directory: `litellm/llms/{provider}/` +2. Create `chat/transformation.py` with class inheriting from `BaseConfig` (`llms/base_llm/chat/transformation.py`) +3. Implement `transform_request()` and `transform_response()` methods +4. Add provider routing in `litellm/main.py` (search for `custom_llm_provider ==`) +5. Add tests in `tests/llm_translation/test_{provider}.py` +6. Update `model_prices_and_context_window.json` with model pricing + +### Adding a New Integration + +1. Create file in `litellm/integrations/{integration}.py` +2. Implement class inheriting from `CustomLogger` (`integrations/custom_logger.py`) +3. Implement `log_success_event()`, `log_failure_event()`, and async variants +4. Register callback name in `litellm/__init__.py` (add to `_known_custom_logger_compatible_callbacks`) +5. Add configuration types in `litellm/types/integrations/` +6. Add tests in `tests/` + +### Adding a New Guardrail + +1. Create directory in `litellm/proxy/guardrails/guardrail_hooks/{guardrail}/` +2. Implement guardrail class with `async_pre_call_hook()` and `async_post_call_hook()` methods +3. Register in `proxy/guardrails/guardrail_registry.py` +4. Add configuration schema in `litellm/types/guardrails.py` +5. Add tests in `tests/proxy_unit_tests/` + +## 12. Security Considerations + +| Feature | Implementation | +|---------|----------------| +| API Key Storage | Keys hashed via `proxy/auth/auth_utils.py` before database storage | +| Secret Management | `litellm/secret_managers/` - AWS Secrets Manager, Azure Key Vault, HashiCorp Vault | +| Input Validation | Request validation in `proxy/litellm_pre_call_utils.py` | +| Rate Limiting | `proxy/hooks/` - per-key, per-user, per-team limits | +| Audit Logging | `proxy/spend_tracking/` - all requests logged with user context | diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 91cb1b8d27a..39008d802be 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1644,7 +1644,7 @@ "cache_read_input_token_cost": 1.4e-07, "input_cost_per_token": 1.38e-06, "litellm_provider": "azure", - "max_input_tokens": 272000, + "max_input_tokens": 128000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -1944,7 +1944,7 @@ "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "azure", - "max_input_tokens": 272000, + "max_input_tokens": 128000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -2893,7 +2893,7 @@ "input_cost_per_token": 1.25e-06, "input_cost_per_token_priority": 2.5e-06, "litellm_provider": "azure", - "max_input_tokens": 272000, + "max_input_tokens": 400000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -2964,7 +2964,7 @@ "input_cost_per_token": 1.25e-06, "input_cost_per_token_priority": 2.5e-06, "litellm_provider": "azure", - "max_input_tokens": 272000, + "max_input_tokens": 400000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", @@ -2997,7 +2997,7 @@ "input_cost_per_token": 2.5e-07, "input_cost_per_token_priority": 4.5e-07, "litellm_provider": "azure", - "max_input_tokens": 272000, + "max_input_tokens": 400000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", @@ -3092,9 +3092,9 @@ "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "azure", - "max_input_tokens": 272000, - "max_output_tokens": 128000, - "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://azure.microsoft.com/en-us/blog/gpt-5-in-azure-ai-foundry-the-future-of-ai-apps-and-agents-starts-here/", @@ -3377,7 +3377,7 @@ "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "azure", - "max_input_tokens": 272000, + "max_input_tokens": 128000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -3440,7 +3440,7 @@ "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "azure", - "max_input_tokens": 400000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", @@ -3500,7 +3500,7 @@ "cache_read_input_token_cost": 1.75e-07, "input_cost_per_token": 1.75e-06, "litellm_provider": "azure", - "max_input_tokens": 400000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -3637,7 +3637,7 @@ "azure/gpt-5.2-pro": { "input_cost_per_token": 2.1e-05, "litellm_provider": "azure", - "max_input_tokens": 400000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", @@ -3668,7 +3668,7 @@ "azure/gpt-5.2-pro-2025-12-11": { "input_cost_per_token": 2.1e-05, "litellm_provider": "azure", - "max_input_tokens": 400000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", @@ -4684,7 +4684,7 @@ "cache_read_input_token_cost": 1.4e-07, "input_cost_per_token": 1.38e-06, "litellm_provider": "azure", - "max_input_tokens": 272000, + "max_input_tokens": 128000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -7813,14 +7813,14 @@ "supports_vision": true }, "deepseek-chat": { - "cache_read_input_token_cost": 6e-08, - "input_cost_per_token": 6e-07, + "cache_read_input_token_cost": 2.8e-08, + "input_cost_per_token": 2.8e-07, "litellm_provider": "deepseek", "max_input_tokens": 131072, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "output_cost_per_token": 1.7e-06, + "output_cost_per_token": 4.2e-07, "source": "https://api-docs.deepseek.com/quick_start/pricing", "supported_endpoints": [ "/v1/chat/completions" @@ -7834,14 +7834,14 @@ "supports_tool_choice": true }, "deepseek-reasoner": { - "cache_read_input_token_cost": 6e-08, - "input_cost_per_token": 6e-07, + "cache_read_input_token_cost": 2.8e-08, + "input_cost_per_token": 2.8e-07, "litellm_provider": "deepseek", "max_input_tokens": 131072, "max_output_tokens": 65536, "max_tokens": 65536, "mode": "chat", - "output_cost_per_token": 1.7e-06, + "output_cost_per_token": 4.2e-07, "source": "https://api-docs.deepseek.com/quick_start/pricing", "supported_endpoints": [ "/v1/chat/completions" @@ -10045,15 +10045,15 @@ }, "deepseek/deepseek-chat": { "cache_creation_input_token_cost": 0.0, - "cache_read_input_token_cost": 7e-08, - "input_cost_per_token": 2.7e-07, - "input_cost_per_token_cache_hit": 7e-08, + "cache_read_input_token_cost": 2.8e-08, + "input_cost_per_token": 2.8e-07, + "input_cost_per_token_cache_hit": 2.8e-08, "litellm_provider": "deepseek", - "max_input_tokens": 65536, + "max_input_tokens": 128000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "output_cost_per_token": 1.1e-06, + "output_cost_per_token": 4.2e-07, "supports_assistant_prefill": true, "supports_function_calling": true, "supports_prompt_caching": true, @@ -10089,14 +10089,15 @@ "supports_tool_choice": true }, "deepseek/deepseek-reasoner": { - "input_cost_per_token": 5.5e-07, - "input_cost_per_token_cache_hit": 1.4e-07, + "cache_read_input_token_cost": 2.8e-08, + "input_cost_per_token": 2.8e-07, + "input_cost_per_token_cache_hit": 2.8e-08, "litellm_provider": "deepseek", - "max_input_tokens": 65536, + "max_input_tokens": 128000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "output_cost_per_token": 2.19e-06, + "output_cost_per_token": 4.2e-07, "supports_assistant_prefill": true, "supports_function_calling": true, "supports_prompt_caching": true, @@ -17481,7 +17482,7 @@ "input_cost_per_token_flex": 6.25e-07, "input_cost_per_token_priority": 2.5e-06, "litellm_provider": "openai", - "max_input_tokens": 272000, + "max_input_tokens": 400000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -17518,7 +17519,7 @@ "input_cost_per_token": 1.25e-06, "input_cost_per_token_priority": 2.5e-06, "litellm_provider": "openai", - "max_input_tokens": 272000, + "max_input_tokens": 400000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -17554,7 +17555,7 @@ "input_cost_per_token": 1.25e-06, "input_cost_per_token_priority": 2.5e-06, "litellm_provider": "openai", - "max_input_tokens": 272000, + "max_input_tokens": 400000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -17863,7 +17864,7 @@ "input_cost_per_token_flex": 6.25e-07, "input_cost_per_token_priority": 2.5e-06, "litellm_provider": "openai", - "max_input_tokens": 272000, + "max_input_tokens": 400000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -17898,9 +17899,9 @@ "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "openai", - "max_input_tokens": 272000, - "max_output_tokens": 128000, - "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "supported_endpoints": [ @@ -17962,7 +17963,7 @@ "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "openai", - "max_input_tokens": 272000, + "max_input_tokens": 400000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", @@ -17994,7 +17995,7 @@ "input_cost_per_token": 1.25e-06, "input_cost_per_token_priority": 2.5e-06, "litellm_provider": "openai", - "max_input_tokens": 272000, + "max_input_tokens": 400000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", @@ -18057,7 +18058,7 @@ "input_cost_per_token": 2.5e-07, "input_cost_per_token_priority": 4.5e-07, "litellm_provider": "openai", - "max_input_tokens": 272000, + "max_input_tokens": 400000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", @@ -18092,7 +18093,7 @@ "input_cost_per_token_flex": 1.25e-07, "input_cost_per_token_priority": 4.5e-07, "litellm_provider": "openai", - "max_input_tokens": 272000, + "max_input_tokens": 400000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -18131,7 +18132,7 @@ "input_cost_per_token_flex": 1.25e-07, "input_cost_per_token_priority": 4.5e-07, "litellm_provider": "openai", - "max_input_tokens": 272000, + "max_input_tokens": 400000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -18169,7 +18170,7 @@ "input_cost_per_token_flex": 2.5e-08, "input_cost_per_token_priority": 2.5e-06, "litellm_provider": "openai", - "max_input_tokens": 272000, + "max_input_tokens": 400000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -18204,7 +18205,7 @@ "input_cost_per_token": 5e-08, "input_cost_per_token_flex": 2.5e-08, "litellm_provider": "openai", - "max_input_tokens": 272000, + "max_input_tokens": 400000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -23217,9 +23218,9 @@ "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "openrouter", - "max_input_tokens": 272000, - "max_output_tokens": 128000, - "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "supported_modalities": [ @@ -23236,7 +23237,7 @@ "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "openrouter", - "max_input_tokens": 272000, + "max_input_tokens": 400000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -23255,7 +23256,7 @@ "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "openrouter", - "max_input_tokens": 272000, + "max_input_tokens": 400000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -23274,7 +23275,7 @@ "cache_read_input_token_cost": 2.5e-08, "input_cost_per_token": 2.5e-07, "litellm_provider": "openrouter", - "max_input_tokens": 272000, + "max_input_tokens": 400000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -23293,7 +23294,7 @@ "cache_read_input_token_cost": 5e-09, "input_cost_per_token": 5e-08, "litellm_provider": "openrouter", - "max_input_tokens": 272000, + "max_input_tokens": 400000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -32563,8 +32564,8 @@ "novita/deepseek/deepseek-v3.2": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 2.69e-03, - "output_cost_per_token": 4e-03, + "input_cost_per_token": 2.69e-7, + "output_cost_per_token": 4e-7, "max_input_tokens": 163840, "max_output_tokens": 65536, "max_tokens": 65536, @@ -32573,15 +32574,15 @@ "supports_tool_choice": true, "supports_system_messages": true, "supports_response_schema": true, - "cache_read_input_token_cost": 1.345e-03, - "input_cost_per_token_cache_hit": 1.345e-03, + "cache_read_input_token_cost": 1.345e-7, + "input_cost_per_token_cache_hit": 1.345e-7, "supports_reasoning": true }, "novita/minimax/minimax-m2.1": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 3e-03, - "output_cost_per_token": 1.2e-02, + "input_cost_per_token": 3e-7, + "output_cost_per_token": 1.2e-06, "max_input_tokens": 204800, "max_output_tokens": 131072, "max_tokens": 131072, @@ -32590,15 +32591,14 @@ "supports_tool_choice": true, "supports_system_messages": true, "supports_response_schema": true, - "cache_read_input_token_cost": 3e-04, - "input_cost_per_token_cache_hit": 3e-04, - "supports_reasoning": true + "cache_read_input_token_cost": 3e-8, + "input_cost_per_token_cache_hit": 3e-8 }, "novita/zai-org/glm-4.7": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 6e-03, - "output_cost_per_token": 2.2e-02, + "input_cost_per_token": 6e-7, + "output_cost_per_token": 2.2e-06, "max_input_tokens": 204800, "max_output_tokens": 131072, "max_tokens": 131072, @@ -32607,15 +32607,15 @@ "supports_tool_choice": true, "supports_system_messages": true, "supports_response_schema": true, - "cache_read_input_token_cost": 1.1e-03, - "input_cost_per_token_cache_hit": 1.1e-03, + "cache_read_input_token_cost": 1.1e-7, + "input_cost_per_token_cache_hit": 1.1e-7, "supports_reasoning": true }, "novita/xiaomimimo/mimo-v2-flash": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 1e-03, - "output_cost_per_token": 3e-03, + "input_cost_per_token": 1e-7, + "output_cost_per_token": 3e-7, "max_input_tokens": 262144, "max_output_tokens": 32000, "max_tokens": 32000, @@ -32624,15 +32624,15 @@ "supports_tool_choice": true, "supports_system_messages": true, "supports_response_schema": true, - "cache_read_input_token_cost": 2e-04, - "input_cost_per_token_cache_hit": 2e-04, + "cache_read_input_token_cost": 2e-8, + "input_cost_per_token_cache_hit": 2e-8, "supports_reasoning": true }, "novita/zai-org/autoglm-phone-9b-multilingual": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 3.5e-04, - "output_cost_per_token": 1.38e-03, + "input_cost_per_token": 3.5e-8, + "output_cost_per_token": 1.38e-7, "max_input_tokens": 65536, "max_output_tokens": 65536, "max_tokens": 65536, @@ -32642,8 +32642,8 @@ "novita/moonshotai/kimi-k2-thinking": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 4.8e-03, - "output_cost_per_token": 2e-02, + "input_cost_per_token": 6e-7, + "output_cost_per_token": 2.5e-06, "max_input_tokens": 262144, "max_output_tokens": 262144, "max_tokens": 262144, @@ -32657,8 +32657,8 @@ "novita/minimax/minimax-m2": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 2.4e-03, - "output_cost_per_token": 9.6e-03, + "input_cost_per_token": 3e-7, + "output_cost_per_token": 1.2e-06, "max_input_tokens": 204800, "max_output_tokens": 131072, "max_tokens": 131072, @@ -32666,15 +32666,15 @@ "supports_parallel_function_calling": true, "supports_tool_choice": true, "supports_system_messages": true, - "cache_read_input_token_cost": 2.4e-04, - "input_cost_per_token_cache_hit": 2.4e-04, + "cache_read_input_token_cost": 3e-8, + "input_cost_per_token_cache_hit": 3e-8, "supports_reasoning": true }, "novita/paddlepaddle/paddleocr-vl": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 1.6e-04, - "output_cost_per_token": 1.6e-04, + "input_cost_per_token": 2e-8, + "output_cost_per_token": 2e-8, "max_input_tokens": 16384, "max_output_tokens": 16384, "max_tokens": 16384, @@ -32684,8 +32684,8 @@ "novita/deepseek/deepseek-v3.2-exp": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 2.16e-03, - "output_cost_per_token": 3.28e-03, + "input_cost_per_token": 2.7e-7, + "output_cost_per_token": 4.1e-7, "max_input_tokens": 163840, "max_output_tokens": 65536, "max_tokens": 65536, @@ -32699,8 +32699,8 @@ "novita/qwen/qwen3-vl-235b-a22b-thinking": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 7.84e-03, - "output_cost_per_token": 3.16e-02, + "input_cost_per_token": 9.8e-7, + "output_cost_per_token": 3.95e-06, "max_input_tokens": 131072, "max_output_tokens": 32768, "max_tokens": 32768, @@ -32711,8 +32711,8 @@ "novita/zai-org/glm-4.6v": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 3e-03, - "output_cost_per_token": 9e-03, + "input_cost_per_token": 3e-7, + "output_cost_per_token": 9e-7, "max_input_tokens": 131072, "max_output_tokens": 32768, "max_tokens": 32768, @@ -32722,15 +32722,15 @@ "supports_vision": true, "supports_system_messages": true, "supports_response_schema": true, - "cache_read_input_token_cost": 5.5e-04, - "input_cost_per_token_cache_hit": 5.5e-04, + "cache_read_input_token_cost": 5.5e-8, + "input_cost_per_token_cache_hit": 5.5e-8, "supports_reasoning": true }, "novita/zai-org/glm-4.6": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 4.4e-03, - "output_cost_per_token": 1.76e-02, + "input_cost_per_token": 5.5e-7, + "output_cost_per_token": 2.2e-06, "max_input_tokens": 204800, "max_output_tokens": 131072, "max_tokens": 131072, @@ -32739,15 +32739,31 @@ "supports_tool_choice": true, "supports_system_messages": true, "supports_response_schema": true, - "cache_read_input_token_cost": 8.8e-04, - "input_cost_per_token_cache_hit": 8.8e-04, + "cache_read_input_token_cost": 1.1e-7, + "input_cost_per_token_cache_hit": 1.1e-7, "supports_reasoning": true }, + "novita/kwaipilot/kat-coder-pro": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 3e-7, + "output_cost_per_token": 1.2e-06, + "max_input_tokens": 256000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_response_schema": true, + "cache_read_input_token_cost": 6e-8, + "input_cost_per_token_cache_hit": 6e-8 + }, "novita/qwen/qwen3-next-80b-a3b-instruct": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 1.2e-03, - "output_cost_per_token": 1.2e-02, + "input_cost_per_token": 1.5e-7, + "output_cost_per_token": 1.5e-06, "max_input_tokens": 131072, "max_output_tokens": 32768, "max_tokens": 32768, @@ -32760,8 +32776,8 @@ "novita/qwen/qwen3-next-80b-a3b-thinking": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 1.2e-03, - "output_cost_per_token": 1.2e-02, + "input_cost_per_token": 1.5e-7, + "output_cost_per_token": 1.5e-06, "max_input_tokens": 131072, "max_output_tokens": 32768, "max_tokens": 32768, @@ -32775,8 +32791,8 @@ "novita/deepseek/deepseek-ocr": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 2.4e-04, - "output_cost_per_token": 2.4e-04, + "input_cost_per_token": 3e-8, + "output_cost_per_token": 3e-8, "max_input_tokens": 8192, "max_output_tokens": 8192, "max_tokens": 8192, @@ -32788,8 +32804,8 @@ "novita/deepseek/deepseek-v3.1-terminus": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 2.16e-03, - "output_cost_per_token": 8e-03, + "input_cost_per_token": 2.7e-7, + "output_cost_per_token": 1e-06, "max_input_tokens": 131072, "max_output_tokens": 32768, "max_tokens": 32768, @@ -32798,15 +32814,15 @@ "supports_tool_choice": true, "supports_system_messages": true, "supports_response_schema": true, - "cache_read_input_token_cost": 1.08e-03, - "input_cost_per_token_cache_hit": 1.08e-03, + "cache_read_input_token_cost": 1.35e-7, + "input_cost_per_token_cache_hit": 1.35e-7, "supports_reasoning": true }, "novita/qwen/qwen3-vl-235b-a22b-instruct": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 2.4e-03, - "output_cost_per_token": 1.2e-02, + "input_cost_per_token": 3e-7, + "output_cost_per_token": 1.5e-06, "max_input_tokens": 131072, "max_output_tokens": 32768, "max_tokens": 32768, @@ -32820,8 +32836,8 @@ "novita/qwen/qwen3-max": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 1.688e-02, - "output_cost_per_token": 6.76e-02, + "input_cost_per_token": 2.11e-06, + "output_cost_per_token": 8.45e-06, "max_input_tokens": 262144, "max_output_tokens": 65536, "max_tokens": 65536, @@ -32834,8 +32850,8 @@ "novita/skywork/r1v4-lite": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 2e-03, - "output_cost_per_token": 6e-03, + "input_cost_per_token": 2e-7, + "output_cost_per_token": 6e-7, "max_input_tokens": 262144, "max_output_tokens": 65536, "max_tokens": 65536, @@ -32847,8 +32863,8 @@ "novita/deepseek/deepseek-v3.1": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 2.16e-03, - "output_cost_per_token": 8e-03, + "input_cost_per_token": 2.7e-7, + "output_cost_per_token": 1e-06, "max_input_tokens": 131072, "max_output_tokens": 32768, "max_tokens": 32768, @@ -32857,15 +32873,15 @@ "supports_tool_choice": true, "supports_system_messages": true, "supports_response_schema": true, - "cache_read_input_token_cost": 1.08e-03, - "input_cost_per_token_cache_hit": 1.08e-03, + "cache_read_input_token_cost": 1.35e-7, + "input_cost_per_token_cache_hit": 1.35e-7, "supports_reasoning": true }, "novita/moonshotai/kimi-k2-0905": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 4.8e-03, - "output_cost_per_token": 2e-02, + "input_cost_per_token": 6e-7, + "output_cost_per_token": 2.5e-06, "max_input_tokens": 262144, "max_output_tokens": 262144, "max_tokens": 262144, @@ -32878,8 +32894,8 @@ "novita/qwen/qwen3-coder-480b-a35b-instruct": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 2.4e-03, - "output_cost_per_token": 1.04e-02, + "input_cost_per_token": 3e-7, + "output_cost_per_token": 1.3e-06, "max_input_tokens": 262144, "max_output_tokens": 65536, "max_tokens": 65536, @@ -32892,8 +32908,8 @@ "novita/qwen/qwen3-coder-30b-a3b-instruct": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 7e-04, - "output_cost_per_token": 2.7e-03, + "input_cost_per_token": 7e-8, + "output_cost_per_token": 2.7e-7, "max_input_tokens": 160000, "max_output_tokens": 32768, "max_tokens": 32768, @@ -32906,8 +32922,8 @@ "novita/openai/gpt-oss-120b": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 4e-04, - "output_cost_per_token": 2e-03, + "input_cost_per_token": 5e-8, + "output_cost_per_token": 2.5e-7, "max_input_tokens": 131072, "max_output_tokens": 32768, "max_tokens": 32768, @@ -32922,8 +32938,8 @@ "novita/moonshotai/kimi-k2-instruct": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 4.56e-03, - "output_cost_per_token": 1.84e-02, + "input_cost_per_token": 5.7e-7, + "output_cost_per_token": 2.3e-06, "max_input_tokens": 131072, "max_output_tokens": 131072, "max_tokens": 131072, @@ -32936,8 +32952,8 @@ "novita/deepseek/deepseek-v3-0324": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 2.16e-03, - "output_cost_per_token": 8.96e-03, + "input_cost_per_token": 2.7e-7, + "output_cost_per_token": 1.12e-06, "max_input_tokens": 163840, "max_output_tokens": 163840, "max_tokens": 163840, @@ -32946,14 +32962,14 @@ "supports_tool_choice": true, "supports_system_messages": true, "supports_response_schema": true, - "cache_read_input_token_cost": 1.08e-03, - "input_cost_per_token_cache_hit": 1.08e-03 + "cache_read_input_token_cost": 1.35e-7, + "input_cost_per_token_cache_hit": 1.35e-7 }, "novita/zai-org/glm-4.5": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 4.8e-03, - "output_cost_per_token": 1.76e-02, + "input_cost_per_token": 6e-7, + "output_cost_per_token": 2.2e-06, "max_input_tokens": 131072, "max_output_tokens": 98304, "max_tokens": 98304, @@ -32961,15 +32977,15 @@ "supports_parallel_function_calling": true, "supports_tool_choice": true, "supports_system_messages": true, - "cache_read_input_token_cost": 8.8e-04, - "input_cost_per_token_cache_hit": 8.8e-04, + "cache_read_input_token_cost": 1.1e-7, + "input_cost_per_token_cache_hit": 1.1e-7, "supports_reasoning": true }, "novita/qwen/qwen3-235b-a22b-thinking-2507": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 2.4e-03, - "output_cost_per_token": 2.4e-02, + "input_cost_per_token": 3e-7, + "output_cost_per_token": 3e-06, "max_input_tokens": 131072, "max_output_tokens": 32768, "max_tokens": 32768, @@ -32982,8 +32998,8 @@ "novita/meta-llama/llama-3.1-8b-instruct": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 2e-04, - "output_cost_per_token": 5e-04, + "input_cost_per_token": 2e-8, + "output_cost_per_token": 5e-8, "max_input_tokens": 16384, "max_output_tokens": 16384, "max_tokens": 16384, @@ -32992,8 +33008,8 @@ "novita/google/gemma-3-12b-it": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 4e-04, - "output_cost_per_token": 8e-04, + "input_cost_per_token": 5e-8, + "output_cost_per_token": 1e-7, "max_input_tokens": 131072, "max_output_tokens": 8192, "max_tokens": 8192, @@ -33005,8 +33021,8 @@ "novita/zai-org/glm-4.5v": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 4.8e-03, - "output_cost_per_token": 1.44e-02, + "input_cost_per_token": 6e-7, + "output_cost_per_token": 1.8e-06, "max_input_tokens": 65536, "max_output_tokens": 16384, "max_tokens": 16384, @@ -33016,15 +33032,15 @@ "supports_vision": true, "supports_system_messages": true, "supports_response_schema": true, - "cache_read_input_token_cost": 8.8e-04, - "input_cost_per_token_cache_hit": 8.8e-04, + "cache_read_input_token_cost": 1.1e-7, + "input_cost_per_token_cache_hit": 1.1e-7, "supports_reasoning": true }, "novita/openai/gpt-oss-20b": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 3.2e-04, - "output_cost_per_token": 1.2e-03, + "input_cost_per_token": 4e-8, + "output_cost_per_token": 1.5e-7, "max_input_tokens": 131072, "max_output_tokens": 32768, "max_tokens": 32768, @@ -33037,8 +33053,8 @@ "novita/qwen/qwen3-235b-a22b-instruct-2507": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 7.2e-04, - "output_cost_per_token": 4.64e-03, + "input_cost_per_token": 9e-8, + "output_cost_per_token": 5.8e-7, "max_input_tokens": 131072, "max_output_tokens": 16384, "max_tokens": 16384, @@ -33051,8 +33067,8 @@ "novita/deepseek/deepseek-r1-distill-qwen-14b": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 1.2e-03, - "output_cost_per_token": 1.2e-03, + "input_cost_per_token": 1.5e-7, + "output_cost_per_token": 1.5e-7, "max_input_tokens": 32768, "max_output_tokens": 16384, "max_tokens": 16384, @@ -33064,8 +33080,8 @@ "novita/meta-llama/llama-3.3-70b-instruct": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 1.08e-03, - "output_cost_per_token": 3.2e-03, + "input_cost_per_token": 1.35e-7, + "output_cost_per_token": 4e-7, "max_input_tokens": 131072, "max_output_tokens": 120000, "max_tokens": 120000, @@ -33077,8 +33093,8 @@ "novita/qwen/qwen-2.5-72b-instruct": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 3.04e-03, - "output_cost_per_token": 3.2e-03, + "input_cost_per_token": 3.8e-7, + "output_cost_per_token": 4e-7, "max_input_tokens": 32000, "max_output_tokens": 8192, "max_tokens": 8192, @@ -33091,8 +33107,8 @@ "novita/mistralai/mistral-nemo": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 3.2e-04, - "output_cost_per_token": 1.36e-03, + "input_cost_per_token": 4e-8, + "output_cost_per_token": 1.7e-7, "max_input_tokens": 60288, "max_output_tokens": 16000, "max_tokens": 16000, @@ -33103,8 +33119,8 @@ "novita/minimaxai/minimax-m1-80k": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 4.4e-03, - "output_cost_per_token": 1.76e-02, + "input_cost_per_token": 5.5e-7, + "output_cost_per_token": 2.2e-06, "max_input_tokens": 1000000, "max_output_tokens": 40000, "max_tokens": 40000, @@ -33117,8 +33133,8 @@ "novita/deepseek/deepseek-r1-0528": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 5.6e-03, - "output_cost_per_token": 2e-02, + "input_cost_per_token": 7e-7, + "output_cost_per_token": 2.5e-06, "max_input_tokens": 163840, "max_output_tokens": 32768, "max_tokens": 32768, @@ -33127,15 +33143,15 @@ "supports_tool_choice": true, "supports_system_messages": true, "supports_response_schema": true, - "cache_read_input_token_cost": 2.8e-03, - "input_cost_per_token_cache_hit": 2.8e-03, + "cache_read_input_token_cost": 3.5e-7, + "input_cost_per_token_cache_hit": 3.5e-7, "supports_reasoning": true }, "novita/deepseek/deepseek-r1-distill-qwen-32b": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 2.4e-03, - "output_cost_per_token": 2.4e-03, + "input_cost_per_token": 3e-7, + "output_cost_per_token": 3e-7, "max_input_tokens": 64000, "max_output_tokens": 32000, "max_tokens": 32000, @@ -33147,8 +33163,8 @@ "novita/meta-llama/llama-3-8b-instruct": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 3.2e-04, - "output_cost_per_token": 3.2e-04, + "input_cost_per_token": 4e-8, + "output_cost_per_token": 4e-8, "max_input_tokens": 8192, "max_output_tokens": 8192, "max_tokens": 8192, @@ -33157,8 +33173,8 @@ "novita/microsoft/wizardlm-2-8x22b": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 4.96e-03, - "output_cost_per_token": 4.96e-03, + "input_cost_per_token": 6.2e-7, + "output_cost_per_token": 6.2e-7, "max_input_tokens": 65535, "max_output_tokens": 8000, "max_tokens": 8000, @@ -33167,8 +33183,8 @@ "novita/deepseek/deepseek-r1-0528-qwen3-8b": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 4.8e-04, - "output_cost_per_token": 7.2e-04, + "input_cost_per_token": 6e-8, + "output_cost_per_token": 9e-8, "max_input_tokens": 128000, "max_output_tokens": 32000, "max_tokens": 32000, @@ -33178,8 +33194,8 @@ "novita/deepseek/deepseek-r1-distill-llama-70b": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 6.4e-03, - "output_cost_per_token": 6.4e-03, + "input_cost_per_token": 8e-7, + "output_cost_per_token": 8e-7, "max_input_tokens": 8192, "max_output_tokens": 8192, "max_tokens": 8192, @@ -33191,8 +33207,8 @@ "novita/meta-llama/llama-3-70b-instruct": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 5.1e-03, - "output_cost_per_token": 7.4e-03, + "input_cost_per_token": 5.1e-7, + "output_cost_per_token": 7.4e-7, "max_input_tokens": 8192, "max_output_tokens": 8000, "max_tokens": 8000, @@ -33203,8 +33219,8 @@ "novita/qwen/qwen3-235b-a22b-fp8": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 1.6e-03, - "output_cost_per_token": 6.4e-03, + "input_cost_per_token": 2e-7, + "output_cost_per_token": 8e-7, "max_input_tokens": 40960, "max_output_tokens": 20000, "max_tokens": 20000, @@ -33214,8 +33230,8 @@ "novita/meta-llama/llama-4-maverick-17b-128e-instruct-fp8": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 1.6e-03, - "output_cost_per_token": 7.2e-03, + "input_cost_per_token": 2.7e-7, + "output_cost_per_token": 8.5e-7, "max_input_tokens": 1048576, "max_output_tokens": 8192, "max_tokens": 8192, @@ -33225,8 +33241,8 @@ "novita/meta-llama/llama-4-scout-17b-16e-instruct": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 8e-04, - "output_cost_per_token": 4e-03, + "input_cost_per_token": 1.8e-7, + "output_cost_per_token": 5.9e-7, "max_input_tokens": 131072, "max_output_tokens": 131072, "max_tokens": 131072, @@ -33236,8 +33252,8 @@ "novita/nousresearch/hermes-2-pro-llama-3-8b": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 1.4e-03, - "output_cost_per_token": 1.4e-03, + "input_cost_per_token": 1.4e-7, + "output_cost_per_token": 1.4e-7, "max_input_tokens": 8192, "max_output_tokens": 8192, "max_tokens": 8192, @@ -33248,8 +33264,8 @@ "novita/qwen/qwen2.5-vl-72b-instruct": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 6.4e-03, - "output_cost_per_token": 6.4e-03, + "input_cost_per_token": 8e-7, + "output_cost_per_token": 8e-7, "max_input_tokens": 32768, "max_output_tokens": 32768, "max_tokens": 32768, @@ -33259,8 +33275,8 @@ "novita/sao10k/l3-70b-euryale-v2.1": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 1.48e-02, - "output_cost_per_token": 1.48e-02, + "input_cost_per_token": 1.48e-06, + "output_cost_per_token": 1.48e-06, "max_input_tokens": 8192, "max_output_tokens": 8192, "max_tokens": 8192, @@ -33272,8 +33288,8 @@ "novita/baidu/ernie-4.5-21B-a3b-thinking": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 5.6e-04, - "output_cost_per_token": 2.24e-03, + "input_cost_per_token": 7e-8, + "output_cost_per_token": 2.8e-7, "max_input_tokens": 131072, "max_output_tokens": 65536, "max_tokens": 65536, @@ -33283,8 +33299,8 @@ "novita/sao10k/l3-8b-lunaris": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 5e-04, - "output_cost_per_token": 5e-04, + "input_cost_per_token": 5e-8, + "output_cost_per_token": 5e-8, "max_input_tokens": 8192, "max_output_tokens": 8192, "max_tokens": 8192, @@ -33295,33 +33311,18 @@ "novita/baichuan/baichuan-m2-32b": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 5.6e-04, - "output_cost_per_token": 5.6e-04, + "input_cost_per_token": 7e-8, + "output_cost_per_token": 7e-8, "max_input_tokens": 131072, "max_output_tokens": 131072, "max_tokens": 131072, - "supports_tool_choice": true, - "supports_system_messages": true, - "supports_response_schema": true, - "supports_reasoning": true - }, - "novita/thudm/glm-4.1v-9b-thinking": { - "litellm_provider": "novita", - "mode": "chat", - "input_cost_per_token": 2.8e-04, - "output_cost_per_token": 1.104e-03, - "max_input_tokens": 65536, - "max_output_tokens": 8000, - "max_tokens": 8000, - "supports_vision": true, - "supports_system_messages": true, - "supports_reasoning": true + "supports_system_messages": true }, "novita/baidu/ernie-4.5-vl-424b-a47b": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 3.36e-03, - "output_cost_per_token": 1e-02, + "input_cost_per_token": 4.2e-7, + "output_cost_per_token": 1.25e-06, "max_input_tokens": 123000, "max_output_tokens": 16000, "max_tokens": 16000, @@ -33332,8 +33333,8 @@ "novita/baidu/ernie-4.5-300b-a47b-paddle": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 2.24e-03, - "output_cost_per_token": 8.8e-03, + "input_cost_per_token": 2.8e-7, + "output_cost_per_token": 1.1e-06, "max_input_tokens": 123000, "max_output_tokens": 12000, "max_tokens": 12000, @@ -33344,8 +33345,8 @@ "novita/deepseek/deepseek-prover-v2-671b": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 5.6e-03, - "output_cost_per_token": 2e-02, + "input_cost_per_token": 7e-7, + "output_cost_per_token": 2.5e-06, "max_input_tokens": 160000, "max_output_tokens": 160000, "max_tokens": 160000, @@ -33354,8 +33355,8 @@ "novita/qwen/qwen3-32b-fp8": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 8e-04, - "output_cost_per_token": 3.6e-03, + "input_cost_per_token": 1e-7, + "output_cost_per_token": 4.5e-7, "max_input_tokens": 40960, "max_output_tokens": 20000, "max_tokens": 20000, @@ -33365,8 +33366,8 @@ "novita/qwen/qwen3-30b-a3b-fp8": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 7.2e-04, - "output_cost_per_token": 3.6e-03, + "input_cost_per_token": 9e-8, + "output_cost_per_token": 4.5e-7, "max_input_tokens": 40960, "max_output_tokens": 20000, "max_tokens": 20000, @@ -33376,8 +33377,8 @@ "novita/google/gemma-3-27b-it": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 9.52e-04, - "output_cost_per_token": 1.6e-03, + "input_cost_per_token": 1.19e-7, + "output_cost_per_token": 2e-7, "max_input_tokens": 98304, "max_output_tokens": 16384, "max_tokens": 16384, @@ -33387,8 +33388,8 @@ "novita/deepseek/deepseek-v3-turbo": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 3.2e-03, - "output_cost_per_token": 1.04e-02, + "input_cost_per_token": 4e-7, + "output_cost_per_token": 1.3e-06, "max_input_tokens": 64000, "max_output_tokens": 16000, "max_tokens": 16000, @@ -33400,8 +33401,8 @@ "novita/deepseek/deepseek-r1-turbo": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 5.6e-03, - "output_cost_per_token": 2e-02, + "input_cost_per_token": 7e-7, + "output_cost_per_token": 2.5e-06, "max_input_tokens": 64000, "max_output_tokens": 16000, "max_tokens": 16000, @@ -33414,8 +33415,8 @@ "novita/Sao10K/L3-8B-Stheno-v3.2": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 5e-04, - "output_cost_per_token": 5e-04, + "input_cost_per_token": 5e-8, + "output_cost_per_token": 5e-8, "max_input_tokens": 8192, "max_output_tokens": 32000, "max_tokens": 32000, @@ -33427,8 +33428,8 @@ "novita/gryphe/mythomax-l2-13b": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 7.2e-04, - "output_cost_per_token": 7.2e-04, + "input_cost_per_token": 9e-8, + "output_cost_per_token": 9e-8, "max_input_tokens": 4096, "max_output_tokens": 3200, "max_tokens": 3200, @@ -33437,8 +33438,8 @@ "novita/baidu/ernie-4.5-vl-28b-a3b-thinking": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 3.9e-03, - "output_cost_per_token": 3.9e-03, + "input_cost_per_token": 3.9e-7, + "output_cost_per_token": 3.9e-7, "max_input_tokens": 131072, "max_output_tokens": 65536, "max_tokens": 65536, @@ -33453,8 +33454,8 @@ "novita/qwen/qwen3-vl-8b-instruct": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 6.4e-04, - "output_cost_per_token": 4e-03, + "input_cost_per_token": 8e-8, + "output_cost_per_token": 5e-7, "max_input_tokens": 131072, "max_output_tokens": 32768, "max_tokens": 32768, @@ -33468,8 +33469,8 @@ "novita/zai-org/glm-4.5-air": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 1.04e-03, - "output_cost_per_token": 6.8e-03, + "input_cost_per_token": 1.3e-7, + "output_cost_per_token": 8.5e-7, "max_input_tokens": 131072, "max_output_tokens": 98304, "max_tokens": 98304, @@ -33482,8 +33483,8 @@ "novita/qwen/qwen3-vl-30b-a3b-instruct": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 1.6e-03, - "output_cost_per_token": 5.6e-03, + "input_cost_per_token": 2e-7, + "output_cost_per_token": 7e-7, "max_input_tokens": 131072, "max_output_tokens": 32768, "max_tokens": 32768, @@ -33497,8 +33498,8 @@ "novita/qwen/qwen3-vl-30b-a3b-thinking": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 1.6e-03, - "output_cost_per_token": 8e-03, + "input_cost_per_token": 2e-7, + "output_cost_per_token": 1e-06, "max_input_tokens": 131072, "max_output_tokens": 32768, "max_tokens": 32768, @@ -33509,11 +33510,45 @@ "supports_system_messages": true, "supports_response_schema": true }, + "novita/qwen/qwen3-omni-30b-a3b-thinking": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 2.5e-7, + "output_cost_per_token": 9.7e-7, + "max_input_tokens": 65536, + "max_output_tokens": 16384, + "max_tokens": 16384, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_system_messages": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_audio_input": true + }, + "novita/qwen/qwen3-omni-30b-a3b-instruct": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 2.5e-7, + "output_cost_per_token": 9.7e-7, + "max_input_tokens": 65536, + "max_output_tokens": 16384, + "max_tokens": 16384, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_system_messages": true, + "supports_response_schema": true, + "supports_audio_input": true, + "supports_audio_output": true + }, "novita/qwen/qwen-mt-plus": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 2e-03, - "output_cost_per_token": 6e-03, + "input_cost_per_token": 2.5e-7, + "output_cost_per_token": 7.5e-7, "max_input_tokens": 16384, "max_output_tokens": 8192, "max_tokens": 8192, @@ -33522,8 +33557,8 @@ "novita/baidu/ernie-4.5-vl-28b-a3b": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 1.12e-03, - "output_cost_per_token": 4.48e-03, + "input_cost_per_token": 1.4e-7, + "output_cost_per_token": 5.6e-7, "max_input_tokens": 30000, "max_output_tokens": 8000, "max_tokens": 8000, @@ -33537,8 +33572,8 @@ "novita/baidu/ernie-4.5-21B-a3b": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 5.6e-04, - "output_cost_per_token": 2.24e-03, + "input_cost_per_token": 7e-8, + "output_cost_per_token": 2.8e-7, "max_input_tokens": 120000, "max_output_tokens": 8000, "max_tokens": 8000, @@ -33550,8 +33585,8 @@ "novita/qwen/qwen3-8b-fp8": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 2.8e-04, - "output_cost_per_token": 1.104e-03, + "input_cost_per_token": 3.5e-8, + "output_cost_per_token": 1.38e-7, "max_input_tokens": 128000, "max_output_tokens": 20000, "max_tokens": 20000, @@ -33561,8 +33596,8 @@ "novita/qwen/qwen3-4b-fp8": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 2.4e-04, - "output_cost_per_token": 2.4e-04, + "input_cost_per_token": 3e-8, + "output_cost_per_token": 3e-8, "max_input_tokens": 128000, "max_output_tokens": 20000, "max_tokens": 20000, @@ -33572,8 +33607,8 @@ "novita/qwen/qwen2.5-7b-instruct": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 5.6e-04, - "output_cost_per_token": 5.6e-04, + "input_cost_per_token": 7e-8, + "output_cost_per_token": 7e-8, "max_input_tokens": 32000, "max_output_tokens": 32000, "max_tokens": 32000, @@ -33586,8 +33621,8 @@ "novita/meta-llama/llama-3.2-3b-instruct": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 2.4e-04, - "output_cost_per_token": 4e-04, + "input_cost_per_token": 3e-8, + "output_cost_per_token": 5e-8, "max_input_tokens": 32768, "max_output_tokens": 32000, "max_tokens": 32000, @@ -33599,8 +33634,8 @@ "novita/sao10k/l31-70b-euryale-v2.2": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 1.48e-02, - "output_cost_per_token": 1.48e-02, + "input_cost_per_token": 1.48e-06, + "output_cost_per_token": 1.48e-06, "max_input_tokens": 8192, "max_output_tokens": 8192, "max_tokens": 8192, @@ -33612,7 +33647,7 @@ "novita/qwen/qwen3-embedding-0.6b": { "litellm_provider": "novita", "mode": "embedding", - "input_cost_per_token": 5.6e-04, + "input_cost_per_token": 7e-8, "output_cost_per_token": 0, "max_input_tokens": 32768, "max_output_tokens": 32768, @@ -33621,7 +33656,7 @@ "novita/qwen/qwen3-embedding-8b": { "litellm_provider": "novita", "mode": "embedding", - "input_cost_per_token": 5.6e-04, + "input_cost_per_token": 7e-8, "output_cost_per_token": 0, "max_input_tokens": 32768, "max_output_tokens": 4096, @@ -33630,8 +33665,8 @@ "novita/baai/bge-m3": { "litellm_provider": "novita", "mode": "embedding", - "input_cost_per_token": 1e-04, - "output_cost_per_token": 1e-04, + "input_cost_per_token": 1e-8, + "output_cost_per_token": 1e-8, "max_input_tokens": 8192, "max_output_tokens": 96000, "max_tokens": 96000 @@ -33639,8 +33674,8 @@ "novita/qwen/qwen3-reranker-8b": { "litellm_provider": "novita", "mode": "rerank", - "input_cost_per_token": 4e-04, - "output_cost_per_token": 4e-04, + "input_cost_per_token": 5e-8, + "output_cost_per_token": 5e-8, "max_input_tokens": 32768, "max_output_tokens": 4096, "max_tokens": 4096 @@ -33648,8 +33683,8 @@ "novita/baai/bge-reranker-v2-m3": { "litellm_provider": "novita", "mode": "rerank", - "input_cost_per_token": 1e-04, - "output_cost_per_token": 1e-04, + "input_cost_per_token": 1e-8, + "output_cost_per_token": 1e-8, "max_input_tokens": 8000, "max_output_tokens": 8000, "max_tokens": 8000