mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-09 22:31:41 +00:00
feature(gigachat): add gigachat passthrough endpoint
This commit is contained in:
KnyazSh
parent
718985144a
commit
bf36953dcb
14 changed files with 859 additions and 0 deletions
122
docs/my-website/docs/pass_through/gigachat.md
Normal file
122
docs/my-website/docs/pass_through/gigachat.md
Normal file
|
|
@ -0,0 +1,122 @@
|
|||
# GigaChat Passthrough
|
||||
|
||||
Pass-through endpoints for direct GigaChat API access via LiteLLM Proxy.
|
||||
|
||||
## Overview
|
||||
|
||||
| Feature | Supported | Notes |
|
||||
|-------|-------|-------|
|
||||
| Cost Tracking | ✅ | Works with proxy cost metadata and router models |
|
||||
| Logging | ✅ | Logs requests and responses across LiteLLM integrations |
|
||||
| Streaming | ✅ | Supported for streaming GigaChat chat completions |
|
||||
|
||||
## When to use this
|
||||
|
||||
- Use the native LiteLLM GigaChat provider for standard chat and embedding calls when possible.
|
||||
- Use `/gigachat` passthrough when you need provider-specific GigaChat endpoints or raw GigaChat request shapes.
|
||||
- This is useful for newer or less common GigaChat API endpoints that LiteLLM does not yet expose natively.
|
||||
|
||||
## How it works
|
||||
|
||||
Any path under `/gigachat` is treated as a provider-specific route and routed through LiteLLM's GigaChat passthrough path.
|
||||
The proxy accepts the same request body shape as GigaChat and forwards it to the GigaChat backend.
|
||||
|
||||
### Proxy base URL mapping
|
||||
|
||||
| Original GigaChat URL | Proxy URL |
|
||||
|-----------------------|-----------|
|
||||
| `https://gigachat.devices.sberbank.ru/api/v1` | `http://0.0.0.0:4000/gigachat/api/v1` |
|
||||
|
||||
## Request format
|
||||
|
||||
The proxy requires a `model` field in the request body. For GigaChat passthrough, use the LightLLM model prefix format such as `gigachat/GigaChat-2-Max`.
|
||||
|
||||
### Example: Chat completion
|
||||
|
||||
```bash
|
||||
curl --request POST \
|
||||
--url http://0.0.0.0:4000/gigachat/api/v1/chat/completions \
|
||||
--header 'accept: application/json' \
|
||||
--header 'content-type: application/json' \
|
||||
--header 'x-api-key: $LITELLM_API_KEY' \
|
||||
--data '{
|
||||
"model": "gigachat/GigaChat-2-Max",
|
||||
"messages": [
|
||||
{"role": "user", "content": "Hello, world"}
|
||||
]
|
||||
}'
|
||||
```
|
||||
|
||||
### Python example
|
||||
|
||||
```python
|
||||
import requests
|
||||
import os
|
||||
|
||||
response = requests.post(
|
||||
"http://0.0.0.0:4000/gigachat/api/v1/chat/completions",
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
"x-api-key": os.environ["LITELLM_API_KEY"],
|
||||
},
|
||||
json={
|
||||
"model": "gigachat/GigaChat-2-Max",
|
||||
"messages": [
|
||||
{"role": "user", "content": "Hello, world"}
|
||||
],
|
||||
},
|
||||
)
|
||||
print(response.json())
|
||||
```
|
||||
|
||||
## Authentication
|
||||
|
||||
- Authenticate to the proxy with `x-api-key: $LITELLM_API_KEY` or `Authorization: Bearer $LITELLM_API_KEY`.
|
||||
- The proxy then uses the configured GigaChat credentials to authenticate with the upstream GigaChat API.
|
||||
|
||||
## Notes
|
||||
|
||||
- GigaChat uses OAuth-style credentials. Configure your GigaChat credentials in LiteLLM using `GIGACHAT_CREDENTIALS` or `GIGACHAT_API_KEY` as described in the main GigaChat provider docs.
|
||||
- The proxy automatically handles GigaChat's self-signed SSL setup when forwarding requests, so you do not need to disable SSL verification from the client side.
|
||||
- The `model` field is required for passthrough requests.
|
||||
|
||||
## Advanced
|
||||
|
||||
### Use with router-backed GigaChat models
|
||||
|
||||
If you define router models in `config.yaml`, you can use the passthrough endpoint with a router-backed GigaChat model:
|
||||
|
||||
```bash
|
||||
curl --request POST \
|
||||
--url http://0.0.0.0:4000/gigachat/api/v1/chat/completions \
|
||||
--header 'Content-Type: application/json' \
|
||||
--header 'x-api-key: $LITELLM_API_KEY' \
|
||||
--data '{
|
||||
"model": "gigachat/GigaChat-2-Max",
|
||||
"messages": [
|
||||
{"role": "user", "content": "Hello, world"}
|
||||
]
|
||||
}'
|
||||
```
|
||||
|
||||
### Sending metadata
|
||||
|
||||
You can attach LiteLLM metadata for cost tracking and tags using `litellm_metadata` in the request body:
|
||||
|
||||
```bash
|
||||
curl --request POST \
|
||||
--url http://0.0.0.0:4000/gigachat/api/v1/chat/completions \
|
||||
--header 'accept: application/json' \
|
||||
--header 'content-type: application/json' \
|
||||
--header 'x-api-key: $LITELLM_API_KEY' \
|
||||
--data '{
|
||||
"model": "gigachat/GigaChat-2-Max",
|
||||
"messages": [
|
||||
{"role": "user", "content": "Hello, world"}
|
||||
],
|
||||
"litellm_metadata": {
|
||||
"tags": ["test-tag"],
|
||||
"user": "test-user"
|
||||
}
|
||||
}'
|
||||
```
|
||||
|
|
@ -30,6 +30,9 @@ _OPTIONAL_KWARGS_KEYS = frozenset(
|
|||
"aws_sts_endpoint",
|
||||
"aws_external_id",
|
||||
"aws_bedrock_runtime_endpoint",
|
||||
"gigachat_scope",
|
||||
"gigachat_auth_url",
|
||||
"gigachat_access_token",
|
||||
"tpm",
|
||||
"rpm",
|
||||
}
|
||||
|
|
|
|||
|
|
@ -324,6 +324,9 @@ def get_llm_provider( # noqa: PLR0915
|
|||
elif endpoint == "https://api.inference.wandb.ai/v1":
|
||||
custom_llm_provider = "wandb"
|
||||
dynamic_api_key = get_secret_str("WANDB_API_KEY")
|
||||
elif endpoint == "https://gigachat.devices.sberbank.ru/api/v1":
|
||||
custom_llm_provider = "gigachat"
|
||||
dynamic_api_key = get_secret_str("GIGACHAT_API_KEY")
|
||||
|
||||
if api_base is not None and not isinstance(api_base, str):
|
||||
raise Exception(
|
||||
|
|
@ -459,6 +462,8 @@ def get_llm_provider( # noqa: PLR0915
|
|||
custom_llm_provider = "amazon_nova"
|
||||
elif model.startswith("sap/"):
|
||||
custom_llm_provider = "sap"
|
||||
elif model in litellm.gigachat_models or model.startswith("gigachat/"):
|
||||
custom_llm_provider = "gigachat"
|
||||
if not custom_llm_provider:
|
||||
if litellm.suppress_debug_info is False:
|
||||
print() # noqa
|
||||
|
|
@ -944,6 +949,13 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915
|
|||
api_base or get_secret_str("MANUS_API_BASE") or "https://api.manus.im"
|
||||
)
|
||||
dynamic_api_key = api_key or get_secret_str("MANUS_API_KEY")
|
||||
elif custom_llm_provider == "gigachat":
|
||||
api_base = (
|
||||
api_base
|
||||
or get_secret_str("GIGACHAT_API_BASE")
|
||||
or "https://gigachat.devices.sberbank.ru/api/v1"
|
||||
)
|
||||
dynamic_api_key = api_key or get_secret_str("GIGACHAT_API_KEY")
|
||||
|
||||
if api_base is not None and not isinstance(api_base, str):
|
||||
raise Exception("api base needs to be a string. api_base={}".format(api_base))
|
||||
|
|
|
|||
|
|
@ -15,9 +15,11 @@ API Documentation: https://developers.sber.ru/docs/ru/gigachat/api/overview
|
|||
|
||||
from .chat.transformation import GigaChatConfig, GigaChatError
|
||||
from .embedding.transformation import GigaChatEmbeddingConfig
|
||||
from .passthrough.transformation import GigaChatPassthroughConfig
|
||||
|
||||
__all__ = [
|
||||
"GigaChatConfig",
|
||||
"GigaChatEmbeddingConfig",
|
||||
"GigaChatError",
|
||||
"GigaChatPassthroughConfig",
|
||||
]
|
||||
|
|
|
|||
7
litellm/llms/gigachat/passthrough/__init__.py
Normal file
7
litellm/llms/gigachat/passthrough/__init__.py
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
"""
|
||||
GigaChat passthrough Module
|
||||
"""
|
||||
|
||||
from .transformation import GigaChatPassthroughConfig
|
||||
|
||||
__all__ = ["GigaChatPassthroughConfig"]
|
||||
196
litellm/llms/gigachat/passthrough/transformation.py
Normal file
196
litellm/llms/gigachat/passthrough/transformation.py
Normal file
|
|
@ -0,0 +1,196 @@
|
|||
import json
|
||||
from typing import TYPE_CHECKING, List, Optional, Tuple, cast
|
||||
|
||||
import httpx
|
||||
|
||||
from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig
|
||||
from litellm.llms.gigachat.authenticator import get_access_token
|
||||
from litellm.llms.gigachat.chat.streaming import GigaChatModelResponseIterator
|
||||
from litellm.llms.gigachat.utils import get_api_base
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from httpx import URL, Response
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.types.utils import CostResponseTypes
|
||||
|
||||
|
||||
class GigaChatPassthroughConfig(BasePassthroughConfig):
|
||||
def is_streaming_request(self, endpoint: str, request_data: dict) -> bool:
|
||||
return request_data.get("stream", False)
|
||||
|
||||
def get_complete_url(
|
||||
self,
|
||||
api_base: Optional[str],
|
||||
api_key: Optional[str],
|
||||
model: str,
|
||||
endpoint: str,
|
||||
request_query_params: Optional[dict],
|
||||
litellm_params: dict,
|
||||
) -> Tuple["URL", str]:
|
||||
"""Get complete API URL for chat completions."""
|
||||
base_target_url = self.get_api_base(api_base)
|
||||
|
||||
if base_target_url is None:
|
||||
raise Exception("GigaChat api base not found")
|
||||
|
||||
litellm_metadata = litellm_params.get("litellm_metadata") or {}
|
||||
model_group = litellm_metadata.get("model_group")
|
||||
if model_group and model_group in endpoint:
|
||||
endpoint = endpoint.replace(model_group, model)
|
||||
|
||||
complete_url = f"{base_target_url}/chat/completions"
|
||||
return (
|
||||
httpx.URL(complete_url),
|
||||
base_target_url,
|
||||
)
|
||||
|
||||
def validate_environment(
|
||||
self,
|
||||
headers: dict,
|
||||
model: str,
|
||||
messages: List[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
api_key: Optional[str] = None,
|
||||
api_base: Optional[str] = None,
|
||||
) -> dict:
|
||||
"""
|
||||
Set up headers with OAuth token.
|
||||
"""
|
||||
# Get access token
|
||||
access_token = get_access_token(
|
||||
credentials=api_key, litellm_params=litellm_params
|
||||
)
|
||||
|
||||
headers["Authorization"] = f"Bearer {access_token}"
|
||||
headers["Content-Type"] = "application/json"
|
||||
headers["Accept"] = "application/json"
|
||||
|
||||
return headers
|
||||
|
||||
def logging_non_streaming_response(
|
||||
self,
|
||||
model: str,
|
||||
custom_llm_provider: str,
|
||||
httpx_response: "Response",
|
||||
request_data: dict,
|
||||
logging_obj: "LiteLLMLoggingObj",
|
||||
endpoint: str,
|
||||
) -> Optional["CostResponseTypes"]:
|
||||
from litellm import encoding
|
||||
from litellm.types.utils import LlmProviders, ModelResponse
|
||||
from litellm.utils import ProviderConfigManager
|
||||
|
||||
provider_chat_config = ProviderConfigManager.get_provider_chat_config(
|
||||
provider=LlmProviders(custom_llm_provider),
|
||||
model=model,
|
||||
)
|
||||
|
||||
if provider_chat_config is None:
|
||||
raise ValueError(f"No provider config found for model: {model}")
|
||||
|
||||
litellm_model_response: ModelResponse = provider_chat_config.transform_response(
|
||||
model=model,
|
||||
messages=request_data.get("messages", []),
|
||||
raw_response=httpx_response,
|
||||
model_response=ModelResponse(),
|
||||
logging_obj=logging_obj,
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
api_key="",
|
||||
request_data=request_data,
|
||||
encoding=encoding,
|
||||
)
|
||||
|
||||
return litellm_model_response
|
||||
|
||||
def handle_logging_collected_chunks(
|
||||
self,
|
||||
all_chunks: List[str],
|
||||
litellm_logging_obj: "LiteLLMLoggingObj",
|
||||
model: str,
|
||||
custom_llm_provider: str,
|
||||
endpoint: str,
|
||||
) -> Optional["CostResponseTypes"]:
|
||||
"""
|
||||
1. Convert all_chunks to a ModelResponseStream
|
||||
2. combine model_response_stream to model_response
|
||||
3. Return the model_response
|
||||
"""
|
||||
|
||||
from litellm.litellm_core_utils.streaming_handler import (
|
||||
convert_generic_chunk_to_model_response_stream,
|
||||
generic_chunk_has_all_required_fields,
|
||||
)
|
||||
from litellm.main import stream_chunk_builder
|
||||
from litellm.types.utils import GenericStreamingChunk, ModelResponseStream
|
||||
|
||||
all_translated_chunks = []
|
||||
|
||||
for chunk in all_chunks:
|
||||
if isinstance(chunk, bytes):
|
||||
chunk = chunk.decode("utf-8", errors="ignore")
|
||||
|
||||
if isinstance(chunk, str):
|
||||
chunk = chunk.strip()
|
||||
if not chunk or chunk == "[DONE]":
|
||||
continue
|
||||
if chunk.startswith("data: "):
|
||||
chunk = chunk[6:]
|
||||
try:
|
||||
message = json.loads(chunk)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
elif isinstance(chunk, dict):
|
||||
message = chunk
|
||||
else:
|
||||
continue
|
||||
|
||||
gigachat_iterator = GigaChatModelResponseIterator(
|
||||
streaming_response=None,
|
||||
sync_stream=False,
|
||||
)
|
||||
translated_chunk = gigachat_iterator.chunk_parser(chunk=message)
|
||||
|
||||
if isinstance(
|
||||
translated_chunk, dict
|
||||
) and generic_chunk_has_all_required_fields(cast(dict, translated_chunk)):
|
||||
chunk_obj = convert_generic_chunk_to_model_response_stream(
|
||||
cast(GenericStreamingChunk, translated_chunk)
|
||||
)
|
||||
elif isinstance(translated_chunk, ModelResponseStream):
|
||||
chunk_obj = translated_chunk
|
||||
else:
|
||||
continue
|
||||
|
||||
all_translated_chunks.append(chunk_obj)
|
||||
|
||||
if len(all_translated_chunks) > 0:
|
||||
model_response = stream_chunk_builder(
|
||||
chunks=all_translated_chunks,
|
||||
logging_obj=litellm_logging_obj,
|
||||
)
|
||||
return model_response
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def get_api_base(api_base: Optional[str] = None) -> Optional[str]:
|
||||
return get_api_base(api_base)
|
||||
|
||||
@staticmethod
|
||||
def get_api_key(
|
||||
api_key: Optional[str] = None,
|
||||
) -> Optional[str]:
|
||||
return api_key or get_secret_str("GIGACHAT_API_KEY")
|
||||
|
||||
@staticmethod
|
||||
def get_base_model(model: str) -> Optional[str]:
|
||||
return model
|
||||
|
||||
def get_models(
|
||||
self, api_key: Optional[str] = None, api_base: Optional[str] = None
|
||||
) -> List[str]:
|
||||
return super().get_models(api_key, api_base)
|
||||
|
|
@ -1590,6 +1590,9 @@ def completion( # type: ignore # noqa: PLR0915
|
|||
litellm_request_debug=kwargs.get("litellm_request_debug", False),
|
||||
tpm=kwargs.get("tpm"),
|
||||
rpm=kwargs.get("rpm"),
|
||||
gigachat_scope=kwargs.get("gigachat_scope"),
|
||||
gigachat_auth_url=kwargs.get("gigachat_auth_url"),
|
||||
gigachat_access_token=kwargs.get("gigachat_access_token"),
|
||||
)
|
||||
cast(LiteLLMLoggingObj, logging).update_environment_variables(
|
||||
model=model,
|
||||
|
|
|
|||
|
|
@ -405,6 +405,7 @@ class LiteLLMRoutes(enum.Enum):
|
|||
"/vllm",
|
||||
"/mistral",
|
||||
"/milvus",
|
||||
"/gigachat",
|
||||
]
|
||||
|
||||
#########################################################
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ from litellm.constants import (
|
|||
ALLOWED_VERTEX_AI_PASSTHROUGH_HEADERS,
|
||||
BEDROCK_AGENT_RUNTIME_PASS_THROUGH_ROUTES,
|
||||
)
|
||||
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
|
||||
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
|
||||
from litellm.proxy._types import *
|
||||
|
|
@ -2369,3 +2370,244 @@ def create_generic_websocket_passthrough_endpoint(
|
|||
_forward_headers=forward_headers,
|
||||
cost_per_request=cost_per_request,
|
||||
)
|
||||
|
||||
|
||||
@router.api_route(
|
||||
"/gigachat/{endpoint:path}",
|
||||
methods=["GET", "POST", "PUT", "DELETE", "PATCH"],
|
||||
tags=["Gigachat Pass-through", "pass-through"],
|
||||
)
|
||||
async def gigachat_proxy_route(
|
||||
endpoint: str,
|
||||
request: Request,
|
||||
fastapi_response: Response,
|
||||
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
|
||||
):
|
||||
"""
|
||||
[Docs](https://docs.litellm.ai/docs/pass_through/gigachat)
|
||||
"""
|
||||
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
|
||||
from litellm.proxy.proxy_server import (
|
||||
general_settings,
|
||||
llm_router,
|
||||
proxy_config,
|
||||
proxy_logging_obj,
|
||||
select_data_generator,
|
||||
user_api_base,
|
||||
user_max_tokens,
|
||||
user_model,
|
||||
user_request_timeout,
|
||||
user_temperature,
|
||||
version,
|
||||
)
|
||||
|
||||
## check for streaming
|
||||
request_body = await get_request_body(request)
|
||||
is_router_model = is_passthrough_request_using_router_model(
|
||||
request_body, llm_router
|
||||
)
|
||||
|
||||
model = request_body.get("model")
|
||||
if not model:
|
||||
msg = "Model is required"
|
||||
raise ValueError(msg)
|
||||
|
||||
# If router model, use dedicated router passthrough handler
|
||||
# This uses the same common processing path as non-router models
|
||||
if is_router_model and llm_router:
|
||||
return await handle_gigachat_passthrough_router_model(
|
||||
model=model,
|
||||
endpoint=endpoint,
|
||||
request=request,
|
||||
request_body=request_body,
|
||||
llm_router=llm_router,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
general_settings=general_settings,
|
||||
proxy_config=proxy_config,
|
||||
select_data_generator=select_data_generator,
|
||||
user_model=user_model,
|
||||
user_temperature=user_temperature,
|
||||
user_request_timeout=user_request_timeout,
|
||||
user_max_tokens=user_max_tokens,
|
||||
user_api_base=user_api_base,
|
||||
version=version,
|
||||
)
|
||||
|
||||
# Fall back to existing implementation for direct GigaChat models
|
||||
verbose_proxy_logger.debug(
|
||||
f"Gigachat passthrough: Using direct Gigachat model '{model}' for endpoint '{endpoint}'"
|
||||
)
|
||||
|
||||
data: Dict[str, Any] = {}
|
||||
|
||||
data["method"] = request.method
|
||||
data["endpoint"] = endpoint
|
||||
data["json"] = request_body
|
||||
data["custom_llm_provider"] = "gigachat"
|
||||
|
||||
client = get_async_httpx_client( # type: ignore
|
||||
llm_provider=LlmProviders.GIGACHAT,
|
||||
params={
|
||||
"timeout": httpx.Timeout(timeout=600.0, connect=5.0),
|
||||
"ssl_verify": False,
|
||||
},
|
||||
)
|
||||
data["http_client"] = client
|
||||
|
||||
base_llm_response_processor = ProxyBaseLLMRequestProcessing(data=data)
|
||||
|
||||
try:
|
||||
result = await base_llm_response_processor.base_passthrough_process_llm_request(
|
||||
request=request,
|
||||
fastapi_response=fastapi_response,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
llm_router=llm_router,
|
||||
general_settings=general_settings,
|
||||
proxy_config=proxy_config,
|
||||
select_data_generator=select_data_generator,
|
||||
model=model,
|
||||
user_model=user_model,
|
||||
user_temperature=user_temperature,
|
||||
user_request_timeout=user_request_timeout,
|
||||
user_max_tokens=user_max_tokens,
|
||||
user_api_base=user_api_base,
|
||||
version=version,
|
||||
)
|
||||
|
||||
return result
|
||||
except Exception as e:
|
||||
raise await base_llm_response_processor._handle_llm_api_exception(
|
||||
e=e,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
)
|
||||
|
||||
|
||||
async def handle_gigachat_passthrough_router_model(
|
||||
model: str,
|
||||
endpoint: str,
|
||||
request: Request,
|
||||
request_body: dict,
|
||||
llm_router: litellm.Router,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
proxy_logging_obj,
|
||||
general_settings: dict,
|
||||
proxy_config,
|
||||
select_data_generator,
|
||||
user_model: Optional[str],
|
||||
user_temperature: Optional[float],
|
||||
user_request_timeout: Optional[float],
|
||||
user_max_tokens: Optional[int],
|
||||
user_api_base: Optional[str],
|
||||
version: Optional[str],
|
||||
) -> Union[Response, StreamingResponse]:
|
||||
"""
|
||||
Handle Gigachat passthrough for router models (models defined in config.yaml).
|
||||
|
||||
Uses the same common processing path as non-router models to ensure
|
||||
metadata and hooks are properly initialized.
|
||||
|
||||
Args:
|
||||
model: The router model name (e.g., "gigachat/gigachat-2")
|
||||
endpoint: The Gigachat endpoint path (e.g., "/chat/completions")
|
||||
request: The FastAPI request object
|
||||
request_body: The parsed request body
|
||||
llm_router: The LiteLLM router instance
|
||||
user_api_key_dict: The user API key authentication dictionary
|
||||
(additional args for common processing)
|
||||
|
||||
Returns:
|
||||
Response or StreamingResponse depending on endpoint type
|
||||
"""
|
||||
from fastapi import Response as FastAPIResponse
|
||||
|
||||
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
|
||||
|
||||
# Detect streaming based on request body
|
||||
is_streaming = request_body.get("stream", False)
|
||||
|
||||
data: Dict[str, Any] = await _read_request_body(request=request)
|
||||
if user_api_key_dict is not None:
|
||||
if data.get("metadata") is None:
|
||||
data["metadata"] = {}
|
||||
if (
|
||||
hasattr(user_api_key_dict, "user_id")
|
||||
and user_api_key_dict.user_id is not None
|
||||
):
|
||||
data["metadata"]["user_api_key_user_id"] = user_api_key_dict.user_id
|
||||
if (
|
||||
hasattr(user_api_key_dict, "team_id")
|
||||
and user_api_key_dict.team_id is not None
|
||||
):
|
||||
data["metadata"]["user_api_key_team_id"] = user_api_key_dict.team_id
|
||||
if (
|
||||
hasattr(user_api_key_dict, "org_id")
|
||||
and user_api_key_dict.org_id is not None
|
||||
):
|
||||
data["metadata"]["user_api_key_org_id"] = user_api_key_dict.org_id
|
||||
if (
|
||||
hasattr(user_api_key_dict, "agent_id")
|
||||
and user_api_key_dict.agent_id is not None
|
||||
):
|
||||
data["metadata"]["agent_id"] = user_api_key_dict.agent_id
|
||||
|
||||
verbose_proxy_logger.debug(
|
||||
f"Gigachat router passthrough: model='{model}', endpoint='{endpoint}', streaming={is_streaming}"
|
||||
)
|
||||
|
||||
# Use the common processing path (same as non-router models)
|
||||
# This ensures all metadata, hooks, and logging are properly initialized
|
||||
|
||||
data["model"] = model
|
||||
data["method"] = request.method
|
||||
data["endpoint"] = endpoint
|
||||
data["json"] = request_body
|
||||
data["custom_llm_provider"] = "gigachat"
|
||||
|
||||
client = get_async_httpx_client( # type: ignore
|
||||
llm_provider=LlmProviders.GIGACHAT,
|
||||
params={
|
||||
"timeout": httpx.Timeout(timeout=600.0, connect=5.0),
|
||||
"ssl_verify": False,
|
||||
},
|
||||
)
|
||||
|
||||
data["http_client"] = client
|
||||
base_llm_response_processor = ProxyBaseLLMRequestProcessing(data=data)
|
||||
|
||||
# Use the common passthrough processing to handle metadata and hooks
|
||||
# This also handles all response formatting (streaming/non-streaming) and exceptions
|
||||
try:
|
||||
result = await base_llm_response_processor.base_passthrough_process_llm_request(
|
||||
request=request,
|
||||
fastapi_response=FastAPIResponse(),
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
llm_router=llm_router,
|
||||
general_settings=general_settings,
|
||||
proxy_config=proxy_config,
|
||||
select_data_generator=select_data_generator,
|
||||
model=model,
|
||||
user_model=user_model,
|
||||
user_temperature=user_temperature,
|
||||
user_request_timeout=user_request_timeout,
|
||||
user_max_tokens=user_max_tokens,
|
||||
user_api_base=user_api_base,
|
||||
version=version,
|
||||
)
|
||||
|
||||
if isinstance(result, StreamingResponse):
|
||||
if result.headers.get("Content-Type") is None:
|
||||
result.headers["Content-Type"] = "text/event-stream; charset=utf-8"
|
||||
return result
|
||||
|
||||
return result
|
||||
except Exception as e:
|
||||
# Use common exception handling
|
||||
raise await base_llm_response_processor._handle_llm_api_exception(
|
||||
e=e,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1172,6 +1172,68 @@
|
|||
],
|
||||
"default_model_placeholder": "gpt-3.5-turbo"
|
||||
},
|
||||
{
|
||||
"provider": "GIGACHAT",
|
||||
"provider_display_name": "GigaChat",
|
||||
"litellm_provider": "gigachat",
|
||||
"credential_fields": [
|
||||
{
|
||||
"key": "api_base",
|
||||
"label": "API Base",
|
||||
"placeholder": null,
|
||||
"tooltip": null,
|
||||
"required": false,
|
||||
"field_type": "text",
|
||||
"options": null,
|
||||
"default_value": null
|
||||
},
|
||||
{
|
||||
"key": "api_key",
|
||||
"label": "API Key",
|
||||
"placeholder": null,
|
||||
"tooltip": null,
|
||||
"required": false,
|
||||
"field_type": "password",
|
||||
"options": null,
|
||||
"default_value": null
|
||||
},
|
||||
{
|
||||
"key": "gigachat_scope",
|
||||
"label": "Scope",
|
||||
"placeholder": null,
|
||||
"tooltip": null,
|
||||
"required": false,
|
||||
"field_type": "select",
|
||||
"options": [
|
||||
"GIGACHAT_API_PERS",
|
||||
"GIGACHAT_API_B2B",
|
||||
"GIGACHAT_API_CORP"
|
||||
],
|
||||
"default_value": "GIGACHAT_API_PERS"
|
||||
},
|
||||
{
|
||||
"key": "gigachat_auth_url",
|
||||
"label": "Auth URL",
|
||||
"placeholder": null,
|
||||
"tooltip": null,
|
||||
"required": false,
|
||||
"field_type": "text",
|
||||
"options": null,
|
||||
"default_value": null
|
||||
},
|
||||
{
|
||||
"key": "gigachat_access_token",
|
||||
"label": "Access token",
|
||||
"placeholder": null,
|
||||
"tooltip": "Disable OAuth, provide value to authorization.",
|
||||
"required": false,
|
||||
"field_type": "password",
|
||||
"options": null,
|
||||
"default_value": null
|
||||
}
|
||||
],
|
||||
"default_model_placeholder": "GigaChat-2"
|
||||
},
|
||||
{
|
||||
"provider": "GITHUB",
|
||||
"provider_display_name": "Github",
|
||||
|
|
|
|||
|
|
@ -8675,6 +8675,12 @@ class ProviderConfigManager:
|
|||
)
|
||||
|
||||
return AzurePassthroughConfig()
|
||||
elif LlmProviders.GIGACHAT == provider:
|
||||
from litellm.llms.gigachat.passthrough.transformation import (
|
||||
GigaChatPassthroughConfig,
|
||||
)
|
||||
|
||||
return GigaChatPassthroughConfig()
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ from unittest.mock import AsyncMock, MagicMock, Mock, patch
|
|||
import httpx
|
||||
import pytest
|
||||
from fastapi import Request, Response
|
||||
from fastapi.responses import StreamingResponse
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
sys.path.insert(
|
||||
|
|
@ -15,12 +16,14 @@ sys.path.insert(
|
|||
) # Adds the parent directory to the system path
|
||||
|
||||
import litellm
|
||||
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
|
||||
from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
|
||||
BaseOpenAIPassThroughHandler,
|
||||
RouteChecks,
|
||||
bedrock_llm_proxy_route,
|
||||
create_pass_through_route,
|
||||
cursor_proxy_route,
|
||||
gigachat_proxy_route,
|
||||
llm_passthrough_factory_proxy_route,
|
||||
milvus_proxy_route,
|
||||
openai_proxy_route,
|
||||
|
|
@ -1466,6 +1469,176 @@ class TestVLLMProxyRoute:
|
|||
mock_factory_route.assert_awaited_once()
|
||||
|
||||
|
||||
class TestGigachatProxyRoute:
|
||||
@pytest.mark.asyncio
|
||||
@patch(
|
||||
"litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.get_request_body",
|
||||
return_value={"model": "router-model", "stream": False},
|
||||
)
|
||||
@patch(
|
||||
"litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.is_passthrough_request_using_router_model",
|
||||
return_value=True,
|
||||
)
|
||||
@patch("litellm.proxy.proxy_server.llm_router")
|
||||
async def test_gigachat_proxy_route_with_router_model(
|
||||
self, mock_llm_router, mock_is_router, mock_get_body
|
||||
):
|
||||
mock_request = MagicMock(spec=Request)
|
||||
mock_request.method = "POST"
|
||||
mock_request.headers = {"content-type": "application/json"}
|
||||
mock_request.query_params = {}
|
||||
mock_fastapi_response = MagicMock(spec=Response)
|
||||
mock_user_api_key_dict = MagicMock()
|
||||
mock_llm_router.allm_passthrough_route = AsyncMock(
|
||||
return_value=httpx.Response(200, json={"response": "success"})
|
||||
)
|
||||
|
||||
result = await gigachat_proxy_route(
|
||||
endpoint="/chat/completions",
|
||||
request=mock_request,
|
||||
fastapi_response=mock_fastapi_response,
|
||||
user_api_key_dict=mock_user_api_key_dict,
|
||||
)
|
||||
|
||||
mock_is_router.assert_called_once()
|
||||
mock_llm_router.allm_passthrough_route.assert_awaited_once()
|
||||
assert isinstance(result, Response)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch(
|
||||
"litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.get_request_body",
|
||||
return_value={"model": "other-model"},
|
||||
)
|
||||
@patch(
|
||||
"litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.is_passthrough_request_using_router_model",
|
||||
return_value=False,
|
||||
)
|
||||
@patch(
|
||||
"litellm.proxy.common_request_processing.ProxyBaseLLMRequestProcessing.base_passthrough_process_llm_request",
|
||||
new_callable=AsyncMock,
|
||||
)
|
||||
async def test_gigachat_proxy_route_fallback_to_http_pass_through(
|
||||
self,
|
||||
mock_base_passthrough,
|
||||
mock_is_router,
|
||||
mock_get_body,
|
||||
):
|
||||
mock_request = MagicMock(spec=Request)
|
||||
mock_fastapi_response = MagicMock(spec=Response)
|
||||
mock_user_api_key_dict = MagicMock()
|
||||
|
||||
expected_response = Response(
|
||||
content=b'{"response": "success"}',
|
||||
status_code=200,
|
||||
media_type="application/json",
|
||||
)
|
||||
mock_base_passthrough.return_value = expected_response
|
||||
|
||||
result = await gigachat_proxy_route(
|
||||
endpoint="/chat/completions",
|
||||
request=mock_request,
|
||||
fastapi_response=mock_fastapi_response,
|
||||
user_api_key_dict=mock_user_api_key_dict,
|
||||
)
|
||||
|
||||
assert isinstance(result, Response)
|
||||
assert result.status_code == 200
|
||||
assert result.body == b'{"response": "success"}'
|
||||
mock_base_passthrough.assert_awaited_once()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_allm_passthrough_streaming_preserves_upstream_headers(self):
|
||||
async def _stream() -> bytes:
|
||||
yield b'data: {"id":"1"}\n\n'
|
||||
|
||||
class MockPassthroughStreamingResponse:
|
||||
def __init__(self):
|
||||
self.status_code = 201
|
||||
self.headers = {
|
||||
"content-type": "text/event-stream; charset=utf-8",
|
||||
"x-request-id": "req-123",
|
||||
"x-ratelimit-remaining-requests": "77",
|
||||
"transfer-encoding": "chunked",
|
||||
"content-encoding": "gzip",
|
||||
}
|
||||
self._iterator = _stream()
|
||||
|
||||
def __aiter__(self):
|
||||
return self
|
||||
|
||||
async def __anext__(self):
|
||||
return await self._iterator.__anext__()
|
||||
|
||||
processor = ProxyBaseLLMRequestProcessing(
|
||||
data={
|
||||
"model": "some-provider/model",
|
||||
"stream": True,
|
||||
"litellm_call_id": "call-123",
|
||||
"litellm_logging_obj": MagicMock(litellm_call_id="call-123"),
|
||||
}
|
||||
)
|
||||
|
||||
mock_request = MagicMock(spec=Request)
|
||||
mock_request.headers = {"content-type": "application/json"}
|
||||
mock_fastapi_response = MagicMock(spec=Response)
|
||||
mock_user_api_key_dict = MagicMock()
|
||||
mock_user_api_key_dict.allowed_model_region = ""
|
||||
mock_user_api_key_dict.spend = 0.0
|
||||
mock_proxy_logging_obj = MagicMock()
|
||||
mock_proxy_logging_obj.during_call_hook = AsyncMock(return_value=None)
|
||||
mock_proxy_logging_obj.update_request_status = AsyncMock(return_value=None)
|
||||
mock_proxy_logging_obj.post_call_response_headers_hook = AsyncMock(
|
||||
return_value={"x-test-callback-header": "callback-value"}
|
||||
)
|
||||
|
||||
streaming_response = MockPassthroughStreamingResponse()
|
||||
|
||||
async def _fake_route_request(*args, **kwargs):
|
||||
async def _inner():
|
||||
return streaming_response
|
||||
|
||||
return _inner()
|
||||
|
||||
with patch.object(
|
||||
processor,
|
||||
"common_processing_pre_call_logic",
|
||||
new=AsyncMock(
|
||||
return_value=(
|
||||
processor.data,
|
||||
processor.data["litellm_logging_obj"],
|
||||
)
|
||||
),
|
||||
), patch(
|
||||
"litellm.proxy.common_request_processing.route_request",
|
||||
new=_fake_route_request,
|
||||
), patch(
|
||||
"litellm.proxy.common_request_processing.ProxyBaseLLMRequestProcessing.get_custom_headers",
|
||||
return_value={"x-litellm-call-id": "call-123"},
|
||||
):
|
||||
result = await processor.base_passthrough_process_llm_request(
|
||||
request=mock_request,
|
||||
fastapi_response=mock_fastapi_response,
|
||||
user_api_key_dict=mock_user_api_key_dict,
|
||||
proxy_logging_obj=mock_proxy_logging_obj,
|
||||
general_settings={},
|
||||
proxy_config=MagicMock(),
|
||||
select_data_generator=MagicMock(),
|
||||
llm_router=None,
|
||||
model="some-provider/model",
|
||||
version="test-version",
|
||||
)
|
||||
|
||||
assert isinstance(result, StreamingResponse)
|
||||
assert result.status_code == 201
|
||||
assert result.headers["content-type"] == "text/event-stream; charset=utf-8"
|
||||
assert result.headers["x-request-id"] == "req-123"
|
||||
assert result.headers["x-ratelimit-remaining-requests"] == "77"
|
||||
assert result.headers["x-litellm-call-id"] == "call-123"
|
||||
assert result.headers["x-test-callback-header"] == "callback-value"
|
||||
assert "transfer-encoding" not in result.headers
|
||||
assert "content-encoding" not in result.headers
|
||||
|
||||
|
||||
class TestForwardHeaders:
|
||||
"""
|
||||
Test cases for _forward_headers parameter in passthrough endpoints
|
||||
|
|
|
|||
27
ui/litellm-dashboard/public/assets/logos/gigachat.svg
Normal file
27
ui/litellm-dashboard/public/assets/logos/gigachat.svg
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<!-- Generator: Adobe Illustrator 28.0.0, SVG Export Plug-In . SVG Version: 6.00 Build 0) -->
|
||||
<svg version="1.1" id="Слой_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
|
||||
viewBox="0 0 1000 1000" enable-background="new 0 0 1000 1000" xml:space="preserve">
|
||||
<path fill-rule="evenodd" clip-rule="evenodd" fill="#000001" d="M500,0c276.1218262,0,500,223.8403168,500,500
|
||||
c0,276.1593628-223.8781738,500-500,500C223.8771973,1000,0,776.1218262,0,500C0.0000314,223.8771973,223.8771973,0.0000314,500,0z
|
||||
M911.703125,378.8250122c-12.6343384,87.6312561-57.1812744,159.9468384-94.3469238,207.0031128
|
||||
c-57.359314,70.65625-134.203064,131.828125-222.3842773,177.03125
|
||||
c-87.2250366,44.7218628-181.578186,71.8875122-272.8406677,78.5562744
|
||||
c-15.4475098,0.890625-29.7947083,1.3311768-43.2412415,1.3311768c-16.0025024,0-30.7562714-0.640625-44.759079-1.8999023
|
||||
c73.1806183,57.3218384,165.3878326,91.499939,265.5440674,91.499939l-0.0562744-0.0562744
|
||||
c238.0687561,0,431.1000061-192.999939,431.1000061-431.0686951c0-41.3156433-5.828125-81.2562561-16.6843872-119.0875244
|
||||
C913.2562256,381.0218811,912.4812622,379.8999939,911.703125,378.8250122z M561.3343506,45.6000023
|
||||
c-147.2843323,0-281.1668701,60.73843-358.1281128,162.4078064l-0.4881287,0.5968781
|
||||
c-33.2709351,40.5328217-55.3865662,84.9996796-63.910614,128.6078033
|
||||
c-4.8540802,26.53125-3.9568939,50.8968811,2.7127991,72.1312561v-0.0562439
|
||||
c15.8215637,47.5031128,54.3062439,85.5968628,97.9543762,96.7875061
|
||||
c18.1562653,4.5218506,35.6100006,6.4124756,51.8393707,5.671875l4.2859497-0.3812256
|
||||
C405.9468689,502.4718628,504.34375,432.9312439,586.046875,366.046875
|
||||
c51.1312256-42.7937622,100.0405884-99.4146729,103.1906128-135.9850006
|
||||
C617.9468384,264.6322021,544.3500366,309.6121826,459.5,370.4156189
|
||||
c-9.4125061,6.7062378-22.2406311,4.828125-29.2437439-4.2875061
|
||||
c-42.1281128-54.5405884-78.8125305-98.8321838-115.53125-139.2943726
|
||||
c-3.9968872-4.4449921-5.9906311-10.4062347-5.3987427-16.3302917c0.555603-5.9284515,3.6299744-11.4168854,8.4081116-14.9740753
|
||||
C430.2281189,111.1612473,542.125,63.3568764,650.6875,53.2768745
|
||||
C621.5718994,48.1650009,591.7124634,45.6006279,561.3343506,45.6000023z"/>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 2.2 KiB |
|
|
@ -40,6 +40,7 @@ export enum Providers {
|
|||
FireworksAI = "Fireworks AI",
|
||||
FRIENDLIAI = "Friendliai",
|
||||
GALADRIEL = "Galadriel",
|
||||
GIGACHAT = "GigaChat",
|
||||
GITHUB_COPILOT = "Github Copilot",
|
||||
Google_AI_Studio = "Google AI Studio",
|
||||
GradientAI = "GradientAI",
|
||||
|
|
@ -146,6 +147,7 @@ export const provider_map: Record<string, string> = {
|
|||
FireworksAI: "fireworks_ai",
|
||||
FRIENDLIAI: "friendliai",
|
||||
GALADRIEL: "galadriel",
|
||||
GIGACHAT: "gigachat",
|
||||
GITHUB_COPILOT: "github_copilot",
|
||||
Google_AI_Studio: "gemini",
|
||||
GradientAI: "gradient_ai",
|
||||
|
|
@ -247,6 +249,7 @@ export const providerLogoMap: Record<string, string> = {
|
|||
[Providers.FEATHERLESS_AI]: `${asset_logos_folder}featherless.svg`,
|
||||
[Providers.FireworksAI]: `${asset_logos_folder}fireworks.svg`,
|
||||
[Providers.FRIENDLIAI]: `${asset_logos_folder}friendli.svg`,
|
||||
[Providers.GIGACHAT]: `${asset_logos_folder}gigachat.svg`,
|
||||
[Providers.GITHUB_COPILOT]: `${asset_logos_folder}github_copilot.svg`,
|
||||
[Providers.Google_AI_Studio]: `${asset_logos_folder}google.svg`,
|
||||
[Providers.GradientAI]: `${asset_logos_folder}gradientai.svg`,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue