mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-13 08:52:02 +00:00
Merge remote-tracking branch 'origin/litellm_internal_staging' into litellm_yj_apr16
This commit is contained in:
Yuneng Jiang
commit
246a5bd158
25 changed files with 606 additions and 179 deletions
|
|
@ -9,6 +9,10 @@ import TabItem from '@theme/TabItem';
|
|||
import NavigationCards from '@site/src/components/NavigationCards';
|
||||
import Image from '@theme/IdealImage';
|
||||
|
||||
:::note Security Update
|
||||
The Trivy supply-chain compromise has been contained :tada: . All affected packages have been deleted and current releases are free of the compromised code/component. Please refer to our [Security Townhall](/blog/security-townhall-updates) for a deeper understanding of the problem, and [CI/CD v2](/blog/ci-cd-v2-improvements) for how we're improving moving forward.
|
||||
:::
|
||||
|
||||
<Image style={{padding: '10px', margin: '0 0 2.5rem'}} img={require('../img/hero.png')} />
|
||||
|
||||
**LiteLLM** is an open-source library that gives you a single, unified interface to call 100+ LLMs — OpenAI, Anthropic, Vertex AI, Bedrock, and more — using the OpenAI format.
|
||||
|
|
|
|||
|
|
@ -192,6 +192,13 @@ export GITHUB_COPILOT_ACCESS_TOKEN_FILE="access-token"
|
|||
|
||||
# Optional: Custom API key file name
|
||||
export GITHUB_COPILOT_API_KEY_FILE="api-key.json"
|
||||
|
||||
# Optional: Custom Copilot endpoints for authentication and usage
|
||||
# (needed when using GitHub Enterprise subscriptions with custom endpoints or self-hosted GitHub servers
|
||||
export GITHUB_COPILOT_API_BASE="https://copilot-api.my-company.ghe.com"
|
||||
export GITHUB_COPILOT_DEVICE_CODE_URL="https://my-company.ghe.com/login/device/code"
|
||||
export GITHUB_COPILOT_ACCESS_TOKEN_URL="https://my-company.ghe.com/login/oauth/access_token"
|
||||
export GITHUB_COPILOT_API_KEY_URL="https://my-company.ghe.com/api/v3/copilot_internal/v2/token"
|
||||
```
|
||||
|
||||
### Headers
|
||||
|
|
|
|||
|
|
@ -720,6 +720,11 @@ router_settings:
|
|||
| GITHUB_COPILOT_TOKEN_DIR | Directory to store GitHub Copilot token for `github_copilot` llm provider
|
||||
| GITHUB_COPILOT_API_KEY_FILE | File to store GitHub Copilot API key for `github_copilot` llm provider
|
||||
| GITHUB_COPILOT_ACCESS_TOKEN_FILE | File to store GitHub Copilot access token for `github_copilot` llm provider
|
||||
| GITHUB_COPILOT_API_BASE | Base URL for GitHub Copilot API. For GitHub Enterprise subscriptions with custom host, it is similar to https://copilot-api.my-company.ghe.com. Default is https://api.githubcopilot.com
|
||||
| GITHUB_COPILOT_DEVICE_CODE_URL | URL for GitHub Copilot device code authentication. For GitHub Enterprise subscriptions with custom host, it is similar to https://my-company.ghe.com/login/device/code. Default is https://github.com/login/device/code
|
||||
| GITHUB_COPILOT_ACCESS_TOKEN_URL | URL for GitHub Copilot access token retrieval. For GitHub Enterprise subscriptions with custom host, it is similar to https://my-company.ghe.com/login/oauth/access_token. Default is https://github.com/login/oauth/access_token
|
||||
| GITHUB_COPILOT_API_KEY_URL | URL for GitHub Copilot API key retrieval. For GitHub Enterprise subscriptions with custom host, it is similar to https://my-company.ghe.com/api/v3/copilot_internal/v2/token. Default is https://api.github.com/copilot_internal/v2/token
|
||||
| GITHUB_COPILOT_CLIENT_ID | Client ID for GitHub Copilot device flow authentication. This is used by the `github_copilot` provider for device code authentication. Default is "Iv1.b507a08c87ecfe98"
|
||||
| GREENSCALE_API_KEY | API key for Greenscale service
|
||||
| GREENSCALE_ENDPOINT | Endpoint URL for Greenscale service
|
||||
| GRAYSWAN_API_BASE | Base URL for GraySwan API. Default is https://api.grayswan.ai
|
||||
|
|
|
|||
|
|
@ -1,6 +1,10 @@
|
|||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
|
||||
:::note Security Update
|
||||
The Trivy supply-chain compromise has been contained :tada: . All affected packages have been deleted and current releases are free of the compromised code/component. Please refer to our [Security Townhall](/blog/security-townhall-updates) for a deeper understanding of the problem, and [CI/CD v2](/blog/ci-cd-v2-improvements) for how we're improving moving forward.
|
||||
:::
|
||||
|
||||
# LiteLLM - Getting Started
|
||||
|
||||
https://github.com/BerriAI/litellm
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[project]
|
||||
name = "litellm-proxy-extras"
|
||||
version = "0.4.65"
|
||||
version = "0.4.66"
|
||||
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.9"
|
||||
|
|
@ -25,7 +25,7 @@ required-version = "==0.10.9"
|
|||
module-root = ""
|
||||
|
||||
[tool.commitizen]
|
||||
version = "0.4.65"
|
||||
version = "0.4.66"
|
||||
version_files = [
|
||||
"pyproject.toml:^version",
|
||||
"../pyproject.toml:litellm-proxy-extras==",
|
||||
|
|
|
|||
|
|
@ -17,11 +17,11 @@ from .common_utils import (
|
|||
RefreshAPIKeyError,
|
||||
)
|
||||
|
||||
# Constants
|
||||
GITHUB_CLIENT_ID = "Iv1.b507a08c87ecfe98"
|
||||
GITHUB_DEVICE_CODE_URL = "https://github.com/login/device/code"
|
||||
GITHUB_ACCESS_TOKEN_URL = "https://github.com/login/oauth/access_token"
|
||||
GITHUB_API_KEY_URL = "https://api.github.com/copilot_internal/v2/token"
|
||||
# Constants (default values — overridable via environment variables at call time)
|
||||
DEFAULT_GITHUB_CLIENT_ID = "Iv1.b507a08c87ecfe98"
|
||||
DEFAULT_GITHUB_DEVICE_CODE_URL = "https://github.com/login/device/code"
|
||||
DEFAULT_GITHUB_ACCESS_TOKEN_URL = "https://github.com/login/oauth/access_token"
|
||||
DEFAULT_GITHUB_API_KEY_URL = "https://api.github.com/copilot_internal/v2/token"
|
||||
|
||||
|
||||
class Authenticator:
|
||||
|
|
@ -161,12 +161,15 @@ class Authenticator:
|
|||
"""
|
||||
access_token = self.get_access_token()
|
||||
headers = self._get_github_headers(access_token)
|
||||
api_key_url = os.getenv(
|
||||
"GITHUB_COPILOT_API_KEY_URL", DEFAULT_GITHUB_API_KEY_URL
|
||||
)
|
||||
|
||||
max_retries = 3
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
sync_client = _get_httpx_client()
|
||||
response = sync_client.get(GITHUB_API_KEY_URL, headers=headers)
|
||||
response = sync_client.get(api_key_url, headers=headers)
|
||||
response.raise_for_status()
|
||||
|
||||
response_json = response.json()
|
||||
|
|
@ -232,10 +235,14 @@ class Authenticator:
|
|||
"""
|
||||
try:
|
||||
sync_client = _get_httpx_client()
|
||||
device_code_url = os.getenv(
|
||||
"GITHUB_COPILOT_DEVICE_CODE_URL", DEFAULT_GITHUB_DEVICE_CODE_URL
|
||||
)
|
||||
client_id = os.getenv("GITHUB_COPILOT_CLIENT_ID", DEFAULT_GITHUB_CLIENT_ID)
|
||||
resp = sync_client.post(
|
||||
GITHUB_DEVICE_CODE_URL,
|
||||
device_code_url,
|
||||
headers=self._get_github_headers(),
|
||||
json={"client_id": GITHUB_CLIENT_ID, "scope": "read:user"},
|
||||
json={"client_id": client_id, "scope": "read:user"},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
resp_json = resp.json()
|
||||
|
|
@ -284,13 +291,20 @@ class Authenticator:
|
|||
sync_client = _get_httpx_client()
|
||||
max_attempts = 12 # 1 minute (12 * 5 seconds)
|
||||
|
||||
access_token_url = os.getenv(
|
||||
"GITHUB_COPILOT_ACCESS_TOKEN_URL", DEFAULT_GITHUB_ACCESS_TOKEN_URL
|
||||
)
|
||||
client_id = os.getenv(
|
||||
"GITHUB_COPILOT_CLIENT_ID", DEFAULT_GITHUB_CLIENT_ID
|
||||
)
|
||||
|
||||
for attempt in range(max_attempts):
|
||||
try:
|
||||
resp = sync_client.post(
|
||||
GITHUB_ACCESS_TOKEN_URL,
|
||||
access_token_url,
|
||||
headers=self._get_github_headers(),
|
||||
json={
|
||||
"client_id": GITHUB_CLIENT_ID,
|
||||
"client_id": client_id,
|
||||
"device_code": device_code,
|
||||
"grant_type": "urn:ietf:params:oauth:grant-type:device_code",
|
||||
},
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
from typing import List, Optional, Tuple
|
||||
|
||||
import os
|
||||
|
||||
from litellm.exceptions import AuthenticationError
|
||||
from litellm.llms.openai.openai import OpenAIConfig
|
||||
|
|
@ -7,7 +8,7 @@ from litellm.types.llms.openai import AllMessageValues
|
|||
|
||||
from ..authenticator import Authenticator
|
||||
from ..common_utils import (
|
||||
GITHUB_COPILOT_API_BASE,
|
||||
DEFAULT_GITHUB_COPILOT_API_BASE,
|
||||
GetAPIKeyError,
|
||||
get_copilot_default_headers,
|
||||
)
|
||||
|
|
@ -30,7 +31,12 @@ class GithubCopilotConfig(OpenAIConfig):
|
|||
api_key: Optional[str],
|
||||
custom_llm_provider: str,
|
||||
) -> Tuple[Optional[str], Optional[str], str]:
|
||||
dynamic_api_base = self.authenticator.get_api_base() or GITHUB_COPILOT_API_BASE
|
||||
dynamic_api_base = (
|
||||
api_base
|
||||
or self.authenticator.get_api_base()
|
||||
or os.getenv("GITHUB_COPILOT_API_BASE")
|
||||
or DEFAULT_GITHUB_COPILOT_API_BASE
|
||||
)
|
||||
try:
|
||||
dynamic_api_key = self.authenticator.get_api_key()
|
||||
except GetAPIKeyError as e:
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
"""
|
||||
Constants for Copilot integration
|
||||
"""
|
||||
|
||||
from typing import Optional, Union
|
||||
from uuid import uuid4
|
||||
|
||||
|
|
@ -13,7 +14,7 @@ COPILOT_VERSION = "0.26.7"
|
|||
EDITOR_PLUGIN_VERSION = f"copilot-chat/{COPILOT_VERSION}"
|
||||
USER_AGENT = f"GitHubCopilotChat/{COPILOT_VERSION}"
|
||||
API_VERSION = "2025-04-01"
|
||||
GITHUB_COPILOT_API_BASE = "https://api.githubcopilot.com"
|
||||
DEFAULT_GITHUB_COPILOT_API_BASE = "https://api.githubcopilot.com"
|
||||
|
||||
|
||||
class GithubCopilotError(BaseLLMException):
|
||||
|
|
|
|||
|
|
@ -6,8 +6,11 @@ This module provides the configuration for GitHub Copilot's Embedding API.
|
|||
Implementation based on analysis of the copilot-api project by caozhiyuan:
|
||||
https://github.com/caozhiyuan/copilot-api
|
||||
"""
|
||||
|
||||
from typing import TYPE_CHECKING, Any, Optional
|
||||
|
||||
import os
|
||||
|
||||
import httpx
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
|
|
@ -20,7 +23,7 @@ from litellm.utils import convert_to_model_response_object
|
|||
from ..authenticator import Authenticator
|
||||
from ..common_utils import (
|
||||
GetAPIKeyError,
|
||||
GITHUB_COPILOT_API_BASE,
|
||||
DEFAULT_GITHUB_COPILOT_API_BASE,
|
||||
get_copilot_default_headers,
|
||||
)
|
||||
|
||||
|
|
@ -99,15 +102,18 @@ class GithubCopilotEmbeddingConfig(BaseEmbeddingConfig):
|
|||
Get the complete URL for GitHub Copilot Embedding API endpoint.
|
||||
"""
|
||||
# Use provided api_base or fall back to authenticator's base or default
|
||||
api_base = (
|
||||
self.authenticator.get_api_base() or api_base or GITHUB_COPILOT_API_BASE
|
||||
effective_api_base = (
|
||||
api_base
|
||||
or self.authenticator.get_api_base()
|
||||
or os.getenv("GITHUB_COPILOT_API_BASE")
|
||||
or DEFAULT_GITHUB_COPILOT_API_BASE
|
||||
)
|
||||
|
||||
# Remove trailing slashes
|
||||
api_base = api_base.rstrip("/")
|
||||
effective_api_base = effective_api_base.rstrip("/")
|
||||
|
||||
# Return the embeddings endpoint
|
||||
return f"{api_base}/embeddings"
|
||||
return f"{effective_api_base}/embeddings"
|
||||
|
||||
def transform_embedding_request(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -7,8 +7,11 @@ which is required for models like gpt-5.1-codex that only support the /responses
|
|||
Implementation based on analysis of the copilot-api project by caozhiyuan:
|
||||
https://github.com/caozhiyuan/copilot-api
|
||||
"""
|
||||
|
||||
from typing import TYPE_CHECKING, Any, Dict, Optional, Union
|
||||
|
||||
import os
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
|
||||
from litellm.exceptions import AuthenticationError
|
||||
|
|
@ -22,7 +25,7 @@ from litellm.types.utils import LlmProviders
|
|||
|
||||
from ..authenticator import Authenticator
|
||||
from ..common_utils import (
|
||||
GITHUB_COPILOT_API_BASE,
|
||||
DEFAULT_GITHUB_COPILOT_API_BASE,
|
||||
GetAPIKeyError,
|
||||
get_copilot_default_headers,
|
||||
)
|
||||
|
|
@ -157,23 +160,20 @@ class GithubCopilotResponsesAPIConfig(OpenAIResponsesAPIConfig):
|
|||
) -> str:
|
||||
"""
|
||||
Get the complete URL for GitHub Copilot Responses API endpoint.
|
||||
|
||||
Returns: https://api.githubcopilot.com/responses
|
||||
|
||||
Note: Currently only supports individual accounts.
|
||||
Business/enterprise accounts (api.business.githubcopilot.com) can be
|
||||
added in the future by detecting account type.
|
||||
"""
|
||||
# Use provided api_base or fall back to authenticator's base or default
|
||||
api_base = (
|
||||
api_base or self.authenticator.get_api_base() or GITHUB_COPILOT_API_BASE
|
||||
effective_api_base = (
|
||||
api_base
|
||||
or self.authenticator.get_api_base()
|
||||
or os.getenv("GITHUB_COPILOT_API_BASE")
|
||||
or DEFAULT_GITHUB_COPILOT_API_BASE
|
||||
)
|
||||
|
||||
# Remove trailing slashes
|
||||
api_base = api_base.rstrip("/")
|
||||
effective_api_base = effective_api_base.rstrip("/")
|
||||
|
||||
# Return the responses endpoint
|
||||
return f"{api_base}/responses"
|
||||
return f"{effective_api_base}/responses"
|
||||
|
||||
def _handle_reasoning_item(self, item: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
"""
|
||||
Mistral OCR transformation implementation.
|
||||
"""
|
||||
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
import httpx
|
||||
|
|
@ -36,8 +37,12 @@ class MistralOCRConfig(BaseOCRConfig):
|
|||
- image_min_size: Minimum size of images to include
|
||||
- bbox_annotation_format: Format for bounding box annotations
|
||||
- document_annotation_format: Format for document annotations
|
||||
- document_annotation_prompt: Prompt for document annotation extraction
|
||||
- extract_header: Whether to extract document header
|
||||
- extract_footer: Whether to extract document footer
|
||||
- table_format: Table output format ("markdown" or "html")
|
||||
- confidence_scores_granularity: Confidence score level ("word" or "page")
|
||||
- id: Request identifier
|
||||
"""
|
||||
return [
|
||||
"pages",
|
||||
|
|
@ -46,8 +51,12 @@ class MistralOCRConfig(BaseOCRConfig):
|
|||
"image_min_size",
|
||||
"bbox_annotation_format",
|
||||
"document_annotation_format",
|
||||
"document_annotation_prompt",
|
||||
"extract_header",
|
||||
"extract_footer",
|
||||
"table_format",
|
||||
"confidence_scores_granularity",
|
||||
"id",
|
||||
]
|
||||
|
||||
def map_ocr_params(
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ Docs: https://docs.together.ai/reference/completions-1
|
|||
|
||||
from typing import Optional
|
||||
|
||||
from litellm.utils import get_model_info
|
||||
from litellm.utils import supports_function_calling
|
||||
from litellm._logging import verbose_logger
|
||||
|
||||
from ..openai.chat.gpt_transformation import OpenAIGPTConfig
|
||||
|
|
@ -21,18 +21,23 @@ class TogetherAIConfig(OpenAIGPTConfig):
|
|||
|
||||
Docs: https://docs.together.ai/docs/json-mode
|
||||
"""
|
||||
supports_function_calling: Optional[bool] = None
|
||||
# Use supports_function_calling() — which reads _get_model_info_helper
|
||||
# directly — instead of get_model_info(). get_model_info() calls
|
||||
# get_supported_openai_params() as its first step, which routes back
|
||||
# into this method for together_ai models, creating a recursion that
|
||||
# only terminates when Python's recursion limit or the "not mapped"
|
||||
# exception in _get_model_info_helper is hit (~332 deep calls).
|
||||
supports_fc: Optional[bool] = None
|
||||
try:
|
||||
model_info = get_model_info(model, custom_llm_provider="together_ai")
|
||||
supports_function_calling = model_info.get(
|
||||
"supports_function_calling", False
|
||||
supports_fc = supports_function_calling(
|
||||
model, custom_llm_provider="together_ai"
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_logger.debug(f"Error getting supported openai params: {e}")
|
||||
pass
|
||||
|
||||
optional_params = super().get_supported_openai_params(model)
|
||||
if supports_function_calling is not True:
|
||||
if supports_fc is not True:
|
||||
verbose_logger.debug(
|
||||
"Only some together models support function calling/response_format. Docs - https://docs.together.ai/docs/function-calling"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -8702,6 +8702,8 @@ class Router:
|
|||
and self.routing_strategy == "latency-based-routing"
|
||||
):
|
||||
_settings_to_return[var] = self.lowestlatency_logger.routing_args.json()
|
||||
elif var == "routing_strategy_args":
|
||||
_settings_to_return[var] = None
|
||||
return _settings_to_return
|
||||
|
||||
def update_settings(self, **kwargs):
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[project]
|
||||
name = "litellm"
|
||||
version = "1.83.8"
|
||||
version = "1.83.9"
|
||||
description = "Library to easily interface with LLM API providers"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.9, <3.14"
|
||||
|
|
@ -52,7 +52,7 @@ proxy = [
|
|||
"azure-identity==1.25.2; python_version >= '3.9'",
|
||||
"azure-storage-blob==12.28.0",
|
||||
"mcp==1.26.0; python_version >= '3.10'",
|
||||
"litellm-proxy-extras==0.4.65",
|
||||
"litellm-proxy-extras==0.4.66",
|
||||
"litellm-enterprise==0.1.37",
|
||||
"RestrictedPython==8.1",
|
||||
"rich==13.9.4",
|
||||
|
|
@ -243,7 +243,7 @@ source-exclude = [
|
|||
profile = "black"
|
||||
|
||||
[tool.commitizen]
|
||||
version = "1.83.8"
|
||||
version = "1.83.9"
|
||||
version_files = [
|
||||
"pyproject.toml:^version",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -16,10 +16,13 @@ import pytest
|
|||
import sys
|
||||
import os
|
||||
import json
|
||||
from typing import Optional
|
||||
from unittest.mock import AsyncMock, Mock, patch
|
||||
|
||||
sys.path.insert(0, os.path.abspath("../.."))
|
||||
import litellm
|
||||
from litellm.llms.bedrock.common_utils import get_bedrock_chat_config
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
|
||||
|
||||
class TestBedrockMoonshotInvoke(BaseLLMChatTest):
|
||||
|
|
@ -27,17 +30,255 @@ class TestBedrockMoonshotInvoke(BaseLLMChatTest):
|
|||
Test suite for Bedrock Moonshot via invoke route.
|
||||
Inherits all standard LLM tests from BaseLLMChatTest.
|
||||
"""
|
||||
|
||||
|
||||
def get_base_completion_call_args(self) -> dict:
|
||||
litellm._turn_on_debug()
|
||||
return {
|
||||
"model": "bedrock/invoke/moonshot.kimi-k2-thinking",
|
||||
}
|
||||
|
||||
|
||||
def test_tool_call_no_arguments(self, tool_call_no_arguments):
|
||||
"""Test that tool calls with no arguments is translated correctly."""
|
||||
pass
|
||||
|
||||
# ---------------------------------------------------------------------
|
||||
# The overrides below replace inherited BaseLLMChatTest tests that would
|
||||
# otherwise make live AWS Bedrock calls. The live versions were
|
||||
# consistently crashing llm_translation xdist workers. Each override
|
||||
# patches the HTTP client's post() so no network request is sent, and
|
||||
# asserts on the outgoing request body (and, where needed, parses a
|
||||
# canned response) — which is what the translation lane is actually
|
||||
# supposed to cover.
|
||||
# ---------------------------------------------------------------------
|
||||
|
||||
@staticmethod
|
||||
def _make_moonshot_response(content: str = "Hi!") -> Mock:
|
||||
"""Build a Mock httpx.Response that AmazonMoonshotConfig.transform_response
|
||||
(which delegates to MoonshotChatConfig → OpenAI) can parse."""
|
||||
body = {
|
||||
"id": "chatcmpl-test",
|
||||
"object": "chat.completion",
|
||||
"created": 1234567890,
|
||||
"model": "moonshot.kimi-k2-thinking",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": content},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
"prompt_tokens": 10,
|
||||
"completion_tokens": 5,
|
||||
"total_tokens": 15,
|
||||
},
|
||||
}
|
||||
mock_resp = Mock()
|
||||
mock_resp.status_code = 200
|
||||
mock_resp.headers = {"Content-Type": "application/json"}
|
||||
mock_resp.text = json.dumps(body)
|
||||
mock_resp.json = lambda: body
|
||||
return mock_resp
|
||||
|
||||
def _invoke_with_mocked_post(
|
||||
self,
|
||||
*,
|
||||
messages: list,
|
||||
extra_kwargs: Optional[dict] = None,
|
||||
response_content: str = "Hi!",
|
||||
) -> "tuple[Mock, object]":
|
||||
"""Run a sync litellm.completion() with HTTPHandler.post patched to
|
||||
return a canned moonshot response. Returns (mock_post, response)."""
|
||||
client = HTTPHandler()
|
||||
mock_resp = self._make_moonshot_response(content=response_content)
|
||||
with patch.object(
|
||||
client, "post", new=Mock(return_value=mock_resp)
|
||||
) as mock_post:
|
||||
response = litellm.completion(
|
||||
model="bedrock/invoke/moonshot.kimi-k2-thinking",
|
||||
messages=messages,
|
||||
aws_access_key_id="fake",
|
||||
aws_secret_access_key="fake",
|
||||
aws_region_name="us-west-2",
|
||||
client=client,
|
||||
**(extra_kwargs or {}),
|
||||
)
|
||||
return mock_post, response
|
||||
|
||||
def test_developer_role_translation(self):
|
||||
"""Verify LiteLLM maps the ``developer`` role to ``system`` on the
|
||||
outgoing Bedrock invoke request, without hitting the network."""
|
||||
mock_post, response = self._invoke_with_mocked_post(
|
||||
messages=[
|
||||
{"role": "developer", "content": "Be a good bot!"},
|
||||
{"role": "user", "content": "Hello, how are you?"},
|
||||
],
|
||||
)
|
||||
mock_post.assert_called_once()
|
||||
body = json.loads(mock_post.call_args.kwargs["data"])
|
||||
assert body["messages"][0]["role"] == "system"
|
||||
assert body["messages"][0]["content"] == "Be a good bot!"
|
||||
assert body["messages"][1]["role"] == "user"
|
||||
assert response.choices[0].message.content is not None
|
||||
|
||||
def test_message_with_name(self):
|
||||
"""Verify a user message carrying a ``name`` field is serialized into
|
||||
the outgoing Bedrock invoke request without breaking the call."""
|
||||
mock_post, response = self._invoke_with_mocked_post(
|
||||
messages=[{"role": "user", "content": "Hello", "name": "test_name"}],
|
||||
)
|
||||
mock_post.assert_called_once()
|
||||
body = json.loads(mock_post.call_args.kwargs["data"])
|
||||
assert body["messages"][0]["role"] == "user"
|
||||
assert body["messages"][0]["content"] == "Hello"
|
||||
assert response is not None
|
||||
|
||||
def test_content_list_handling(self):
|
||||
"""Verify the inherited content-list-handling test passes against a
|
||||
mocked moonshot response (no network)."""
|
||||
mock_post, response = self._invoke_with_mocked_post(
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "text", "text": "Hello, how are you?"}],
|
||||
}
|
||||
],
|
||||
)
|
||||
mock_post.assert_called_once()
|
||||
assert response.choices[0].message.content is not None
|
||||
|
||||
def test_pydantic_model_input(self):
|
||||
"""Verify a completion call with a pydantic ``Message`` as input does
|
||||
not raise and produces a parseable response."""
|
||||
from litellm import Message
|
||||
|
||||
mock_post, response = self._invoke_with_mocked_post(
|
||||
messages=[Message(content="Hello, how are you?", role="user")],
|
||||
)
|
||||
mock_post.assert_called_once()
|
||||
assert response is not None
|
||||
|
||||
@pytest.mark.parametrize("response_format", [{"type": "text"}])
|
||||
def test_response_format_type_text_with_tool_calls_no_tool_choice(
|
||||
self, response_format
|
||||
):
|
||||
"""Verify response_format + tools + drop_params sends a valid request
|
||||
and produces a response object."""
|
||||
tools = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_current_weather",
|
||||
"description": "Get the current weather in a given location",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {
|
||||
"type": "string",
|
||||
"description": "The city and state, e.g. San Francisco, CA",
|
||||
},
|
||||
"unit": {
|
||||
"type": "string",
|
||||
"enum": ["celsius", "fahrenheit"],
|
||||
},
|
||||
},
|
||||
"required": ["location"],
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
mock_post, response = self._invoke_with_mocked_post(
|
||||
messages=[
|
||||
{"role": "user", "content": "What's the weather like in Boston today?"}
|
||||
],
|
||||
extra_kwargs={
|
||||
"response_format": response_format,
|
||||
"tools": tools,
|
||||
"drop_params": True,
|
||||
},
|
||||
)
|
||||
mock_post.assert_called_once()
|
||||
body = json.loads(mock_post.call_args.kwargs["data"])
|
||||
assert "tools" in body
|
||||
assert body["tools"][0]["function"]["name"] == "get_current_weather"
|
||||
assert response is not None
|
||||
|
||||
def test_streaming(self):
|
||||
"""Verify stream=True routes to the invoke-with-response-stream
|
||||
endpoint with the messages body. Iteration of the stream itself is
|
||||
not exercised here — moonshot streaming delegates to the OpenAI
|
||||
parser and is covered by the OpenAI test suite.
|
||||
|
||||
Note: bedrock invoke streaming cannot be intercepted by patching
|
||||
the caller-supplied client, because ``CustomStreamWrapper.fetch_sync_stream``
|
||||
at streaming_handler.py invokes the stored ``make_call`` partial with
|
||||
``client=litellm.module_level_client``, which overrides any client the
|
||||
caller passed. Patch ``make_sync_call`` at its import site in
|
||||
``base_invoke_transformation`` so we observe the exact kwargs the
|
||||
partial was built with at stream-wrapper construction time.
|
||||
"""
|
||||
from litellm.utils import CustomStreamWrapper
|
||||
|
||||
captured: dict = {}
|
||||
|
||||
def fake_make_sync_call(**kwargs):
|
||||
captured.update(kwargs)
|
||||
# Return an empty iterator so the stream wrapper's iteration
|
||||
# doesn't try to parse real bytes.
|
||||
return iter([])
|
||||
|
||||
with patch(
|
||||
"litellm.llms.bedrock.chat.invoke_transformations."
|
||||
"base_invoke_transformation.make_sync_call",
|
||||
new=fake_make_sync_call,
|
||||
):
|
||||
response = litellm.completion(
|
||||
model="bedrock/invoke/moonshot.kimi-k2-thinking",
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "text", "text": "Hello, how are you?"}],
|
||||
}
|
||||
],
|
||||
stream=True,
|
||||
aws_access_key_id="fake",
|
||||
aws_secret_access_key="fake",
|
||||
aws_region_name="us-west-2",
|
||||
)
|
||||
assert isinstance(response, CustomStreamWrapper)
|
||||
# Trigger fetch_sync_stream → make_call(...) → fake_make_sync_call.
|
||||
try:
|
||||
next(iter(response))
|
||||
except StopIteration:
|
||||
pass
|
||||
|
||||
assert captured, "make_sync_call was never invoked"
|
||||
assert captured["api_base"].endswith("/invoke-with-response-stream")
|
||||
body = json.loads(captured["data"])
|
||||
# Bedrock invoke does not put stream=true in the body (the URL
|
||||
# carries the streaming flag); verify the user message is present.
|
||||
assert body["messages"][0]["role"] == "user"
|
||||
|
||||
async def test_completion_cost(self):
|
||||
"""Verify LiteLLM computes a positive cost from a mocked Bedrock
|
||||
Moonshot response, using the local model cost map."""
|
||||
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
|
||||
litellm.model_cost = litellm.get_model_cost_map(url="")
|
||||
|
||||
mock_response = self._make_moonshot_response()
|
||||
client = AsyncHTTPHandler()
|
||||
with patch.object(client, "post", new=AsyncMock(return_value=mock_response)):
|
||||
response = await litellm.acompletion(
|
||||
model="bedrock/invoke/moonshot.kimi-k2-thinking",
|
||||
messages=[{"role": "user", "content": "Hello, how are you?"}],
|
||||
aws_access_key_id="fake",
|
||||
aws_secret_access_key="fake",
|
||||
aws_region_name="us-west-2",
|
||||
client=client,
|
||||
)
|
||||
|
||||
assert response._hidden_params["response_cost"] > 0
|
||||
|
||||
|
||||
class TestBedrockMoonshotBasic:
|
||||
"""Unit tests for Bedrock Moonshot configuration and transformations."""
|
||||
|
|
@ -47,7 +288,7 @@ class TestBedrockMoonshotBasic:
|
|||
config = get_bedrock_chat_config("bedrock/invoke/moonshot.kimi-k2-thinking")
|
||||
assert config is not None
|
||||
assert config.__class__.__name__ == "AmazonMoonshotConfig"
|
||||
|
||||
|
||||
def test_provider_detection_converse(self):
|
||||
"""Test that Bedrock Moonshot converse models are correctly detected."""
|
||||
config = get_bedrock_chat_config("bedrock/moonshot.kimi-k2-thinking")
|
||||
|
|
@ -62,8 +303,10 @@ class TestBedrockMoonshotBasic:
|
|||
def test_supported_params(self):
|
||||
"""Test that supported OpenAI params are correctly defined."""
|
||||
config = get_bedrock_chat_config("invoke/moonshot.kimi-k2-thinking")
|
||||
supported_params = config.get_supported_openai_params("moonshot.kimi-k2-thinking")
|
||||
|
||||
supported_params = config.get_supported_openai_params(
|
||||
"moonshot.kimi-k2-thinking"
|
||||
)
|
||||
|
||||
# Should support these params
|
||||
assert "temperature" in supported_params
|
||||
assert "max_tokens" in supported_params
|
||||
|
|
@ -71,10 +314,10 @@ class TestBedrockMoonshotBasic:
|
|||
assert "stream" in supported_params
|
||||
assert "tools" in supported_params
|
||||
assert "tool_choice" in supported_params
|
||||
|
||||
|
||||
# Should NOT support stop sequences on Bedrock
|
||||
assert "stop" not in supported_params
|
||||
|
||||
|
||||
# Should NOT support functions (use tools instead)
|
||||
assert "functions" not in supported_params
|
||||
|
||||
|
|
@ -83,20 +326,20 @@ class TestBedrockMoonshotBasic:
|
|||
from litellm.llms.bedrock.chat.invoke_transformations.amazon_moonshot_transformation import (
|
||||
AmazonMoonshotConfig,
|
||||
)
|
||||
|
||||
|
||||
config = AmazonMoonshotConfig()
|
||||
|
||||
|
||||
messages = [{"role": "user", "content": "Hello"}]
|
||||
|
||||
|
||||
# Test that bedrock/invoke/ prefix is stripped
|
||||
transformed = config.transform_request(
|
||||
model="bedrock/invoke/moonshot.kimi-k2-thinking",
|
||||
messages=messages,
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
headers={}
|
||||
headers={},
|
||||
)
|
||||
|
||||
|
||||
# The model ID in the request body should be stripped
|
||||
assert transformed["model"] == "moonshot.kimi-k2-thinking"
|
||||
|
||||
|
|
@ -109,21 +352,27 @@ class TestBedrockMoonshotReasoningContent:
|
|||
from litellm.llms.bedrock.chat.invoke_transformations.amazon_moonshot_transformation import (
|
||||
AmazonMoonshotConfig,
|
||||
)
|
||||
|
||||
|
||||
config = AmazonMoonshotConfig()
|
||||
|
||||
|
||||
# Test with reasoning tags
|
||||
content_with_reasoning = "<reasoning>This is my thought process</reasoning>This is the answer"
|
||||
reasoning, content = config._extract_reasoning_from_content(content_with_reasoning)
|
||||
|
||||
content_with_reasoning = (
|
||||
"<reasoning>This is my thought process</reasoning>This is the answer"
|
||||
)
|
||||
reasoning, content = config._extract_reasoning_from_content(
|
||||
content_with_reasoning
|
||||
)
|
||||
|
||||
assert reasoning == "This is my thought process"
|
||||
assert content == "This is the answer"
|
||||
assert "<reasoning>" not in content
|
||||
|
||||
|
||||
# Test without reasoning tags
|
||||
content_without_reasoning = "This is just a regular answer"
|
||||
reasoning, content = config._extract_reasoning_from_content(content_without_reasoning)
|
||||
|
||||
reasoning, content = config._extract_reasoning_from_content(
|
||||
content_without_reasoning
|
||||
)
|
||||
|
||||
assert reasoning is None
|
||||
assert content == "This is just a regular answer"
|
||||
|
||||
|
|
@ -134,8 +383,10 @@ class TestBedrockMoonshotToolCalling:
|
|||
def test_tool_calling_supported(self):
|
||||
"""Test that tool calling is supported for Kimi K2 Thinking model."""
|
||||
config = get_bedrock_chat_config("invoke/moonshot.kimi-k2-thinking")
|
||||
supported_params = config.get_supported_openai_params("moonshot.kimi-k2-thinking")
|
||||
|
||||
supported_params = config.get_supported_openai_params(
|
||||
"moonshot.kimi-k2-thinking"
|
||||
)
|
||||
|
||||
# Kimi K2 Thinking DOES support tool calls (unlike kimi-thinking-preview)
|
||||
assert "tools" in supported_params
|
||||
assert "tool_choice" in supported_params
|
||||
|
|
@ -145,13 +396,11 @@ class TestBedrockMoonshotToolCalling:
|
|||
from litellm.llms.bedrock.chat.invoke_transformations.amazon_moonshot_transformation import (
|
||||
AmazonMoonshotConfig,
|
||||
)
|
||||
|
||||
|
||||
config = AmazonMoonshotConfig()
|
||||
|
||||
messages = [
|
||||
{"role": "user", "content": "What's the weather in San Francisco?"}
|
||||
]
|
||||
|
||||
|
||||
messages = [{"role": "user", "content": "What's the weather in San Francisco?"}]
|
||||
|
||||
optional_params = {
|
||||
"tools": [
|
||||
{
|
||||
|
|
@ -161,27 +410,25 @@ class TestBedrockMoonshotToolCalling:
|
|||
"description": "Get the current weather",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {"type": "string"}
|
||||
},
|
||||
"required": ["location"]
|
||||
}
|
||||
}
|
||||
"properties": {"location": {"type": "string"}},
|
||||
"required": ["location"],
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
transformed = config.transform_request(
|
||||
model="bedrock/invoke/moonshot.kimi-k2-thinking",
|
||||
messages=messages,
|
||||
optional_params=optional_params,
|
||||
litellm_params={},
|
||||
headers={}
|
||||
headers={},
|
||||
)
|
||||
|
||||
|
||||
# Verify model ID is stripped
|
||||
assert transformed["model"] == "moonshot.kimi-k2-thinking"
|
||||
|
||||
|
||||
# Verify tools are included
|
||||
assert "tools" in transformed
|
||||
assert len(transformed["tools"]) == 1
|
||||
|
|
@ -193,9 +440,9 @@ class TestBedrockMoonshotToolCalling:
|
|||
tool_response_message = {
|
||||
"role": "tool",
|
||||
"tool_call_id": "call_123",
|
||||
"content": json.dumps({"temperature": 72, "condition": "sunny"})
|
||||
"content": json.dumps({"temperature": 72, "condition": "sunny"}),
|
||||
}
|
||||
|
||||
|
||||
# Verify the message structure
|
||||
assert tool_response_message["role"] == "tool"
|
||||
assert "tool_call_id" in tool_response_message
|
||||
|
|
@ -208,8 +455,10 @@ class TestBedrockMoonshotParameterValidation:
|
|||
def test_stop_sequences_not_supported(self):
|
||||
"""Test that stop sequences are correctly excluded from supported params."""
|
||||
config = get_bedrock_chat_config("invoke/moonshot.kimi-k2-thinking")
|
||||
supported_params = config.get_supported_openai_params("moonshot.kimi-k2-thinking")
|
||||
|
||||
supported_params = config.get_supported_openai_params(
|
||||
"moonshot.kimi-k2-thinking"
|
||||
)
|
||||
|
||||
# Bedrock Moonshot doesn't support stopSequences field
|
||||
assert "stop" not in supported_params
|
||||
|
||||
|
|
@ -218,10 +467,12 @@ class TestBedrockMoonshotParameterValidation:
|
|||
# Moonshot models support temperature 0-1
|
||||
# This is handled by the parent MoonshotChatConfig class
|
||||
config = get_bedrock_chat_config("invoke/moonshot.kimi-k2-thinking")
|
||||
|
||||
|
||||
# Verify config exists and can handle temperature
|
||||
assert config is not None
|
||||
supported_params = config.get_supported_openai_params("moonshot.kimi-k2-thinking")
|
||||
supported_params = config.get_supported_openai_params(
|
||||
"moonshot.kimi-k2-thinking"
|
||||
)
|
||||
assert "temperature" in supported_params
|
||||
|
||||
|
||||
|
|
@ -233,34 +484,31 @@ class TestBedrockMoonshotTransformations:
|
|||
from litellm.llms.bedrock.chat.invoke_transformations.amazon_moonshot_transformation import (
|
||||
AmazonMoonshotConfig,
|
||||
)
|
||||
|
||||
|
||||
config = AmazonMoonshotConfig()
|
||||
|
||||
|
||||
messages = [
|
||||
{"role": "system", "content": "You are a helpful assistant."},
|
||||
{"role": "user", "content": "Hello!"}
|
||||
{"role": "user", "content": "Hello!"},
|
||||
]
|
||||
|
||||
optional_params = {
|
||||
"temperature": 0.7,
|
||||
"max_tokens": 100
|
||||
}
|
||||
|
||||
|
||||
optional_params = {"temperature": 0.7, "max_tokens": 100}
|
||||
|
||||
transformed = config.transform_request(
|
||||
model="bedrock/invoke/moonshot.kimi-k2-thinking",
|
||||
messages=messages,
|
||||
optional_params=optional_params,
|
||||
litellm_params={},
|
||||
headers={}
|
||||
headers={},
|
||||
)
|
||||
|
||||
|
||||
# Verify model ID is stripped
|
||||
assert transformed["model"] == "moonshot.kimi-k2-thinking"
|
||||
|
||||
|
||||
# Verify messages are included
|
||||
assert "messages" in transformed
|
||||
assert len(transformed["messages"]) >= 1
|
||||
|
||||
|
||||
# Verify optional params are included
|
||||
assert transformed["temperature"] == 0.7
|
||||
assert transformed["max_tokens"] == 100
|
||||
|
|
@ -270,21 +518,21 @@ class TestBedrockMoonshotTransformations:
|
|||
from litellm.llms.bedrock.chat.invoke_transformations.amazon_moonshot_transformation import (
|
||||
AmazonMoonshotConfig,
|
||||
)
|
||||
|
||||
|
||||
config = AmazonMoonshotConfig()
|
||||
|
||||
|
||||
messages = [
|
||||
{"role": "system", "content": "You are a helpful assistant."},
|
||||
{"role": "user", "content": "Hello!"}
|
||||
{"role": "user", "content": "Hello!"},
|
||||
]
|
||||
|
||||
|
||||
transformed = config.transform_request(
|
||||
model="moonshot.kimi-k2-thinking",
|
||||
messages=messages,
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
headers={}
|
||||
headers={},
|
||||
)
|
||||
|
||||
|
||||
# System messages should be supported
|
||||
assert "messages" in transformed
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ import pytest
|
|||
class TestTogetherAI(BaseLLMChatTest):
|
||||
def get_base_completion_call_args(self) -> dict:
|
||||
litellm.set_verbose = True
|
||||
return {"model": "together_ai/Qwen/Qwen3.5-9B"}
|
||||
return {"model": "together_ai/Qwen/Qwen2.5-7B-Instruct-Turbo"}
|
||||
|
||||
def test_tool_call_no_arguments(self, tool_call_no_arguments):
|
||||
"""Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833"""
|
||||
|
|
|
|||
|
|
@ -65,7 +65,7 @@ def test_completion_custom_provider_model_name():
|
|||
try:
|
||||
litellm.cache = None
|
||||
response = completion(
|
||||
model="together_ai/Qwen/Qwen3.5-9B",
|
||||
model="together_ai/Qwen/Qwen2.5-7B-Instruct-Turbo",
|
||||
messages=messages,
|
||||
logger_fn=logger_fn,
|
||||
)
|
||||
|
|
@ -2815,7 +2815,7 @@ def test_customprompt_together_ai():
|
|||
print(litellm.success_callback)
|
||||
print(litellm._async_success_callback)
|
||||
response = completion(
|
||||
model="together_ai/Qwen/Qwen3.5-9B",
|
||||
model="together_ai/Qwen/Qwen2.5-7B-Instruct-Turbo",
|
||||
messages=messages,
|
||||
roles={
|
||||
"system": {
|
||||
|
|
@ -3682,7 +3682,7 @@ def test_completion_together_ai_stream():
|
|||
messages = [{"content": user_message, "role": "user"}]
|
||||
try:
|
||||
response = completion(
|
||||
model="together_ai/Qwen/Qwen3.5-9B",
|
||||
model="together_ai/Qwen/Qwen2.5-7B-Instruct-Turbo",
|
||||
messages=messages,
|
||||
stream=True,
|
||||
max_tokens=5,
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ model_list = [
|
|||
{
|
||||
"model_name": "mistral-7b-instruct",
|
||||
"litellm_params": { # params for litellm completion/embedding call
|
||||
"model": "together_ai/Qwen/Qwen3.5-9B",
|
||||
"model": "together_ai/Qwen/Qwen2.5-7B-Instruct-Turbo",
|
||||
"api_key": os.getenv("TOGETHERAI_API_KEY"),
|
||||
},
|
||||
},
|
||||
|
|
|
|||
|
|
@ -4034,7 +4034,7 @@ def test_async_text_completion_together_ai():
|
|||
async def test_get_response():
|
||||
try:
|
||||
response = await litellm.atext_completion(
|
||||
model="together_ai/Qwen/Qwen3.5-9B",
|
||||
model="together_ai/Qwen/Qwen2.5-7B-Instruct-Turbo",
|
||||
prompt="good morning",
|
||||
max_tokens=10,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -316,7 +316,10 @@ def test_normalize_tool_input_schema_types_for_bedrock_invoke():
|
|||
"type": "custom",
|
||||
"additionalProperties": False,
|
||||
"properties": {
|
||||
"nested": {"type": "custom", "properties": {"x": {"type": "string"}}}
|
||||
"nested": {
|
||||
"type": "custom",
|
||||
"properties": {"x": {"type": "string"}},
|
||||
}
|
||||
},
|
||||
"required": ["nested"],
|
||||
},
|
||||
|
|
@ -385,34 +388,6 @@ def test_bedrock_invoke_messages_transform_adds_name_when_tool_missing_name():
|
|||
assert result["tools"][0]["name"] == "litellm_unnamed_tool_0"
|
||||
|
||||
|
||||
def test_bedrock_invoke_messages_injects_thinking_for_clear_thinking_context_management():
|
||||
"""
|
||||
Bedrock requires extended thinking when ``clear_thinking_20251015`` appears in
|
||||
``context_management`` (Claude Code sends CM without ``thinking``).
|
||||
"""
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
||||
cfg = AmazonAnthropicClaudeMessagesConfig()
|
||||
optional_params = {
|
||||
"max_tokens": 32000,
|
||||
"stream": False,
|
||||
"context_management": {
|
||||
"edits": [{"type": "clear_thinking_20251015", "keep": "all"}]
|
||||
},
|
||||
}
|
||||
result = cfg.transform_anthropic_messages_request(
|
||||
model="global.anthropic.claude-sonnet-4-6-v1:0",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
anthropic_messages_optional_request_params=copy.deepcopy(optional_params),
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers={},
|
||||
)
|
||||
assert result["thinking"]["type"] == "enabled"
|
||||
assert result["thinking"]["budget_tokens"] == BEDROCK_MIN_THINKING_BUDGET_TOKENS
|
||||
betas = result.get("anthropic_beta") or []
|
||||
assert "interleaved-thinking-2025-05-14" in betas
|
||||
|
||||
|
||||
def test_bedrock_invoke_messages_skips_thinking_injection_when_already_enabled():
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
||||
|
|
|
|||
|
|
@ -189,3 +189,65 @@ class TestGitHubCopilotAuthenticator:
|
|||
with patch("builtins.open", mock_open(read_data=mock_api_key_data)):
|
||||
api_base = authenticator.get_api_base()
|
||||
assert api_base == "https://api.enterprise.githubcopilot.com"
|
||||
|
||||
def test_get_device_code_with_custom_url(self, authenticator, mock_http_client):
|
||||
"""GITHUB_COPILOT_DEVICE_CODE_URL env var must be used by _get_device_code at call time."""
|
||||
mock_client, mock_response = mock_http_client
|
||||
custom_url = "https://custom.example.com/device"
|
||||
mock_response.json.return_value = {
|
||||
"device_code": "dc",
|
||||
"user_code": "UC",
|
||||
"verification_uri": "https://example.com",
|
||||
}
|
||||
with patch.dict(os.environ, {"GITHUB_COPILOT_DEVICE_CODE_URL": custom_url}), \
|
||||
patch("litellm.llms.github_copilot.authenticator._get_httpx_client", return_value=mock_client):
|
||||
authenticator._get_device_code()
|
||||
assert mock_client.post.call_args[0][0] == custom_url
|
||||
|
||||
def test_get_device_code_with_custom_client_id(self, authenticator, mock_http_client):
|
||||
"""GITHUB_COPILOT_CLIENT_ID env var must appear as client_id in the device-code request body."""
|
||||
mock_client, mock_response = mock_http_client
|
||||
custom_id = "custom_client_id"
|
||||
mock_response.json.return_value = {
|
||||
"device_code": "dc",
|
||||
"user_code": "UC",
|
||||
"verification_uri": "https://example.com",
|
||||
}
|
||||
with patch.dict(os.environ, {"GITHUB_COPILOT_CLIENT_ID": custom_id}), \
|
||||
patch("litellm.llms.github_copilot.authenticator._get_httpx_client", return_value=mock_client):
|
||||
authenticator._get_device_code()
|
||||
assert mock_client.post.call_args[1]["json"]["client_id"] == custom_id
|
||||
|
||||
def test_poll_for_access_token_with_custom_url(self, authenticator, mock_http_client):
|
||||
"""GITHUB_COPILOT_ACCESS_TOKEN_URL env var must be used by _poll_for_access_token at call time."""
|
||||
mock_client, mock_response = mock_http_client
|
||||
custom_url = "https://custom.example.com/token"
|
||||
mock_response.json.return_value = {"access_token": "tok"}
|
||||
with patch.dict(os.environ, {"GITHUB_COPILOT_ACCESS_TOKEN_URL": custom_url}), \
|
||||
patch("litellm.llms.github_copilot.authenticator._get_httpx_client", return_value=mock_client), \
|
||||
patch("time.sleep"):
|
||||
authenticator._poll_for_access_token("dc")
|
||||
assert mock_client.post.call_args[0][0] == custom_url
|
||||
|
||||
def test_poll_for_access_token_with_custom_client_id(self, authenticator, mock_http_client):
|
||||
"""GITHUB_COPILOT_CLIENT_ID env var must appear as client_id in the polling request body."""
|
||||
mock_client, mock_response = mock_http_client
|
||||
custom_id = "custom_client_id"
|
||||
mock_response.json.return_value = {"access_token": "tok"}
|
||||
with patch.dict(os.environ, {"GITHUB_COPILOT_CLIENT_ID": custom_id}), \
|
||||
patch("litellm.llms.github_copilot.authenticator._get_httpx_client", return_value=mock_client), \
|
||||
patch("time.sleep"):
|
||||
authenticator._poll_for_access_token("dc")
|
||||
assert mock_client.post.call_args[1]["json"]["client_id"] == custom_id
|
||||
|
||||
def test_refresh_api_key_with_custom_url(self, authenticator, mock_http_client):
|
||||
"""GITHUB_COPILOT_API_KEY_URL env var must be used by _refresh_api_key at call time."""
|
||||
mock_client, mock_response = mock_http_client
|
||||
custom_url = "https://custom.example.com/api-key"
|
||||
mock_response.json.return_value = {"token": "api-tok", "expires_at": 9999999999}
|
||||
with patch.dict(os.environ, {"GITHUB_COPILOT_API_KEY_URL": custom_url}), \
|
||||
patch("litellm.llms.github_copilot.authenticator._get_httpx_client", return_value=mock_client), \
|
||||
patch.object(authenticator, "get_access_token", return_value="access-tok"):
|
||||
authenticator._refresh_api_key()
|
||||
assert mock_client.get.call_args[0][0] == custom_url
|
||||
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ Unit tests for MistralOCRConfig transformation.
|
|||
Tests the supported OCR parameters and their mapping behaviour.
|
||||
No real API calls are made — all tests are fully mocked/local.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.llms.mistral.ocr.transformation import MistralOCRConfig
|
||||
|
|
@ -39,7 +40,9 @@ class TestGetSupportedOcrParams:
|
|||
"bbox_annotation_format",
|
||||
"document_annotation_format",
|
||||
]:
|
||||
assert param in supported, f"Previously supported param '{param}' is missing"
|
||||
assert (
|
||||
param in supported
|
||||
), f"Previously supported param '{param}' is missing"
|
||||
|
||||
|
||||
class TestMapOcrParams:
|
||||
|
|
@ -79,3 +82,97 @@ class TestMapOcrParams:
|
|||
)
|
||||
assert "extract_header" in result
|
||||
assert "unsupported_param" not in result
|
||||
|
||||
|
||||
class TestNewSupportedParams:
|
||||
"""Verify the newly added params are in the supported list."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"param_name",
|
||||
[
|
||||
"table_format",
|
||||
"confidence_scores_granularity",
|
||||
"document_annotation_prompt",
|
||||
"id",
|
||||
],
|
||||
)
|
||||
def test_new_param_in_supported_list(
|
||||
self, config: MistralOCRConfig, param_name: str
|
||||
) -> None:
|
||||
supported = config.get_supported_ocr_params(model=MODEL)
|
||||
assert param_name in supported
|
||||
|
||||
|
||||
class TestNewParamsMapOcr:
|
||||
"""Verify the newly added params survive map_ocr_params."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"param_name,param_value",
|
||||
[
|
||||
("table_format", "html"),
|
||||
("table_format", "markdown"),
|
||||
("confidence_scores_granularity", "word"),
|
||||
("confidence_scores_granularity", "page"),
|
||||
("document_annotation_prompt", "Extract all invoice line items"),
|
||||
("id", "req-123"),
|
||||
],
|
||||
)
|
||||
def test_new_param_passed_through(
|
||||
self, config: MistralOCRConfig, param_name: str, param_value: str
|
||||
) -> None:
|
||||
result = config.map_ocr_params(
|
||||
non_default_params={param_name: param_value},
|
||||
optional_params={},
|
||||
model=MODEL,
|
||||
)
|
||||
assert result == {param_name: param_value}
|
||||
|
||||
|
||||
class TestTransformOcrRequest:
|
||||
"""Verify params end up in the final request body via transform_ocr_request."""
|
||||
|
||||
SAMPLE_DOCUMENT = {
|
||||
"type": "document_url",
|
||||
"document_url": "https://example.com/doc.pdf",
|
||||
}
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"param_name,param_value",
|
||||
[
|
||||
("table_format", "html"),
|
||||
("confidence_scores_granularity", "word"),
|
||||
("document_annotation_prompt", "Extract all invoice line items"),
|
||||
("id", "req-123"),
|
||||
("extract_header", True),
|
||||
("pages", [0, 1]),
|
||||
],
|
||||
)
|
||||
def test_param_included_in_request_body(
|
||||
self, config: MistralOCRConfig, param_name: str, param_value
|
||||
) -> None:
|
||||
result = config.transform_ocr_request(
|
||||
model=MODEL,
|
||||
document=self.SAMPLE_DOCUMENT,
|
||||
optional_params={param_name: param_value},
|
||||
headers={},
|
||||
)
|
||||
assert result.data[param_name] == param_value
|
||||
assert result.data["model"] == MODEL
|
||||
assert result.data["document"] == self.SAMPLE_DOCUMENT
|
||||
assert result.files is None
|
||||
|
||||
def test_multiple_new_params_together(self, config: MistralOCRConfig) -> None:
|
||||
"""Multiple new params can be passed together in a single request."""
|
||||
optional_params = {
|
||||
"table_format": "html",
|
||||
"confidence_scores_granularity": "page",
|
||||
"extract_header": True,
|
||||
}
|
||||
result = config.transform_ocr_request(
|
||||
model=MODEL,
|
||||
document=self.SAMPLE_DOCUMENT,
|
||||
optional_params=optional_params,
|
||||
headers={},
|
||||
)
|
||||
for key, value in optional_params.items():
|
||||
assert result.data[key] == value
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ import {
|
|||
serverRootPath,
|
||||
} from "@/components/networking";
|
||||
import { extractErrorMessage } from "@/utils/errorUtils";
|
||||
import { generateCodeChallenge, generateCodeVerifier } from "@/utils/pkce";
|
||||
import { getSecureItem, setSecureItem } from "@/utils/secureStorage";
|
||||
|
||||
export type McpOAuthStatus = "idle" | "authorizing" | "exchanging" | "success" | "error";
|
||||
|
|
@ -34,25 +35,6 @@ interface UseMcpOAuthFlowResult {
|
|||
tokenResponse: Record<string, any> | null;
|
||||
}
|
||||
|
||||
const base64UrlEncode = (buffer: ArrayBuffer) => {
|
||||
const bytes = new Uint8Array(buffer);
|
||||
let binary = "";
|
||||
bytes.forEach((b) => (binary += String.fromCharCode(b)));
|
||||
return btoa(binary).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, "");
|
||||
};
|
||||
|
||||
const generateCodeVerifier = () => {
|
||||
const array = new Uint8Array(32);
|
||||
window.crypto.getRandomValues(array);
|
||||
return base64UrlEncode(array.buffer);
|
||||
};
|
||||
|
||||
const generateCodeChallenge = async (verifier: string) => {
|
||||
const data = new TextEncoder().encode(verifier);
|
||||
const digest = await window.crypto.subtle.digest("SHA-256", data);
|
||||
return base64UrlEncode(digest);
|
||||
};
|
||||
|
||||
export const useMcpOAuthFlow = ({
|
||||
accessToken,
|
||||
getCredentials,
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ import {
|
|||
} from "@/components/networking";
|
||||
import NotificationsManager from "@/components/molecules/notifications_manager";
|
||||
import { extractErrorMessage } from "@/utils/errorUtils";
|
||||
import { generateCodeChallenge, generateCodeVerifier } from "@/utils/pkce";
|
||||
import { getSecureItem, setSecureItem } from "@/utils/secureStorage";
|
||||
|
||||
export type UserMcpOAuthStatus = "idle" | "authorizing" | "exchanging" | "success" | "error";
|
||||
|
|
@ -60,25 +61,6 @@ type StoredFlowState = {
|
|||
scopes?: string[];
|
||||
};
|
||||
|
||||
const b64url = (buf: ArrayBuffer) => {
|
||||
const bytes = new Uint8Array(buf);
|
||||
let s = "";
|
||||
bytes.forEach((b) => (s += String.fromCharCode(b)));
|
||||
return btoa(s).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, "");
|
||||
};
|
||||
|
||||
const genVerifier = () => {
|
||||
const arr = new Uint8Array(32);
|
||||
window.crypto.getRandomValues(arr);
|
||||
return b64url(arr.buffer);
|
||||
};
|
||||
|
||||
const genChallenge = async (verifier: string) => {
|
||||
const data = new TextEncoder().encode(verifier);
|
||||
const digest = await window.crypto.subtle.digest("SHA-256", data);
|
||||
return b64url(digest);
|
||||
};
|
||||
|
||||
const setStorage = (key: string, value: string) => {
|
||||
setSecureItem(key, value);
|
||||
};
|
||||
|
|
@ -144,8 +126,8 @@ export const useUserMcpOAuthFlow = ({
|
|||
}
|
||||
}
|
||||
|
||||
const verifier = genVerifier();
|
||||
const challenge = await genChallenge(verifier);
|
||||
const verifier = generateCodeVerifier();
|
||||
const challenge = await generateCodeChallenge(verifier);
|
||||
const state = crypto.randomUUID();
|
||||
const redirectUri = buildCallbackUrl();
|
||||
const scopeString = scopes?.filter((s) => s.trim()).join(" ");
|
||||
|
|
|
|||
18
ui/litellm-dashboard/src/utils/pkce.ts
Normal file
18
ui/litellm-dashboard/src/utils/pkce.ts
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
const base64UrlEncode = (buffer: ArrayBuffer) => {
|
||||
const bytes = new Uint8Array(buffer);
|
||||
let binary = "";
|
||||
bytes.forEach((b) => (binary += String.fromCharCode(b)));
|
||||
return btoa(binary).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, "");
|
||||
};
|
||||
|
||||
export const generateCodeVerifier = () => {
|
||||
const array = new Uint8Array(32);
|
||||
window.crypto.getRandomValues(array);
|
||||
return base64UrlEncode(array.buffer);
|
||||
};
|
||||
|
||||
export const generateCodeChallenge = async (verifier: string) => {
|
||||
const data = new TextEncoder().encode(verifier);
|
||||
const digest = await window.crypto.subtle.digest("SHA-256", data);
|
||||
return base64UrlEncode(digest);
|
||||
};
|
||||
Loading…
Add table
Reference in a new issue