mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-10 03:28:53 +00:00
Merge origin/litellm_internal_staging into litellm_mantle_codex_additional_tools (resolve overlap with #33228 hoist)
This commit is contained in:
mateo-berri
commit
fadec17a0f
6 changed files with 383 additions and 1 deletions
|
|
@ -26,7 +26,10 @@ from litellm.llms.bedrock_mantle.common_utils import (
|
|||
)
|
||||
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams
|
||||
from litellm.types.llms.openai import (
|
||||
ResponseInputParam,
|
||||
ResponsesAPIOptionalRequestParams,
|
||||
)
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import LlmProviders
|
||||
|
||||
|
|
@ -45,6 +48,8 @@ _BEDROCK_MANTLE_SUPPORTED_RESPONSE_TOOL_TYPES = frozenset({"function", "mcp", "c
|
|||
|
||||
_BEDROCK_MANTLE_SUPPORTED_SERVICE_TIERS = frozenset({"auto", "default"})
|
||||
|
||||
_CODEX_ADDITIONAL_TOOLS_INPUT_ITEM_TYPE = "additional_tools"
|
||||
|
||||
|
||||
class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPIConfig):
|
||||
def __init__(
|
||||
|
|
@ -141,6 +146,70 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI
|
|||
)
|
||||
return {key: value for key, value in params.items() if key != "service_tier"}
|
||||
|
||||
def transform_responses_api_request(
|
||||
self,
|
||||
model: str,
|
||||
input: "str | ResponseInputParam",
|
||||
response_api_optional_request_params: dict,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: dict,
|
||||
) -> dict:
|
||||
remaining_input, hoisted_tools = self._hoist_codex_additional_tools(input)
|
||||
request_params = (
|
||||
{
|
||||
**response_api_optional_request_params,
|
||||
"tools": [
|
||||
*(response_api_optional_request_params.get("tools") or []),
|
||||
*hoisted_tools,
|
||||
],
|
||||
}
|
||||
if hoisted_tools
|
||||
else response_api_optional_request_params
|
||||
)
|
||||
return super().transform_responses_api_request(
|
||||
model=model,
|
||||
input=remaining_input,
|
||||
response_api_optional_request_params=request_params,
|
||||
litellm_params=litellm_params,
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _is_codex_additional_tools_item(item: Any) -> bool:
|
||||
return isinstance(item, dict) and item.get("type") == _CODEX_ADDITIONAL_TOOLS_INPUT_ITEM_TYPE
|
||||
|
||||
@staticmethod
|
||||
def _tools_of_additional_tools_item(item: "dict[str, Any]") -> "list[Any]":
|
||||
tools = item.get("tools")
|
||||
return tools if isinstance(tools, list) else []
|
||||
|
||||
@classmethod
|
||||
def _hoist_codex_additional_tools(
|
||||
cls,
|
||||
input: "str | ResponseInputParam",
|
||||
) -> "tuple[str | ResponseInputParam, list[Any]]":
|
||||
"""Codex's "responses lite" wire mode ships tool definitions inside
|
||||
`input` as {"type": "additional_tools", "role": "developer",
|
||||
"tools": [...]} items. api.openai.com accepts that item type; Mantle
|
||||
rejects the whole request with 400 "Invalid 'input': value did not
|
||||
match any expected variant" but accepts the same tools at the top
|
||||
level, so move them there and strip the items from `input`.
|
||||
"""
|
||||
if not isinstance(input, list):
|
||||
return input, []
|
||||
additional_tools_items = [item for item in input if cls._is_codex_additional_tools_item(item)]
|
||||
if not additional_tools_items:
|
||||
return input, []
|
||||
remaining_input = [item for item in input if not cls._is_codex_additional_tools_item(item)]
|
||||
hoisted_tools = [tool for item in additional_tools_items for tool in cls._tools_of_additional_tools_item(item)]
|
||||
verbose_logger.debug(
|
||||
"Bedrock Mantle Responses API: hoisting %d tool(s) out of %d 'additional_tools' input item(s) "
|
||||
"into the top-level tools param (Mantle rejects that input item type).",
|
||||
len(hoisted_tools),
|
||||
len(additional_tools_items),
|
||||
)
|
||||
return remaining_input, cls._filter_unsupported_tools(hoisted_tools)
|
||||
|
||||
def map_openai_params(
|
||||
self,
|
||||
response_api_optional_params: ResponsesAPIOptionalRequestParams,
|
||||
|
|
|
|||
|
|
@ -92,6 +92,89 @@
|
|||
{ "name": "trash_message", "description": "Move a message to trash" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "google_sheets",
|
||||
"title": "Google Sheets",
|
||||
"description": "Read, write, and format data in Google Sheets spreadsheets",
|
||||
"icon_url": "https://cdn.simpleicons.org/googlesheets",
|
||||
"spec_url": "https://raw.githubusercontent.com/APIs-guru/openapi-directory/main/APIs/googleapis.com/sheets/v4/openapi.yaml",
|
||||
"oauth": {
|
||||
"authorization_url": "https://accounts.google.com/o/oauth2/v2/auth",
|
||||
"token_url": "https://oauth2.googleapis.com/token",
|
||||
"pkce": true,
|
||||
"docs_url": "https://developers.google.com/sheets/api/guides/authorizing"
|
||||
},
|
||||
"key_tools": [
|
||||
{ "name": "create_spreadsheet", "description": "Create a new spreadsheet" },
|
||||
{ "name": "get_spreadsheet", "description": "Get spreadsheet metadata and sheet properties" },
|
||||
{ "name": "get_values", "description": "Read cell values from a range" },
|
||||
{ "name": "update_values", "description": "Write cell values to a range" },
|
||||
{ "name": "append_values", "description": "Append rows of values to a range" },
|
||||
{ "name": "clear_values", "description": "Clear cell values in a range" },
|
||||
{ "name": "batch_update", "description": "Apply batched formatting and structural updates" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "google_drive",
|
||||
"title": "Google Drive",
|
||||
"description": "List, read, upload, and manage files in Google Drive",
|
||||
"icon_url": "https://cdn.simpleicons.org/googledrive",
|
||||
"spec_url": "https://raw.githubusercontent.com/APIs-guru/openapi-directory/main/APIs/googleapis.com/drive/v3/openapi.yaml",
|
||||
"oauth": {
|
||||
"authorization_url": "https://accounts.google.com/o/oauth2/v2/auth",
|
||||
"token_url": "https://oauth2.googleapis.com/token",
|
||||
"pkce": true,
|
||||
"docs_url": "https://developers.google.com/drive/api/guides/api-specific-auth"
|
||||
},
|
||||
"key_tools": [
|
||||
{ "name": "list_files", "description": "List and search files" },
|
||||
{ "name": "get_file", "description": "Get file metadata" },
|
||||
{ "name": "create_file", "description": "Create a file or folder" },
|
||||
{ "name": "update_file", "description": "Update file metadata or content" },
|
||||
{ "name": "copy_file", "description": "Copy a file" },
|
||||
{ "name": "delete_file", "description": "Delete a file" },
|
||||
{ "name": "list_permissions", "description": "List sharing permissions on a file" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "google_calendar",
|
||||
"title": "Google Calendar",
|
||||
"description": "Read and manage Google Calendar events and calendars",
|
||||
"icon_url": "https://cdn.simpleicons.org/googlecalendar",
|
||||
"spec_url": "https://raw.githubusercontent.com/APIs-guru/openapi-directory/main/APIs/googleapis.com/calendar/v3/openapi.yaml",
|
||||
"oauth": {
|
||||
"authorization_url": "https://accounts.google.com/o/oauth2/v2/auth",
|
||||
"token_url": "https://oauth2.googleapis.com/token",
|
||||
"pkce": true,
|
||||
"docs_url": "https://developers.google.com/workspace/calendar/api/guides/auth"
|
||||
},
|
||||
"key_tools": [
|
||||
{ "name": "list_events", "description": "List events on a calendar" },
|
||||
{ "name": "get_event", "description": "Get a single event" },
|
||||
{ "name": "insert_event", "description": "Create an event" },
|
||||
{ "name": "update_event", "description": "Update an event" },
|
||||
{ "name": "delete_event", "description": "Delete an event" },
|
||||
{ "name": "query_freebusy", "description": "Query free/busy availability" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "google_docs",
|
||||
"title": "Google Docs",
|
||||
"description": "Create, read, and edit Google Docs documents",
|
||||
"icon_url": "https://cdn.simpleicons.org/googledocs",
|
||||
"spec_url": "https://raw.githubusercontent.com/APIs-guru/openapi-directory/main/APIs/googleapis.com/docs/v1/openapi.yaml",
|
||||
"oauth": {
|
||||
"authorization_url": "https://accounts.google.com/o/oauth2/v2/auth",
|
||||
"token_url": "https://oauth2.googleapis.com/token",
|
||||
"pkce": true,
|
||||
"docs_url": "https://developers.google.com/docs/api/how-tos/authorizing"
|
||||
},
|
||||
"key_tools": [
|
||||
{ "name": "create_document", "description": "Create a new document" },
|
||||
{ "name": "get_document", "description": "Get a document's full content" },
|
||||
{ "name": "batch_update_document", "description": "Apply batched edits to a document" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stripe",
|
||||
"title": "Stripe",
|
||||
|
|
|
|||
|
|
@ -280,6 +280,8 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
|
|||
"realtime",
|
||||
]
|
||||
]
|
||||
supported_endpoints: Optional[List[str]]
|
||||
use_openai_responses_path: Optional[bool]
|
||||
tpm: Optional[int]
|
||||
rpm: Optional[int]
|
||||
provider_specific_entry: Optional[Dict[str, float]]
|
||||
|
|
|
|||
|
|
@ -494,6 +494,144 @@ class TestBedrockMantleCodexRequestEndToEnd:
|
|||
assert body["tool_choice"] == "auto"
|
||||
|
||||
|
||||
class TestBedrockMantleCodexAdditionalTools:
|
||||
"""Codex CLI's "responses lite" wire mode ships tool definitions inside
|
||||
`input` as {"type": "additional_tools", "role": "developer", "tools": [...]}
|
||||
items instead of the top-level `tools` param. api.openai.com accepts that
|
||||
item; Mantle 400s the whole request with "Invalid 'input': value did not
|
||||
match any expected variant" but accepts the same tools at the top level
|
||||
(verified against bedrock-mantle.us-east-2.api.aws with openai.gpt-5.6-sol),
|
||||
so the config must hoist them."""
|
||||
|
||||
_USER_MESSAGE = {
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [{"type": "input_text", "text": "Say hi in one word."}],
|
||||
}
|
||||
_DEVELOPER_MESSAGE = {
|
||||
"type": "message",
|
||||
"role": "developer",
|
||||
"content": [{"type": "input_text", "text": "You are Codex."}],
|
||||
}
|
||||
_CODEX_TOOLS = [
|
||||
{"type": "custom", "name": "exec", "format": {"type": "grammar", "syntax": "lark", "definition": "start: X"}},
|
||||
{"type": "function", "name": "wait", "parameters": {"type": "object"}},
|
||||
{"type": "namespace", "name": "collaboration", "tools": [{"type": "function", "name": "spawn_agent"}]},
|
||||
]
|
||||
|
||||
def _transform(self, input, params=None):
|
||||
cfg = BedrockMantleResponsesAPIConfig()
|
||||
return cfg.transform_responses_api_request(
|
||||
model="openai.gpt-5.6-sol",
|
||||
input=input,
|
||||
response_api_optional_request_params=params if params is not None else {},
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers={},
|
||||
)
|
||||
|
||||
def test_additional_tools_item_hoisted_to_top_level_tools(self):
|
||||
body = self._transform(
|
||||
input=[
|
||||
{"type": "additional_tools", "role": "developer", "tools": self._CODEX_TOOLS},
|
||||
self._DEVELOPER_MESSAGE,
|
||||
self._USER_MESSAGE,
|
||||
]
|
||||
)
|
||||
assert body["input"] == [self._DEVELOPER_MESSAGE, self._USER_MESSAGE]
|
||||
assert body["tools"] == self._CODEX_TOOLS
|
||||
|
||||
def test_hoisted_tools_append_after_existing_tools(self):
|
||||
existing_tool = {"type": "function", "name": "preexisting"}
|
||||
body = self._transform(
|
||||
input=[
|
||||
{"type": "additional_tools", "role": "developer", "tools": self._CODEX_TOOLS},
|
||||
self._USER_MESSAGE,
|
||||
],
|
||||
params={"tools": [existing_tool]},
|
||||
)
|
||||
assert body["tools"] == [existing_tool, *self._CODEX_TOOLS]
|
||||
|
||||
def test_unsupported_hoisted_tool_types_are_dropped(self):
|
||||
body = self._transform(
|
||||
input=[
|
||||
{
|
||||
"type": "additional_tools",
|
||||
"role": "developer",
|
||||
"tools": [
|
||||
{"type": "web_search"},
|
||||
{"type": "function", "name": "wait"},
|
||||
],
|
||||
},
|
||||
self._USER_MESSAGE,
|
||||
]
|
||||
)
|
||||
assert body["tools"] == [{"type": "function", "name": "wait"}]
|
||||
|
||||
def test_item_stripped_even_when_no_hoisted_tool_survives(self):
|
||||
body = self._transform(
|
||||
input=[
|
||||
{"type": "additional_tools", "role": "developer", "tools": [{"type": "web_search"}]},
|
||||
self._USER_MESSAGE,
|
||||
]
|
||||
)
|
||||
assert body["input"] == [self._USER_MESSAGE]
|
||||
assert "tools" not in body
|
||||
|
||||
def test_multiple_additional_tools_items_merge_in_order(self):
|
||||
first = {"type": "function", "name": "first"}
|
||||
second = {"type": "function", "name": "second"}
|
||||
body = self._transform(
|
||||
input=[
|
||||
{"type": "additional_tools", "role": "developer", "tools": [first]},
|
||||
self._USER_MESSAGE,
|
||||
{"type": "additional_tools", "role": "developer", "tools": [second]},
|
||||
]
|
||||
)
|
||||
assert body["input"] == [self._USER_MESSAGE]
|
||||
assert body["tools"] == [first, second]
|
||||
|
||||
def test_string_input_passes_through(self):
|
||||
body = self._transform(input="hello")
|
||||
assert body["input"] == "hello"
|
||||
assert "tools" not in body
|
||||
|
||||
def test_input_without_additional_tools_is_unchanged(self):
|
||||
codex_agentic_items = [
|
||||
self._USER_MESSAGE,
|
||||
{"type": "reasoning", "summary": [], "encrypted_content": "gAAAA=="},
|
||||
{"type": "function_call", "name": "wait", "arguments": "{}", "call_id": "call_1"},
|
||||
{"type": "function_call_output", "call_id": "call_1", "output": "done"},
|
||||
]
|
||||
body = self._transform(input=list(codex_agentic_items))
|
||||
assert body["input"] == codex_agentic_items
|
||||
assert "tools" not in body
|
||||
|
||||
def test_malformed_additional_tools_item_without_tools_list_is_stripped(self):
|
||||
body = self._transform(
|
||||
input=[
|
||||
{"type": "additional_tools", "role": "developer"},
|
||||
self._USER_MESSAGE,
|
||||
]
|
||||
)
|
||||
assert body["input"] == [self._USER_MESSAGE]
|
||||
assert "tools" not in body
|
||||
|
||||
def test_hoist_is_logged_at_debug_level(self):
|
||||
from unittest.mock import patch
|
||||
|
||||
with patch(
|
||||
"litellm.llms.bedrock_mantle.responses.transformation.verbose_logger.debug"
|
||||
) as mock_debug:
|
||||
self._transform(
|
||||
input=[
|
||||
{"type": "additional_tools", "role": "developer", "tools": self._CODEX_TOOLS},
|
||||
self._USER_MESSAGE,
|
||||
]
|
||||
)
|
||||
assert mock_debug.call_count == 1
|
||||
assert "additional_tools" in str(mock_debug.call_args)
|
||||
|
||||
|
||||
class TestBedrockMantleResponsesRegistry:
|
||||
def test_registry_returns_config_for_gpt_5_5(self, local_cost_map):
|
||||
# gpt-5.x advertises /v1/responses in supported_endpoints (capability)
|
||||
|
|
|
|||
|
|
@ -5376,3 +5376,41 @@ async def test_edit_mcp_server_snapshot_failure_skips_purge_but_edit_succeeds():
|
|||
|
||||
assert result.server_id == server_id
|
||||
mock_purge.assert_not_awaited()
|
||||
|
||||
|
||||
def test_bundled_openapi_registry_parses_and_entries_are_well_formed():
|
||||
"""The OpenAPI quick-picker registry ships as a bundled JSON file; a malformed file or entry
|
||||
silently degrades the picker to empty (the endpoint swallows load errors), so pin the file's
|
||||
shape here: it must parse, and every entry needs the fields the create-form prefill reads.
|
||||
OAuth-capable entries must carry both endpoint URLs; a catalog entry with a blank
|
||||
authorization_url would recreate the exact 400 ("authorization url is not set") the catalog
|
||||
exists to prevent for spec-only servers, which never run OAuth endpoint discovery."""
|
||||
import json
|
||||
import os
|
||||
|
||||
registry_path = os.path.join(
|
||||
os.path.dirname(os.path.abspath(__file__)),
|
||||
"..", "..", "..", "..", "litellm", "proxy", "openapi_registry.json",
|
||||
)
|
||||
with open(registry_path) as f:
|
||||
registry = json.load(f)
|
||||
|
||||
apis = registry["apis"]
|
||||
assert apis, "registry must not be empty"
|
||||
names = [entry["name"] for entry in apis]
|
||||
assert len(names) == len(set(names)), "duplicate registry entry names"
|
||||
for google_entry in ("google_sheets", "google_drive", "google_calendar", "google_docs"):
|
||||
assert google_entry in names, f"LIT-4629: {google_entry} must be in the catalog"
|
||||
|
||||
for entry in apis:
|
||||
for required in ("name", "title", "description", "icon_url", "spec_url"):
|
||||
assert entry.get(required), f"{entry.get('name')}: missing {required}"
|
||||
assert entry["spec_url"].startswith("https://"), f"{entry['name']}: non-https spec_url"
|
||||
oauth = entry.get("oauth")
|
||||
if oauth is not None:
|
||||
for required in ("authorization_url", "token_url"):
|
||||
assert oauth.get(required, "").startswith("https://"), (
|
||||
f"{entry['name']}: oauth.{required} must be a non-empty https URL"
|
||||
)
|
||||
for tool in entry.get("key_tools", []):
|
||||
assert tool.get("name") and tool.get("description"), f"{entry['name']}: malformed key_tool"
|
||||
|
|
|
|||
|
|
@ -803,6 +803,8 @@ def test_shared_backend_model_info_keeps_schema_fields_and_drops_the_rest():
|
|||
"litellm_provider": "openai",
|
||||
"max_tokens": 128000,
|
||||
"supports_vision": True,
|
||||
"supported_endpoints": ["/v1/responses"],
|
||||
"use_openai_responses_path": True,
|
||||
"input_cost_per_token": 0.99,
|
||||
"output_cost_per_token": 0.99,
|
||||
"id": "deploy-a",
|
||||
|
|
@ -818,9 +820,59 @@ def test_shared_backend_model_info_keeps_schema_fields_and_drops_the_rest():
|
|||
"litellm_provider": "openai",
|
||||
"max_tokens": 128000,
|
||||
"supports_vision": True,
|
||||
"supported_endpoints": ["/v1/responses"],
|
||||
"use_openai_responses_path": True,
|
||||
}
|
||||
|
||||
|
||||
def test_capability_flags_propagate_from_deployment_model_info_to_shared_key():
|
||||
"""Backend-model capability facts (supported_endpoints,
|
||||
use_openai_responses_path) declared in a deployment's model_info must reach
|
||||
the shared backend key: the Bedrock Mantle routing gates read them raw off
|
||||
litellm.model_cost and document proxy model_info as an override path for
|
||||
models missing from the built-in cost map.
|
||||
"""
|
||||
from litellm.llms.bedrock_mantle.common_utils import (
|
||||
mantle_base_segment,
|
||||
mantle_supports_responses,
|
||||
)
|
||||
|
||||
bare_model = "somelab.lit4544-unmapped-model"
|
||||
backend_model = f"bedrock_mantle/{bare_model}"
|
||||
deploy_id = "lit4544-mantle-deploy"
|
||||
|
||||
model_keys = {
|
||||
key: copy.deepcopy(litellm.model_cost.get(key))
|
||||
for key in (bare_model, backend_model, deploy_id)
|
||||
}
|
||||
try:
|
||||
Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "mantle-alias",
|
||||
"litellm_params": {
|
||||
"model": backend_model,
|
||||
"api_key": "fake-key",
|
||||
},
|
||||
"model_info": {
|
||||
"id": deploy_id,
|
||||
"supported_endpoints": ["/v1/responses"],
|
||||
"use_openai_responses_path": True,
|
||||
},
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
shared_entry = litellm.model_cost.get(backend_model) or {}
|
||||
assert shared_entry.get("supported_endpoints") == ["/v1/responses"]
|
||||
assert shared_entry.get("use_openai_responses_path") is True
|
||||
assert "id" not in shared_entry
|
||||
assert mantle_supports_responses(bare_model, litellm.model_cost) is True
|
||||
assert mantle_base_segment(bare_model, litellm.model_cost) == "openai/v1"
|
||||
finally:
|
||||
_restore_model_cost_entries(model_keys)
|
||||
|
||||
|
||||
def test_wildcard_zero_cost_request_does_not_poison_named_deployment_pricing():
|
||||
"""LIT-3991 end to end: a proxy has a named text-embedding-3-small
|
||||
deployment relying on built-in pricing plus an ``openai/*`` wildcard with
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue