fix(providers): generate Tsubasa endpoint discovery

This commit is contained in:
cenab 2026-10-28 11:31:16 -06:00
commit dc473cb257
17 changed files with 658 additions and 71 deletions

View file

@ -1670,6 +1670,7 @@ MANAGED_BACKGROUND_RESPONSE_METADATA_KEY: Final = "_managed_background_response"
SESSION_ID_GENERATED_METADATA_KEY: Final = "litellm_session_id_generated"
SESSION_ID_OMITTED_METADATA_KEY: Final = "litellm_session_id_omitted"
CALLER_SESSION_ID_METADATA_KEY: Final = "litellm_caller_session_id"
COMPLEXITY_ROUTER_CHILD_SESSION_ID_METADATA_KEY: Final = "_complexity_router_child_session_id"
LITELLM_TRUNCATED_PAYLOAD_FIELD: Final = "litellm_truncated"
LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE: Final = (
"Truncation is a DB storage safeguard. "

View file

@ -77007,13 +77007,13 @@
"supports_vision": true
},
"refuseless/glm-5.3-abliterated": {
"cache_read_input_token_cost": 2.5e-07,
"input_cost_per_token": 1.25e-06,
"cache_read_input_token_cost": 1.5e-07,
"input_cost_per_token": 1e-06,
"litellm_provider": "refuseless",
"max_input_tokens": 1048576,
"max_output_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 2.25e-06,
"output_cost_per_token": 2e-06,
"source": "https://refuseless.com/pricing",
"supports_function_calling": true,
"supports_prompt_caching": true,
@ -77028,13 +77028,13 @@
]
},
"refuseless/glm-5.3-flash-abliterated": {
"cache_read_input_token_cost": 1.5e-07,
"input_cost_per_token": 2.5e-07,
"cache_read_input_token_cost": 4e-08,
"input_cost_per_token": 1.2e-07,
"litellm_provider": "refuseless",
"max_input_tokens": 1048576,
"max_output_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 8e-07,
"output_cost_per_token": 5e-07,
"source": "https://refuseless.com/pricing",
"supports_function_calling": true,
"supports_prompt_caching": true,
@ -77049,13 +77049,13 @@
]
},
"refuseless/deepseek-v4.1-flash-abliterated": {
"cache_read_input_token_cost": 4.5e-08,
"input_cost_per_token": 2.2e-07,
"cache_read_input_token_cost": 3.5e-08,
"input_cost_per_token": 1.4e-07,
"litellm_provider": "refuseless",
"max_input_tokens": 1048576,
"max_output_tokens": 130000,
"mode": "chat",
"output_cost_per_token": 7.5e-07,
"output_cost_per_token": 5e-07,
"source": "https://refuseless.com/pricing",
"supports_function_calling": true,
"supports_prompt_caching": true,

View file

@ -4180,7 +4180,26 @@
"display_name": "Tsubasa (`tsubasa`)",
"url": "https://docs.litellm.ai/docs/providers/tsubasa",
"endpoints": {
"chat_completions": true
"chat_completions": true,
"responses": true,
"text_completion": true,
"embeddings": false,
"image_generations": false,
"image_edits": false,
"audio_transcriptions": false,
"audio_speech": false,
"rerank": false,
"ocr": false,
"search": false,
"moderations": false,
"realtime": false,
"video_generations": false,
"batches": false,
"messages": true,
"interactions": true,
"systemone": false,
"a2a": true,
"generateContent": true
}
},
"v0": {

View file

@ -32,6 +32,7 @@ from litellm.constants import (
BILLED_RETRY_ATTEMPT_METADATA_KEY,
CALLER_SESSION_ID_METADATA_KEY,
CLIENT_OUTPUT_CEILING_METADATA_KEY,
COMPLEXITY_ROUTER_CHILD_SESSION_ID_METADATA_KEY,
CONSUMED_REQUEST_TAGS_METADATA_KEY,
INTERNAL_CALL_ORIGIN_METADATA_KEY,
LITELLM_PROXY_MASTER_KEY_ALIAS,
@ -210,13 +211,46 @@ def _caller_session_beside_trace_header(
return get_chain_id_from_headers(session_headers) or _caller_body_session_id(data, metadata_variable_name)
def _drop_client_caller_session(request_body: Mapping[str, object]) -> None:
def _child_tier_session_metadata(
normalized_headers: Mapping[str, str], data: Mapping[str, object], metadata_variable_name: str
) -> Mapping[str, str | None]:
parent: Final = normalized_headers.get("x-parent-session-id")
if not parent or not _SESSION_ID_VALUE_RE.fullmatch(parent):
return {}
own_headers: Final = {
key: value
for key, value in normalized_headers.items()
if key not in ("x-parent-session-id", "x-litellm-trace-id")
}
active: Final = data.get(metadata_variable_name)
inactive: Final = data.get("litellm_metadata")
inactive_session: Final = (
_caller_body_session_id(data, "litellm_metadata")
if metadata_variable_name == "metadata"
and not (isinstance(active, Mapping) and "session_id" in active)
and isinstance(inactive, Mapping)
else None
)
own_session: Final = (
get_chain_id_from_headers(own_headers)
or _get_anthropic_session_id_from_metadata(data.get("metadata"))
or _caller_body_session_id(data, metadata_variable_name)
or inactive_session
or _session_id_from_baggage(normalized_headers.get("baggage", ""))
)
return {COMPLEXITY_ROUTER_CHILD_SESSION_ID_METADATA_KEY: own_session if own_session != parent else None}
def _drop_client_session_metadata(request_body: Mapping[str, object]) -> None:
for metadata_key in ("metadata", "litellm_metadata"):
if isinstance(metadata := request_body.get(metadata_key), dict):
metadata.pop(CALLER_SESSION_ID_METADATA_KEY, None)
for key in (CALLER_SESSION_ID_METADATA_KEY, COMPLEXITY_ROUTER_CHILD_SESSION_ID_METADATA_KEY):
cast(dict[str, object], metadata).pop(key, None) # cast-ok: request metadata is a str-keyed JSON object
_PROXY_SET_METADATA_KEYS: Final = frozenset({CALLER_SESSION_ID_METADATA_KEY, BAGGAGE_MAPPED_TAGS_METADATA_KEY})
_PROXY_SET_METADATA_KEYS: Final = frozenset(
{CALLER_SESSION_ID_METADATA_KEY, BAGGAGE_MAPPED_TAGS_METADATA_KEY, COMPLEXITY_ROUTER_CHILD_SESSION_ID_METADATA_KEY}
)
_METADATA_TAGS_ADAPTER: Final = TypeAdapter(tuple[object, ...])
@ -1855,7 +1889,7 @@ class LiteLLMProxyRequestSetup:
metadata_from_headers: Final = LitellmMetadataFromRequestHeaders()
request_body: Final = cast(Mapping[str, object], data) # cast-ok: request body is a str-keyed JSON object
_drop_client_caller_session(request_body)
_drop_client_session_metadata(request_body)
spend_logs_metadata: Final = LiteLLMProxyRequestSetup._get_spend_logs_metadata_from_request_headers(headers)
if spend_logs_metadata is not None:
metadata_from_headers["spend_logs_metadata"] = spend_logs_metadata
@ -1870,6 +1904,9 @@ class LiteLLMProxyRequestSetup:
agent_id_from_header: Final = headers.get("x-litellm-agent-id")
# Explicit litellm headers take precedence; fall back to any x-*-session-id header.
chain_id: Final = get_chain_id_from_headers(dict(headers))
child_tier_metadata: Final = _child_tier_session_metadata(
normalized_headers, request_body, _metadata_variable_name
)
if agent_id_from_header:
metadata_from_headers["agent_id"] = agent_id_from_header
@ -1939,9 +1976,15 @@ class LiteLLMProxyRequestSetup:
if isinstance(data[_metadata_variable_name], dict):
data[_metadata_variable_name].update(
{**metadata_from_headers, CALLER_SESSION_ID_METADATA_KEY: caller_session_beside_trace}
if caller_session_beside_trace
else metadata_from_headers
{
**metadata_from_headers,
**child_tier_metadata,
**(
{CALLER_SESSION_ID_METADATA_KEY: caller_session_beside_trace}
if caller_session_beside_trace
else {}
),
}
)
return data

View file

@ -477,6 +477,12 @@ unless `modality_routing` is also on.
### Session pin retention
With `classification_mode: user_turn`, a child request carrying `x-parent-session-id` classifies
its own new asks and stores the tier under its own session ID. Its continuations reuse that tier
without replacing the parent's tier. Existing trace grouping, per-tier model affinity and deployment
affinity retain their session identities. A child without a distinct, usable session ID classifies
without storing a tier pin. `session_affinity: true` retains precedence and the existing shared-pin behavior
`session_affinity_ttl_seconds` is the idle window for both the model pin selected by session affinity and the deployment pin. Every request that reuses a pin refreshes its TTL, so a session actively sending requests stays pinned. After the window passes with no pin reuse, the next request classifies again and creates a fresh pin. Omit the setting to track the default of 3600 seconds.
### Mid-task stall escalation

View file

@ -37,6 +37,7 @@ from litellm._internal_context import with_service_target
from litellm._logging import verbose_router_logger
from litellm.caching.affinity_cache import ROUTER_SESSION_PINS_TARGET, claim_affinity_pin, read_affinity_pin
from litellm.constants import (
COMPLEXITY_ROUTER_CHILD_SESSION_ID_METADATA_KEY,
EMPTY_MAPPING,
INTERNAL_CALL_ORIGIN_METADATA_KEY,
OUTPUT_TOKEN_CEILING_PARAMS,
@ -4358,17 +4359,17 @@ class ComplexityRouter(CustomLogger):
return newest_ask is not None and newest_ask.startswith(_CODEX_COMPACTION_SUMMARY_PREFIX)
@staticmethod
def _iter_metadata_dicts(request_kwargs: dict) -> list[dict]:
def _iter_metadata_dicts(request_kwargs: Mapping[str, object]) -> list[Mapping[str, object]]:
"""Metadata may land on `metadata` or `litellm_metadata` depending on the
endpoint, mirroring DeploymentAffinityCheck's precedence."""
return [
metadata
cast(Mapping[str, object], metadata) # cast-ok: request metadata is a str-keyed JSON object
for metadata_key in ("litellm_metadata", "metadata")
if isinstance(metadata := request_kwargs.get(metadata_key), dict)
]
@staticmethod
def _get_session_id_from_request_kwargs(request_kwargs: dict) -> str | None:
def _get_session_id_from_request_kwargs(request_kwargs: Mapping[str, object]) -> str | None:
"""Resolve a client-supplied session_id."""
for metadata in ComplexityRouter._iter_metadata_dicts(request_kwargs):
session_id = metadata.get("session_id")
@ -4376,6 +4377,21 @@ class ComplexityRouter(CustomLogger):
return str(session_id)
return None
def _get_tier_pin_session_id(self, request_kwargs: Mapping[str, object]) -> str | None:
if self.config.classification_mode == "user_turn" and not self.config.session_affinity:
child_metadata: Final = next(
(
metadata
for metadata in self._iter_metadata_dicts(request_kwargs)
if COMPLEXITY_ROUTER_CHILD_SESSION_ID_METADATA_KEY in metadata
),
None,
)
if child_metadata is not None:
child_session: Final = child_metadata[COMPLEXITY_ROUTER_CHILD_SESSION_ID_METADATA_KEY]
return child_session if isinstance(child_session, str) and child_session else None
return self._get_session_id_from_request_kwargs(request_kwargs)
@staticmethod
def _get_user_api_key_hash_from_request_kwargs(request_kwargs: dict) -> str | None:
"""Resolve the proxy-derived API key hash, the same trust boundary
@ -4460,7 +4476,7 @@ class ComplexityRouter(CustomLogger):
conversation_continuing: Final = _conversation_is_continuing(resolved_messages)
use_session_affinity: Final = self._uses_tier_pin
session_id: Final = self._get_session_id_from_request_kwargs(request_kwargs) if use_session_affinity else None
session_id: Final = self._get_tier_pin_session_id(request_kwargs) if use_session_affinity else None
cache_key = self._get_session_affinity_cache_key(session_id, request_kwargs) if session_id is not None else None
# In 'user_turn' mode a held pin is replayed only on continuation turns; a new human

View file

@ -77007,13 +77007,13 @@
"supports_vision": true
},
"refuseless/glm-5.3-abliterated": {
"cache_read_input_token_cost": 2.5e-07,
"input_cost_per_token": 1.25e-06,
"cache_read_input_token_cost": 1.5e-07,
"input_cost_per_token": 1e-06,
"litellm_provider": "refuseless",
"max_input_tokens": 1048576,
"max_output_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 2.25e-06,
"output_cost_per_token": 2e-06,
"source": "https://refuseless.com/pricing",
"supports_function_calling": true,
"supports_prompt_caching": true,
@ -77028,13 +77028,13 @@
]
},
"refuseless/glm-5.3-flash-abliterated": {
"cache_read_input_token_cost": 1.5e-07,
"input_cost_per_token": 2.5e-07,
"cache_read_input_token_cost": 4e-08,
"input_cost_per_token": 1.2e-07,
"litellm_provider": "refuseless",
"max_input_tokens": 1048576,
"max_output_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 8e-07,
"output_cost_per_token": 5e-07,
"source": "https://refuseless.com/pricing",
"supports_function_calling": true,
"supports_prompt_caching": true,
@ -77049,13 +77049,13 @@
]
},
"refuseless/deepseek-v4.1-flash-abliterated": {
"cache_read_input_token_cost": 4.5e-08,
"input_cost_per_token": 2.2e-07,
"cache_read_input_token_cost": 3.5e-08,
"input_cost_per_token": 1.4e-07,
"litellm_provider": "refuseless",
"max_input_tokens": 1048576,
"max_output_tokens": 130000,
"mode": "chat",
"output_cost_per_token": 7.5e-07,
"output_cost_per_token": 5e-07,
"source": "https://refuseless.com/pricing",
"supports_function_calling": true,
"supports_prompt_caching": true,

View file

@ -4180,7 +4180,26 @@
"display_name": "Tsubasa (`tsubasa`)",
"url": "https://docs.litellm.ai/docs/providers/tsubasa",
"endpoints": {
"chat_completions": true
"chat_completions": true,
"responses": true,
"text_completion": true,
"embeddings": false,
"image_generations": false,
"image_edits": false,
"audio_transcriptions": false,
"audio_speech": false,
"rerank": false,
"ocr": false,
"search": false,
"moderations": false,
"realtime": false,
"video_generations": false,
"batches": false,
"messages": true,
"interactions": true,
"systemone": false,
"a2a": true,
"generateContent": true
}
},
"v0": {

View file

@ -33,6 +33,14 @@ async def test_tsubasa_dashboard_fields_resolve_to_a_registered_model() -> None:
}
assert endpoints_response.status_code == 200, endpoints_response.text
endpoints: Final = SupportedEndpointsResponse.model_validate(endpoints_response.json()).endpoints
assert [
assert {
endpoint.key for endpoint in endpoints if any(p.slug == provider.litellm_provider for p in endpoint.providers)
] == ["chat_completions"]
} == {
"chat_completions",
"responses",
"text_completion",
"messages",
"interactions",
"a2a",
"generateContent",
}

View file

@ -19,6 +19,7 @@ from starlette.datastructures import Headers
import litellm
from litellm.constants import (
CALLER_SESSION_ID_METADATA_KEY,
COMPLEXITY_ROUTER_CHILD_SESSION_ID_METADATA_KEY,
ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY,
SERVER_STREAMING_CLASSIFICATION_KEY,
SERVER_STREAMING_CLASSIFICATION_MARKER,
@ -3684,6 +3685,110 @@ async def test_spend_log_caller_session_stays_out_of_requester_metadata(monkeypa
assert CALLER_SESSION_ID_METADATA_KEY not in updated["metadata"]["requester_metadata"]
@pytest.mark.asyncio
@pytest.mark.parametrize(
"surface,headers,body,child_id,trace_id",
[
(
"chat/completions", {"X-Session-Id": "child-01"}, {"metadata": {"session_id": "body-001"}},
"child-01", "parent-01",
),
("responses", {"x-vendor-session-id": "child-01"}, {}, "child-01", "parent-01"),
("messages", {"session-id": "child-01", "user-agent": "codex_cli_rs/1.0"}, {}, "child-01", "parent-01"),
("chat/completions", {"x-litellm-session-id": "short", "x-session-id": "child-01"}, {}, "short", "short"),
("responses", {"x-litellm-trace-id": "trace-01", "x-session-id": "child-01"}, {}, "child-01", "trace-01"),
(
"messages", {"x-litellm-trace-id": "trace-01", "x-litellm-session-id": "child-01"},
{}, "child-01", "trace-01",
),
("responses", {}, {"metadata": {"session_id": "body-001"}}, "body-001", "parent-01"),
("messages", {}, {"litellm_metadata": {"session_id": "body-001"}}, "body-001", "parent-01"),
("chat/completions", {}, {"litellm_metadata": {"session_id": "body-001"}}, "body-001", "parent-01"),
("messages", {"baggage": "session.id=child-01"}, {}, "child-01", "parent-01"),
(
"responses", {"baggage": "session.id=child-01"}, {"metadata": {"session_id": "body-001"}},
"body-001", "parent-01",
),
(
"chat/completions", {}, {"litellm_session_id": "root-001", "metadata": {"session_id": "body-001"}},
"root-001", "parent-01",
),
(
"responses", {}, {"metadata": {"session_id": "body-001"}, "litellm_metadata": {"session_id": "active-1"}},
"active-1", "parent-01",
),
(
"chat/completions", {},
{"metadata": {"session_id": "active-1"}, "litellm_metadata": {"session_id": "body-001"}},
"active-1", "parent-01",
),
(
"chat/completions", {}, {"metadata": {"session_id": ""}, "litellm_metadata": {"session_id": "body-001"}},
None, "parent-01",
),
(
"messages", {}, {"metadata": {"session_id": "body-001"}, "litellm_metadata": {"session_id": ""}},
None, "parent-01",
),
(
"messages", {}, {"litellm_session_id": "root-001", "litellm_metadata": {"session_id": "active-1"},
"metadata": {"session_id": "body-001", "user_id": "user_abc_account__session_anthro-1"}},
"anthro-1", "parent-01",
),
("messages", {"x-session-id": "child-01"}, {"metadata": {"user_id": {"session_id": "body-001"}}},
"child-01", "parent-01"),
("chat/completions", {}, {}, None, "parent-01"),
("responses", {"x-session-id": "bad"}, {}, None, "parent-01"),
("messages", {}, {"metadata": {"session_id": 123}}, None, "parent-01"),
("chat/completions", {"x-session-id": "parent-01"}, {}, None, "parent-01"),
],
)
async def test_child_tier_identity_preserves_trace_and_body_session_precedence(
surface: str, headers: dict[str, str], body: dict[str, object], child_id: str | None, trace_id: str
) -> None:
request: Final = _make_request_mock(f"/v1/{surface}", {"x-parent-session-id": "parent-01", **headers})
updated: Final = await add_litellm_data_to_request(
data={"model": "test-router", "messages": [], **copy.deepcopy(body)},
request=request,
user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"),
proxy_config=MagicMock(),
general_settings={},
)
metadata: Final = updated[get_metadata_variable_name(request)]
assert metadata[COMPLEXITY_ROUTER_CHILD_SESSION_ID_METADATA_KEY] == child_id
assert metadata["session_id"] == metadata["trace_id"] == trace_id
assert updated["litellm_session_id"] == updated["litellm_trace_id"] == trace_id
expected_caller: Final = (
headers.get("x-litellm-session-id", "parent-01") if "x-litellm-trace-id" in headers else None
)
assert metadata.get(CALLER_SESSION_ID_METADATA_KEY) == expected_caller
assert COMPLEXITY_ROUTER_CHILD_SESSION_ID_METADATA_KEY not in metadata.get("requester_metadata", {})
@pytest.mark.asyncio
@pytest.mark.parametrize("surface,parent_id", [("chat/completions", None), ("responses", None), ("messages", "bad")])
async def test_non_child_requests_cannot_forge_a_child_tier_identity(surface: str, parent_id: str | None) -> None:
headers: Final = {"x-session-id": "child-01", **({"x-parent-session-id": parent_id} if parent_id else {})}
request: Final = _make_request_mock(f"/v1/{surface}", headers)
updated: Final = await add_litellm_data_to_request(
data={
"model": "test-router",
"messages": [],
"metadata": {COMPLEXITY_ROUTER_CHILD_SESSION_ID_METADATA_KEY: "forged-1"},
"litellm_metadata": {COMPLEXITY_ROUTER_CHILD_SESSION_ID_METADATA_KEY: "forged-2"},
},
request=request,
user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"),
proxy_config=MagicMock(),
general_settings={},
)
for bucket in ("metadata", "litellm_metadata"):
metadata: Final = updated.get(bucket, {})
assert COMPLEXITY_ROUTER_CHILD_SESSION_ID_METADATA_KEY not in metadata
assert COMPLEXITY_ROUTER_CHILD_SESSION_ID_METADATA_KEY not in metadata.get("requester_metadata", {})
assert updated[get_metadata_variable_name(request)]["session_id"] == "child-01"
def test_add_litellm_metadata_from_request_headers_x_litellm_session_id_sets_chain_id():
"""x-litellm-session-id sets both metadata and top-level litellm_session_id/litellm_trace_id for call chaining."""
headers = {"x-litellm-session-id": "bar"}

View file

@ -7714,6 +7714,176 @@ class _DummyPlugin:
return context
class TestChildSessionTierPins:
@staticmethod
def _router(session_affinity: bool = False) -> Router:
return Router(
model_list=[
{
"model_name": "child-tier-router",
"litellm_params": {
"model": "auto_router/complexity_router",
"complexity_router_config": {
"tiers": {"SIMPLE": "simple", "REASONING": ["reasoning", "reasoning-peer"]},
"classification_mode": "user_turn",
"session_affinity": session_affinity,
"keyword_tier_rules": [{"keywords": ["reason carefully"], "tier": "REASONING"}],
},
},
},
*[
{
"model_name": group,
"litellm_params": {"model": "openai/test-model", "api_key": "test-only"},
"model_info": {"id": f"{group}-{index}"},
}
for group, index in (
("simple", 0), ("simple", 1), ("reasoning", 0),
("reasoning", 1), ("reasoning-peer", 0), ("reasoning-peer", 1),
)
],
],
num_retries=0,
)
@staticmethod
def _normalized_request(
surface: str, session_id: str | None, caller: str = "caller-a", child: bool = True
) -> tuple[dict[str, object], str]:
from litellm.proxy.litellm_pre_call_utils import (
LiteLLMProxyRequestSetup,
metadata_variable_name_for_route,
)
route: Final = {
"chat": "/v1/chat/completions", "responses": "/v1/responses", "messages": "/v1/messages"
}[surface]
bucket: Final = metadata_variable_name_for_route(route)
data: Final[dict[str, object]] = {bucket: {"user_api_key_hash": caller, "user_api_key_request_route": route}}
LiteLLMProxyRequestSetup.add_litellm_metadata_from_request_headers(
headers={
**({"X-Session-Id": session_id} if session_id is not None else {}),
**({"x-parent-session-id": "parent-session"} if child else {}),
},
data=data,
_metadata_variable_name=bucket,
)
return data, bucket
async def _route(
self, router: Router, surface: str, session_id: str | None, prompt: str | None,
expected_tier: str, expected_cause: str, caller: str = "caller-a", child: bool = True,
expected_deployment: str | None = None,
) -> str:
from litellm.router_utils.pre_call_checks.deployment_affinity_check import DeploymentAffinityCheck
data, bucket = self._normalized_request(surface, session_id, caller, child)
turn: Final[dict[str, object]] = (
{"role": "user", "content": prompt} if prompt is not None else
{"role": "user", "content": [{"type": "tool_result", "tool_use_id": "call_1", "content": "done"}]}
if surface == "messages" else
{"type": "function_call_output", "call_id": "call_1", "output": "done"}
if surface == "responses" else
{"role": "tool", "tool_call_id": "call_1", "content": "done"}
)
tool_call: Final = (
{"type": "function_call", "call_id": "call_1", "name": "read_file", "arguments": "{}"}
if surface == "responses" else
{"role": "assistant", "content": [{"type": "tool_use", "id": "call_1", "name": "read_file", "input": {}}]}
if surface == "messages" else TestClassificationMode.TOOL_CALL_1
)
turns: Final = [turn] if prompt is not None else [TestClassificationMode.SIMPLE_ASK, tool_call, turn]
if surface == "responses":
data["input"] = turns
selected: Final = await router.async_get_available_deployment(
model="child-tier-router", request_kwargs=data,
messages=None if surface == "responses" else turns,
input=turns if surface == "responses" else None,
)
metadata: Final = data[bucket]
assert isinstance(metadata, dict)
assert (metadata["session_id"], metadata["trace_id"], data["litellm_session_id"], data["litellm_trace_id"]) == (
"parent-session", "parent-session", "parent-session", "parent-session"
)
assert (metadata["routing_decision"]["tier"], metadata["routing_decision"]["cause"]) == (
expected_tier, expected_cause
)
affinity: Final = next(cb for cb in router.optional_callbacks or () if isinstance(cb, DeploymentAffinityCheck))
if expected_deployment is not None:
filtered: Final = await affinity.async_filter_deployments(
model=selected["model_name"], healthy_deployments=router.get_model_list(selected["model_name"]) or [],
messages=None, request_kwargs=data,
)
assert [deployment["model_info"]["id"] for deployment in filtered] == [expected_deployment]
assert selected["model_info"]["id"] == expected_deployment
await affinity.async_pre_call_deployment_hook(
kwargs={
**data, "model_info": selected["model_info"],
bucket: {**metadata, "deployment_model_name": selected["model_name"]},
},
call_type=None,
)
return str(selected["model_info"]["id"])
@pytest.mark.asyncio
@pytest.mark.parametrize("surface", ["chat", "responses", "messages"])
async def test_children_keep_their_tier_without_replacing_the_parent(self, surface: str) -> None:
router: Final = self._router()
try:
parent: Final = await self._route(
router, surface, "parent-session", "reason carefully", "REASONING", "literal_keyword_match", child=False
)
for session, prompt, tier, cause, caller, child in (
("child-session-a", "Hello!", "SIMPLE", "heuristic_scorer", "caller-a", True),
("child-session-a", None, "SIMPLE", "user_turn_continuation", "caller-a", True),
("parent-session", None, "REASONING", "user_turn_continuation", "caller-a", False),
("child-session-b", None, "SIMPLE", "heuristic_scorer", "caller-a", True),
("child-session-b", "reason carefully", "REASONING", "literal_keyword_match", "caller-a", True),
("child-session-a", None, "SIMPLE", "user_turn_continuation", "caller-a", True),
("child-session-a", "reason carefully", "REASONING", "literal_keyword_match", "caller-b", True),
("child-session-a", None, "SIMPLE", "user_turn_continuation", "caller-a", True),
(None, "Hello!", "SIMPLE", "heuristic_scorer", "caller-a", True),
(None, None, "SIMPLE", "heuristic_scorer", "caller-a", True),
):
await self._route(
router, surface, session, prompt, tier, cause, caller, child,
expected_deployment=parent if tier == "REASONING" and caller == "caller-a" else None,
)
assert await self._route(
router, surface, "parent-session", None, "REASONING", "user_turn_continuation",
child=False, expected_deployment=parent,
) == parent
finally:
router.discard()
@pytest.mark.asyncio
async def test_session_affinity_still_shares_the_parent_tier(self) -> None:
router: Final = self._router(session_affinity=True)
try:
parent: Final = await self._route(
router, "chat", "parent-session", "reason carefully", "REASONING", "literal_keyword_match", child=False
)
assert await self._route(
router, "chat", "child-session-a", "Hello!", "REASONING", "session_affinity_pin",
expected_deployment=parent,
) == parent
finally:
router.discard()
@pytest.mark.asyncio
async def test_child_tier_model_affinity_keeps_using_the_parent_identity(self) -> None:
router: Final = self._router()
try:
strategy: Final = router.complexity_routers["child-tier-router"][0].strategy
parent, _ = self._normalized_request("chat", "parent-session", child=False)
child, _ = self._normalized_request("chat", "child-session-a")
candidates: Final = ("reasoning", "reasoning-peer")
assert await strategy._pin_model_for_tier("REASONING", "reasoning", candidates, parent) == "reasoning"
assert await strategy._pin_model_for_tier("REASONING", "reasoning-peer", candidates, child) == "reasoning"
finally:
router.discard()
class TestClassificationMode:
"""Test classification_mode='user_turn': classify only requests whose newest turn is a new
human ask; tool-loop continuation turns replay the session's held routing decision."""

View file

@ -37,6 +37,7 @@ import {
} from "./tier_rows";
import React from "react";
import { ModelGroup } from "@/components/llm_calls/fetch_models";
import { routingModelGroups } from "./auto_setup";
import ClassificationMethodConfig, { InactiveHeuristicV2Threshold } from "./ClassificationMethodConfig";
import ComplexityRouterAdvancedSections from "./ComplexityRouterAdvancedSections";
import { type TierSetAction, applyTierSetAction, setFallbackTier } from "./tier_set_actions";
@ -657,13 +658,10 @@ const ComplexityRouterConfig: React.FC<ComplexityRouterConfigProps> = ({
);
const classifierEffortOptionsByModel = classifierEffortOptionsForModels(modelInfo);
// Embedding models can't serve a chat-completion role, so they're excluded here.
const modelOptions = modelInfo
.filter((model) => model.mode !== "embedding")
.map((model) => ({
value: model.model_group,
label: model.model_group,
}));
const modelOptions = routingModelGroups(modelInfo).map((model) => ({
value: model.model_group,
label: model.model_group,
}));
const handleTierModelParamChange = (tier: string, model: string, change: TierModelParamChange) =>
onChange({

View file

@ -319,6 +319,27 @@ describe("AddAutoRouterTab", () => {
},
);
it("carries Responses model choices from simple setup into Advanced tiers", async () => {
const models = ["gpt-6-luna", "gpt-6-astra"];
mockFetchAvailableModels.mockResolvedValue(models.map((model_group) => ({ model_group, mode: "responses" })));
mockFetchAllModelDeployments.mockResolvedValue(
models.map((model_name) => ({
model_name,
litellm_params: { model: `chatgpt/${model_name}` },
model_info: { mode: "responses" },
})),
);
renderWithProviders(<Harness />);
await waitFor(() =>
expect(screen.getByRole("combobox", { name: "Everyday model" })).toHaveTextContent("gpt-6-luna"),
);
expect(screen.getByRole("combobox", { name: "Hard tasks model" })).toHaveTextContent("gpt-6-astra");
expect(screen.queryByText(/Two deployed models are needed/)).not.toBeInTheDocument();
fireEvent.click(screen.getByRole("button", { name: "Advanced configuration", exact: true }));
expectTierModel("Simple", "gpt-6-luna");
expectTierModel("Reasoning", "gpt-6-astra");
});
it("keeps manual solver choices on goal changes and carries them into Advanced", async () => {
mockFetchAvailableModels.mockResolvedValue(
["claude-haiku-5-5", "custom-fast", "claude-opus-5-5"].map((model_group) => ({

View file

@ -7,6 +7,7 @@ import {
solverPairConfig,
nextRouterName,
simpleSetupError,
routingModelGroups,
} from "./auto_setup";
const deployment = (name: string, model: string, extra: Partial<AutoRouterDeployment> = {}): AutoRouterDeployment => ({
@ -76,7 +77,7 @@ describe("deployedRouterModels", () => {
]);
});
it("uses only caller-visible chat models and excludes blocked and recursive targets", () => {
it("matches Advanced choices while keeping blocked and recursive targets out of suggestions", () => {
const rows = [
deployment("hidden", "anthropic/claude-opus-6"),
deployment("blocked", "anthropic/claude-opus-5", { model_info: { blocked: true } }),
@ -94,9 +95,40 @@ describe("deployedRouterModels", () => {
],
rows,
).map((model) => model.name),
).toEqual(["ready"]);
).toEqual(["blocked", "ready", "recursive"]);
expect(
recommendSolverPair(
deployedRouterModels(
rows
.filter((row) => row.model_name !== "hidden" && row.model_name !== "embedding")
.map((row) => ({ model_group: row.model_name! })),
rows,
),
"balanced",
"auto",
),
).toBeNull();
});
it.each(["chat", "responses", "completion", "unknown", undefined])(
"offers the same routing choices as Advanced for mode %s",
(mode) => {
const groups = [
{ model_group: "everyday", mode },
{ model_group: "hard", mode },
{ model_group: "embedding", mode: "embedding" },
];
const rows = [
deployment("everyday", "chatgpt/gpt-6-luna", { model_info: { mode } }),
deployment("hard", "chatgpt/gpt-6-astra", { model_info: { mode } }),
];
const models = deployedRouterModels(groups, rows);
expect(models.map((model) => model.name)).toEqual(routingModelGroups(groups).map((group) => group.model_group));
expect(models.map((model) => model.name)).toEqual(["everyday", "hard"]);
expect(names(recommendSolverPair(models, "balanced", "openai"))).toEqual(["everyday", "hard"]);
},
);
it("does not claim a mixed-version or unknown model group is a known version", () => {
const rows = [
deployment("mixed", "anthropic/claude-opus-5"),
@ -112,7 +144,7 @@ describe("deployedRouterModels", () => {
],
rows,
);
expect(models.map((model) => model.name)).toEqual(["claude-sonnet-5", "mixed"]);
expect(models.map((model) => model.name)).toEqual(["claude-sonnet-5", "mixed", "wildcard"]);
expect(models.every((model) => model.family === undefined)).toBe(true);
expect(recommendSolverPair(models, "balanced", "auto")).toBeNull();
});

View file

@ -146,24 +146,21 @@ const compareVersion = (left: readonly number[], right: readonly number[]): numb
const compareRelease = (left: ModelIdentity, right: ModelIdentity): number =>
compareVersion(left.version, right.version) || left.snapshot - right.snapshot;
export const routingModelGroups = (groups: readonly ModelGroup[]): ModelGroup[] =>
groups.filter((group) => group.mode !== "embedding");
export const deployedRouterModels = (
groups: readonly ModelGroup[],
deployments: readonly AutoRouterDeployment[],
): SolverModel[] =>
groups
.flatMap((group) => {
if (group.mode && group.mode !== "chat") return [];
routingModelGroups(groups)
.map((group) => {
const rows = deployments.filter((row) => row.model_name === group.model_group);
if (
rows.some(isAutoRouterDeployment) ||
group.model_group.includes("*") ||
group.model_group.startsWith("auto_router/")
)
return [];
const active = rows.filter((row) => !row.model_info?.blocked);
if (rows.length > 0 && active.length === 0) return [];
if (active.some((row) => row.model_info?.mode && row.model_info.mode !== "chat")) return [];
if (active.some((row) => row.litellm_params?.model?.includes("*"))) return [];
const isRouter = rows.some(isAutoRouterDeployment) || group.model_group.startsWith("auto_router/");
const hasWildcard =
group.model_group.includes("*") || active.some((row) => row.litellm_params?.model?.includes("*"));
const manualOnly = isRouter || hasWildcard || (rows.length > 0 && active.length === 0);
const identities = active.length > 0 ? active.map(identityOf) : [parseModel(group.model_group)];
const first = identities[0];
const homogeneous =
@ -173,19 +170,17 @@ export const deployedRouterModels = (
const sameLine = identity.family === first.family && identity.line === first.line;
return sameLine && compareRelease(identity, first) === 0;
});
const identity = homogeneous ? first : undefined;
const identity = homogeneous && !manualOnly ? first : undefined;
const providers =
active.length > 0
? active.map((row) => providerOf(row, group, identity?.family))
: [providerOf({}, group, identity?.family)];
return [
{
...identity,
name: group.model_group,
model: identity?.model ?? group.model_group,
provider: [...new Set(providers)].toSorted().join(", "),
},
];
return {
...identity,
name: group.model_group,
model: identity?.model ?? group.model_group,
provider: [...new Set(providers)].toSorted().join(", "),
};
})
.toSorted((a, b) => a.name.localeCompare(b.name));

View file

@ -490,6 +490,137 @@ describe("responses_api", () => {
});
});
describe("responses_api reasoning", () => {
type Run = { reasoning: string; answer: string; timeToFirstToken: number[]; reasoningBeforeFirstToken: string };
const run = async (streamingEnabled: boolean): Promise<Run> => {
const reasoningChunks: string[] = [];
const answerChunks: string[] = [];
const timeToFirstToken: number[] = [];
const reasoningAtFirstToken: string[] = [];
await makeOpenAIResponsesRequest(
[{ role: "user", content: "What is 3+4?" }],
(_role, delta) => answerChunks.push(delta),
"qwen3",
"test-token",
undefined,
undefined,
(content) => reasoningChunks.push(content),
(ttft) => {
timeToFirstToken.push(ttft);
reasoningAtFirstToken.push(reasoningChunks.join("") + "|" + answerChunks.join(""));
},
undefined,
undefined,
undefined,
undefined,
undefined,
undefined,
undefined,
undefined,
undefined,
undefined,
undefined,
undefined,
undefined,
undefined,
undefined,
streamingEnabled,
);
return {
reasoning: reasoningChunks.join(""),
answer: answerChunks.join(""),
timeToFirstToken,
reasoningBeforeFirstToken: reasoningAtFirstToken[0] ?? "",
};
};
const streamOf = (events: Record<string, unknown>[]) => {
async function* mockStream() {
for (const event of events) {
yield event;
}
yield { type: "response.completed", response: { id: "resp_reasoning" } };
}
mockResponsesCreate.mockResolvedValue(mockStream());
};
afterEach(() => {
vi.clearAllMocks();
});
it("shows streamed reasoning_text deltas and starts the first-token timer on them", async () => {
streamOf([
{ type: "response.reasoning_text.delta", item_id: "rs_1", content_index: 0, delta: "Add " },
{ type: "response.reasoning_text.delta", item_id: "rs_1", content_index: 0, delta: "them" },
{ type: "response.output_text.delta", delta: "7" },
]);
const result = await run(true);
expect(result.reasoning).toBe("Add them");
expect(result.answer).toBe("7");
expect(result.timeToFirstToken).toHaveLength(1);
expect(result.reasoningBeforeFirstToken).toBe("Add |");
});
it("separates streamed reasoning summary parts and items with a blank line", async () => {
streamOf([
{ type: "response.reasoning_summary_text.delta", item_id: "rs_1", summary_index: 0, delta: "First" },
{ type: "response.reasoning_summary_text.delta", item_id: "rs_1", summary_index: 0, delta: " part" },
{ type: "response.reasoning_summary_text.delta", item_id: "rs_1", summary_index: 1, delta: "Second" },
{ type: "response.reasoning_summary_text.delta", item_id: "rs_2", summary_index: 0, delta: "Third" },
]);
await expect(run(true)).resolves.toMatchObject({ reasoning: "First part\n\nSecond\n\nThird" });
});
it("still shows the legacy reasoning.delta event", async () => {
streamOf([{ type: "response.reasoning.delta", delta: "Legacy thought" }]);
await expect(run(true)).resolves.toMatchObject({ reasoning: "Legacy thought" });
});
it("shows non-streamed reasoning content when the summary is empty", async () => {
mockResponsesCreate.mockReturnValueOnce(
nonStreamingResponse({
id: "resp_content",
output: [
{
type: "reasoning",
summary: [],
content: [
{ type: "reasoning_text", text: "Add" },
{ type: "reasoning_text", text: "them" },
],
},
{ type: "message", content: [{ type: "output_text", text: "7" }] },
],
}),
);
await expect(run(false)).resolves.toMatchObject({ reasoning: "Add\n\nthem", answer: "7" });
});
it("prefers the non-streamed reasoning summary over its content", async () => {
mockResponsesCreate.mockReturnValueOnce(
nonStreamingResponse({
id: "resp_summary",
output: [
{
type: "reasoning",
summary: [{ type: "summary_text", text: "Short summary" }],
content: [{ type: "reasoning_text", text: "Raw chain" }],
},
{ type: "reasoning", content: [{ type: "output_text", text: "Second item" }] },
],
}),
);
await expect(run(false)).resolves.toMatchObject({ reasoning: "Short summary\n\nSecond item" });
});
});
describe("responses_api prompt cache usage", () => {
const captureUsage = async (usage: Record<string, unknown>): Promise<TokenUsage> => {
async function* mockStream() {

View file

@ -44,6 +44,22 @@ type SynthesizedResponseEvent =
| { type: "response.output_text.delta"; delta: string }
| { type: "response.completed"; response: NonStreamedResponse };
const REASONING_PART_SEPARATOR = "\n\n";
const REASONING_DELTA_EVENT_TYPES: ReadonlySet<string> = new Set([
"response.reasoning_text.delta",
"response.reasoning_summary_text.delta",
"response.reasoning.delta",
]);
const nonEmptyTexts = (parts: ResponseOutputPart[] | undefined): string[] =>
(parts ?? []).map((part) => part.text ?? "").filter((text) => text.length > 0);
const reasoningItemText = (item: ResponseOutputItem): string => {
const summary = nonEmptyTexts(item.summary);
return (summary.length > 0 ? summary : nonEmptyTexts(item.content)).join(REASONING_PART_SEPARATOR);
};
const responseAsEvents = (response: NonStreamedResponse): SynthesizedResponseEvent[] => {
const outputItems = response.output ?? [];
const outputText = outputItems
@ -54,9 +70,9 @@ const responseAsEvents = (response: NonStreamedResponse): SynthesizedResponseEve
.join("");
const reasoningText = outputItems
.filter((item) => item.type === "reasoning")
.flatMap((item) => item.summary ?? [])
.map((part) => part.text ?? "")
.join("");
.map(reasoningItemText)
.filter((text) => text.length > 0)
.join(REASONING_PART_SEPARATOR);
return [
...outputItems.map((item) => ({ type: "response.output_item.done" as const, item })),
@ -124,6 +140,7 @@ export async function makeOpenAIResponsesRequest(
try {
const startTime = Date.now();
let firstTokenReceived = false;
let reasoningPartKey: string | null = null;
let servedFromResponseCache = false;
const formattedInput = messages.flatMap((message) => {
@ -298,10 +315,16 @@ export async function makeOpenAIResponsesRequest(
}
// Handle reasoning content
if (event.type === "response.reasoning.delta" && "delta" in event) {
const delta = event.delta;
if (typeof delta === "string" && onReasoningContent) {
onReasoningContent(delta);
if (REASONING_DELTA_EVENT_TYPES.has(event.type) && typeof event.delta === "string" && event.delta.length > 0) {
const partKey = `${event.item_id ?? ""}:${event.summary_index ?? event.content_index ?? 0}`;
const separator = reasoningPartKey !== null && reasoningPartKey !== partKey ? REASONING_PART_SEPARATOR : "";
reasoningPartKey = partKey;
onReasoningContent?.(separator + event.delta);
const isFirstToken = !firstTokenReceived;
firstTokenReceived = true;
if (isFirstToken && onTimingData && streamingEnabled) {
onTimingData(Date.now() - startTime);
}
}