mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(providers): generate Tsubasa endpoint discovery
This commit is contained in:
commit
dc473cb257
17 changed files with 658 additions and 71 deletions
|
|
@ -1670,6 +1670,7 @@ MANAGED_BACKGROUND_RESPONSE_METADATA_KEY: Final = "_managed_background_response"
|
|||
SESSION_ID_GENERATED_METADATA_KEY: Final = "litellm_session_id_generated"
|
||||
SESSION_ID_OMITTED_METADATA_KEY: Final = "litellm_session_id_omitted"
|
||||
CALLER_SESSION_ID_METADATA_KEY: Final = "litellm_caller_session_id"
|
||||
COMPLEXITY_ROUTER_CHILD_SESSION_ID_METADATA_KEY: Final = "_complexity_router_child_session_id"
|
||||
LITELLM_TRUNCATED_PAYLOAD_FIELD: Final = "litellm_truncated"
|
||||
LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE: Final = (
|
||||
"Truncation is a DB storage safeguard. "
|
||||
|
|
|
|||
|
|
@ -77007,13 +77007,13 @@
|
|||
"supports_vision": true
|
||||
},
|
||||
"refuseless/glm-5.3-abliterated": {
|
||||
"cache_read_input_token_cost": 2.5e-07,
|
||||
"input_cost_per_token": 1.25e-06,
|
||||
"cache_read_input_token_cost": 1.5e-07,
|
||||
"input_cost_per_token": 1e-06,
|
||||
"litellm_provider": "refuseless",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 131072,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2.25e-06,
|
||||
"output_cost_per_token": 2e-06,
|
||||
"source": "https://refuseless.com/pricing",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
|
|
@ -77028,13 +77028,13 @@
|
|||
]
|
||||
},
|
||||
"refuseless/glm-5.3-flash-abliterated": {
|
||||
"cache_read_input_token_cost": 1.5e-07,
|
||||
"input_cost_per_token": 2.5e-07,
|
||||
"cache_read_input_token_cost": 4e-08,
|
||||
"input_cost_per_token": 1.2e-07,
|
||||
"litellm_provider": "refuseless",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 131072,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 8e-07,
|
||||
"output_cost_per_token": 5e-07,
|
||||
"source": "https://refuseless.com/pricing",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
|
|
@ -77049,13 +77049,13 @@
|
|||
]
|
||||
},
|
||||
"refuseless/deepseek-v4.1-flash-abliterated": {
|
||||
"cache_read_input_token_cost": 4.5e-08,
|
||||
"input_cost_per_token": 2.2e-07,
|
||||
"cache_read_input_token_cost": 3.5e-08,
|
||||
"input_cost_per_token": 1.4e-07,
|
||||
"litellm_provider": "refuseless",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 130000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 7.5e-07,
|
||||
"output_cost_per_token": 5e-07,
|
||||
"source": "https://refuseless.com/pricing",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
|
|
|
|||
|
|
@ -4180,7 +4180,26 @@
|
|||
"display_name": "Tsubasa (`tsubasa`)",
|
||||
"url": "https://docs.litellm.ai/docs/providers/tsubasa",
|
||||
"endpoints": {
|
||||
"chat_completions": true
|
||||
"chat_completions": true,
|
||||
"responses": true,
|
||||
"text_completion": true,
|
||||
"embeddings": false,
|
||||
"image_generations": false,
|
||||
"image_edits": false,
|
||||
"audio_transcriptions": false,
|
||||
"audio_speech": false,
|
||||
"rerank": false,
|
||||
"ocr": false,
|
||||
"search": false,
|
||||
"moderations": false,
|
||||
"realtime": false,
|
||||
"video_generations": false,
|
||||
"batches": false,
|
||||
"messages": true,
|
||||
"interactions": true,
|
||||
"systemone": false,
|
||||
"a2a": true,
|
||||
"generateContent": true
|
||||
}
|
||||
},
|
||||
"v0": {
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ from litellm.constants import (
|
|||
BILLED_RETRY_ATTEMPT_METADATA_KEY,
|
||||
CALLER_SESSION_ID_METADATA_KEY,
|
||||
CLIENT_OUTPUT_CEILING_METADATA_KEY,
|
||||
COMPLEXITY_ROUTER_CHILD_SESSION_ID_METADATA_KEY,
|
||||
CONSUMED_REQUEST_TAGS_METADATA_KEY,
|
||||
INTERNAL_CALL_ORIGIN_METADATA_KEY,
|
||||
LITELLM_PROXY_MASTER_KEY_ALIAS,
|
||||
|
|
@ -210,13 +211,46 @@ def _caller_session_beside_trace_header(
|
|||
return get_chain_id_from_headers(session_headers) or _caller_body_session_id(data, metadata_variable_name)
|
||||
|
||||
|
||||
def _drop_client_caller_session(request_body: Mapping[str, object]) -> None:
|
||||
def _child_tier_session_metadata(
|
||||
normalized_headers: Mapping[str, str], data: Mapping[str, object], metadata_variable_name: str
|
||||
) -> Mapping[str, str | None]:
|
||||
parent: Final = normalized_headers.get("x-parent-session-id")
|
||||
if not parent or not _SESSION_ID_VALUE_RE.fullmatch(parent):
|
||||
return {}
|
||||
own_headers: Final = {
|
||||
key: value
|
||||
for key, value in normalized_headers.items()
|
||||
if key not in ("x-parent-session-id", "x-litellm-trace-id")
|
||||
}
|
||||
active: Final = data.get(metadata_variable_name)
|
||||
inactive: Final = data.get("litellm_metadata")
|
||||
inactive_session: Final = (
|
||||
_caller_body_session_id(data, "litellm_metadata")
|
||||
if metadata_variable_name == "metadata"
|
||||
and not (isinstance(active, Mapping) and "session_id" in active)
|
||||
and isinstance(inactive, Mapping)
|
||||
else None
|
||||
)
|
||||
own_session: Final = (
|
||||
get_chain_id_from_headers(own_headers)
|
||||
or _get_anthropic_session_id_from_metadata(data.get("metadata"))
|
||||
or _caller_body_session_id(data, metadata_variable_name)
|
||||
or inactive_session
|
||||
or _session_id_from_baggage(normalized_headers.get("baggage", ""))
|
||||
)
|
||||
return {COMPLEXITY_ROUTER_CHILD_SESSION_ID_METADATA_KEY: own_session if own_session != parent else None}
|
||||
|
||||
|
||||
def _drop_client_session_metadata(request_body: Mapping[str, object]) -> None:
|
||||
for metadata_key in ("metadata", "litellm_metadata"):
|
||||
if isinstance(metadata := request_body.get(metadata_key), dict):
|
||||
metadata.pop(CALLER_SESSION_ID_METADATA_KEY, None)
|
||||
for key in (CALLER_SESSION_ID_METADATA_KEY, COMPLEXITY_ROUTER_CHILD_SESSION_ID_METADATA_KEY):
|
||||
cast(dict[str, object], metadata).pop(key, None) # cast-ok: request metadata is a str-keyed JSON object
|
||||
|
||||
|
||||
_PROXY_SET_METADATA_KEYS: Final = frozenset({CALLER_SESSION_ID_METADATA_KEY, BAGGAGE_MAPPED_TAGS_METADATA_KEY})
|
||||
_PROXY_SET_METADATA_KEYS: Final = frozenset(
|
||||
{CALLER_SESSION_ID_METADATA_KEY, BAGGAGE_MAPPED_TAGS_METADATA_KEY, COMPLEXITY_ROUTER_CHILD_SESSION_ID_METADATA_KEY}
|
||||
)
|
||||
_METADATA_TAGS_ADAPTER: Final = TypeAdapter(tuple[object, ...])
|
||||
|
||||
|
||||
|
|
@ -1855,7 +1889,7 @@ class LiteLLMProxyRequestSetup:
|
|||
|
||||
metadata_from_headers: Final = LitellmMetadataFromRequestHeaders()
|
||||
request_body: Final = cast(Mapping[str, object], data) # cast-ok: request body is a str-keyed JSON object
|
||||
_drop_client_caller_session(request_body)
|
||||
_drop_client_session_metadata(request_body)
|
||||
spend_logs_metadata: Final = LiteLLMProxyRequestSetup._get_spend_logs_metadata_from_request_headers(headers)
|
||||
if spend_logs_metadata is not None:
|
||||
metadata_from_headers["spend_logs_metadata"] = spend_logs_metadata
|
||||
|
|
@ -1870,6 +1904,9 @@ class LiteLLMProxyRequestSetup:
|
|||
agent_id_from_header: Final = headers.get("x-litellm-agent-id")
|
||||
# Explicit litellm headers take precedence; fall back to any x-*-session-id header.
|
||||
chain_id: Final = get_chain_id_from_headers(dict(headers))
|
||||
child_tier_metadata: Final = _child_tier_session_metadata(
|
||||
normalized_headers, request_body, _metadata_variable_name
|
||||
)
|
||||
|
||||
if agent_id_from_header:
|
||||
metadata_from_headers["agent_id"] = agent_id_from_header
|
||||
|
|
@ -1939,9 +1976,15 @@ class LiteLLMProxyRequestSetup:
|
|||
|
||||
if isinstance(data[_metadata_variable_name], dict):
|
||||
data[_metadata_variable_name].update(
|
||||
{**metadata_from_headers, CALLER_SESSION_ID_METADATA_KEY: caller_session_beside_trace}
|
||||
if caller_session_beside_trace
|
||||
else metadata_from_headers
|
||||
{
|
||||
**metadata_from_headers,
|
||||
**child_tier_metadata,
|
||||
**(
|
||||
{CALLER_SESSION_ID_METADATA_KEY: caller_session_beside_trace}
|
||||
if caller_session_beside_trace
|
||||
else {}
|
||||
),
|
||||
}
|
||||
)
|
||||
return data
|
||||
|
||||
|
|
|
|||
|
|
@ -477,6 +477,12 @@ unless `modality_routing` is also on.
|
|||
|
||||
### Session pin retention
|
||||
|
||||
With `classification_mode: user_turn`, a child request carrying `x-parent-session-id` classifies
|
||||
its own new asks and stores the tier under its own session ID. Its continuations reuse that tier
|
||||
without replacing the parent's tier. Existing trace grouping, per-tier model affinity and deployment
|
||||
affinity retain their session identities. A child without a distinct, usable session ID classifies
|
||||
without storing a tier pin. `session_affinity: true` retains precedence and the existing shared-pin behavior
|
||||
|
||||
`session_affinity_ttl_seconds` is the idle window for both the model pin selected by session affinity and the deployment pin. Every request that reuses a pin refreshes its TTL, so a session actively sending requests stays pinned. After the window passes with no pin reuse, the next request classifies again and creates a fresh pin. Omit the setting to track the default of 3600 seconds.
|
||||
|
||||
### Mid-task stall escalation
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@ from litellm._internal_context import with_service_target
|
|||
from litellm._logging import verbose_router_logger
|
||||
from litellm.caching.affinity_cache import ROUTER_SESSION_PINS_TARGET, claim_affinity_pin, read_affinity_pin
|
||||
from litellm.constants import (
|
||||
COMPLEXITY_ROUTER_CHILD_SESSION_ID_METADATA_KEY,
|
||||
EMPTY_MAPPING,
|
||||
INTERNAL_CALL_ORIGIN_METADATA_KEY,
|
||||
OUTPUT_TOKEN_CEILING_PARAMS,
|
||||
|
|
@ -4358,17 +4359,17 @@ class ComplexityRouter(CustomLogger):
|
|||
return newest_ask is not None and newest_ask.startswith(_CODEX_COMPACTION_SUMMARY_PREFIX)
|
||||
|
||||
@staticmethod
|
||||
def _iter_metadata_dicts(request_kwargs: dict) -> list[dict]:
|
||||
def _iter_metadata_dicts(request_kwargs: Mapping[str, object]) -> list[Mapping[str, object]]:
|
||||
"""Metadata may land on `metadata` or `litellm_metadata` depending on the
|
||||
endpoint, mirroring DeploymentAffinityCheck's precedence."""
|
||||
return [
|
||||
metadata
|
||||
cast(Mapping[str, object], metadata) # cast-ok: request metadata is a str-keyed JSON object
|
||||
for metadata_key in ("litellm_metadata", "metadata")
|
||||
if isinstance(metadata := request_kwargs.get(metadata_key), dict)
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _get_session_id_from_request_kwargs(request_kwargs: dict) -> str | None:
|
||||
def _get_session_id_from_request_kwargs(request_kwargs: Mapping[str, object]) -> str | None:
|
||||
"""Resolve a client-supplied session_id."""
|
||||
for metadata in ComplexityRouter._iter_metadata_dicts(request_kwargs):
|
||||
session_id = metadata.get("session_id")
|
||||
|
|
@ -4376,6 +4377,21 @@ class ComplexityRouter(CustomLogger):
|
|||
return str(session_id)
|
||||
return None
|
||||
|
||||
def _get_tier_pin_session_id(self, request_kwargs: Mapping[str, object]) -> str | None:
|
||||
if self.config.classification_mode == "user_turn" and not self.config.session_affinity:
|
||||
child_metadata: Final = next(
|
||||
(
|
||||
metadata
|
||||
for metadata in self._iter_metadata_dicts(request_kwargs)
|
||||
if COMPLEXITY_ROUTER_CHILD_SESSION_ID_METADATA_KEY in metadata
|
||||
),
|
||||
None,
|
||||
)
|
||||
if child_metadata is not None:
|
||||
child_session: Final = child_metadata[COMPLEXITY_ROUTER_CHILD_SESSION_ID_METADATA_KEY]
|
||||
return child_session if isinstance(child_session, str) and child_session else None
|
||||
return self._get_session_id_from_request_kwargs(request_kwargs)
|
||||
|
||||
@staticmethod
|
||||
def _get_user_api_key_hash_from_request_kwargs(request_kwargs: dict) -> str | None:
|
||||
"""Resolve the proxy-derived API key hash, the same trust boundary
|
||||
|
|
@ -4460,7 +4476,7 @@ class ComplexityRouter(CustomLogger):
|
|||
conversation_continuing: Final = _conversation_is_continuing(resolved_messages)
|
||||
|
||||
use_session_affinity: Final = self._uses_tier_pin
|
||||
session_id: Final = self._get_session_id_from_request_kwargs(request_kwargs) if use_session_affinity else None
|
||||
session_id: Final = self._get_tier_pin_session_id(request_kwargs) if use_session_affinity else None
|
||||
cache_key = self._get_session_affinity_cache_key(session_id, request_kwargs) if session_id is not None else None
|
||||
|
||||
# In 'user_turn' mode a held pin is replayed only on continuation turns; a new human
|
||||
|
|
|
|||
|
|
@ -77007,13 +77007,13 @@
|
|||
"supports_vision": true
|
||||
},
|
||||
"refuseless/glm-5.3-abliterated": {
|
||||
"cache_read_input_token_cost": 2.5e-07,
|
||||
"input_cost_per_token": 1.25e-06,
|
||||
"cache_read_input_token_cost": 1.5e-07,
|
||||
"input_cost_per_token": 1e-06,
|
||||
"litellm_provider": "refuseless",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 131072,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2.25e-06,
|
||||
"output_cost_per_token": 2e-06,
|
||||
"source": "https://refuseless.com/pricing",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
|
|
@ -77028,13 +77028,13 @@
|
|||
]
|
||||
},
|
||||
"refuseless/glm-5.3-flash-abliterated": {
|
||||
"cache_read_input_token_cost": 1.5e-07,
|
||||
"input_cost_per_token": 2.5e-07,
|
||||
"cache_read_input_token_cost": 4e-08,
|
||||
"input_cost_per_token": 1.2e-07,
|
||||
"litellm_provider": "refuseless",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 131072,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 8e-07,
|
||||
"output_cost_per_token": 5e-07,
|
||||
"source": "https://refuseless.com/pricing",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
|
|
@ -77049,13 +77049,13 @@
|
|||
]
|
||||
},
|
||||
"refuseless/deepseek-v4.1-flash-abliterated": {
|
||||
"cache_read_input_token_cost": 4.5e-08,
|
||||
"input_cost_per_token": 2.2e-07,
|
||||
"cache_read_input_token_cost": 3.5e-08,
|
||||
"input_cost_per_token": 1.4e-07,
|
||||
"litellm_provider": "refuseless",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 130000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 7.5e-07,
|
||||
"output_cost_per_token": 5e-07,
|
||||
"source": "https://refuseless.com/pricing",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
|
|
|
|||
|
|
@ -4180,7 +4180,26 @@
|
|||
"display_name": "Tsubasa (`tsubasa`)",
|
||||
"url": "https://docs.litellm.ai/docs/providers/tsubasa",
|
||||
"endpoints": {
|
||||
"chat_completions": true
|
||||
"chat_completions": true,
|
||||
"responses": true,
|
||||
"text_completion": true,
|
||||
"embeddings": false,
|
||||
"image_generations": false,
|
||||
"image_edits": false,
|
||||
"audio_transcriptions": false,
|
||||
"audio_speech": false,
|
||||
"rerank": false,
|
||||
"ocr": false,
|
||||
"search": false,
|
||||
"moderations": false,
|
||||
"realtime": false,
|
||||
"video_generations": false,
|
||||
"batches": false,
|
||||
"messages": true,
|
||||
"interactions": true,
|
||||
"systemone": false,
|
||||
"a2a": true,
|
||||
"generateContent": true
|
||||
}
|
||||
},
|
||||
"v0": {
|
||||
|
|
|
|||
|
|
@ -33,6 +33,14 @@ async def test_tsubasa_dashboard_fields_resolve_to_a_registered_model() -> None:
|
|||
}
|
||||
assert endpoints_response.status_code == 200, endpoints_response.text
|
||||
endpoints: Final = SupportedEndpointsResponse.model_validate(endpoints_response.json()).endpoints
|
||||
assert [
|
||||
assert {
|
||||
endpoint.key for endpoint in endpoints if any(p.slug == provider.litellm_provider for p in endpoint.providers)
|
||||
] == ["chat_completions"]
|
||||
} == {
|
||||
"chat_completions",
|
||||
"responses",
|
||||
"text_completion",
|
||||
"messages",
|
||||
"interactions",
|
||||
"a2a",
|
||||
"generateContent",
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ from starlette.datastructures import Headers
|
|||
import litellm
|
||||
from litellm.constants import (
|
||||
CALLER_SESSION_ID_METADATA_KEY,
|
||||
COMPLEXITY_ROUTER_CHILD_SESSION_ID_METADATA_KEY,
|
||||
ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY,
|
||||
SERVER_STREAMING_CLASSIFICATION_KEY,
|
||||
SERVER_STREAMING_CLASSIFICATION_MARKER,
|
||||
|
|
@ -3684,6 +3685,110 @@ async def test_spend_log_caller_session_stays_out_of_requester_metadata(monkeypa
|
|||
assert CALLER_SESSION_ID_METADATA_KEY not in updated["metadata"]["requester_metadata"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"surface,headers,body,child_id,trace_id",
|
||||
[
|
||||
(
|
||||
"chat/completions", {"X-Session-Id": "child-01"}, {"metadata": {"session_id": "body-001"}},
|
||||
"child-01", "parent-01",
|
||||
),
|
||||
("responses", {"x-vendor-session-id": "child-01"}, {}, "child-01", "parent-01"),
|
||||
("messages", {"session-id": "child-01", "user-agent": "codex_cli_rs/1.0"}, {}, "child-01", "parent-01"),
|
||||
("chat/completions", {"x-litellm-session-id": "short", "x-session-id": "child-01"}, {}, "short", "short"),
|
||||
("responses", {"x-litellm-trace-id": "trace-01", "x-session-id": "child-01"}, {}, "child-01", "trace-01"),
|
||||
(
|
||||
"messages", {"x-litellm-trace-id": "trace-01", "x-litellm-session-id": "child-01"},
|
||||
{}, "child-01", "trace-01",
|
||||
),
|
||||
("responses", {}, {"metadata": {"session_id": "body-001"}}, "body-001", "parent-01"),
|
||||
("messages", {}, {"litellm_metadata": {"session_id": "body-001"}}, "body-001", "parent-01"),
|
||||
("chat/completions", {}, {"litellm_metadata": {"session_id": "body-001"}}, "body-001", "parent-01"),
|
||||
("messages", {"baggage": "session.id=child-01"}, {}, "child-01", "parent-01"),
|
||||
(
|
||||
"responses", {"baggage": "session.id=child-01"}, {"metadata": {"session_id": "body-001"}},
|
||||
"body-001", "parent-01",
|
||||
),
|
||||
(
|
||||
"chat/completions", {}, {"litellm_session_id": "root-001", "metadata": {"session_id": "body-001"}},
|
||||
"root-001", "parent-01",
|
||||
),
|
||||
(
|
||||
"responses", {}, {"metadata": {"session_id": "body-001"}, "litellm_metadata": {"session_id": "active-1"}},
|
||||
"active-1", "parent-01",
|
||||
),
|
||||
(
|
||||
"chat/completions", {},
|
||||
{"metadata": {"session_id": "active-1"}, "litellm_metadata": {"session_id": "body-001"}},
|
||||
"active-1", "parent-01",
|
||||
),
|
||||
(
|
||||
"chat/completions", {}, {"metadata": {"session_id": ""}, "litellm_metadata": {"session_id": "body-001"}},
|
||||
None, "parent-01",
|
||||
),
|
||||
(
|
||||
"messages", {}, {"metadata": {"session_id": "body-001"}, "litellm_metadata": {"session_id": ""}},
|
||||
None, "parent-01",
|
||||
),
|
||||
(
|
||||
"messages", {}, {"litellm_session_id": "root-001", "litellm_metadata": {"session_id": "active-1"},
|
||||
"metadata": {"session_id": "body-001", "user_id": "user_abc_account__session_anthro-1"}},
|
||||
"anthro-1", "parent-01",
|
||||
),
|
||||
("messages", {"x-session-id": "child-01"}, {"metadata": {"user_id": {"session_id": "body-001"}}},
|
||||
"child-01", "parent-01"),
|
||||
("chat/completions", {}, {}, None, "parent-01"),
|
||||
("responses", {"x-session-id": "bad"}, {}, None, "parent-01"),
|
||||
("messages", {}, {"metadata": {"session_id": 123}}, None, "parent-01"),
|
||||
("chat/completions", {"x-session-id": "parent-01"}, {}, None, "parent-01"),
|
||||
],
|
||||
)
|
||||
async def test_child_tier_identity_preserves_trace_and_body_session_precedence(
|
||||
surface: str, headers: dict[str, str], body: dict[str, object], child_id: str | None, trace_id: str
|
||||
) -> None:
|
||||
request: Final = _make_request_mock(f"/v1/{surface}", {"x-parent-session-id": "parent-01", **headers})
|
||||
updated: Final = await add_litellm_data_to_request(
|
||||
data={"model": "test-router", "messages": [], **copy.deepcopy(body)},
|
||||
request=request,
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"),
|
||||
proxy_config=MagicMock(),
|
||||
general_settings={},
|
||||
)
|
||||
metadata: Final = updated[get_metadata_variable_name(request)]
|
||||
assert metadata[COMPLEXITY_ROUTER_CHILD_SESSION_ID_METADATA_KEY] == child_id
|
||||
assert metadata["session_id"] == metadata["trace_id"] == trace_id
|
||||
assert updated["litellm_session_id"] == updated["litellm_trace_id"] == trace_id
|
||||
expected_caller: Final = (
|
||||
headers.get("x-litellm-session-id", "parent-01") if "x-litellm-trace-id" in headers else None
|
||||
)
|
||||
assert metadata.get(CALLER_SESSION_ID_METADATA_KEY) == expected_caller
|
||||
assert COMPLEXITY_ROUTER_CHILD_SESSION_ID_METADATA_KEY not in metadata.get("requester_metadata", {})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("surface,parent_id", [("chat/completions", None), ("responses", None), ("messages", "bad")])
|
||||
async def test_non_child_requests_cannot_forge_a_child_tier_identity(surface: str, parent_id: str | None) -> None:
|
||||
headers: Final = {"x-session-id": "child-01", **({"x-parent-session-id": parent_id} if parent_id else {})}
|
||||
request: Final = _make_request_mock(f"/v1/{surface}", headers)
|
||||
updated: Final = await add_litellm_data_to_request(
|
||||
data={
|
||||
"model": "test-router",
|
||||
"messages": [],
|
||||
"metadata": {COMPLEXITY_ROUTER_CHILD_SESSION_ID_METADATA_KEY: "forged-1"},
|
||||
"litellm_metadata": {COMPLEXITY_ROUTER_CHILD_SESSION_ID_METADATA_KEY: "forged-2"},
|
||||
},
|
||||
request=request,
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"),
|
||||
proxy_config=MagicMock(),
|
||||
general_settings={},
|
||||
)
|
||||
for bucket in ("metadata", "litellm_metadata"):
|
||||
metadata: Final = updated.get(bucket, {})
|
||||
assert COMPLEXITY_ROUTER_CHILD_SESSION_ID_METADATA_KEY not in metadata
|
||||
assert COMPLEXITY_ROUTER_CHILD_SESSION_ID_METADATA_KEY not in metadata.get("requester_metadata", {})
|
||||
assert updated[get_metadata_variable_name(request)]["session_id"] == "child-01"
|
||||
|
||||
|
||||
def test_add_litellm_metadata_from_request_headers_x_litellm_session_id_sets_chain_id():
|
||||
"""x-litellm-session-id sets both metadata and top-level litellm_session_id/litellm_trace_id for call chaining."""
|
||||
headers = {"x-litellm-session-id": "bar"}
|
||||
|
|
|
|||
|
|
@ -7714,6 +7714,176 @@ class _DummyPlugin:
|
|||
return context
|
||||
|
||||
|
||||
class TestChildSessionTierPins:
|
||||
@staticmethod
|
||||
def _router(session_affinity: bool = False) -> Router:
|
||||
return Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "child-tier-router",
|
||||
"litellm_params": {
|
||||
"model": "auto_router/complexity_router",
|
||||
"complexity_router_config": {
|
||||
"tiers": {"SIMPLE": "simple", "REASONING": ["reasoning", "reasoning-peer"]},
|
||||
"classification_mode": "user_turn",
|
||||
"session_affinity": session_affinity,
|
||||
"keyword_tier_rules": [{"keywords": ["reason carefully"], "tier": "REASONING"}],
|
||||
},
|
||||
},
|
||||
},
|
||||
*[
|
||||
{
|
||||
"model_name": group,
|
||||
"litellm_params": {"model": "openai/test-model", "api_key": "test-only"},
|
||||
"model_info": {"id": f"{group}-{index}"},
|
||||
}
|
||||
for group, index in (
|
||||
("simple", 0), ("simple", 1), ("reasoning", 0),
|
||||
("reasoning", 1), ("reasoning-peer", 0), ("reasoning-peer", 1),
|
||||
)
|
||||
],
|
||||
],
|
||||
num_retries=0,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _normalized_request(
|
||||
surface: str, session_id: str | None, caller: str = "caller-a", child: bool = True
|
||||
) -> tuple[dict[str, object], str]:
|
||||
from litellm.proxy.litellm_pre_call_utils import (
|
||||
LiteLLMProxyRequestSetup,
|
||||
metadata_variable_name_for_route,
|
||||
)
|
||||
|
||||
route: Final = {
|
||||
"chat": "/v1/chat/completions", "responses": "/v1/responses", "messages": "/v1/messages"
|
||||
}[surface]
|
||||
bucket: Final = metadata_variable_name_for_route(route)
|
||||
data: Final[dict[str, object]] = {bucket: {"user_api_key_hash": caller, "user_api_key_request_route": route}}
|
||||
LiteLLMProxyRequestSetup.add_litellm_metadata_from_request_headers(
|
||||
headers={
|
||||
**({"X-Session-Id": session_id} if session_id is not None else {}),
|
||||
**({"x-parent-session-id": "parent-session"} if child else {}),
|
||||
},
|
||||
data=data,
|
||||
_metadata_variable_name=bucket,
|
||||
)
|
||||
return data, bucket
|
||||
|
||||
async def _route(
|
||||
self, router: Router, surface: str, session_id: str | None, prompt: str | None,
|
||||
expected_tier: str, expected_cause: str, caller: str = "caller-a", child: bool = True,
|
||||
expected_deployment: str | None = None,
|
||||
) -> str:
|
||||
from litellm.router_utils.pre_call_checks.deployment_affinity_check import DeploymentAffinityCheck
|
||||
|
||||
data, bucket = self._normalized_request(surface, session_id, caller, child)
|
||||
turn: Final[dict[str, object]] = (
|
||||
{"role": "user", "content": prompt} if prompt is not None else
|
||||
{"role": "user", "content": [{"type": "tool_result", "tool_use_id": "call_1", "content": "done"}]}
|
||||
if surface == "messages" else
|
||||
{"type": "function_call_output", "call_id": "call_1", "output": "done"}
|
||||
if surface == "responses" else
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": "done"}
|
||||
)
|
||||
tool_call: Final = (
|
||||
{"type": "function_call", "call_id": "call_1", "name": "read_file", "arguments": "{}"}
|
||||
if surface == "responses" else
|
||||
{"role": "assistant", "content": [{"type": "tool_use", "id": "call_1", "name": "read_file", "input": {}}]}
|
||||
if surface == "messages" else TestClassificationMode.TOOL_CALL_1
|
||||
)
|
||||
turns: Final = [turn] if prompt is not None else [TestClassificationMode.SIMPLE_ASK, tool_call, turn]
|
||||
if surface == "responses":
|
||||
data["input"] = turns
|
||||
selected: Final = await router.async_get_available_deployment(
|
||||
model="child-tier-router", request_kwargs=data,
|
||||
messages=None if surface == "responses" else turns,
|
||||
input=turns if surface == "responses" else None,
|
||||
)
|
||||
metadata: Final = data[bucket]
|
||||
assert isinstance(metadata, dict)
|
||||
assert (metadata["session_id"], metadata["trace_id"], data["litellm_session_id"], data["litellm_trace_id"]) == (
|
||||
"parent-session", "parent-session", "parent-session", "parent-session"
|
||||
)
|
||||
assert (metadata["routing_decision"]["tier"], metadata["routing_decision"]["cause"]) == (
|
||||
expected_tier, expected_cause
|
||||
)
|
||||
affinity: Final = next(cb for cb in router.optional_callbacks or () if isinstance(cb, DeploymentAffinityCheck))
|
||||
if expected_deployment is not None:
|
||||
filtered: Final = await affinity.async_filter_deployments(
|
||||
model=selected["model_name"], healthy_deployments=router.get_model_list(selected["model_name"]) or [],
|
||||
messages=None, request_kwargs=data,
|
||||
)
|
||||
assert [deployment["model_info"]["id"] for deployment in filtered] == [expected_deployment]
|
||||
assert selected["model_info"]["id"] == expected_deployment
|
||||
await affinity.async_pre_call_deployment_hook(
|
||||
kwargs={
|
||||
**data, "model_info": selected["model_info"],
|
||||
bucket: {**metadata, "deployment_model_name": selected["model_name"]},
|
||||
},
|
||||
call_type=None,
|
||||
)
|
||||
return str(selected["model_info"]["id"])
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("surface", ["chat", "responses", "messages"])
|
||||
async def test_children_keep_their_tier_without_replacing_the_parent(self, surface: str) -> None:
|
||||
router: Final = self._router()
|
||||
try:
|
||||
parent: Final = await self._route(
|
||||
router, surface, "parent-session", "reason carefully", "REASONING", "literal_keyword_match", child=False
|
||||
)
|
||||
for session, prompt, tier, cause, caller, child in (
|
||||
("child-session-a", "Hello!", "SIMPLE", "heuristic_scorer", "caller-a", True),
|
||||
("child-session-a", None, "SIMPLE", "user_turn_continuation", "caller-a", True),
|
||||
("parent-session", None, "REASONING", "user_turn_continuation", "caller-a", False),
|
||||
("child-session-b", None, "SIMPLE", "heuristic_scorer", "caller-a", True),
|
||||
("child-session-b", "reason carefully", "REASONING", "literal_keyword_match", "caller-a", True),
|
||||
("child-session-a", None, "SIMPLE", "user_turn_continuation", "caller-a", True),
|
||||
("child-session-a", "reason carefully", "REASONING", "literal_keyword_match", "caller-b", True),
|
||||
("child-session-a", None, "SIMPLE", "user_turn_continuation", "caller-a", True),
|
||||
(None, "Hello!", "SIMPLE", "heuristic_scorer", "caller-a", True),
|
||||
(None, None, "SIMPLE", "heuristic_scorer", "caller-a", True),
|
||||
):
|
||||
await self._route(
|
||||
router, surface, session, prompt, tier, cause, caller, child,
|
||||
expected_deployment=parent if tier == "REASONING" and caller == "caller-a" else None,
|
||||
)
|
||||
assert await self._route(
|
||||
router, surface, "parent-session", None, "REASONING", "user_turn_continuation",
|
||||
child=False, expected_deployment=parent,
|
||||
) == parent
|
||||
finally:
|
||||
router.discard()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_affinity_still_shares_the_parent_tier(self) -> None:
|
||||
router: Final = self._router(session_affinity=True)
|
||||
try:
|
||||
parent: Final = await self._route(
|
||||
router, "chat", "parent-session", "reason carefully", "REASONING", "literal_keyword_match", child=False
|
||||
)
|
||||
assert await self._route(
|
||||
router, "chat", "child-session-a", "Hello!", "REASONING", "session_affinity_pin",
|
||||
expected_deployment=parent,
|
||||
) == parent
|
||||
finally:
|
||||
router.discard()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_child_tier_model_affinity_keeps_using_the_parent_identity(self) -> None:
|
||||
router: Final = self._router()
|
||||
try:
|
||||
strategy: Final = router.complexity_routers["child-tier-router"][0].strategy
|
||||
parent, _ = self._normalized_request("chat", "parent-session", child=False)
|
||||
child, _ = self._normalized_request("chat", "child-session-a")
|
||||
candidates: Final = ("reasoning", "reasoning-peer")
|
||||
assert await strategy._pin_model_for_tier("REASONING", "reasoning", candidates, parent) == "reasoning"
|
||||
assert await strategy._pin_model_for_tier("REASONING", "reasoning-peer", candidates, child) == "reasoning"
|
||||
finally:
|
||||
router.discard()
|
||||
|
||||
|
||||
class TestClassificationMode:
|
||||
"""Test classification_mode='user_turn': classify only requests whose newest turn is a new
|
||||
human ask; tool-loop continuation turns replay the session's held routing decision."""
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@ import {
|
|||
} from "./tier_rows";
|
||||
import React from "react";
|
||||
import { ModelGroup } from "@/components/llm_calls/fetch_models";
|
||||
import { routingModelGroups } from "./auto_setup";
|
||||
import ClassificationMethodConfig, { InactiveHeuristicV2Threshold } from "./ClassificationMethodConfig";
|
||||
import ComplexityRouterAdvancedSections from "./ComplexityRouterAdvancedSections";
|
||||
import { type TierSetAction, applyTierSetAction, setFallbackTier } from "./tier_set_actions";
|
||||
|
|
@ -657,13 +658,10 @@ const ComplexityRouterConfig: React.FC<ComplexityRouterConfigProps> = ({
|
|||
);
|
||||
const classifierEffortOptionsByModel = classifierEffortOptionsForModels(modelInfo);
|
||||
|
||||
// Embedding models can't serve a chat-completion role, so they're excluded here.
|
||||
const modelOptions = modelInfo
|
||||
.filter((model) => model.mode !== "embedding")
|
||||
.map((model) => ({
|
||||
value: model.model_group,
|
||||
label: model.model_group,
|
||||
}));
|
||||
const modelOptions = routingModelGroups(modelInfo).map((model) => ({
|
||||
value: model.model_group,
|
||||
label: model.model_group,
|
||||
}));
|
||||
|
||||
const handleTierModelParamChange = (tier: string, model: string, change: TierModelParamChange) =>
|
||||
onChange({
|
||||
|
|
|
|||
|
|
@ -319,6 +319,27 @@ describe("AddAutoRouterTab", () => {
|
|||
},
|
||||
);
|
||||
|
||||
it("carries Responses model choices from simple setup into Advanced tiers", async () => {
|
||||
const models = ["gpt-6-luna", "gpt-6-astra"];
|
||||
mockFetchAvailableModels.mockResolvedValue(models.map((model_group) => ({ model_group, mode: "responses" })));
|
||||
mockFetchAllModelDeployments.mockResolvedValue(
|
||||
models.map((model_name) => ({
|
||||
model_name,
|
||||
litellm_params: { model: `chatgpt/${model_name}` },
|
||||
model_info: { mode: "responses" },
|
||||
})),
|
||||
);
|
||||
renderWithProviders(<Harness />);
|
||||
await waitFor(() =>
|
||||
expect(screen.getByRole("combobox", { name: "Everyday model" })).toHaveTextContent("gpt-6-luna"),
|
||||
);
|
||||
expect(screen.getByRole("combobox", { name: "Hard tasks model" })).toHaveTextContent("gpt-6-astra");
|
||||
expect(screen.queryByText(/Two deployed models are needed/)).not.toBeInTheDocument();
|
||||
fireEvent.click(screen.getByRole("button", { name: "Advanced configuration", exact: true }));
|
||||
expectTierModel("Simple", "gpt-6-luna");
|
||||
expectTierModel("Reasoning", "gpt-6-astra");
|
||||
});
|
||||
|
||||
it("keeps manual solver choices on goal changes and carries them into Advanced", async () => {
|
||||
mockFetchAvailableModels.mockResolvedValue(
|
||||
["claude-haiku-5-5", "custom-fast", "claude-opus-5-5"].map((model_group) => ({
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import {
|
|||
solverPairConfig,
|
||||
nextRouterName,
|
||||
simpleSetupError,
|
||||
routingModelGroups,
|
||||
} from "./auto_setup";
|
||||
|
||||
const deployment = (name: string, model: string, extra: Partial<AutoRouterDeployment> = {}): AutoRouterDeployment => ({
|
||||
|
|
@ -76,7 +77,7 @@ describe("deployedRouterModels", () => {
|
|||
]);
|
||||
});
|
||||
|
||||
it("uses only caller-visible chat models and excludes blocked and recursive targets", () => {
|
||||
it("matches Advanced choices while keeping blocked and recursive targets out of suggestions", () => {
|
||||
const rows = [
|
||||
deployment("hidden", "anthropic/claude-opus-6"),
|
||||
deployment("blocked", "anthropic/claude-opus-5", { model_info: { blocked: true } }),
|
||||
|
|
@ -94,9 +95,40 @@ describe("deployedRouterModels", () => {
|
|||
],
|
||||
rows,
|
||||
).map((model) => model.name),
|
||||
).toEqual(["ready"]);
|
||||
).toEqual(["blocked", "ready", "recursive"]);
|
||||
expect(
|
||||
recommendSolverPair(
|
||||
deployedRouterModels(
|
||||
rows
|
||||
.filter((row) => row.model_name !== "hidden" && row.model_name !== "embedding")
|
||||
.map((row) => ({ model_group: row.model_name! })),
|
||||
rows,
|
||||
),
|
||||
"balanced",
|
||||
"auto",
|
||||
),
|
||||
).toBeNull();
|
||||
});
|
||||
|
||||
it.each(["chat", "responses", "completion", "unknown", undefined])(
|
||||
"offers the same routing choices as Advanced for mode %s",
|
||||
(mode) => {
|
||||
const groups = [
|
||||
{ model_group: "everyday", mode },
|
||||
{ model_group: "hard", mode },
|
||||
{ model_group: "embedding", mode: "embedding" },
|
||||
];
|
||||
const rows = [
|
||||
deployment("everyday", "chatgpt/gpt-6-luna", { model_info: { mode } }),
|
||||
deployment("hard", "chatgpt/gpt-6-astra", { model_info: { mode } }),
|
||||
];
|
||||
const models = deployedRouterModels(groups, rows);
|
||||
expect(models.map((model) => model.name)).toEqual(routingModelGroups(groups).map((group) => group.model_group));
|
||||
expect(models.map((model) => model.name)).toEqual(["everyday", "hard"]);
|
||||
expect(names(recommendSolverPair(models, "balanced", "openai"))).toEqual(["everyday", "hard"]);
|
||||
},
|
||||
);
|
||||
|
||||
it("does not claim a mixed-version or unknown model group is a known version", () => {
|
||||
const rows = [
|
||||
deployment("mixed", "anthropic/claude-opus-5"),
|
||||
|
|
@ -112,7 +144,7 @@ describe("deployedRouterModels", () => {
|
|||
],
|
||||
rows,
|
||||
);
|
||||
expect(models.map((model) => model.name)).toEqual(["claude-sonnet-5", "mixed"]);
|
||||
expect(models.map((model) => model.name)).toEqual(["claude-sonnet-5", "mixed", "wildcard"]);
|
||||
expect(models.every((model) => model.family === undefined)).toBe(true);
|
||||
expect(recommendSolverPair(models, "balanced", "auto")).toBeNull();
|
||||
});
|
||||
|
|
|
|||
|
|
@ -146,24 +146,21 @@ const compareVersion = (left: readonly number[], right: readonly number[]): numb
|
|||
const compareRelease = (left: ModelIdentity, right: ModelIdentity): number =>
|
||||
compareVersion(left.version, right.version) || left.snapshot - right.snapshot;
|
||||
|
||||
export const routingModelGroups = (groups: readonly ModelGroup[]): ModelGroup[] =>
|
||||
groups.filter((group) => group.mode !== "embedding");
|
||||
|
||||
export const deployedRouterModels = (
|
||||
groups: readonly ModelGroup[],
|
||||
deployments: readonly AutoRouterDeployment[],
|
||||
): SolverModel[] =>
|
||||
groups
|
||||
.flatMap((group) => {
|
||||
if (group.mode && group.mode !== "chat") return [];
|
||||
routingModelGroups(groups)
|
||||
.map((group) => {
|
||||
const rows = deployments.filter((row) => row.model_name === group.model_group);
|
||||
if (
|
||||
rows.some(isAutoRouterDeployment) ||
|
||||
group.model_group.includes("*") ||
|
||||
group.model_group.startsWith("auto_router/")
|
||||
)
|
||||
return [];
|
||||
const active = rows.filter((row) => !row.model_info?.blocked);
|
||||
if (rows.length > 0 && active.length === 0) return [];
|
||||
if (active.some((row) => row.model_info?.mode && row.model_info.mode !== "chat")) return [];
|
||||
if (active.some((row) => row.litellm_params?.model?.includes("*"))) return [];
|
||||
const isRouter = rows.some(isAutoRouterDeployment) || group.model_group.startsWith("auto_router/");
|
||||
const hasWildcard =
|
||||
group.model_group.includes("*") || active.some((row) => row.litellm_params?.model?.includes("*"));
|
||||
const manualOnly = isRouter || hasWildcard || (rows.length > 0 && active.length === 0);
|
||||
const identities = active.length > 0 ? active.map(identityOf) : [parseModel(group.model_group)];
|
||||
const first = identities[0];
|
||||
const homogeneous =
|
||||
|
|
@ -173,19 +170,17 @@ export const deployedRouterModels = (
|
|||
const sameLine = identity.family === first.family && identity.line === first.line;
|
||||
return sameLine && compareRelease(identity, first) === 0;
|
||||
});
|
||||
const identity = homogeneous ? first : undefined;
|
||||
const identity = homogeneous && !manualOnly ? first : undefined;
|
||||
const providers =
|
||||
active.length > 0
|
||||
? active.map((row) => providerOf(row, group, identity?.family))
|
||||
: [providerOf({}, group, identity?.family)];
|
||||
return [
|
||||
{
|
||||
...identity,
|
||||
name: group.model_group,
|
||||
model: identity?.model ?? group.model_group,
|
||||
provider: [...new Set(providers)].toSorted().join(", "),
|
||||
},
|
||||
];
|
||||
return {
|
||||
...identity,
|
||||
name: group.model_group,
|
||||
model: identity?.model ?? group.model_group,
|
||||
provider: [...new Set(providers)].toSorted().join(", "),
|
||||
};
|
||||
})
|
||||
.toSorted((a, b) => a.name.localeCompare(b.name));
|
||||
|
||||
|
|
|
|||
|
|
@ -490,6 +490,137 @@ describe("responses_api", () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe("responses_api reasoning", () => {
|
||||
type Run = { reasoning: string; answer: string; timeToFirstToken: number[]; reasoningBeforeFirstToken: string };
|
||||
|
||||
const run = async (streamingEnabled: boolean): Promise<Run> => {
|
||||
const reasoningChunks: string[] = [];
|
||||
const answerChunks: string[] = [];
|
||||
const timeToFirstToken: number[] = [];
|
||||
const reasoningAtFirstToken: string[] = [];
|
||||
await makeOpenAIResponsesRequest(
|
||||
[{ role: "user", content: "What is 3+4?" }],
|
||||
(_role, delta) => answerChunks.push(delta),
|
||||
"qwen3",
|
||||
"test-token",
|
||||
undefined,
|
||||
undefined,
|
||||
(content) => reasoningChunks.push(content),
|
||||
(ttft) => {
|
||||
timeToFirstToken.push(ttft);
|
||||
reasoningAtFirstToken.push(reasoningChunks.join("") + "|" + answerChunks.join(""));
|
||||
},
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
streamingEnabled,
|
||||
);
|
||||
return {
|
||||
reasoning: reasoningChunks.join(""),
|
||||
answer: answerChunks.join(""),
|
||||
timeToFirstToken,
|
||||
reasoningBeforeFirstToken: reasoningAtFirstToken[0] ?? "",
|
||||
};
|
||||
};
|
||||
|
||||
const streamOf = (events: Record<string, unknown>[]) => {
|
||||
async function* mockStream() {
|
||||
for (const event of events) {
|
||||
yield event;
|
||||
}
|
||||
yield { type: "response.completed", response: { id: "resp_reasoning" } };
|
||||
}
|
||||
mockResponsesCreate.mockResolvedValue(mockStream());
|
||||
};
|
||||
|
||||
afterEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
it("shows streamed reasoning_text deltas and starts the first-token timer on them", async () => {
|
||||
streamOf([
|
||||
{ type: "response.reasoning_text.delta", item_id: "rs_1", content_index: 0, delta: "Add " },
|
||||
{ type: "response.reasoning_text.delta", item_id: "rs_1", content_index: 0, delta: "them" },
|
||||
{ type: "response.output_text.delta", delta: "7" },
|
||||
]);
|
||||
|
||||
const result = await run(true);
|
||||
|
||||
expect(result.reasoning).toBe("Add them");
|
||||
expect(result.answer).toBe("7");
|
||||
expect(result.timeToFirstToken).toHaveLength(1);
|
||||
expect(result.reasoningBeforeFirstToken).toBe("Add |");
|
||||
});
|
||||
|
||||
it("separates streamed reasoning summary parts and items with a blank line", async () => {
|
||||
streamOf([
|
||||
{ type: "response.reasoning_summary_text.delta", item_id: "rs_1", summary_index: 0, delta: "First" },
|
||||
{ type: "response.reasoning_summary_text.delta", item_id: "rs_1", summary_index: 0, delta: " part" },
|
||||
{ type: "response.reasoning_summary_text.delta", item_id: "rs_1", summary_index: 1, delta: "Second" },
|
||||
{ type: "response.reasoning_summary_text.delta", item_id: "rs_2", summary_index: 0, delta: "Third" },
|
||||
]);
|
||||
|
||||
await expect(run(true)).resolves.toMatchObject({ reasoning: "First part\n\nSecond\n\nThird" });
|
||||
});
|
||||
|
||||
it("still shows the legacy reasoning.delta event", async () => {
|
||||
streamOf([{ type: "response.reasoning.delta", delta: "Legacy thought" }]);
|
||||
|
||||
await expect(run(true)).resolves.toMatchObject({ reasoning: "Legacy thought" });
|
||||
});
|
||||
|
||||
it("shows non-streamed reasoning content when the summary is empty", async () => {
|
||||
mockResponsesCreate.mockReturnValueOnce(
|
||||
nonStreamingResponse({
|
||||
id: "resp_content",
|
||||
output: [
|
||||
{
|
||||
type: "reasoning",
|
||||
summary: [],
|
||||
content: [
|
||||
{ type: "reasoning_text", text: "Add" },
|
||||
{ type: "reasoning_text", text: "them" },
|
||||
],
|
||||
},
|
||||
{ type: "message", content: [{ type: "output_text", text: "7" }] },
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(run(false)).resolves.toMatchObject({ reasoning: "Add\n\nthem", answer: "7" });
|
||||
});
|
||||
|
||||
it("prefers the non-streamed reasoning summary over its content", async () => {
|
||||
mockResponsesCreate.mockReturnValueOnce(
|
||||
nonStreamingResponse({
|
||||
id: "resp_summary",
|
||||
output: [
|
||||
{
|
||||
type: "reasoning",
|
||||
summary: [{ type: "summary_text", text: "Short summary" }],
|
||||
content: [{ type: "reasoning_text", text: "Raw chain" }],
|
||||
},
|
||||
{ type: "reasoning", content: [{ type: "output_text", text: "Second item" }] },
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(run(false)).resolves.toMatchObject({ reasoning: "Short summary\n\nSecond item" });
|
||||
});
|
||||
});
|
||||
|
||||
describe("responses_api prompt cache usage", () => {
|
||||
const captureUsage = async (usage: Record<string, unknown>): Promise<TokenUsage> => {
|
||||
async function* mockStream() {
|
||||
|
|
|
|||
|
|
@ -44,6 +44,22 @@ type SynthesizedResponseEvent =
|
|||
| { type: "response.output_text.delta"; delta: string }
|
||||
| { type: "response.completed"; response: NonStreamedResponse };
|
||||
|
||||
const REASONING_PART_SEPARATOR = "\n\n";
|
||||
|
||||
const REASONING_DELTA_EVENT_TYPES: ReadonlySet<string> = new Set([
|
||||
"response.reasoning_text.delta",
|
||||
"response.reasoning_summary_text.delta",
|
||||
"response.reasoning.delta",
|
||||
]);
|
||||
|
||||
const nonEmptyTexts = (parts: ResponseOutputPart[] | undefined): string[] =>
|
||||
(parts ?? []).map((part) => part.text ?? "").filter((text) => text.length > 0);
|
||||
|
||||
const reasoningItemText = (item: ResponseOutputItem): string => {
|
||||
const summary = nonEmptyTexts(item.summary);
|
||||
return (summary.length > 0 ? summary : nonEmptyTexts(item.content)).join(REASONING_PART_SEPARATOR);
|
||||
};
|
||||
|
||||
const responseAsEvents = (response: NonStreamedResponse): SynthesizedResponseEvent[] => {
|
||||
const outputItems = response.output ?? [];
|
||||
const outputText = outputItems
|
||||
|
|
@ -54,9 +70,9 @@ const responseAsEvents = (response: NonStreamedResponse): SynthesizedResponseEve
|
|||
.join("");
|
||||
const reasoningText = outputItems
|
||||
.filter((item) => item.type === "reasoning")
|
||||
.flatMap((item) => item.summary ?? [])
|
||||
.map((part) => part.text ?? "")
|
||||
.join("");
|
||||
.map(reasoningItemText)
|
||||
.filter((text) => text.length > 0)
|
||||
.join(REASONING_PART_SEPARATOR);
|
||||
|
||||
return [
|
||||
...outputItems.map((item) => ({ type: "response.output_item.done" as const, item })),
|
||||
|
|
@ -124,6 +140,7 @@ export async function makeOpenAIResponsesRequest(
|
|||
try {
|
||||
const startTime = Date.now();
|
||||
let firstTokenReceived = false;
|
||||
let reasoningPartKey: string | null = null;
|
||||
let servedFromResponseCache = false;
|
||||
|
||||
const formattedInput = messages.flatMap((message) => {
|
||||
|
|
@ -298,10 +315,16 @@ export async function makeOpenAIResponsesRequest(
|
|||
}
|
||||
|
||||
// Handle reasoning content
|
||||
if (event.type === "response.reasoning.delta" && "delta" in event) {
|
||||
const delta = event.delta;
|
||||
if (typeof delta === "string" && onReasoningContent) {
|
||||
onReasoningContent(delta);
|
||||
if (REASONING_DELTA_EVENT_TYPES.has(event.type) && typeof event.delta === "string" && event.delta.length > 0) {
|
||||
const partKey = `${event.item_id ?? ""}:${event.summary_index ?? event.content_index ?? 0}`;
|
||||
const separator = reasoningPartKey !== null && reasoningPartKey !== partKey ? REASONING_PART_SEPARATOR : "";
|
||||
reasoningPartKey = partKey;
|
||||
onReasoningContent?.(separator + event.delta);
|
||||
|
||||
const isFirstToken = !firstTokenReceived;
|
||||
firstTokenReceived = true;
|
||||
if (isFirstToken && onTimingData && streamingEnabled) {
|
||||
onTimingData(Date.now() - startTime);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue