mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-21 00:21:49 +00:00
Merge remote-tracking branch 'origin/main' into litellm_fix_integration_conftest_import
This commit is contained in:
Yuneng Jiang
commit
5bd27d1d99
16 changed files with 582 additions and 62 deletions
|
|
@ -32,6 +32,9 @@ from litellm.litellm_core_utils.llm_response_utils.response_metadata import (
|
|||
)
|
||||
from litellm.litellm_core_utils.thread_pool_executor import executor
|
||||
from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig
|
||||
from litellm.responses.litellm_completion_transformation.transformation import (
|
||||
LiteLLMCompletionResponsesConfig,
|
||||
)
|
||||
from litellm.responses.utils import ResponseAPILoggingUtils, ResponsesAPIRequestUtils
|
||||
from litellm.types.integrations.custom_logger import converted_stream_requested
|
||||
from litellm.types.llms.openai import (
|
||||
|
|
@ -257,6 +260,7 @@ class BaseResponsesAPIStreamingIterator:
|
|||
self._failure_handled = False # Track if failure handler has been called
|
||||
self._yielded_first_chunk = False
|
||||
self._generated_content = ""
|
||||
self._generated_tool_arguments = ""
|
||||
self._completed_response_cached = False
|
||||
self._completed_response_logged = False
|
||||
self._completed_response_cache_hit: bool | None = None
|
||||
|
|
@ -352,6 +356,10 @@ class BaseResponsesAPIStreamingIterator:
|
|||
_delta: Final = getattr(openai_responses_api_chunk, "delta", None)
|
||||
if isinstance(_delta, str):
|
||||
self._generated_content += _delta
|
||||
elif _event_type in _TOOL_ARGUMENTS_DELTA_EVENTS:
|
||||
_args_delta: Final = getattr(openai_responses_api_chunk, "delta", None)
|
||||
if isinstance(_args_delta, str):
|
||||
self._generated_tool_arguments += _args_delta
|
||||
_stream_model_id: Final = _model_id_from_metadata(self.litellm_metadata)
|
||||
if _event_type in (
|
||||
ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED,
|
||||
|
|
@ -419,14 +427,41 @@ class BaseResponsesAPIStreamingIterator:
|
|||
openai_types.ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE,
|
||||
openai_types.ResponsesAPIStreamEvents.RESPONSE_FAILED,
|
||||
):
|
||||
self.completed_response = openai_responses_api_chunk
|
||||
_stamp_responses_usage_cost(getattr(openai_responses_api_chunk, "response", None), self.logging_obj)
|
||||
_response_obj: Final[object] = getattr(openai_responses_api_chunk, "response", None)
|
||||
_estimate_wanted: Final[bool] = _chunk_type in (
|
||||
openai_types.ResponsesAPIStreamEvents.RESPONSE_COMPLETED,
|
||||
openai_types.ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE,
|
||||
)
|
||||
_billed_response: Final[ResponsesAPIResponse | None] = _billed_terminal_response(
|
||||
_response_obj,
|
||||
(
|
||||
lambda: (
|
||||
_estimate_usage_safely(
|
||||
self.model or "",
|
||||
self.request_data.get("input"),
|
||||
self.request_data,
|
||||
self._generated_content + self._generated_tool_arguments,
|
||||
)
|
||||
if _estimate_wanted
|
||||
else None
|
||||
)
|
||||
),
|
||||
)
|
||||
_terminal_chunk: Final = (
|
||||
openai_responses_api_chunk
|
||||
if _billed_response is None or _billed_response is _response_obj
|
||||
else openai_responses_api_chunk.model_copy(update={"response": _billed_response})
|
||||
)
|
||||
self.completed_response = _terminal_chunk
|
||||
_stamp_responses_usage_cost(_billed_response, self.logging_obj)
|
||||
|
||||
if _chunk_type == openai_types.ResponsesAPIStreamEvents.RESPONSE_FAILED:
|
||||
self._handle_logging_failed_response()
|
||||
else:
|
||||
self._handle_logging_completed_response()
|
||||
|
||||
return _terminal_chunk
|
||||
|
||||
return openai_responses_api_chunk
|
||||
|
||||
return None
|
||||
|
|
@ -655,7 +690,9 @@ class BaseResponsesAPIStreamingIterator:
|
|||
if cache is None:
|
||||
return
|
||||
|
||||
cached_response: Final = response_obj.model_dump_json()
|
||||
cached_response: Final = _dump_json_safely(response_obj)
|
||||
if cached_response is None:
|
||||
return
|
||||
if is_async:
|
||||
from litellm.caching.caching_handler import create_cache_write_task
|
||||
|
||||
|
|
@ -1301,6 +1338,31 @@ def _add_text_like_part_events(
|
|||
)
|
||||
|
||||
|
||||
def _billed_terminal_response(
|
||||
response_obj: object, estimate: Callable[[], ResponseAPIUsage | None] | None
|
||||
) -> ResponsesAPIResponse | None:
|
||||
if isinstance(response_obj, ResponsesAPIResponse):
|
||||
return (
|
||||
response_obj
|
||||
if response_obj.usage is not None or estimate is None
|
||||
else response_obj.model_copy(update={"usage": estimate()})
|
||||
)
|
||||
if not isinstance(response_obj, dict):
|
||||
return None
|
||||
usage: Final[object] = response_obj.get("usage") # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # a model_constructed terminal event leaves response as an untyped dict
|
||||
return ResponsesAPIResponse.model_construct(
|
||||
**{**response_obj, "usage": usage if usage is not None or estimate is None else estimate()} # pyright: ignore[reportUnknownArgumentType, reportArgumentType] # same untyped dict spread
|
||||
)
|
||||
|
||||
|
||||
def _dump_json_safely(response: BaseModel) -> str | None:
|
||||
try:
|
||||
return response.model_dump_json()
|
||||
except Exception as exc:
|
||||
verbose_logger.debug("could not serialize completed response for cache: %s", exc)
|
||||
return None
|
||||
|
||||
|
||||
def _logging_copy(event: object) -> object:
|
||||
"""Hand logging callbacks a copy, so their usage rewrite (Responses shape to chat shape) never
|
||||
reaches the event the caller is iterating. The round trip through ``model_dump`` sidesteps the
|
||||
|
|
@ -1332,6 +1394,56 @@ def _usage_as_model(usage: object) -> ResponseAPIUsage | None:
|
|||
return None
|
||||
|
||||
|
||||
_TOOL_ARGUMENTS_DELTA_EVENTS: Final = frozenset(
|
||||
{
|
||||
ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA,
|
||||
ResponsesAPIStreamEvents.CUSTOM_TOOL_CALL_INPUT_DELTA,
|
||||
ResponsesAPIStreamEvents.MCP_CALL_ARGUMENTS_DELTA,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _estimate_usage_from_text(
|
||||
model: str,
|
||||
request_input: object,
|
||||
responses_api_request: Mapping[str, object],
|
||||
generated_text: str,
|
||||
) -> ResponseAPIUsage:
|
||||
messages: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( # pyright: ignore[reportUnknownMemberType] # the transformer's signature is partially untyped
|
||||
input=request_input, # pyright: ignore[reportArgumentType] # the raw Responses API input is a str or ResponseInputParam list, matching the helper's declared union
|
||||
responses_api_request=dict(responses_api_request),
|
||||
)
|
||||
input_tokens: Final = litellm.token_counter( # pyright: ignore[reportUnknownMemberType] # token_counter's public signature is untyped
|
||||
model=model, messages=messages
|
||||
)
|
||||
output_tokens: Final = litellm.token_counter( # pyright: ignore[reportUnknownMemberType] # token_counter's public signature is untyped
|
||||
model=model, text=generated_text, count_response_tokens=True
|
||||
)
|
||||
return ResponseAPIUsage(
|
||||
input_tokens=input_tokens,
|
||||
output_tokens=output_tokens,
|
||||
total_tokens=input_tokens + output_tokens,
|
||||
)
|
||||
|
||||
|
||||
def _estimate_usage_safely(
|
||||
model: str,
|
||||
request_input: object,
|
||||
responses_api_request: Mapping[str, object],
|
||||
generated_text: str,
|
||||
) -> ResponseAPIUsage | None:
|
||||
try:
|
||||
return _estimate_usage_from_text(
|
||||
model=model,
|
||||
request_input=request_input,
|
||||
responses_api_request=responses_api_request,
|
||||
generated_text=generated_text,
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_logger.debug("Could not estimate usage from stream text, billing $0: %s", e)
|
||||
return None
|
||||
|
||||
|
||||
def _stamp_responses_usage_cost(
|
||||
response_obj: ResponsesAPIResponse | None, logging_obj: LiteLLMLoggingObj | None
|
||||
) -> None:
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ class Provider(ThreadingHTTPServer):
|
|||
delay: float = 0
|
||||
stream: bool = False
|
||||
truncated: bool = False
|
||||
cookie: str = ""
|
||||
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
|
|
@ -62,6 +63,8 @@ class Handler(BaseHTTPRequestHandler):
|
|||
return
|
||||
self.send_header("content-type", "application/json")
|
||||
self.send_header("content-length", str(len(server.response)))
|
||||
if server.cookie:
|
||||
self.send_header("set-cookie", server.cookie)
|
||||
self.end_headers()
|
||||
self.wfile.write(server.response)
|
||||
|
||||
|
|
@ -172,6 +175,14 @@ def test_failed_provider_responses_never_enter_cache(store: RedisResponseStore,
|
|||
assert len(provider.hits) == 2
|
||||
|
||||
|
||||
def test_cookie_setting_success_is_reused_without_the_cookie(store: RedisResponseStore, provider: Provider) -> None:
|
||||
provider.cookie = "__cf_bm=synthetic-bot-management; Path=/; HttpOnly; Secure"
|
||||
with edge(CacheEdge(store, SECRET), provider) as url:
|
||||
replies: Final = tuple(call(url) for _ in range(2))
|
||||
assert len(provider.hits) == 1
|
||||
assert all(reply.body == SUCCESS and "set-cookie" not in reply.headers for reply in replies)
|
||||
|
||||
|
||||
def test_expiry_does_not_slide(store: RedisResponseStore, provider: Provider) -> None:
|
||||
short: Final = replace(store, lifetime_ms=250)
|
||||
with edge(CacheEdge(short, SECRET), provider) as url:
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
`E2E_PROVIDER_CACHE=1` enables automatic response reuse in the live E2E mode. Standard OpenAI and Anthropic model registrations use the provider edge. Existing custom API bases, named credentials, mocked models and realtime WebSocket deployments keep their existing routing. Other provider protocols remain live
|
||||
|
||||
The edge caches complete successful POST responses for `/v1/chat/completions` and `/v1/messages`, including streams. Unsupported endpoints pass through. It matches the method, original URL, effective outbound headers (including authentication and HTTP-library defaults), body presence and exact body bytes using a full keyed digest. It sends the same prepared request used for matching. No prompts, random markers, JSON values or credentials are normalized away
|
||||
The edge caches complete successful POST responses for `/v1/chat/completions` and `/v1/messages`, including streams. Unsupported endpoints pass through. It matches the method, original URL, effective outbound headers (including authentication and HTTP-library defaults), body presence and exact body bytes using a full keyed digest. It sends the same prepared request used for matching. No prompts, random markers, JSON values or credentials are normalized away. Provider `Set-Cookie` headers are dropped before validation and never recorded: the edge already withholds them from the proxy, and OpenAI responses always carry Cloudflare bot-management cookies
|
||||
|
||||
An eligible miss calls the provider. A complete successful response is stored immediately even if a later test assertion fails. Provider errors, malformed responses, truncated streams and cancelled captures are not stored. Cache reads, writes and lease failures fall through to normal provider behavior; they introduce no provider retry. An already-started response cannot be restarted after a delivery failure
|
||||
|
||||
|
|
@ -20,7 +20,7 @@ The trusted runner receives:
|
|||
|
||||
Do not give cache credentials to candidate deployments. Counter artifacts contain no recorded payloads or credentials. Hits count shared-cache responses; upstream attempts count actual forwards from the edge. Existing application-cache observations still count requests arriving at the edge, including shared-cache hits
|
||||
|
||||
Tests that require real provider timing, limits or state use `@pytest.mark.provider_live`. The marker keeps newly registered models on live routes without weakening their assertions. Ordinary assertion failures still fail E2E. The shared cache does not modify provider response IDs or make the proxy aware of replay
|
||||
Tests that require real provider timing, limits or state use `@pytest.mark.provider_live`. The marker keeps newly registered models on live routes without weakening their assertions. The provider prompt-caching tests carry it because a replayed priming response reports cache creation rather than a cache read. Ordinary assertion failures still fail E2E. The shared cache does not modify provider response IDs or make the proxy aware of replay
|
||||
|
||||
## Recorded response semantics
|
||||
|
||||
|
|
|
|||
|
|
@ -44,7 +44,7 @@ from models import ChatBody, ChatMessage, ChatResponse, LiteLLMParamsBody, Usage
|
|||
from passthrough_client import PassthroughClient
|
||||
import os
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
pytestmark = [pytest.mark.e2e, pytest.mark.provider_live]
|
||||
|
||||
BEDROCK_MODEL = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0"
|
||||
VERTEX_MODEL = "vertex_ai/gemini-2.5-flash"
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter, ValidationEr
|
|||
LIFETIME_SECONDS: Final = 86_400
|
||||
MAX_REQUEST_BYTES: Final = 256 * 1024
|
||||
MAX_RESPONSE_BYTES: Final = 8 * 1024 * 1024
|
||||
UNRECORDED_RESPONSE_HEADERS: Final = frozenset({"set-cookie"})
|
||||
JSON_VALUE: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
|
||||
|
||||
|
||||
|
|
@ -104,8 +105,6 @@ def cacheable_endpoint(method: str, url: str, body: bytes | None) -> bool:
|
|||
def successful_response(url: str, status: int, headers: Mapping[str, str], body: bytes) -> bool:
|
||||
if not 200 <= status < 300 or len(body) > MAX_RESPONSE_BYTES:
|
||||
return False
|
||||
if any(name.lower() == "set-cookie" for name in headers):
|
||||
return False
|
||||
streaming: Final = "text/event-stream" in headers.get("content-type", "").lower()
|
||||
if streaming:
|
||||
try:
|
||||
|
|
@ -283,11 +282,14 @@ class CacheEdge:
|
|||
yield step
|
||||
capture.observe(step)
|
||||
chunks: Final = capture.chunks() if capture.eligible else ()
|
||||
if not capture.eligible or not successful_response(url, head.status_code, head.headers, b"".join(chunks)):
|
||||
headers: Final = {
|
||||
name: value for name, value in head.headers.items() if name.lower() not in UNRECORDED_RESPONSE_HEADERS
|
||||
}
|
||||
if not capture.eligible or not successful_response(url, head.status_code, headers, b"".join(chunks)):
|
||||
self.counters.increment("rejected")
|
||||
return
|
||||
response: Final = CachedResponse(
|
||||
request_key=key, status_code=head.status_code, headers=head.headers,
|
||||
request_key=key, status_code=head.status_code, headers=headers,
|
||||
chunks=tuple(base64.b64encode(chunk).decode("ascii") for chunk in chunks),
|
||||
)
|
||||
published: Final = self.store.publish(key, lease, encode_response(self.secret, response))
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ from models import (
|
|||
)
|
||||
from quota_client import QuotaClient
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
pytestmark = [pytest.mark.e2e, pytest.mark.provider_live]
|
||||
|
||||
# Anthropic prompt caching (host has ANTHROPIC_API_KEY; Bedrock was "Operation not allowed").
|
||||
ANTHROPIC_MODEL = "anthropic/claude-haiku-4-5-20251001"
|
||||
|
|
|
|||
|
|
@ -1254,7 +1254,8 @@ class TestHandleEdgeRequestPure:
|
|||
|
||||
|
||||
class TestApiBaseSeam:
|
||||
def test_live_mode_returns_none(self, tmp_path: Path) -> None:
|
||||
def test_live_mode_returns_none(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("E2E_PROVIDER_CACHE", raising=False)
|
||||
for mode_raw in ("live", ""):
|
||||
assert (
|
||||
provider_edge_api_base(
|
||||
|
|
|
|||
|
|
@ -5,17 +5,20 @@ completion_start_time = end_time."""
|
|||
|
||||
import json
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
from typing import Final, Optional
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from pydantic_core import PydanticSerializationError
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig
|
||||
from litellm.responses.streaming_iterator import (
|
||||
ResponsesAPIStreamingIterator,
|
||||
SyncResponsesAPIStreamingIterator,
|
||||
_estimate_usage_from_text,
|
||||
)
|
||||
from litellm.types.llms.openai import (
|
||||
ResponseAPIUsage,
|
||||
|
|
@ -31,16 +34,23 @@ def _sse_event(payload: dict) -> bytes:
|
|||
|
||||
def _mock_config() -> Mock:
|
||||
mock_config = Mock(spec=BaseResponsesAPIConfig)
|
||||
mock_responses_api_response = Mock(spec=ResponsesAPIResponse)
|
||||
mock_responses_api_response.id = "resp_ttft"
|
||||
mock_responses_api_response = ResponsesAPIResponse(
|
||||
id="resp_ttft",
|
||||
created_at=0,
|
||||
status="completed",
|
||||
model="gpt-4o-mini",
|
||||
object="response",
|
||||
output=[],
|
||||
usage=ResponseAPIUsage(input_tokens=1, output_tokens=1, total_tokens=2),
|
||||
)
|
||||
|
||||
def _transform(model, parsed_chunk, logging_obj):
|
||||
evt_type = parsed_chunk.get("type")
|
||||
if evt_type == "response.completed":
|
||||
completed = Mock(spec=ResponseCompletedEvent)
|
||||
completed.type = ResponsesAPIStreamEvents.RESPONSE_COMPLETED
|
||||
completed.response = mock_responses_api_response
|
||||
return completed
|
||||
return ResponseCompletedEvent(
|
||||
type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED,
|
||||
response=mock_responses_api_response,
|
||||
)
|
||||
stub = Mock()
|
||||
stub.type = evt_type
|
||||
return stub
|
||||
|
|
@ -54,6 +64,8 @@ def _make_iterator(
|
|||
sse_events: list[bytes],
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
trailing_error: Optional[Exception] = None,
|
||||
config: Mock | None = None,
|
||||
request_data: dict | None = None,
|
||||
) -> ResponsesAPIStreamingIterator:
|
||||
async def aiter_bytes():
|
||||
for evt in sse_events:
|
||||
|
|
@ -68,10 +80,11 @@ def _make_iterator(
|
|||
return ResponsesAPIStreamingIterator(
|
||||
response=mock_response,
|
||||
model="gpt-4o-mini",
|
||||
responses_api_provider_config=_mock_config(),
|
||||
responses_api_provider_config=config or _mock_config(),
|
||||
logging_obj=logging_obj,
|
||||
litellm_metadata={},
|
||||
custom_llm_provider="openai",
|
||||
request_data=request_data,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -329,6 +342,88 @@ def test_run_post_success_hooks_does_not_report_generation_time_as_overhead():
|
|||
assert "litellm_overhead_time_ms" not in iterator.completed_response._hidden_params
|
||||
|
||||
|
||||
def _mock_config_with_completed_response(response: ResponsesAPIResponse) -> Mock:
|
||||
mock_config = Mock(spec=BaseResponsesAPIConfig)
|
||||
|
||||
def _transform(model, parsed_chunk, logging_obj):
|
||||
evt_type = parsed_chunk.get("type")
|
||||
if evt_type == "response.completed":
|
||||
return ResponseCompletedEvent(
|
||||
type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED,
|
||||
response=response,
|
||||
)
|
||||
stub = Mock()
|
||||
stub.type = evt_type
|
||||
if "delta" in parsed_chunk:
|
||||
stub.delta = parsed_chunk.get("delta")
|
||||
if "item" in parsed_chunk:
|
||||
stub.item = parsed_chunk.get("item")
|
||||
return stub
|
||||
|
||||
mock_config.transform_streaming_response.side_effect = _transform
|
||||
return mock_config
|
||||
|
||||
|
||||
def _responses_api_response_without_usage() -> ResponsesAPIResponse:
|
||||
return ResponsesAPIResponse(
|
||||
id="resp_no_usage",
|
||||
created_at=int(datetime(2025, 1, 1).timestamp()),
|
||||
status="completed",
|
||||
model="gpt-4o-mini",
|
||||
object="response",
|
||||
output=[],
|
||||
usage=None,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completed_event_without_usage_gets_text_estimate():
|
||||
"""A response.completed event carrying usage: null still bills: the
|
||||
iterator estimates usage from the request input and generated text."""
|
||||
response = _responses_api_response_without_usage()
|
||||
iterator = _make_iterator(
|
||||
sse_events=[
|
||||
_sse_event({"type": "response.output_text.delta", "delta": "hello world"}),
|
||||
_sse_event({"type": "response.completed", "response": {}}),
|
||||
],
|
||||
logging_obj=_logging_obj_stub(),
|
||||
config=_mock_config_with_completed_response(response),
|
||||
request_data={"input": "count these input tokens please"},
|
||||
)
|
||||
|
||||
async for _ in iterator:
|
||||
pass
|
||||
|
||||
usage = iterator.completed_response.response.usage
|
||||
assert usage is not None
|
||||
assert usage.input_tokens > 0
|
||||
assert usage.output_tokens > 0
|
||||
assert usage.total_tokens == usage.input_tokens + usage.output_tokens
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completed_event_with_usage_is_left_untouched():
|
||||
"""Provider-reported usage on response.completed wins over the estimate."""
|
||||
response = _responses_api_response_with_usage()
|
||||
iterator = _make_iterator(
|
||||
sse_events=[
|
||||
_sse_event({"type": "response.output_text.delta", "delta": "hello world"}),
|
||||
_sse_event({"type": "response.completed", "response": {}}),
|
||||
],
|
||||
logging_obj=_logging_obj_stub(),
|
||||
config=_mock_config_with_completed_response(response),
|
||||
request_data={"input": "count these input tokens please"},
|
||||
)
|
||||
|
||||
async for _ in iterator:
|
||||
pass
|
||||
|
||||
usage = iterator.completed_response.response.usage
|
||||
assert usage.input_tokens == 20
|
||||
assert usage.output_tokens == 60
|
||||
assert usage.total_tokens == 80
|
||||
|
||||
|
||||
def _responses_api_response_with_usage() -> ResponsesAPIResponse:
|
||||
return ResponsesAPIResponse(
|
||||
id="resp_lit6427",
|
||||
|
|
@ -628,3 +723,222 @@ async def test_streaming_logging_copy_keeps_client_usage_when_response_fails_val
|
|||
assert isinstance(client_usage, ResponseAPIUsage)
|
||||
assert client_usage.input_tokens == 29
|
||||
assert client_usage.cost == pytest.approx(0.0001)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completed_event_without_usage_counts_tool_call_arguments():
|
||||
"""A function-call-only stream still bills output tokens: streamed
|
||||
function_call_arguments deltas feed the text estimate."""
|
||||
response = _responses_api_response_without_usage()
|
||||
iterator = _make_iterator(
|
||||
sse_events=[
|
||||
_sse_event(
|
||||
{
|
||||
"type": "response.output_item.added",
|
||||
"item": {"type": "function_call", "name": "get_weather", "call_id": "call_1"},
|
||||
}
|
||||
),
|
||||
_sse_event(
|
||||
{
|
||||
"type": "response.function_call_arguments.delta",
|
||||
"delta": '{"location": "San Francisco", "unit": "celsius"}',
|
||||
}
|
||||
),
|
||||
_sse_event({"type": "response.completed", "response": {}}),
|
||||
],
|
||||
logging_obj=_logging_obj_stub(),
|
||||
config=_mock_config_with_completed_response(response),
|
||||
request_data={"input": "what is the weather in san francisco"},
|
||||
)
|
||||
|
||||
async for _ in iterator:
|
||||
pass
|
||||
|
||||
usage = iterator.completed_response.response.usage
|
||||
assert usage is not None
|
||||
assert usage.output_tokens > 0
|
||||
assert usage.total_tokens == usage.input_tokens + usage.output_tokens
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completed_event_without_usage_counts_multimodal_input_as_messages():
|
||||
"""Multimodal request input is counted as chat messages, not as a JSON blob:
|
||||
a huge base64 image must not inflate the estimated input tokens."""
|
||||
image_input: Final = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "input_text", "text": "what is in this image"},
|
||||
{
|
||||
"type": "input_image",
|
||||
"image_url": "data:image/png;base64," + "A" * 4000,
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
json_count: Final = litellm.token_counter(model="gpt-4o-mini", text=json.dumps(image_input))
|
||||
response = _responses_api_response_without_usage()
|
||||
iterator = _make_iterator(
|
||||
sse_events=[
|
||||
_sse_event({"type": "response.output_text.delta", "delta": "it is a cat"}),
|
||||
_sse_event({"type": "response.completed", "response": {}}),
|
||||
],
|
||||
logging_obj=_logging_obj_stub(),
|
||||
config=_mock_config_with_completed_response(response),
|
||||
request_data={"input": image_input},
|
||||
)
|
||||
|
||||
async for _ in iterator:
|
||||
pass
|
||||
|
||||
usage = iterator.completed_response.response.usage
|
||||
assert usage is not None
|
||||
assert usage.input_tokens < json_count / 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completed_event_survives_a_failing_usage_estimate():
|
||||
"""A malformed request input that makes the message transformer raise must not
|
||||
break a stream that previously completed: the estimate is best-effort and
|
||||
falls back to usage None."""
|
||||
malformed_input: Final = [{"type": "message", "role": "user", "content": 42}]
|
||||
with pytest.raises(ValueError, match="Invalid content type"):
|
||||
_estimate_usage_from_text("gpt-4o-mini", malformed_input, {"input": malformed_input}, "hello world")
|
||||
|
||||
response = _responses_api_response_without_usage()
|
||||
iterator = _make_iterator(
|
||||
sse_events=[
|
||||
_sse_event({"type": "response.output_text.delta", "delta": "hello world"}),
|
||||
_sse_event({"type": "response.completed", "response": {}}),
|
||||
],
|
||||
logging_obj=_logging_obj_stub(),
|
||||
config=_mock_config_with_completed_response(response),
|
||||
request_data={"input": malformed_input},
|
||||
)
|
||||
|
||||
yielded: list = []
|
||||
async for chunk in iterator:
|
||||
yielded.append(chunk)
|
||||
|
||||
assert yielded
|
||||
assert iterator.completed_response.response.usage is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"tool_delta_event_type",
|
||||
["response.custom_tool_call_input.delta", "response.mcp_call_arguments.delta"],
|
||||
)
|
||||
async def test_completed_event_without_usage_counts_tool_input_deltas(tool_delta_event_type):
|
||||
"""Custom-tool and MCP argument deltas feed the streamed usage fallback the
|
||||
same way function_call_arguments deltas do."""
|
||||
response = _responses_api_response_without_usage()
|
||||
iterator = _make_iterator(
|
||||
sse_events=[
|
||||
_sse_event({"type": tool_delta_event_type, "delta": '{"query": "weather in sf"}'}),
|
||||
_sse_event({"type": "response.completed", "response": {}}),
|
||||
],
|
||||
logging_obj=_logging_obj_stub(),
|
||||
config=_mock_config_with_completed_response(response),
|
||||
request_data={"input": "what is the weather in san francisco"},
|
||||
)
|
||||
|
||||
async for _ in iterator:
|
||||
pass
|
||||
|
||||
usage = iterator.completed_response.response.usage
|
||||
assert usage is not None
|
||||
assert usage.output_tokens > 0
|
||||
assert usage.total_tokens == usage.input_tokens + usage.output_tokens
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completed_event_with_a_dict_response_is_typed_and_billed():
|
||||
"""transform_streaming_response can model_construct a terminal event whose
|
||||
response stays a plain dict; the iterator must type it so the estimated
|
||||
usage reaches the cost stamping path."""
|
||||
dict_response: Final = {
|
||||
"id": "resp_dict",
|
||||
"model": "gpt-4o-mini",
|
||||
"object": "response",
|
||||
"output": [],
|
||||
"usage": None,
|
||||
}
|
||||
|
||||
def _transform(model, parsed_chunk, logging_obj):
|
||||
if parsed_chunk.get("type") == "response.completed":
|
||||
return ResponseCompletedEvent.model_construct(type="response.completed", response=dict_response)
|
||||
stub: Final = Mock()
|
||||
stub.type = parsed_chunk.get("type")
|
||||
if "delta" in parsed_chunk:
|
||||
stub.delta = parsed_chunk.get("delta")
|
||||
return stub
|
||||
|
||||
config: Final = Mock(spec=BaseResponsesAPIConfig)
|
||||
config.transform_streaming_response.side_effect = _transform
|
||||
logging_obj: Final = _logging_obj_stub()
|
||||
logging_obj._response_cost_calculator.return_value = 0.000704
|
||||
iterator: Final = _make_iterator(
|
||||
sse_events=[
|
||||
_sse_event({"type": "response.output_text.delta", "delta": "hello world"}),
|
||||
_sse_event({"type": "response.completed", "response": {}}),
|
||||
],
|
||||
logging_obj=logging_obj,
|
||||
config=config,
|
||||
request_data={"input": "count these input tokens please"},
|
||||
)
|
||||
|
||||
yielded: Final = [chunk async for chunk in iterator]
|
||||
|
||||
terminal_event: Final = iterator.completed_response
|
||||
assert yielded[-1] is terminal_event
|
||||
completed_response: Final = terminal_event.response
|
||||
assert isinstance(completed_response, ResponsesAPIResponse)
|
||||
usage: Final = completed_response.usage
|
||||
assert usage is not None
|
||||
assert usage.input_tokens > 0
|
||||
assert usage.output_tokens > 0
|
||||
assert usage.cost == pytest.approx(0.000704)
|
||||
logging_obj._response_cost_calculator.assert_any_call(result=completed_response)
|
||||
|
||||
|
||||
def test_billed_terminal_response_keeps_a_response_that_already_has_usage():
|
||||
from litellm.responses.streaming_iterator import _billed_terminal_response
|
||||
|
||||
response: Final = _responses_api_response_with_usage()
|
||||
|
||||
assert _billed_terminal_response(response, None) is response
|
||||
|
||||
|
||||
def test_billed_terminal_response_copies_when_estimating_and_leaves_the_original_untouched():
|
||||
from litellm.responses.streaming_iterator import _billed_terminal_response
|
||||
|
||||
response: Final = _responses_api_response_without_usage()
|
||||
estimated: Final = ResponseAPIUsage(input_tokens=3, output_tokens=4, total_tokens=7)
|
||||
|
||||
billed: Final = _billed_terminal_response(response, lambda: estimated)
|
||||
|
||||
assert billed is not response
|
||||
assert billed.usage is estimated
|
||||
assert response.usage is None
|
||||
|
||||
|
||||
def test_persist_completed_response_to_cache_survives_an_unserializable_response(monkeypatch):
|
||||
bad_response: Final = ResponsesAPIResponse.model_construct(id="r", output=[object()], usage=None)
|
||||
with pytest.raises(PydanticSerializationError):
|
||||
bad_response.model_dump_json()
|
||||
|
||||
logging_obj: Final = _logging_obj_stub()
|
||||
caching_handler: Final = Mock()
|
||||
caching_handler.request_kwargs = {"stream": True}
|
||||
logging_obj._llm_caching_handler = caching_handler
|
||||
iterator: Final = _make_iterator(sse_events=[], logging_obj=logging_obj)
|
||||
iterator.completed_response = ResponseCompletedEvent.model_construct(
|
||||
type="response.completed", response=bad_response
|
||||
)
|
||||
cache: Final = Mock()
|
||||
monkeypatch.setattr(litellm, "cache", cache)
|
||||
|
||||
iterator._persist_completed_response_to_cache(is_async=False)
|
||||
|
||||
cache.add_cache.assert_not_called()
|
||||
|
|
|
|||
|
|
@ -794,19 +794,15 @@ const ComplexityRouterConfig: React.FC<ComplexityRouterConfigProps> = ({
|
|||
},
|
||||
]
|
||||
: []),
|
||||
...(value.classifier_type !== "llm_v2"
|
||||
? [
|
||||
{
|
||||
key: "adaptive",
|
||||
label: <strong className="text-foreground font-semibold">Advanced: Adaptive Routing</strong>,
|
||||
children: (
|
||||
<Restricted by={restrictedBy(value, "adaptive")}>
|
||||
<AdaptiveRoutingConfig value={value} onChange={onChange} />
|
||||
</Restricted>
|
||||
),
|
||||
},
|
||||
]
|
||||
: []),
|
||||
{
|
||||
key: "adaptive",
|
||||
label: <strong className="text-foreground font-semibold">Advanced: Adaptive Routing</strong>,
|
||||
children: (
|
||||
<Restricted by={restrictedBy(value, "adaptive")}>
|
||||
<AdaptiveRoutingConfig value={value} onChange={onChange} />
|
||||
</Restricted>
|
||||
),
|
||||
},
|
||||
{
|
||||
key: "affinity",
|
||||
label: <strong className="text-foreground font-semibold">Advanced: Affinity</strong>,
|
||||
|
|
@ -906,15 +902,17 @@ const ComplexityRouterConfig: React.FC<ComplexityRouterConfigProps> = ({
|
|||
},
|
||||
]
|
||||
: []),
|
||||
].map(({ key, label, children }) => (
|
||||
<Collapsible key={key} className="border-b border-border last:border-b-0">
|
||||
<CollapsibleTrigger className="group flex w-full items-center gap-2 px-4 py-3 text-left">
|
||||
<ChevronRight className="size-4 shrink-0 text-muted-foreground transition-transform group-data-panel-open:rotate-90" />
|
||||
{label}
|
||||
</CollapsibleTrigger>
|
||||
<CollapsibleContent className="px-4 pb-4">{children}</CollapsibleContent>
|
||||
</Collapsible>
|
||||
))}
|
||||
]
|
||||
.filter(({ key }) => !forecast || !["adaptive", "context-window", "escalation"].includes(key))
|
||||
.map(({ key, label, children }) => (
|
||||
<Collapsible key={key} className="border-b border-border last:border-b-0">
|
||||
<CollapsibleTrigger className="group flex w-full items-center gap-2 px-4 py-3 text-left">
|
||||
<ChevronRight className="size-4 shrink-0 text-muted-foreground transition-transform group-data-panel-open:rotate-90" />
|
||||
{label}
|
||||
</CollapsibleTrigger>
|
||||
<CollapsibleContent className="px-4 pb-4">{children}</CollapsibleContent>
|
||||
</Collapsible>
|
||||
))}
|
||||
</div>
|
||||
</RoutingOptions>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -105,7 +105,7 @@ describe("forecast classifier form", () => {
|
|||
expect(output).toHaveTextContent('"REASONING":["capable"]');
|
||||
expect(output).toHaveTextContent('"reasoning_effort":"low","speed":"fast","max_tokens":1024');
|
||||
expect(output).toHaveTextContent('"reasoning_effort":"high"');
|
||||
expect(output).toHaveTextContent('"adaptive":true');
|
||||
expect(output).toHaveTextContent('"adaptive":false');
|
||||
expect(output).not.toHaveTextContent("leftover-medium");
|
||||
expect(output).not.toHaveTextContent("leftover-complex");
|
||||
expect(output).not.toHaveTextContent('"plan_mode_min_tier"');
|
||||
|
|
|
|||
|
|
@ -213,17 +213,24 @@ describe("AddAutoRouterTab", () => {
|
|||
expect(screen.queryByRole("alert")).not.toBeInTheDocument();
|
||||
expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeEnabled();
|
||||
await user.click(screen.getByRole("button", { name: "Advanced routing options" }));
|
||||
if (capability) expect(screen.getByText("Advanced: Adaptive Routing")).toBeInTheDocument();
|
||||
else expect(screen.queryByText("Advanced: Adaptive Routing")).not.toBeInTheDocument();
|
||||
for (const label of ["Adaptive Routing", "Context Window Escalation", "Escalation Keywords"]) {
|
||||
expect(screen.queryByText(`Advanced: ${label}`)).not.toBeInTheDocument();
|
||||
}
|
||||
expect(screen.getByText("Advanced: Stalled Task Escalation")).toBeInTheDocument();
|
||||
expect(screen.getByText("Advanced: Response Format")).toBeInTheDocument();
|
||||
expect(screen.queryByText("Advanced: Classification Method")).not.toBeInTheDocument();
|
||||
expect(screen.getByText("Advanced: Affinity")).toBeInTheDocument();
|
||||
await user.click(screen.getByRole("button", { name: "Add Auto Router" }));
|
||||
await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalledTimes(1));
|
||||
expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls[0][0].complexity_router_config).toMatchObject({
|
||||
const expected = {
|
||||
classifier_type: capability ? "capability" : "llm_v2",
|
||||
adaptive: false,
|
||||
enable_context_window_escalation: false,
|
||||
escalation_keywords: [],
|
||||
tiers: { SIMPLE: ["efficient"], REASONING: ["capable"] },
|
||||
classifier_llm_config: { model: "judge" },
|
||||
});
|
||||
};
|
||||
expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls[0][0].complexity_router_config).toMatchObject(expected);
|
||||
},
|
||||
);
|
||||
|
||||
|
|
@ -239,6 +246,9 @@ describe("AddAutoRouterTab", () => {
|
|||
expect(screen.getByTestId("template-selector")).toBeInTheDocument();
|
||||
expandDetailedConfiguration();
|
||||
expect(screen.getByText("Complexity Tier Configuration")).toBeInTheDocument();
|
||||
for (const label of ["Adaptive Routing", "Context Window Escalation", "Escalation Keywords"]) {
|
||||
expect(screen.getByText(`Advanced: ${label}`)).toBeInTheDocument();
|
||||
}
|
||||
await user.click(screen.getByText("Advanced: Classification Method"));
|
||||
expect(screen.queryByRole("radio", { name: /^Capability/ })).not.toBeInTheDocument();
|
||||
expect(screen.queryByRole("radio", { name: /^Fuse v2/ })).not.toBeInTheDocument();
|
||||
|
|
|
|||
|
|
@ -48,6 +48,37 @@ const baseParams: BuildComplexityRouterConfigParams = {
|
|||
};
|
||||
|
||||
describe("buildComplexityRouterConfig", () => {
|
||||
it.each(["capability", "llm_v2", "heuristic"] as const)(
|
||||
"disables the removed overrides only for forecast creates: %s",
|
||||
(classifierType) => {
|
||||
const forecast = classifierType !== "heuristic";
|
||||
const params = {
|
||||
...baseParams,
|
||||
classifierType,
|
||||
adaptive: true,
|
||||
enableContextWindowEscalation: true,
|
||||
contextWindowEscalationBuffer: 0.9,
|
||||
};
|
||||
const config = buildComplexityRouterConfig(params);
|
||||
expect(config.adaptive).toBe(!forecast);
|
||||
expect(config.enable_context_window_escalation).toBe(!forecast);
|
||||
expect(config.escalation_keywords).toEqual(forecast ? [] : ["LITELLM ESCALATE"]);
|
||||
for (const key of [
|
||||
"adaptive_weights",
|
||||
"adaptive_eligible",
|
||||
"tier_distance_penalty",
|
||||
"context_window_escalation_buffer",
|
||||
]) {
|
||||
expect(Object.hasOwn(config, key)).toBe(!forecast);
|
||||
}
|
||||
if (forecast) {
|
||||
const untouched = buildComplexityRouterConfig({ ...baseParams, classifierType });
|
||||
expect(untouched.enable_context_window_escalation).toBe(false);
|
||||
expect(untouched.escalation_keywords).toEqual([]);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it("carries Fast and reasoning overrides independently into a new router payload", () => {
|
||||
const params = { speed: "fast", reasoning_effort: "high", max_tokens: 1024 };
|
||||
const config = buildComplexityRouterConfig({
|
||||
|
|
|
|||
|
|
@ -628,13 +628,11 @@ export const buildComplexityRouterConfig = ({
|
|||
// An edited tier set forces the LLM classifier, so llm-only inputs must survive a classifier_type
|
||||
// the form never rewrote. The UI gates the same controls on this, not on the raw value.
|
||||
const effectiveType: ClassifierType = customTierSet ? "llm" : classifierType;
|
||||
const forecast = isForecastClassifier(effectiveType);
|
||||
|
||||
const supportsOpeningPrompt =
|
||||
!customTierSet && !isForecastClassifier(effectiveType) && usesLlmClassifier(effectiveType);
|
||||
const supportsOpeningPrompt = !customTierSet && !forecast && usesLlmClassifier(effectiveType);
|
||||
const payload: ComplexityRouterConfigPayload = {
|
||||
tiers: isForecastClassifier(effectiveType)
|
||||
? Object.fromEntries(Object.entries(tiers).filter(([, models]) => models.length > 0))
|
||||
: tiers,
|
||||
tiers: forecast ? Object.fromEntries(Object.entries(tiers).filter(([, models]) => models.length > 0)) : tiers,
|
||||
// The backend rejects the flag beside a custom tier set.
|
||||
...(!customTierSet && enableNonReasoningTier && { enable_non_reasoning_tier: true }),
|
||||
...(serializedTierModelConfigs && { tier_model_configs: serializedTierModelConfigs }),
|
||||
|
|
@ -645,7 +643,8 @@ export const buildComplexityRouterConfig = ({
|
|||
...classifierWireFields(effectiveType, classifierInputs),
|
||||
...(effectiveType === "capability" &&
|
||||
capabilityClassifierConfig && { capability_classifier_config: capabilityClassifierConfig }),
|
||||
...(effectiveType === "llm_v2" && { llm_v2_config: llmV2Config, adaptive: false }),
|
||||
...(effectiveType === "llm_v2" && { llm_v2_config: llmV2Config }),
|
||||
...(forecast && { adaptive: false }),
|
||||
// A built-in router's opening instructions. Suppressed beside a legacy whole-prompt override,
|
||||
// which the backend rejects as a second override of the same prompt.
|
||||
...(supportsOpeningPrompt &&
|
||||
|
|
@ -660,7 +659,7 @@ export const buildComplexityRouterConfig = ({
|
|||
modality_pin_override: modalityPinOverride ?? false,
|
||||
...(customTechnicalKeywords.length > 0 && { custom_technical_keywords: customTechnicalKeywords }),
|
||||
...(cleanedKeywordTierRules.length > 0 && { keyword_tier_rules: cleanedKeywordTierRules }),
|
||||
escalation_keywords: cleanedEscalationKeywords,
|
||||
escalation_keywords: forecast ? [] : cleanedEscalationKeywords,
|
||||
// Only written when on: the backend rejects it alongside session_affinity, user_turn mode and
|
||||
// a custom tier set, so an off router must not carry the key into any of those saves.
|
||||
...(stallEscalationEnabled && {
|
||||
|
|
@ -676,19 +675,21 @@ export const buildComplexityRouterConfig = ({
|
|||
match_threshold: matchThreshold,
|
||||
}),
|
||||
...(adaptive &&
|
||||
effectiveType !== "llm_v2" && {
|
||||
!forecast && {
|
||||
adaptive: true,
|
||||
adaptive_weights: adaptiveWeights,
|
||||
...(adaptiveEligible === "all" && { tier_distance_penalty: tierDistancePenalty }),
|
||||
adaptive_eligible: adaptiveEligible,
|
||||
}),
|
||||
...(returnRawModelName && { return_raw_model_name: true }),
|
||||
...(enableContextWindowEscalation !== undefined && {
|
||||
enable_context_window_escalation: enableContextWindowEscalation,
|
||||
}),
|
||||
...(contextWindowEscalationBuffer !== undefined && {
|
||||
context_window_escalation_buffer: contextWindowEscalationBuffer,
|
||||
// Omission enables the backend default, so hidden forecast controls need an explicit opt-out.
|
||||
...((forecast || enableContextWindowEscalation !== undefined) && {
|
||||
enable_context_window_escalation: forecast ? false : enableContextWindowEscalation,
|
||||
}),
|
||||
...(!forecast &&
|
||||
contextWindowEscalationBuffer !== undefined && {
|
||||
context_window_escalation_buffer: contextWindowEscalationBuffer,
|
||||
}),
|
||||
...(sessionAffinityTtlSeconds !== undefined && {
|
||||
session_affinity_ttl_seconds: sessionAffinityTtlSeconds,
|
||||
}),
|
||||
|
|
|
|||
|
|
@ -264,7 +264,7 @@ describe("forecast classifier configuration", () => {
|
|||
});
|
||||
expect(saved.tiers).toEqual({ SIMPLE: ["efficient"], MEDIUM: ["middle"], REASONING: ["capable"] });
|
||||
expect(saved.tier_model_configs).toEqual(stored.tier_model_configs);
|
||||
expect(saved.adaptive).toBe(true);
|
||||
expect(saved.adaptive).toBe(false);
|
||||
expect(saved.plan_mode_min_tier).toBe("MEDIUM");
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -46,6 +46,43 @@ const hydratedState: KeywordMatchingState = {
|
|||
};
|
||||
|
||||
describe("buildUpdatedComplexityRouterConfig keyword matching", () => {
|
||||
it.each(["capability", "llm_v2", "heuristic"] as const)(
|
||||
"handles enabled stored overrides when editing %s with or without keyword form state",
|
||||
(classifier_type) => {
|
||||
const stored = {
|
||||
...STORED,
|
||||
classifier_type,
|
||||
adaptive: classifier_type !== "llm_v2",
|
||||
adaptive_weights: { quality: 0.6, cost: 0.4 },
|
||||
adaptive_eligible: "all",
|
||||
tier_distance_penalty: 0.8,
|
||||
enable_context_window_escalation: true,
|
||||
context_window_escalation_buffer: 0.9,
|
||||
};
|
||||
const value = hydrateComplexityRouterConfig(stored, undefined);
|
||||
for (const keywordState of [undefined, hydratedState]) {
|
||||
const saved = buildUpdatedComplexityRouterConfig(stored, value, undefined, keywordState);
|
||||
const forecast = classifier_type !== "heuristic";
|
||||
expect(saved.adaptive).toBe(!forecast);
|
||||
expect(saved.enable_context_window_escalation).toBe(!forecast);
|
||||
expect(saved.escalation_keywords).toEqual(forecast ? [] : stored.escalation_keywords);
|
||||
for (const key of [
|
||||
"adaptive_weights",
|
||||
"adaptive_eligible",
|
||||
"tier_distance_penalty",
|
||||
"context_window_escalation_buffer",
|
||||
]) {
|
||||
expect(Object.hasOwn(saved, key)).toBe(!forecast);
|
||||
}
|
||||
expect(saved.keyword_tier_rules).toEqual(STORED.keyword_tier_rules);
|
||||
expect(saved.semantic_keyword_matching).toBe(true);
|
||||
expect(saved.some_future_backend_key).toEqual(STORED.some_future_backend_key);
|
||||
}
|
||||
expect(value.enable_context_window_escalation).toBe(true);
|
||||
expect(stored.escalation_keywords).toEqual(["urgent", "outage"]);
|
||||
},
|
||||
);
|
||||
|
||||
it("round-trips an untouched edit without changing any keyword-matching value", () => {
|
||||
// Opening the modal hydrates state from STORED; saving with nothing changed must be a
|
||||
// no-op. These keys are now MANAGED, so a hydration bug silently wipes them.
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import type { StoredComplexityRouterConfig } from "../add_model/build_complexity
|
|||
export type { StoredComplexityRouterConfig } from "../add_model/build_complexity_router_config";
|
||||
import {
|
||||
getForecastConfigError,
|
||||
isForecastClassifier,
|
||||
capabilitySettingsSchema,
|
||||
fuseSettingsSchema,
|
||||
} from "../add_model/forecast_classifier_config";
|
||||
|
|
@ -71,6 +72,7 @@ import {
|
|||
} from "../add_model/heuristic_scoring_knobs";
|
||||
import ComplexityRouterConfig, {
|
||||
ComplexityRouterConfigValue,
|
||||
effectiveClassifierType,
|
||||
heuristicScoringRole,
|
||||
DEFAULT_ADAPTIVE_WEIGHTS,
|
||||
DEFAULT_SESSION_AFFINITY,
|
||||
|
|
@ -305,6 +307,7 @@ export const buildUpdatedComplexityRouterConfig = (
|
|||
): Record<string, unknown> => {
|
||||
const isManaged = (key: string): boolean => {
|
||||
if (MANAGED_COMPLEXITY_ROUTER_KEYS.has(key)) return true;
|
||||
if (key === "escalation_keywords" && isForecastClassifier(effectiveClassifierType(value))) return true;
|
||||
if (keywordMatching !== undefined && KEYWORD_MATCHING_KEYS.has(key)) return true;
|
||||
return customTechnicalKeywords !== undefined && key === "custom_technical_keywords";
|
||||
};
|
||||
|
|
@ -365,7 +368,7 @@ export const buildUpdatedComplexityRouterConfig = (
|
|||
|
||||
// Keys this call does not own stay as the stored config left them.
|
||||
const unowned: readonly string[] = [
|
||||
...(keywordMatching === undefined ? KEYWORD_MATCHING_KEYS : []),
|
||||
...(keywordMatching === undefined ? [...KEYWORD_MATCHING_KEYS].filter((key) => !isManaged(key)) : []),
|
||||
...(customTechnicalKeywords === undefined ? ["custom_technical_keywords"] : []),
|
||||
];
|
||||
return {
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue