From 6e52ee2818b74690d289d1187b45fe6cdfaab8d5 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 30 Oct 2026 21:55:12 +0000 Subject: [PATCH 01/15] feat(proxy): add POST /health/test_model live two-turn model checks Runs one tool-calling conversation per selected endpoint (/v1/chat/completions, /v1/responses, /v1/messages) against a saved deployment or unsaved litellm_params and reports conversation, prompt_caching and cost_tracking verdicts, including whether LiteLLM_SpendLogs rows landed --- litellm/constants.py | 2 + .../health_endpoints/_health_endpoints.py | 529 ++++++++++--- .../proxy/health_endpoints/model_checks.py | 729 ++++++++++++++++++ .../management/test_model_health_check.py | 147 +++- .../health_endpoints/test_model_checks.py | 373 +++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 152 ++++ 6 files changed, 1840 insertions(+), 92 deletions(-) create mode 100644 litellm/proxy/health_endpoints/model_checks.py create mode 100644 tests/unit/proxy/health_endpoints/test_model_checks.py diff --git a/litellm/constants.py b/litellm/constants.py index c586a5f767a..a943316b6af 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1967,6 +1967,8 @@ BACKGROUND_INTERACTION_COST_POLLING_ENABLED: Final = _background_interaction_cos BACKGROUND_INTERACTION_SETTLEMENT_CLAIM_LEASE_SECONDS: Final = 300.0 PROXY_BUDGET_RESCHEDULER_MAX_TIME: Final = get_env_int("PROXY_BUDGET_RESCHEDULER_MAX_TIME", 605) PROXY_BATCH_WRITE_AT: Final = get_env_int("PROXY_BATCH_WRITE_AT", 10) # in seconds, increased from 10 +TEST_MODEL_SPEND_LOG_WAIT_SECONDS: Final = get_env_int("TEST_MODEL_SPEND_LOG_WAIT_SECONDS", PROXY_BATCH_WRITE_AT + 5) +TEST_MODEL_COST_WAIT_SECONDS: Final = get_env_float("TEST_MODEL_COST_WAIT_SECONDS", 3.0) PROXY_CONFIG_RELOAD_INTERVAL_SECONDS: Final = get_env_int("PROXY_CONFIG_RELOAD_INTERVAL_SECONDS", 30) # APScheduler Configuration - MEMORY LEAK FIX diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index 8545e0609db..727d2f0309b 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -6,12 +6,13 @@ import os import secrets import time import traceback -from collections.abc import Iterable, Mapping, Sequence +import uuid +from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence from dataclasses import dataclass from datetime import datetime, timedelta, timezone from itertools import chain, groupby from types import MappingProxyType -from typing import Final, Literal, TypedDict +from typing import Annotated, Final, Literal, TypeAlias, TypedDict import fastapi from fastapi import APIRouter, Depends, HTTPException, Request, Response, status @@ -20,13 +21,19 @@ from typing_extensions import ReadOnly import litellm from litellm._logging import verbose_logger, verbose_proxy_logger -from litellm.constants import HEALTH_CHECK_TIMEOUT_SECONDS, PROXY_DB_LOOKUP_STALL_WINDOW_SECONDS +from litellm.constants import ( + HEALTH_CHECK_TIMEOUT_SECONDS, + PROXY_DB_LOOKUP_STALL_WINDOW_SECONDS, + TEST_MODEL_COST_WAIT_SECONDS, + TEST_MODEL_SPEND_LOG_WAIT_SECONDS, +) from litellm.integrations.SlackAlerting.ms_teams import ( MS_TEAMS_ALERT_HEADERS, build_ms_teams_payload, get_ms_teams_webhook_url, ) from litellm.litellm_core_utils.custom_logger_registry import CustomLoggerRegistry +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.proxy._types import ( AlertType, @@ -85,6 +92,19 @@ from litellm.proxy.health_check import ( # noqa: F401 # legacy module exports run_with_timeout, update_litellm_params_for_health_check, ) +from litellm.proxy.health_endpoints.model_checks import ( + ENDPOINT_FOR_MODE, + SCRIPTS, + CalledTurn, + EndpointResult, + EndpointScript, + TestModelEndpoint, + TestModelResponse, + endpoint_result_payload, + overall_status, + response_cost, + run_endpoint_check, +) from litellm.proxy.middleware.admission_control_middleware import ( get_admission_control_stats, ) @@ -101,7 +121,7 @@ from litellm.router_utils.clientside_credential_handler import ( from litellm.router_utils.common_utils import resolve_model_group_alias from litellm.secret_managers.main import get_secret_bool from litellm.types.proxy.litellm_pre_call_utils import RedactedDict, SecretFields -from litellm.types.router import reject_server_owned_wif_fields +from litellm.types.router import Deployment, reject_server_owned_wif_fields from litellm.types.utils import AdminApiKeySecretReference, admin_api_key_secret_reference #### Health ENDPOINTS #### @@ -2487,6 +2507,410 @@ async def health_liveliness_options(): return Response(headers=response_headers, status_code=200) +@dataclass(frozen=True, slots=True) +class _ResolvedConnection: + """A connection test's probe parameters, resolved the way ``/health/test_connection`` always has.""" + + request_litellm_params: Mapping[str, object] + config_litellm_params: Mapping[str, object] + litellm_params: dict[str, object] # mutable-ok: the health check helpers mutate the probe params in place + resolved_model_info: dict[str, object] | None # mutable-ok: handed to Deployment, which takes a plain dict + probe_model_info: Mapping[str, object] + deployment_by_id: Deployment | None + stored_model: str | None + + +def _resolve_connection_params( + request_litellm_params: Mapping[str, object], + model_info: Mapping[str, object] | None, + llm_router: Router | None, + general_settings: Mapping[str, object], +) -> _ResolvedConnection: + # Reject request-supplied os.environ/ references. Config values are + # already resolved before reaching this endpoint; any remaining + # reference must have come from the request body. + _reject_os_environ_references(dict(request_litellm_params)) + if model_info: + _reject_os_environ_references(dict(model_info)) + requested_model: Final = request_litellm_params.get("model") + model_name: Final = requested_model if isinstance(requested_model, str) else None + + # Look up model configuration from router if model name is provided + # This gets the litellm_params from proxy config (with resolved env vars) + config_litellm_params: dict = {} + loaded_model_info: dict | None = None + deployment_by_id: Deployment | None = None + if llm_router is not None: + # Prefer disambiguation by deployment id (`model_info.id`) when + # the caller supplies it. This is required when multiple + # deployments share a `model_name` (e.g. wildcard `openai/*` + # with multiple `api_base` values for failover): the UI's + # "Test Connection" button targets a specific row, and that + # row's id is the only thing that uniquely identifies which + # deployment to probe. Without this, all duplicates collapse + # onto `deployments[0]`. + requested_id: Final = (model_info or {}).get("id") + request_model_id: Final = requested_id if isinstance(requested_id, str) else None + try: + if request_model_id: + deployment_by_id = llm_router.registry().deployment(request_model_id) + + if deployment_by_id is not None: + config_litellm_params = deployment_by_id.litellm_params.model_dump(exclude_none=True) + loaded_model_info = deployment_by_id.model_info.model_dump(exclude_none=True) + elif model_name: + # Fall back to model_name lookup for callers (e.g. the + # "Add Model" wizard, or curl) that don't supply an id. + # First try to find by proxy model_name (e.g., "gpt-4o") + deployments = llm_router.registry().deployments(model_name) + + # If not found, try to find by litellm model name + # (e.g., "azure/gpt-4o") + if not deployments or len(deployments) == 0: + all_deployments: Final = llm_router.registry().deployments(None) + if all_deployments: + for deployment in all_deployments: + if deployment.get("litellm_params", {}).get("model") == model_name: + deployments = [deployment] + break + + if deployments and len(deployments) > 0: + # Use the first deployment's litellm_params as base + # config. These already have resolved environment + # variables from proxy config. + config_litellm_params = dict(deployments[0].get("litellm_params", {})) + loaded_model_info = dict(deployments[0].get("model_info") or {}) + except Exception as e: + verbose_proxy_logger.debug( + "Could not find model %s in router: %s. Proceeding with request params only.", model_name, e + ) + + selected_by_id: Final = deployment_by_id is not None + reject_server_owned_wif_fields( + MappingProxyType( + { + key: value + for key, value in _OBJECT_MAPPING.validate_python(request_litellm_params).items() + if key != "chatgpt_auth_file" + } + ) + ) + # Merge: config params (from proxy config) as base, request params override + merged_litellm_params: Final = { + **_config_base_for_health_check( + config_litellm_params, + request_litellm_params, + allow_client_side_credentials=general_settings.get("allow_client_side_credentials") is True, + selected_by_id=selected_by_id, + ), + **request_litellm_params, + } + + resolved_model_info: Final = ( + loaded_model_info if loaded_model_info is not None else (dict(model_info) if model_info is not None else None) + ) + probe_model_info: Final = _model_info_for_mode_resolution( + _OBJECT_MAPPING.validate_python(resolved_model_info or {}), + stored_params=_OBJECT_MAPPING.validate_python(config_litellm_params), + request_params=_OBJECT_MAPPING.validate_python(request_litellm_params), + ) + stored_model: Final = _OBJECT_MAPPING.validate_python(config_litellm_params).get("model") + return _ResolvedConnection( + request_litellm_params=_OBJECT_MAPPING.validate_python(request_litellm_params), + config_litellm_params=_OBJECT_MAPPING.validate_python(config_litellm_params), + litellm_params=merged_litellm_params, + resolved_model_info=resolved_model_info, + probe_model_info=probe_model_info, + deployment_by_id=deployment_by_id, + stored_model=stored_model if isinstance(stored_model, str) else None, + ) + + +_RawQuery: TypeAlias = Callable[..., Awaitable[Sequence[object]]] + + +class _SpendLogRow(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + tag: str + spend: float + + +_SPEND_LOG_ROWS: Final = TypeAdapter(list[_SpendLogRow]) +_TEST_MODEL_TAG_PREFIX: Final = "litellm_test_model:" +_SPEND_LOG_BY_TAG_SQL: Final = """ + SELECT tag, "spend" + FROM "LiteLLM_SpendLogs" sl, jsonb_array_elements_text(sl."request_tags"::jsonb) AS tag + WHERE tag IN (SELECT jsonb_array_elements_text($1::jsonb)) +""" + + +def _test_model_tag(call_id: str) -> str: + return f"{_TEST_MODEL_TAG_PREFIX}{call_id}" + + +async def _spend_rows_once(run_query: _RawQuery, request_ids: frozenset[str]) -> Mapping[str, float]: + """Spend per request id. The row's request_id is the provider's response id, so each turn is found by the + tag the check put on it, which the proxy writes to request_tags like any client tag.""" + rows: Final = _SPEND_LOG_ROWS.validate_python( + await run_query(_SPEND_LOG_BY_TAG_SQL, json.dumps(sorted(_test_model_tag(call_id) for call_id in request_ids))) + ) + return MappingProxyType({row.tag.removeprefix(_TEST_MODEL_TAG_PREFIX): row.spend for row in rows}) + + +async def _spend_log_rows( + run_query: _RawQuery, + request_ids: Sequence[str], + *, + wait_seconds: float = TEST_MODEL_SPEND_LOG_WAIT_SECONDS, + poll_seconds: float = 1.0, +) -> Mapping[str, float]: + """Spend rows for ``request_ids``, polled until every row landed or ``wait_seconds`` passed. + + The proxy flushes spend logs in batches every ``PROXY_BATCH_WRITE_AT`` seconds, so a row for a request + that just finished is expected to take that long to appear. + """ + deadline: Final = time.monotonic() + wait_seconds + wanted: Final = frozenset(request_ids) + while not wanted <= (found := await _spend_rows_once(run_query, wanted)).keys() and time.monotonic() < deadline: + await asyncio.sleep(poll_seconds) + return found + + +def _test_model_endpoints( + requested: Sequence[TestModelEndpoint] | None, mode: str | None +) -> tuple[TestModelEndpoint, ...]: + if requested: + return tuple(dict.fromkeys(requested)) + native: Final = ENDPOINT_FOR_MODE.get(mode or "chat") + if native is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={ + "error": f"/health/test_model covers chat models only and this model's mode is {mode!r}. " + "Use /health/test_connection for it." + }, + ) + return (native,) + + +async def _logged_cost(logging_obj: LiteLLMLoggingObj, fallback: float | None) -> float | None: + """The cost the success handler logged for the call, which lands a moment after the call returns.""" + deadline: Final = time.monotonic() + TEST_MODEL_COST_WAIT_SECONDS + while (logged := _OBJECT_MAPPING.validate_python(logging_obj.model_call_details).get("response_cost")) is None and ( + time.monotonic() < deadline + ): + await asyncio.sleep(0.05) + return float(logged) if isinstance(logged, int | float) else fallback + + +def _turn_caller( + llm_router: Router | None, + *, + pinned_model_id: str | None, + call_params: Mapping[str, object], + proxy_metadata: Mapping[str, object], +) -> Callable[[EndpointScript, Mapping[str, object], str], Awaitable[CalledTurn]]: + """The two ways a turn reaches the provider: pinned to a saved deployment through the router, so the + deployment's own credentials and pricing apply, or straight through litellm with the submitted params. + Either way the call carries the proxy's own logging object, the way every proxied request does, so the + cost read back is the one the spend log row gets.""" + + async def call(script: EndpointScript, body: Mapping[str, object], call_id: str) -> CalledTurn: + target_params: Final = {"model": pinned_model_id} if pinned_model_id is not None else call_params + logging_obj, prepared = litellm.utils.function_setup( + original_function=script.route_type, + rules_obj=litellm.utils.Rules(), + start_time=datetime.now(), # noqa: DTZ005 # Logging pairs it with the naive end_time it records + **{ # pyright: ignore[reportArgumentType] # takes the untyped request dict, like the proxy + **target_params, + **body, + script.metadata_key: {**proxy_metadata, "tags": [_test_model_tag(call_id)]}, + "litellm_call_id": call_id, + "drop_params": True, + }, + ) + prepared["litellm_logging_obj"] = logging_obj + target: Final = llm_router if pinned_model_id is not None and llm_router is not None else litellm + response: Final[object] = await asyncio.wait_for( + getattr(target, script.route_type)(**prepared), HEALTH_CHECK_TIMEOUT_SECONDS + ) + return CalledTurn(response=response, cost=await _logged_cost(logging_obj, response_cost(response))) + + return call + + +@router.post( + "/health/test_model", + tags=["health"], + dependencies=[Depends(user_api_key_auth)], +) +async def test_model( + request: Request, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], + model_id: Annotated[ + str | None, + fastapi.Body( + description="The deployment to test, by its model_info.id. The saved credentials and pricing are used." + ), + ] = None, + litellm_params: Annotated[ + Mapping[str, object] | None, + fastapi.Body( + description=( + "Parameters of a model that is not saved yet (model, api_key, api_base, api_version), as sent to " + "/model/new. When model_id is also given these override the saved deployment's values." + ) + ), + ] = None, + model_info: Annotated[ + Mapping[str, object] | None, + fastapi.Body(description="Model info of an unsaved model, for example mode or custom pricing."), + ] = None, + endpoints: Annotated[ + Sequence[TestModelEndpoint] | None, + fastapi.Body( + description=( + "Which endpoints to test: /v1/chat/completions, /v1/responses, /v1/messages. " + "Defaults to the model's native endpoint." + ) + ), + ] = None, +) -> TestModelResponse: + """ + Run the live checks on a model, the way a client would use it, and report what works. + + Each selected endpoint gets one two-turn conversation with a long system prompt and a tool. Turn 1 asks + a question the tool answers, turn 2 sends the tool result back. Three checks come out of it: + + - `conversation`: both turns parse, turn 1 calls the tool, turn 2 uses the tool result + - `prompt_caching`: turn 2 reports cached input tokens, with no cache markers in the request + - `cost_tracking`: every turn carries a non-zero response cost, and the spend log row lands when a + database is connected + + Example: + ```bash + curl -X POST 'http://localhost:4000/health/test_model' \\ + -H "Authorization: Bearer $LITELLM_MASTER_KEY" \\ + -H 'Content-Type: application/json' \\ + -d '{"model_id": "a1b2c3", "endpoints": ["/v1/chat/completions", "/v1/responses"]}' + ``` + + Returns `status` (`success`, `warning` or `error`), the tested `model`, and one entry per endpoint + with its `status`, the three `checks` (`name`, `status`, `detail`), `cache_read_tokens`, + `cache_read_percent`, `spend_usd` and `duration_ms`. + """ + from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup + from litellm.proxy.management_endpoints.model_management_endpoints import ( + ModelManagementAuthChecks, + ) + from litellm.proxy.proxy_server import ( + general_settings, + llm_router, + premium_user, + prisma_client, + ) + from litellm.types.router import Deployment, LiteLLM_Params + + try: + request_litellm_params: Final = _OBJECT_MAPPING.validate_python(litellm_params or {}) + request_model_info: Final[Mapping[str, object]] = { + **(model_info or {}), + **({"id": model_id} if model_id else {}), + } + if not model_id and not request_litellm_params.get("model"): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={"error": "Send model_id for a saved deployment or litellm_params.model for a new one."}, + ) + resolved: Final = _resolve_connection_params( + request_litellm_params=request_litellm_params, + model_info=request_model_info, + llm_router=llm_router, + general_settings=_OBJECT_MAPPING.validate_python(general_settings), + ) + if model_id and resolved.deployment_by_id is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail={"error": f"No deployment with model_info.id {model_id!r} is loaded on this proxy."}, + ) + if prisma_client is not None: + await ModelManagementAuthChecks.can_user_make_model_call( + model_params=Deployment( + model_name="test_model", + litellm_params=LiteLLM_Params.model_validate(resolved.litellm_params), + model_info=resolved.resolved_model_info, + ), + user_api_key_dict=user_api_key_dict, + prisma_client=prisma_client, + premium_user=premium_user, + incoming_params=resolved.request_litellm_params, + ) + elif not _is_proxy_admin(user_api_key_dict): + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, detail={"error": "Only proxy admins can test models."} + ) + + call_params: Final = {key: value for key, value in resolved.litellm_params.items() if key != "mode"} + mode: Final = _string_mode_or_bad_request(resolved.litellm_params.get("mode")) or resolve_health_check_mode( + resolved.probe_model_info, _OBJECT_MAPPING.validate_python(call_params) + ) + selected: Final = _test_model_endpoints(endpoints, mode) + overrides_saved_deployment: Final = any(key != "model" for key in resolved.request_litellm_params) + pinned_model_id: Final = ( + resolved.deployment_by_id.model_info.id + if resolved.deployment_by_id is not None and not overrides_saved_deployment + else None + ) + proxy_metadata: Final = { + **LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key(user_api_key_dict=user_api_key_dict), + "user_api_key": LiteLLMProxyRequestSetup.get_logged_api_key(user_api_key_dict), + "endpoint": "/health/test_model", + } + call: Final = _turn_caller( + llm_router, + pinned_model_id=pinned_model_id, + call_params={ + **_OBJECT_MAPPING.validate_python(call_params), + **_admin_api_key_from_request( + resolved.request_litellm_params, _OBJECT_MAPPING.validate_python(call_params) + ), + }, + proxy_metadata=proxy_metadata, + ) + run_query: Final[_RawQuery | None] = prisma_client.db.query_raw if prisma_client is not None else None + + async def spend_lookup(request_ids: Sequence[str]) -> Mapping[str, float]: + return await _spend_log_rows(run_query, request_ids) if run_query is not None else {} + + results: Final[Sequence[EndpointResult]] = await asyncio.gather( + *( + run_endpoint_check( + SCRIPTS[endpoint], + call, + new_call_id=lambda: str(uuid.uuid4()), + spend_lookup=spend_lookup if prisma_client is not None else None, + ) + for endpoint in selected + ) + ) + tested_model: Final = resolved.litellm_params.get("model") + return TestModelResponse( + status=overall_status(results), + model=tested_model if isinstance(tested_model, str) else None, + model_id=pinned_model_id, + endpoints=[endpoint_result_payload(result) for result in results], + ) + except (HTTPException, ProxyException): + raise + except Exception as e: # noqa: BLE001 # any other failure becomes the endpoint's 500, like test_connection + verbose_proxy_logger.debug("litellm.proxy.health_endpoints.test_model(): Exception occurred - %s", e) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail={"error": f"Failed to test model: {e}"}, + ) + + @router.post( "/health/test_connection", tags=["health"], @@ -2599,94 +3023,17 @@ async def test_model_connection( detail={"error": CommonProxyErrors.db_not_connected_error.value}, ) - # Get model name from litellm_params - request_litellm_params: Final = litellm_params or {} - # Reject request-supplied os.environ/ references. Config values are - # already resolved before reaching this endpoint; any remaining - # reference must have come from the request body. - _reject_os_environ_references(request_litellm_params) - if model_info: - _reject_os_environ_references(model_info) - model_name: Final = request_litellm_params.get("model") - - # Look up model configuration from router if model name is provided - # This gets the litellm_params from proxy config (with resolved env vars) - config_litellm_params: dict = {} - loaded_model_info: dict | None = None - deployment_by_id: Deployment | None = None - if llm_router is not None: - # Prefer disambiguation by deployment id (`model_info.id`) when - # the caller supplies it. This is required when multiple - # deployments share a `model_name` (e.g. wildcard `openai/*` - # with multiple `api_base` values for failover): the UI's - # "Test Connection" button targets a specific row, and that - # row's id is the only thing that uniquely identifies which - # deployment to probe. Without this, all duplicates collapse - # onto `deployments[0]`. - request_model_info: Final = model_info or {} - request_model_id: Final = request_model_info.get("id") - try: - if request_model_id: - deployment_by_id = llm_router.registry().deployment(request_model_id) - - if deployment_by_id is not None: - config_litellm_params = deployment_by_id.litellm_params.model_dump(exclude_none=True) - loaded_model_info = deployment_by_id.model_info.model_dump(exclude_none=True) - elif model_name: - # Fall back to model_name lookup for callers (e.g. the - # "Add Model" wizard, or curl) that don't supply an id. - # First try to find by proxy model_name (e.g., "gpt-4o") - deployments = llm_router.registry().deployments(model_name) - - # If not found, try to find by litellm model name - # (e.g., "azure/gpt-4o") - if not deployments or len(deployments) == 0: - all_deployments: Final = llm_router.registry().deployments(None) - if all_deployments: - for deployment in all_deployments: - if deployment.get("litellm_params", {}).get("model") == model_name: - deployments = [deployment] - break - - if deployments and len(deployments) > 0: - # Use the first deployment's litellm_params as base - # config. These already have resolved environment - # variables from proxy config. - config_litellm_params = dict(deployments[0].get("litellm_params", {})) - loaded_model_info = dict(deployments[0].get("model_info") or {}) - except Exception as e: - verbose_proxy_logger.debug( - "Could not find model %s in router: %s. Proceeding with request params only.", model_name, e - ) - - selected_by_id: Final = deployment_by_id is not None - reject_server_owned_wif_fields( - MappingProxyType( - { - key: value - for key, value in _OBJECT_MAPPING.validate_python(request_litellm_params).items() - if key != "chatgpt_auth_file" - } - ) + resolved: Final = _resolve_connection_params( + request_litellm_params=_OBJECT_MAPPING.validate_python(litellm_params or {}), + model_info=_OBJECT_MAPPING.validate_python(model_info) if model_info else None, + llm_router=llm_router, + general_settings=_OBJECT_MAPPING.validate_python(general_settings), ) - # Merge: config params (from proxy config) as base, request params override - litellm_params = { - **_config_base_for_health_check( - config_litellm_params, - request_litellm_params, - allow_client_side_credentials=general_settings.get("allow_client_side_credentials") is True, - selected_by_id=selected_by_id, - ), - **request_litellm_params, - } - - resolved_model_info: Final = loaded_model_info if loaded_model_info is not None else model_info - probe_model_info: Final = _model_info_for_mode_resolution( - _OBJECT_MAPPING.validate_python(resolved_model_info or {}), - stored_params=_OBJECT_MAPPING.validate_python(config_litellm_params), - request_params=_OBJECT_MAPPING.validate_python(request_litellm_params), - ) - stored_model: Final = _OBJECT_MAPPING.validate_python(config_litellm_params).get("model") + request_litellm_params: Final = resolved.request_litellm_params + resolved_model_info: Final = resolved.resolved_model_info + probe_model_info: Final = resolved.probe_model_info + stored_model: Final = resolved.stored_model + litellm_params = resolved.litellm_params litellm_params = update_litellm_params_for_health_check( model_info=dict(probe_model_info), litellm_params=litellm_params, diff --git a/litellm/proxy/health_endpoints/model_checks.py b/litellm/proxy/health_endpoints/model_checks.py new file mode 100644 index 00000000000..719243cc032 --- /dev/null +++ b/litellm/proxy/health_endpoints/model_checks.py @@ -0,0 +1,729 @@ +"""Live checks behind ``POST /health/test_model``. + +One two-turn tool-call conversation per selected endpoint, scripted in that endpoint's native request shape +with no prompt-cache markers, so the second turn can only report cached tokens when the provider caches the +repeated prefix on its own. Three verdicts come out of the two turns: the conversation itself (both turns +parse, the tool call lands, the tool result is used), prompt caching, and cost tracking. +""" + +from __future__ import annotations + +import json +import time +from collections.abc import Awaitable, Callable, Mapping, Sequence +from dataclasses import dataclass +from types import MappingProxyType +from typing import Final, Literal, TypeAlias + +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError + +from litellm.router_utils.add_retry_fallback_headers import get_hidden_params_dict + +TestModelEndpoint: TypeAlias = Literal["/v1/chat/completions", "/v1/responses", "/v1/messages"] +TestModelRouteType: TypeAlias = Literal["acompletion", "aresponses", "anthropic_messages"] +Verdict: TypeAlias = Literal["pass", "warn", "fail"] +EndpointStatus: TypeAlias = Literal["success", "warning", "error"] + +TEST_MODEL_ENDPOINTS: Final[tuple[TestModelEndpoint, ...]] = ("/v1/chat/completions", "/v1/responses", "/v1/messages") +WEATHER_TOOL_NAME: Final = "get_weather" +WEATHER_TOOL_RESULT: Final = "72F and sunny" +WEATHER_QUESTION: Final = "What is the weather in Boston right now? Use the get_weather tool to find out." +MAX_OUTPUT_TOKENS: Final = 2048 +THINKING_BUDGET_TOKENS: Final = 1024 + +_WEATHER_PARAMETERS: Final[Mapping[str, object]] = MappingProxyType( + { + "type": "object", + "properties": { + "location": {"type": "string", "description": "City name, for example Boston"}, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, + }, + "required": ["location"], + } +) +_WEATHER_DESCRIPTION: Final = "Get the current weather for a city" + +_CITIES: Final = ("Boston", "Denver", "Lisbon", "Osaka", "Nairobi", "Perth", "Quito", "Reykjavik") +_SECTION_SENTENCES: Final = ( + "Section {n}: deliveries routed through the {city} hub are scanned at intake, weighed on the calibrated " + "scale, and matched against the manifest before any label is printed or any pallet leaves the dock.", + "Agents handling {city} tickets confirm the tracking number, the declared value, and the delivery window " + "with the sender before promising a reroute, and record the promise in the ticket with a timestamp.", + "If a parcel bound for {city} misses its scheduled scan by more than {hours} hours, the agent opens a " + "trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift.", + "Refunds for {city} shipments above {amount} dollars need a supervisor code, a photo of the damage when " + "one exists, and the original invoice number, and are paid to the original payment method within ten days.", +) +_SECTION_COUNT: Final = 32 + +SYSTEM_PROMPT: Final = "\n\n".join( + ( + "You are the support assistant for Northwind Logistics. Follow the playbook below exactly.", + *( + " ".join( + sentence.format(n=n, city=_CITIES[n % len(_CITIES)], hours=4 + (n % 5), amount=100 * (1 + (n % 7))) + for sentence in _SECTION_SENTENCES + ) + for n in range(1, _SECTION_COUNT + 1) + ), + f"When the user asks about the weather, call the {WEATHER_TOOL_NAME} tool with the city name. After a " + "tool result arrives, answer in one short sentence that repeats the temperature exactly as the tool " + "reported it.", + ) +) + + +@dataclass(frozen=True, slots=True) +class ToolCallSeen: + id: str + name: str + raw_arguments: str + arguments: Mapping[str, object] | None + + +@dataclass(frozen=True, slots=True) +class ParsedTurn: + tool_call: ToolCallSeen | None + text: str + input_tokens: int + cache_read_tokens: int + assistant_echo: object + + +@dataclass(frozen=True, slots=True) +class EndpointScript: + endpoint: TestModelEndpoint + route_type: TestModelRouteType + metadata_key: Literal["metadata", "litellm_metadata"] + turn_one: Callable[[], Mapping[str, object]] + turn_two: Callable[[ParsedTurn], Mapping[str, object]] + parse: Callable[[object], ParsedTurn | str] + + +@dataclass(frozen=True, slots=True) +class CheckResult: + name: Literal["conversation", "prompt_caching", "cost_tracking"] + status: Verdict + detail: str + + +@dataclass(frozen=True, slots=True) +class EndpointResult: + endpoint: TestModelEndpoint + status: EndpointStatus + checks: tuple[CheckResult, ...] + cache_read_tokens: int + cache_read_percent: float | None + spend_usd: float + duration_ms: int + + +@dataclass(frozen=True, slots=True) +class CalledTurn: + """One turn's provider response and the cost the proxy logged for it (what the spend log row carries).""" + + response: object + cost: float | None + + +@dataclass(frozen=True, slots=True) +class TurnOutcome: + response: object | None + cost: float | None + error: str | None + call_id: str + + +TurnCaller: TypeAlias = Callable[[EndpointScript, Mapping[str, object], str], Awaitable[CalledTurn]] +SpendLookup: TypeAlias = Callable[[Sequence[str]], Awaitable[Mapping[str, float]]] +CallIdFactory: TypeAlias = Callable[[], str] + + +class _Loose(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + +class _ChatFunction(_Loose): + name: str + arguments: str = "" + + +class _ChatToolCall(_Loose): + id: str = "" + function: _ChatFunction + + +class _PromptTokensDetails(_Loose): + cached_tokens: int | None = None + + +class _ChatUsage(_Loose): + prompt_tokens: int = 0 + prompt_tokens_details: _PromptTokensDetails | None = None + + +class _ChatMessage(_Loose): + content: str | Sequence[object] | None = None + tool_calls: Sequence[_ChatToolCall] | None = None + + +class _ChatChoice(_Loose): + message: _ChatMessage + + +class _ChatResponse(_Loose): + choices: Sequence[_ChatChoice] + usage: _ChatUsage | None = None + + +class _ResponsesContent(_Loose): + type: str = "" + text: str = "" + + +class _ResponsesItem(_Loose): + type: str = "" + name: str = "" + arguments: str = "" + call_id: str = "" + content: Sequence[_ResponsesContent] | None = None + + +class _InputTokensDetails(_Loose): + cached_tokens: int | None = None + + +class _ResponsesUsage(_Loose): + input_tokens: int = 0 + input_tokens_details: _InputTokensDetails | None = None + + +class _ResponsesResponse(_Loose): + output: Sequence[_ResponsesItem] + usage: _ResponsesUsage | None = None + + +class _MessagesBlock(_Loose): + type: str = "" + text: str = "" + id: str = "" + name: str = "" + input: Mapping[str, object] | None = None + + +class _MessagesUsage(_Loose): + input_tokens: int = 0 + cache_read_input_tokens: int | None = None + cache_creation_input_tokens: int | None = None + + +class _MessagesResponse(_Loose): + content: Sequence[_MessagesBlock] + usage: _MessagesUsage + + +_CHAT: Final = TypeAdapter(_ChatResponse) +_RESPONSES: Final = TypeAdapter(_ResponsesResponse) +_MESSAGES: Final = TypeAdapter(_MessagesResponse) +_PLAIN_OBJECT: Final = TypeAdapter(dict[str, object]) +_OBJECT_LIST: Final = TypeAdapter(list[object]) +_CHOICES: Final = TypeAdapter(list[dict[str, object]]) +_JSON_DOCUMENT: Final = TypeAdapter(object) +_EMPTY_ASSISTANT: Final[Mapping[str, object]] = MappingProxyType({"role": "assistant", "content": None}) + + +def _plain(response: object) -> Mapping[str, object] | None: + if isinstance(response, BaseModel): + return _PLAIN_OBJECT.validate_python(response.model_dump(exclude_none=True)) + if isinstance(response, dict): + return _PLAIN_OBJECT.validate_python(response) + return None + + +def _validation_detail(error: ValidationError) -> str: + first: Final = error.errors()[0] + location: Final = ".".join(str(part) for part in first["loc"]) + return f"{location}: {first['msg']}" if location else first["msg"] + + +def _arguments(raw: str) -> Mapping[str, object] | None: + try: + parsed: Final = _JSON_DOCUMENT.validate_json(raw) + except ValidationError: + return None + return MappingProxyType(_PLAIN_OBJECT.validate_python(parsed)) if isinstance(parsed, dict) else None + + +def _chat_turn_one() -> Mapping[str, object]: + return { + "messages": [ + {"role": "system", "content": SYSTEM_PROMPT}, + {"role": "user", "content": WEATHER_QUESTION}, + ], + "tools": [ + { + "type": "function", + "function": { + "name": WEATHER_TOOL_NAME, + "description": _WEATHER_DESCRIPTION, + "parameters": dict(_WEATHER_PARAMETERS), + }, + } + ], + "reasoning_effort": "low", + "max_tokens": MAX_OUTPUT_TOKENS, + } + + +def _chat_turn_two(turn_one: ParsedTurn) -> Mapping[str, object]: + first: Final = _chat_turn_one() + tool_call_id: Final = turn_one.tool_call.id if turn_one.tool_call is not None else "" + return { + **first, + "messages": [ + *_message_list(first["messages"]), + turn_one.assistant_echo, + {"role": "tool", "tool_call_id": tool_call_id, "content": WEATHER_TOOL_RESULT}, + ], + } + + +def _message_list(messages: object) -> Sequence[object]: + return _OBJECT_LIST.validate_python(messages) if isinstance(messages, list) else [] + + +def _first_choice_message(plain: Mapping[str, object]) -> object: + try: + choices: Final = _CHOICES.validate_python(plain.get("choices")) + except ValidationError: + return dict(_EMPTY_ASSISTANT) + return choices[0].get("message", dict(_EMPTY_ASSISTANT)) if choices else dict(_EMPTY_ASSISTANT) + + +def _parse_chat(response: object) -> ParsedTurn | str: + plain: Final = _plain(response) + if plain is None: + return f"response is a {type(response).__name__}, not a chat completion" + try: + parsed: Final = _CHAT.validate_python(plain) + except ValidationError as error: + return f"chat completion did not parse: {_validation_detail(error)}" + if not parsed.choices: + return "chat completion returned no choices" + message: Final = parsed.choices[0].message + first_call: Final = message.tool_calls[0] if message.tool_calls else None + usage: Final = parsed.usage + details: Final = usage.prompt_tokens_details if usage is not None else None + return ParsedTurn( + tool_call=( + ToolCallSeen( + id=first_call.id, + name=first_call.function.name, + raw_arguments=first_call.function.arguments, + arguments=_arguments(first_call.function.arguments), + ) + if first_call is not None + else None + ), + text=message.content if isinstance(message.content, str) else "", + input_tokens=usage.prompt_tokens if usage is not None else 0, + cache_read_tokens=(details.cached_tokens or 0) if details is not None else 0, + assistant_echo=_first_choice_message(plain), + ) + + +def _responses_turn_one() -> Mapping[str, object]: + return { + "instructions": SYSTEM_PROMPT, + "input": [{"role": "user", "content": WEATHER_QUESTION}], + "tools": [ + { + "type": "function", + "name": WEATHER_TOOL_NAME, + "description": _WEATHER_DESCRIPTION, + "parameters": dict(_WEATHER_PARAMETERS), + } + ], + "reasoning": {"effort": "low"}, + "max_output_tokens": MAX_OUTPUT_TOKENS, + } + + +def _responses_turn_two(turn_one: ParsedTurn) -> Mapping[str, object]: + first: Final = _responses_turn_one() + call_id: Final = turn_one.tool_call.id if turn_one.tool_call is not None else "" + return { + **first, + "input": [ + *_message_list(first["input"]), + *_message_list(turn_one.assistant_echo), + {"type": "function_call_output", "call_id": call_id, "output": WEATHER_TOOL_RESULT}, + ], + } + + +def _parse_responses(response: object) -> ParsedTurn | str: + plain: Final = _plain(response) + if plain is None: + return f"response is a {type(response).__name__}, not a Responses API response" + try: + parsed: Final = _RESPONSES.validate_python(plain) + except ValidationError as error: + return f"Responses API response did not parse: {_validation_detail(error)}" + function_call: Final = next((item for item in parsed.output if item.type == "function_call"), None) + texts: Final = tuple( + content.text + for item in parsed.output + if item.type == "message" and item.content is not None + for content in item.content # comprehension-ok: flattening message content blocks + if content.type == "output_text" + ) + usage: Final = parsed.usage + details: Final = usage.input_tokens_details if usage is not None else None + return ParsedTurn( + tool_call=( + ToolCallSeen( + id=function_call.call_id, + name=function_call.name, + raw_arguments=function_call.arguments, + arguments=_arguments(function_call.arguments), + ) + if function_call is not None + else None + ), + text=" ".join(texts), + input_tokens=usage.input_tokens if usage is not None else 0, + cache_read_tokens=(details.cached_tokens or 0) if details is not None else 0, + assistant_echo=plain.get("output", []), + ) + + +def _messages_turn_one() -> Mapping[str, object]: + return { + "system": SYSTEM_PROMPT, + "messages": [{"role": "user", "content": WEATHER_QUESTION}], + "tools": [ + { + "name": WEATHER_TOOL_NAME, + "description": _WEATHER_DESCRIPTION, + "input_schema": dict(_WEATHER_PARAMETERS), + } + ], + "thinking": {"type": "enabled", "budget_tokens": THINKING_BUDGET_TOKENS}, + "max_tokens": MAX_OUTPUT_TOKENS, + } + + +def _messages_turn_two(turn_one: ParsedTurn) -> Mapping[str, object]: + first: Final = _messages_turn_one() + tool_use_id: Final = turn_one.tool_call.id if turn_one.tool_call is not None else "" + return { + **first, + "messages": [ + *_message_list(first["messages"]), + {"role": "assistant", "content": turn_one.assistant_echo}, + { + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": tool_use_id, "content": WEATHER_TOOL_RESULT}], + }, + ], + } + + +def _parse_messages(response: object) -> ParsedTurn | str: + plain: Final = _plain(response) + if plain is None: + return f"response is a {type(response).__name__}, not a Messages API response" + try: + parsed: Final = _MESSAGES.validate_python(plain) + except ValidationError as error: + return f"Messages API response did not parse: {_validation_detail(error)}" + tool_use: Final = next((block for block in parsed.content if block.type == "tool_use"), None) + cache_read: Final = parsed.usage.cache_read_input_tokens or 0 + cache_creation: Final = parsed.usage.cache_creation_input_tokens or 0 + return ParsedTurn( + tool_call=( + ToolCallSeen( + id=tool_use.id, + name=tool_use.name, + raw_arguments=json.dumps(tool_use.input or {}), + arguments=MappingProxyType(dict(tool_use.input or {})), + ) + if tool_use is not None + else None + ), + text=" ".join(block.text for block in parsed.content if block.type == "text"), + input_tokens=parsed.usage.input_tokens + cache_read + cache_creation, + cache_read_tokens=cache_read, + assistant_echo=plain.get("content", []), + ) + + +CHAT_COMPLETIONS_SCRIPT: Final = EndpointScript( + endpoint="/v1/chat/completions", + route_type="acompletion", + metadata_key="metadata", + turn_one=_chat_turn_one, + turn_two=_chat_turn_two, + parse=_parse_chat, +) +RESPONSES_SCRIPT: Final = EndpointScript( + endpoint="/v1/responses", + route_type="aresponses", + metadata_key="litellm_metadata", + turn_one=_responses_turn_one, + turn_two=_responses_turn_two, + parse=_parse_responses, +) +MESSAGES_SCRIPT: Final = EndpointScript( + endpoint="/v1/messages", + route_type="anthropic_messages", + metadata_key="litellm_metadata", + turn_one=_messages_turn_one, + turn_two=_messages_turn_two, + parse=_parse_messages, +) +SCRIPTS: Final[Mapping[TestModelEndpoint, EndpointScript]] = MappingProxyType( + {script.endpoint: script for script in (CHAT_COMPLETIONS_SCRIPT, RESPONSES_SCRIPT, MESSAGES_SCRIPT)} +) +ENDPOINT_FOR_MODE: Final[Mapping[str, TestModelEndpoint]] = MappingProxyType( + {"chat": "/v1/chat/completions", "responses": "/v1/responses", "anthropic_messages": "/v1/messages"} +) + + +def response_cost(response: object) -> float | None: + cost: Final = get_hidden_params_dict(response).get("response_cost") + return float(cost) if isinstance(cost, int | float) else None + + +async def _turn(script: EndpointScript, body: Mapping[str, object], call: TurnCaller, call_id: str) -> TurnOutcome: + try: + called: Final = await call(script, body, call_id) + except Exception as error: # noqa: BLE001 # a provider or translation failure is the check's finding + return TurnOutcome(response=None, cost=None, error=f"{type(error).__name__}: {error}", call_id=call_id) + return TurnOutcome(response=called.response, cost=called.cost, error=None, call_id=call_id) + + +def _conversation_check( + turn_one: TurnOutcome, + parsed_one: ParsedTurn | str | None, + turn_two: TurnOutcome | None, + parsed_two: ParsedTurn | str | None, +) -> CheckResult: + if turn_one.error is not None: + return CheckResult("conversation", "fail", f"turn 1 request failed: {turn_one.error}") + if isinstance(parsed_one, str): + return CheckResult("conversation", "fail", f"turn 1 {parsed_one}") + if parsed_one is None or parsed_one.tool_call is None: + shown: Final = parsed_one.text[:160] if parsed_one is not None else "" + return CheckResult("conversation", "fail", f"turn 1 returned no tool call, the model answered: {shown!r}") + call: Final = parsed_one.tool_call + if call.name != WEATHER_TOOL_NAME: + return CheckResult("conversation", "fail", f"turn 1 called {call.name!r} instead of {WEATHER_TOOL_NAME!r}") + if call.arguments is None: + return CheckResult( + "conversation", "fail", f"turn 1 tool arguments are not a JSON object: {call.raw_arguments!r}" + ) + if "location" not in call.arguments: + return CheckResult("conversation", "fail", f"turn 1 tool arguments lack 'location': {call.raw_arguments!r}") + if turn_two is None or turn_two.error is not None: + reason: Final = turn_two.error if turn_two is not None else "not sent" + return CheckResult("conversation", "fail", f"turn 2 request with the tool result failed: {reason}") + if isinstance(parsed_two, str): + return CheckResult("conversation", "fail", f"turn 2 {parsed_two}") + if parsed_two is None or "72" not in parsed_two.text: + answer: Final = parsed_two.text[:160] if parsed_two is not None else "" + return CheckResult( + "conversation", "fail", f"turn 2 did not use the tool result {WEATHER_TOOL_RESULT!r}: {answer!r}" + ) + return CheckResult( + "conversation", + "pass", + f"turn 1 called {WEATHER_TOOL_NAME}({call.raw_arguments}), turn 2 answered {parsed_two.text[:120]!r}", + ) + + +def _cache_percent(parsed_two: ParsedTurn | None) -> float | None: + if parsed_two is None or parsed_two.input_tokens <= 0: + return None + return round(100.0 * parsed_two.cache_read_tokens / parsed_two.input_tokens, 1) + + +def _missed_cache(parsed: ParsedTurn | str | None) -> bool: + return isinstance(parsed, ParsedTurn) and parsed.cache_read_tokens == 0 + + +def _hit_cache(parsed: ParsedTurn | str | None) -> bool: + return isinstance(parsed, ParsedTurn) and parsed.cache_read_tokens > 0 + + +def _prompt_caching_check(parsed_two: ParsedTurn | None, attempts: int) -> CheckResult: + if parsed_two is None: + return CheckResult("prompt_caching", "fail", "not checked, the conversation did not reach turn 2") + percent: Final = _cache_percent(parsed_two) + attempt_note: Final = " on the second attempt" if attempts > 1 else "" + if parsed_two.cache_read_tokens > 0: + return CheckResult( + "prompt_caching", + "pass", + f"turn 2 read {parsed_two.cache_read_tokens} of {parsed_two.input_tokens} input tokens from cache " + f"({percent}%){attempt_note}", + ) + return CheckResult( + "prompt_caching", + "fail", + f"turn 2 reported 0 cached tokens on {parsed_two.input_tokens} input tokens in {attempts} attempts, the " + "provider did not reuse the repeated system prompt", + ) + + +def _cost_verdict(turn_number: int, cost: float | None) -> CheckResult | None: + if cost is None: + return CheckResult( + "cost_tracking", + "fail", + f"turn {turn_number} carried no response_cost, no price resolved for this model (not in the cost map and " + "no custom pricing on the deployment)", + ) + if cost == 0: + return CheckResult( + "cost_tracking", "warn", f"turn {turn_number} response_cost is $0, this model is priced at zero" + ) + return None + + +def _cost_tracking_check( + turn_one: TurnOutcome, turn_two: TurnOutcome | None, spend_rows: Mapping[str, float] | None +) -> CheckResult: + turns: Final = tuple(turn for turn in (turn_one, turn_two) if turn is not None and turn.response is not None) + if not turns: + return CheckResult("cost_tracking", "fail", "not checked, no turn returned a response") + costs: Final = tuple(turn.cost for turn in turns) + failed: Final = next( + (verdict for number, cost in enumerate(costs, start=1) if (verdict := _cost_verdict(number, cost)) is not None), + None, + ) + if failed is not None: + return failed + priced: Final = ", ".join(f"turn {number} ${cost:.6f}" for number, cost in enumerate(costs, start=1)) + if spend_rows is None: + return CheckResult("cost_tracking", "pass", f"{priced}, spend log not checked (no database)") + missing: Final = tuple(turn.call_id for turn in turns if turn.call_id not in spend_rows) + if missing: + return CheckResult( + "cost_tracking", + "fail", + f"{priced}, but no LiteLLM_SpendLogs row landed for request_id {', '.join(missing)}", + ) + logged: Final = ", ".join(f"${spend_rows[turn.call_id]:.6f}" for turn in turns) + return CheckResult("cost_tracking", "pass", f"{priced}, spend log rows written ({logged})") + + +def _status(checks: Sequence[CheckResult]) -> EndpointStatus: + statuses: Final = frozenset(check.status for check in checks) + if "fail" in statuses: + return "error" + return "warning" if "warn" in statuses else "success" + + +def overall_status(results: Sequence[EndpointResult]) -> EndpointStatus: + statuses: Final = frozenset(result.status for result in results) + if "error" in statuses: + return "error" + return "warning" if "warning" in statuses else "success" + + +async def run_endpoint_check( + script: EndpointScript, + call: TurnCaller, + *, + new_call_id: CallIdFactory, + spend_lookup: SpendLookup | None, +) -> EndpointResult: + started: Final = time.monotonic() + turn_one: Final = await _turn(script, script.turn_one(), call, new_call_id()) + parsed_one: Final = script.parse(turn_one.response) if turn_one.response is not None else None + ready_for_turn_two: Final = isinstance(parsed_one, ParsedTurn) and parsed_one.tool_call is not None + turn_two: Final = ( + await _turn(script, script.turn_two(parsed_one), call, new_call_id()) + if ready_for_turn_two and isinstance(parsed_one, ParsedTurn) + else None + ) + first_parsed_two: Final = ( + script.parse(turn_two.response) if turn_two is not None and turn_two.response is not None else None + ) + retried_turn_two: Final = ( + await _turn(script, script.turn_two(parsed_one), call, new_call_id()) + if isinstance(parsed_one, ParsedTurn) and _missed_cache(first_parsed_two) + else None + ) + retried_parsed_two: Final = ( + script.parse(retried_turn_two.response) + if retried_turn_two is not None and retried_turn_two.response is not None + else None + ) + turn_two_used: Final = retried_turn_two if _hit_cache(retried_parsed_two) else turn_two + parsed_two: Final = retried_parsed_two if _hit_cache(retried_parsed_two) else first_parsed_two + cache_attempts: Final = 1 if retried_turn_two is None else 2 + answered: Final = tuple( + turn for turn in (turn_one, turn_two_used) if turn is not None and turn.response is not None + ) + spend_rows: Final = ( + await spend_lookup(tuple(turn.call_id for turn in answered)) if spend_lookup is not None and answered else None + ) + second: Final = parsed_two if isinstance(parsed_two, ParsedTurn) else None + checks: Final = ( + _conversation_check(turn_one, parsed_one, turn_two_used, parsed_two), + _prompt_caching_check(second, cache_attempts), + _cost_tracking_check(turn_one, turn_two_used, spend_rows), + ) + return EndpointResult( + endpoint=script.endpoint, + status=_status(checks), + checks=checks, + cache_read_tokens=second.cache_read_tokens if second is not None else 0, + cache_read_percent=_cache_percent(second), + spend_usd=sum(turn.cost for turn in answered if turn.cost is not None), + duration_ms=int((time.monotonic() - started) * 1000), + ) + + +class TestModelCheck(BaseModel): + model_config = ConfigDict(frozen=True) + + name: Literal["conversation", "prompt_caching", "cost_tracking"] + status: Verdict + detail: str + + +class TestModelEndpointResult(BaseModel): + model_config = ConfigDict(frozen=True) + + endpoint: TestModelEndpoint + status: EndpointStatus + checks: Sequence[TestModelCheck] + cache_read_tokens: int + cache_read_percent: float | None + spend_usd: float + duration_ms: int + + +class TestModelResponse(BaseModel): + model_config = ConfigDict(frozen=True) + + status: EndpointStatus + model: str | None + model_id: str | None + endpoints: Sequence[TestModelEndpointResult] + + +def endpoint_result_payload(result: EndpointResult) -> TestModelEndpointResult: + return TestModelEndpointResult( + endpoint=result.endpoint, + status=result.status, + checks=[TestModelCheck(name=check.name, status=check.status, detail=check.detail) for check in result.checks], + cache_read_tokens=result.cache_read_tokens, + cache_read_percent=result.cache_read_percent, + spend_usd=result.spend_usd, + duration_ms=result.duration_ms, + ) diff --git a/tests/integration/management/test_model_health_check.py b/tests/integration/management/test_model_health_check.py index 992c1e8e875..36572d9d5b8 100644 --- a/tests/integration/management/test_model_health_check.py +++ b/tests/integration/management/test_model_health_check.py @@ -1,6 +1,7 @@ import json import os import uuid +from collections.abc import Mapping from email.message import Message from email.parser import BytesParser from email.policy import HTTP @@ -10,7 +11,15 @@ from typing import Final import httpx import pytest import yaml -from integration._support.client import JSON_OBJECT, Gateway, Scenario, eventually, object_value, string_value +from integration._support.client import ( + JSON_OBJECT, + Gateway, + Scenario, + eventually, + list_value, + object_value, + string_value, +) from integration._support.database import scratch_database from integration._support.process import owned_proxy from integration._support.upstream import ScenarioHandle, delete_scenario, register_scenario @@ -1426,3 +1435,139 @@ def test_health_check_of_gpt_transcribe_probes_without_verbose_json_and_reports_ (request.method, request.target, tuple(sorted(_multipart_text_fields(_multipart_parts(request)).items()))) for request in requests } == {("POST", "/v1/audio/transcriptions", (("model", "gpt-transcribe"),))} + + +_TEST_MODEL_TOOL_CALL_ID: Final = "call_boston_wire" + + +def _test_model_chat_reply(request: Request, *, cached_tokens: int) -> Reply: + assert request.target == "/v1/chat/completions", request.target + body: Final = JSON_OBJECT.validate_json(request.body) + assert "cache_control" not in request.body.decode(), request.body.decode() + messages: Final = body["messages"] + assert isinstance(messages, list) and isinstance(messages[0], dict), body + assert messages[0]["role"] == "system", messages[0] + tool_turn: Final = any(isinstance(message, dict) and message.get("role") == "tool" for message in messages) + if tool_turn: + assert isinstance(messages[-1], dict), messages + assert messages[-1]["tool_call_id"] == _TEST_MODEL_TOOL_CALL_ID, messages[-1] + message: dict[str, JsonValue] = {"role": "assistant", "content": "Boston is 72F and sunny."} + else: + assert body["tools"], body + message = { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": _TEST_MODEL_TOOL_CALL_ID, + "type": "function", + "function": {"name": "get_weather", "arguments": json.dumps({"location": "Boston"})}, + } + ], + } + return Reply( + body=json.dumps( + { + "id": f"chatcmpl-{uuid.uuid4().hex}", + "object": "chat.completion", + "created": 1, + "model": body["model"], + "choices": [{"index": 0, "finish_reason": "stop", "message": message}], + "usage": { + "prompt_tokens": 5000, + "completion_tokens": 12, + "total_tokens": 5012, + "prompt_tokens_details": {"cached_tokens": cached_tokens if tool_turn else 0}, + }, + } + ).encode() + ) + + +def _check_statuses(endpoint_result: Mapping[str, JsonValue]) -> dict[str, tuple[str, str]]: + return { + string_value(object_value(check)["name"]): ( + string_value(object_value(check)["status"]), + string_value(object_value(check)["detail"]), + ) + for check in list_value(endpoint_result["checks"]) + } + + +@pytest.mark.timeout(240) +def test_health_test_model_runs_the_tool_conversation_against_a_saved_deployment_and_finds_its_spend_rows( + gateway: Gateway, +) -> None: + with ( + gateway.scenario() as scenario, + wire_server(lambda request: _test_model_chat_reply(request, cached_tokens=4800)) as wire, + ): + model_name: Final = scenario.model(api_base=f"{wire.url}/v1", api_key="synthetic-test-model-key") + model_id: Final = next( + string_value(object_value(info["model_info"])["id"]) + for info in (object_value(entry) for entry in list_value(gateway.get("/v1/model/info")["data"])) + if info["model_name"] == model_name + ) + + response: Final = gateway.request( + "POST", "/health/test_model", {"model_id": model_id, "endpoints": ["/v1/chat/completions"]} + ) + + assert response.status_code == 200, response.text + payload: Final = JSON_OBJECT.validate_json(response.content) + assert payload["status"] == "success", response.text + assert payload["model_id"] == model_id, response.text + endpoint_result: Final = object_value(list_value(payload["endpoints"])[0]) + assert endpoint_result["endpoint"] == "/v1/chat/completions", response.text + checks: Final = _check_statuses(endpoint_result) + assert checks["conversation"][0] == "pass", response.text + assert checks["prompt_caching"] == ("pass", "turn 2 read 4800 of 5000 input tokens from cache (96.0%)"), ( + response.text + ) + assert checks["cost_tracking"][0] == "pass", response.text + assert "spend log rows written" in checks["cost_tracking"][1], response.text + assert endpoint_result["cache_read_tokens"] == 4800, response.text + requests: Final = wire.drain() + assert [request.headers["authorization"] for request in requests] == ["Bearer synthetic-test-model-key"] * 2, ( + response.text + ) + rows: Final = gateway.rows( + 'SELECT "spend", "request_tags"::text AS tags FROM "LiteLLM_SpendLogs" WHERE "model_id" = %s', + (model_id,), + ) + assert len(rows) == 2, rows + assert all(isinstance(row["spend"], int | float) and row["spend"] > 0 for row in rows), rows + assert all("litellm_test_model:" in string_value(row["tags"]) for row in rows), rows + + +@pytest.mark.timeout(240) +def test_health_test_model_names_the_check_that_failed_for_an_unsaved_model_that_never_caches( + gateway: Gateway, +) -> None: + with wire_server(lambda request: _test_model_chat_reply(request, cached_tokens=0)) as wire: + response: Final = gateway.request( + "POST", + "/health/test_model", + { + "litellm_params": { + "model": f"openai/unpriced-{uuid.uuid4().hex}", + "api_base": f"{wire.url}/v1", + "api_key": "synthetic-unsaved-key", + }, + "endpoints": ["/v1/chat/completions"], + }, + ) + + assert response.status_code == 200, response.text + payload: Final = JSON_OBJECT.validate_json(response.content) + assert payload["status"] == "error", response.text + checks: Final = _check_statuses(object_value(list_value(payload["endpoints"])[0])) + assert checks["conversation"][0] == "pass", response.text + assert checks["prompt_caching"] == ( + "fail", + "turn 2 reported 0 cached tokens on 5000 input tokens in 2 attempts, the provider did not reuse the " + "repeated system prompt", + ), response.text + assert checks["cost_tracking"][0] == "fail", response.text + assert "no price resolved for this model" in checks["cost_tracking"][1], response.text + assert len(wire.drain()) >= 2, response.text diff --git a/tests/unit/proxy/health_endpoints/test_model_checks.py b/tests/unit/proxy/health_endpoints/test_model_checks.py new file mode 100644 index 00000000000..35094b8f32e --- /dev/null +++ b/tests/unit/proxy/health_endpoints/test_model_checks.py @@ -0,0 +1,373 @@ +import asyncio +import json +from collections.abc import Mapping, Sequence +from typing import Final + +import pytest + +from litellm.proxy.health_endpoints.model_checks import ( + CHAT_COMPLETIONS_SCRIPT, + MESSAGES_SCRIPT, + RESPONSES_SCRIPT, + WEATHER_TOOL_RESULT, + CalledTurn, + EndpointScript, + endpoint_result_payload, + overall_status, + run_endpoint_check, +) +from litellm.types.llms.openai import ResponsesAPIResponse +from litellm.types.utils import ModelResponse + +_TOOL_CALL_ID: Final = "call_boston" + + +def _chat_reply(*, tool_call: bool, text: str = "", cached_tokens: int = 0, prompt_tokens: int = 5000) -> ModelResponse: + message: Final = ( + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": _TOOL_CALL_ID, + "type": "function", + "function": {"name": "get_weather", "arguments": json.dumps({"location": "Boston"})}, + } + ], + } + if tool_call + else {"role": "assistant", "content": text} + ) + return ModelResponse( + id="chatcmpl-1", + model="gpt-test", + choices=[{"index": 0, "finish_reason": "tool_calls" if tool_call else "stop", "message": message}], + usage={ + "prompt_tokens": prompt_tokens, + "completion_tokens": 12, + "total_tokens": prompt_tokens + 12, + "prompt_tokens_details": {"cached_tokens": cached_tokens}, + }, + ) + + +def _responses_reply(*, tool_call: bool, text: str = "", cached_tokens: int = 0) -> ResponsesAPIResponse: + output: Final = ( + [ + {"type": "reasoning", "id": "rs_1", "summary": []}, + { + "type": "function_call", + "id": "fc_1", + "call_id": _TOOL_CALL_ID, + "name": "get_weather", + "arguments": json.dumps({"location": "Boston"}), + "status": "completed", + }, + ] + if tool_call + else [ + { + "type": "message", + "id": "msg_1", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": text, "annotations": []}], + } + ] + ) + return ResponsesAPIResponse.model_validate( + { + "id": "resp_1", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "gpt-test", + "output": output, + "parallel_tool_calls": True, + "tool_choice": "auto", + "tools": [], + "usage": { + "input_tokens": 5000, + "output_tokens": 12, + "total_tokens": 5012, + "input_tokens_details": {"cached_tokens": cached_tokens}, + }, + } + ) + + +def _messages_reply(*, tool_call: bool, text: str = "", cached_tokens: int = 0) -> Mapping[str, object]: + content: Final = ( + [ + {"type": "thinking", "thinking": "look it up", "signature": "sig"}, + {"type": "tool_use", "id": _TOOL_CALL_ID, "name": "get_weather", "input": {"location": "Boston"}}, + ] + if tool_call + else [{"type": "text", "text": text}] + ) + return { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-test", + "content": content, + "stop_reason": "tool_use" if tool_call else "end_turn", + "usage": { + "input_tokens": 5000 - cached_tokens, + "output_tokens": 12, + "cache_read_input_tokens": cached_tokens, + "cache_creation_input_tokens": 0, + }, + } + + +class _ScriptedProvider: + """Answers turn 1 with a tool call and every later turn from ``second_turns``, recording what was sent.""" + + def __init__( + self, + first: CalledTurn, + second_turns: Sequence[CalledTurn | Exception], + ) -> None: + self.first = first + self.second_turns = list(second_turns) + self.sent: list[Mapping[str, object]] = [] + + async def __call__(self, script: EndpointScript, body: Mapping[str, object], call_id: str) -> CalledTurn: + self.sent.append(body) + if len(self.sent) == 1: + return self.first + outcome: Final = self.second_turns.pop(0) + if isinstance(outcome, Exception): + raise outcome + return outcome + + +def _ids() -> Sequence[str]: + return ("turn-1", "turn-2", "turn-2-retry") + + +def _run(script: EndpointScript, provider: _ScriptedProvider, spend_rows: Mapping[str, float] | None = None): + ids: Final = iter(_ids()) + + async def spend_lookup(request_ids: Sequence[str]) -> Mapping[str, float]: + return spend_rows or {} + + return asyncio.run( + run_endpoint_check( + script, + provider, + new_call_id=lambda: next(ids), + spend_lookup=spend_lookup if spend_rows is not None else None, + ) + ) + + +def _checks(result) -> Mapping[str, tuple[str, str]]: + return {check.name: (check.status, check.detail) for check in result.checks} + + +@pytest.mark.parametrize( + ("script", "first", "second"), + [ + ( + CHAT_COMPLETIONS_SCRIPT, + _chat_reply(tool_call=True), + _chat_reply(tool_call=False, text="Boston is 72F and sunny.", cached_tokens=4800), + ), + ( + RESPONSES_SCRIPT, + _responses_reply(tool_call=True), + _responses_reply(tool_call=False, text="Boston is 72F and sunny.", cached_tokens=4800), + ), + ( + MESSAGES_SCRIPT, + _messages_reply(tool_call=True), + _messages_reply(tool_call=False, text="Boston is 72F and sunny.", cached_tokens=4800), + ), + ], + ids=["chat", "responses", "messages"], +) +def test_every_endpoint_passes_all_three_checks_and_feeds_turn_one_back_in_its_own_shape(script, first, second): + provider: Final = _ScriptedProvider(CalledTurn(first, 0.004), [CalledTurn(second, 0.0005)]) + + result: Final = _run(script, provider, spend_rows={"turn-1": 0.004, "turn-2": 0.0005}) + + assert result.status == "success", result + checks: Final = _checks(result) + assert checks["conversation"][0] == "pass", checks + assert checks["prompt_caching"] == ("pass", "turn 2 read 4800 of 5000 input tokens from cache (96.0%)"), checks + assert checks["cost_tracking"] == ( + "pass", + "turn 1 $0.004000, turn 2 $0.000500, spend log rows written ($0.004000, $0.000500)", + ), checks + assert result.cache_read_tokens == 4800 + assert result.cache_read_percent == 96.0 + assert result.spend_usd == pytest.approx(0.0045) + assert len(provider.sent) == 2 + turn_two: Final = provider.sent[1] + sent_json: Final = json.dumps(turn_two) + assert _TOOL_CALL_ID in sent_json and WEATHER_TOOL_RESULT in sent_json, turn_two + assert "cache_control" not in sent_json and "cachePoint" not in sent_json, turn_two + if script is CHAT_COMPLETIONS_SCRIPT: + tool_message: Final = turn_two["messages"][-1] + assert tool_message == {"role": "tool", "tool_call_id": _TOOL_CALL_ID, "content": WEATHER_TOOL_RESULT} + assert turn_two["messages"][-2]["tool_calls"][0]["id"] == _TOOL_CALL_ID + elif script is RESPONSES_SCRIPT: + assert turn_two["input"][-1] == { + "type": "function_call_output", + "call_id": _TOOL_CALL_ID, + "output": WEATHER_TOOL_RESULT, + } + assert [item["type"] for item in turn_two["input"][1:-1]] == ["reasoning", "function_call"] + else: + assert turn_two["messages"][-1]["content"] == [ + {"type": "tool_result", "tool_use_id": _TOOL_CALL_ID, "content": WEATHER_TOOL_RESULT} + ] + assert turn_two["messages"][-2]["content"][0]["type"] == "thinking" + + +def test_a_model_that_answers_in_text_instead_of_calling_the_tool_fails_the_conversation_and_stops(): + provider: Final = _ScriptedProvider(CalledTurn(_chat_reply(tool_call=False, text="It is sunny."), 0.004), []) + + result: Final = _run(CHAT_COMPLETIONS_SCRIPT, provider) + + assert result.status == "error" + checks: Final = _checks(result) + assert checks["conversation"] == ("fail", "turn 1 returned no tool call, the model answered: 'It is sunny.'") + assert checks["prompt_caching"] == ("fail", "not checked, the conversation did not reach turn 2") + assert checks["cost_tracking"] == ("pass", "turn 1 $0.004000, spend log not checked (no database)") + assert len(provider.sent) == 1 + + +def test_a_turn_two_provider_error_is_reported_as_the_conversation_failure(): + provider: Final = _ScriptedProvider( + CalledTurn(_chat_reply(tool_call=True), 0.004), [RuntimeError("400 invalid tool message")] + ) + + result: Final = _run(CHAT_COMPLETIONS_SCRIPT, provider) + + checks: Final = _checks(result) + assert checks["conversation"] == ( + "fail", + "turn 2 request with the tool result failed: RuntimeError: 400 invalid tool message", + ) + assert result.status == "error" + + +def test_an_answer_that_ignores_the_tool_result_fails_the_conversation(): + provider: Final = _ScriptedProvider( + CalledTurn(_chat_reply(tool_call=True), 0.004), + [CalledTurn(_chat_reply(tool_call=False, text="I cannot tell.", cached_tokens=4800), 0.0005)], + ) + + checks: Final = _checks(_run(CHAT_COMPLETIONS_SCRIPT, provider)) + + assert checks["conversation"] == ("fail", "turn 2 did not use the tool result '72F and sunny': 'I cannot tell.'") + assert checks["prompt_caching"][0] == "pass" + + +def test_zero_cached_tokens_is_retried_once_and_fails_when_the_retry_misses_too(): + provider: Final = _ScriptedProvider( + CalledTurn(_chat_reply(tool_call=True), 0.004), + [ + CalledTurn(_chat_reply(tool_call=False, text="72F", cached_tokens=0), 0.004), + CalledTurn(_chat_reply(tool_call=False, text="72F", cached_tokens=0), 0.004), + ], + ) + + result: Final = _run(CHAT_COMPLETIONS_SCRIPT, provider) + + assert len(provider.sent) == 3 + assert _checks(result)["prompt_caching"] == ( + "fail", + "turn 2 reported 0 cached tokens on 5000 input tokens in 2 attempts, the provider did not reuse the " + "repeated system prompt", + ) + assert result.status == "error" + assert result.cache_read_tokens == 0 + + +def test_a_cache_hit_on_the_retry_passes_and_says_so(): + provider: Final = _ScriptedProvider( + CalledTurn(_chat_reply(tool_call=True), 0.004), + [ + CalledTurn(_chat_reply(tool_call=False, text="72F", cached_tokens=0), 0.004), + CalledTurn(_chat_reply(tool_call=False, text="72F", cached_tokens=5000), 0.0005), + ], + ) + + result: Final = _run(CHAT_COMPLETIONS_SCRIPT, provider) + + assert _checks(result)["prompt_caching"] == ( + "pass", + "turn 2 read 5000 of 5000 input tokens from cache (100.0%) on the second attempt", + ) + assert result.spend_usd == pytest.approx(0.0045) + + +def test_a_model_without_a_price_fails_cost_tracking_and_a_zero_priced_model_only_warns(): + unpriced: Final = _ScriptedProvider( + CalledTurn(_chat_reply(tool_call=True), None), + [CalledTurn(_chat_reply(tool_call=False, text="72F", cached_tokens=4800), None)], + ) + free: Final = _ScriptedProvider( + CalledTurn(_chat_reply(tool_call=True), 0.0), + [CalledTurn(_chat_reply(tool_call=False, text="72F", cached_tokens=4800), 0.0)], + ) + + unpriced_result: Final = _run(CHAT_COMPLETIONS_SCRIPT, unpriced) + free_result: Final = _run(CHAT_COMPLETIONS_SCRIPT, free) + + assert _checks(unpriced_result)["cost_tracking"] == ( + "fail", + "turn 1 carried no response_cost, no price resolved for this model (not in the cost map and no custom " + "pricing on the deployment)", + ) + assert unpriced_result.status == "error" + assert _checks(free_result)["cost_tracking"] == ( + "warn", + "turn 1 response_cost is $0, this model is priced at zero", + ) + assert free_result.status == "warning" + assert overall_status([unpriced_result, free_result]) == "error" + assert overall_status([free_result]) == "warning" + + +def test_a_missing_spend_log_row_fails_cost_tracking_even_when_the_response_was_priced(): + provider: Final = _ScriptedProvider( + CalledTurn(_chat_reply(tool_call=True), 0.004), + [CalledTurn(_chat_reply(tool_call=False, text="72F", cached_tokens=4800), 0.0005)], + ) + + checks: Final = _checks(_run(CHAT_COMPLETIONS_SCRIPT, provider, spend_rows={"turn-1": 0.004})) + + assert checks["cost_tracking"] == ( + "fail", + "turn 1 $0.004000, turn 2 $0.000500, but no LiteLLM_SpendLogs row landed for request_id turn-2", + ) + + +def test_a_response_that_is_not_the_endpoint_shape_fails_the_conversation_with_the_parse_error(): + provider: Final = _ScriptedProvider(CalledTurn({"id": "x", "object": "nonsense"}, 0.001), []) + + checks: Final = _checks(_run(MESSAGES_SCRIPT, provider)) + + assert checks["conversation"][0] == "fail" + assert checks["conversation"][1].startswith("turn 1 Messages API response did not parse: content"), checks + + +def test_payload_lists_the_checks_in_order_with_their_details(): + provider: Final = _ScriptedProvider( + CalledTurn(_chat_reply(tool_call=True), 0.004), + [CalledTurn(_chat_reply(tool_call=False, text="72F", cached_tokens=4800), 0.0005)], + ) + + payload: Final = endpoint_result_payload(_run(CHAT_COMPLETIONS_SCRIPT, provider)).model_dump() + + assert payload["endpoint"] == "/v1/chat/completions" + assert payload["status"] == "success" + assert [check["name"] for check in payload["checks"]] == ["conversation", "prompt_caching", "cost_tracking"] + assert all(check["status"] == "pass" for check in payload["checks"]) + assert payload["cache_read_percent"] == 96.0 + assert isinstance(payload["duration_ms"], int) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 96c7c34621b..cfdee1cf59f 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -7967,6 +7967,46 @@ export interface paths { patch?: never; trace?: never; }; + "/health/test_model": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Test Model + * @description Run the live checks on a model, the way a client would use it, and report what works. + * + * Each selected endpoint gets one two-turn conversation with a long system prompt and a tool. Turn 1 asks + * a question the tool answers, turn 2 sends the tool result back. Three checks come out of it: + * + * - `conversation`: both turns parse, turn 1 calls the tool, turn 2 uses the tool result + * - `prompt_caching`: turn 2 reports cached input tokens, with no cache markers in the request + * - `cost_tracking`: every turn carries a non-zero response cost, and the spend log row lands when a + * database is connected + * + * Example: + * ```bash + * curl -X POST 'http://localhost:4000/health/test_model' \ + * -H "Authorization: Bearer $LITELLM_MASTER_KEY" \ + * -H 'Content-Type: application/json' \ + * -d '{"model_id": "a1b2c3", "endpoints": ["/v1/chat/completions", "/v1/responses"]}' + * ``` + * + * Returns `status` (`success`, `warning` or `error`), the tested `model`, and one entry per endpoint + * with its `status`, the three `checks` (`name`, `status`, `detail`), `cache_read_tokens`, + * `cache_read_percent`, `spend_usd` and `duration_ms`. + */ + post: operations["test_model_health_test_model_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/images/edits": { parameters: { query?: never; @@ -27753,6 +27793,33 @@ export interface components { [key: string]: unknown; }; }; + /** Body_test_model_health_test_model_post */ + Body_test_model_health_test_model_post: { + /** + * Endpoints + * @description Which endpoints to test: /v1/chat/completions, /v1/responses, /v1/messages. Defaults to the model's native endpoint. + */ + endpoints?: ("/v1/chat/completions" | "/v1/responses" | "/v1/messages")[] | null; + /** + * Litellm Params + * @description Parameters of a model that is not saved yet (model, api_key, api_base, api_version), as sent to /model/new. When model_id is also given these override the saved deployment's values. + */ + litellm_params?: { + [key: string]: unknown; + } | null; + /** + * Model Id + * @description The deployment to test, by its model_info.id. The saved credentials and pricing are used. + */ + model_id?: string | null; + /** + * Model Info + * @description Model info of an unsaved model, for example mode or custom pricing. + */ + model_info?: { + [key: string]: unknown; + } | null; + }; /** Body_token_endpoint__mcp_server_name__token_post */ Body_token_endpoint__mcp_server_name__token_post: { /** Assertion */ @@ -47363,6 +47430,58 @@ export interface components { /** Success */ success: boolean; }; + /** TestModelCheck */ + TestModelCheck: { + /** Detail */ + detail: string; + /** + * Name + * @enum {string} + */ + name: "conversation" | "prompt_caching" | "cost_tracking"; + /** + * Status + * @enum {string} + */ + status: "pass" | "warn" | "fail"; + }; + /** TestModelEndpointResult */ + TestModelEndpointResult: { + /** Cache Read Percent */ + cache_read_percent: number | null; + /** Cache Read Tokens */ + cache_read_tokens: number; + /** Checks */ + checks: components["schemas"]["TestModelCheck"][]; + /** Duration Ms */ + duration_ms: number; + /** + * Endpoint + * @enum {string} + */ + endpoint: "/v1/chat/completions" | "/v1/responses" | "/v1/messages"; + /** Spend Usd */ + spend_usd: number; + /** + * Status + * @enum {string} + */ + status: "success" | "warning" | "error"; + }; + /** TestModelResponse */ + TestModelResponse: { + /** Endpoints */ + endpoints: components["schemas"]["TestModelEndpointResult"][]; + /** Model */ + model: string | null; + /** Model Id */ + model_id: string | null; + /** + * Status + * @enum {string} + */ + status: "success" | "warning" | "error"; + }; /** * TestPoliciesAndGuardrailsRequest * @description Request body for POST /utils/test_policies_and_guardrails. @@ -63001,6 +63120,39 @@ export interface operations { }; }; }; + test_model_health_test_model_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: { + content: { + "application/json": components["schemas"]["Body_test_model_health_test_model_post"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["TestModelResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; image_edit_api_images_edits_post: { parameters: { query?: { From 8396465a16f1ad4bf111bd712d18679a94605cdd Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 31 Oct 2026 00:47:04 +0000 Subject: [PATCH 02/15] refactor(proxy): load test_model turn 1 request bodies from JSON files --- .../proxy/health_endpoints/model_checks.py | 133 ++++-------------- .../test_model_requests/chat_completions.json | 42 ++++++ .../test_model_requests/messages.json | 39 +++++ .../test_model_requests/responses.json | 39 +++++ pyproject.toml | 1 + .../management/test_model_health_check.py | 9 +- .../health_endpoints/test_model_checks.py | 16 +++ 7 files changed, 169 insertions(+), 110 deletions(-) create mode 100644 litellm/proxy/health_endpoints/test_model_requests/chat_completions.json create mode 100644 litellm/proxy/health_endpoints/test_model_requests/messages.json create mode 100644 litellm/proxy/health_endpoints/test_model_requests/responses.json diff --git a/litellm/proxy/health_endpoints/model_checks.py b/litellm/proxy/health_endpoints/model_checks.py index 719243cc032..4296d798ca7 100644 --- a/litellm/proxy/health_endpoints/model_checks.py +++ b/litellm/proxy/health_endpoints/model_checks.py @@ -1,9 +1,10 @@ """Live checks behind ``POST /health/test_model``. -One two-turn tool-call conversation per selected endpoint, scripted in that endpoint's native request shape -with no prompt-cache markers, so the second turn can only report cached tokens when the provider caches the -repeated prefix on its own. Three verdicts come out of the two turns: the conversation itself (both turns -parse, the tool call lands, the tool result is used), prompt caching, and cost tracking. +One two-turn tool-call conversation per selected endpoint. Turn 1 is the JSON body in ``test_model_requests/`` +for that endpoint, in its native request shape with no prompt-cache markers, so the second turn can only report +cached tokens when the provider caches the repeated prefix on its own. Turn 2 is built from turn 1's reply, since +it has to carry the tool call id the model returned. Three verdicts come out of the two turns: the conversation +itself (both turns parse, the tool call lands, the tool result is used), prompt caching, and cost tracking. """ from __future__ import annotations @@ -12,6 +13,7 @@ import json import time from collections.abc import Awaitable, Callable, Mapping, Sequence from dataclasses import dataclass +from pathlib import Path from types import MappingProxyType from typing import Final, Literal, TypeAlias @@ -27,50 +29,7 @@ EndpointStatus: TypeAlias = Literal["success", "warning", "error"] TEST_MODEL_ENDPOINTS: Final[tuple[TestModelEndpoint, ...]] = ("/v1/chat/completions", "/v1/responses", "/v1/messages") WEATHER_TOOL_NAME: Final = "get_weather" WEATHER_TOOL_RESULT: Final = "72F and sunny" -WEATHER_QUESTION: Final = "What is the weather in Boston right now? Use the get_weather tool to find out." -MAX_OUTPUT_TOKENS: Final = 2048 -THINKING_BUDGET_TOKENS: Final = 1024 - -_WEATHER_PARAMETERS: Final[Mapping[str, object]] = MappingProxyType( - { - "type": "object", - "properties": { - "location": {"type": "string", "description": "City name, for example Boston"}, - "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, - }, - "required": ["location"], - } -) -_WEATHER_DESCRIPTION: Final = "Get the current weather for a city" - -_CITIES: Final = ("Boston", "Denver", "Lisbon", "Osaka", "Nairobi", "Perth", "Quito", "Reykjavik") -_SECTION_SENTENCES: Final = ( - "Section {n}: deliveries routed through the {city} hub are scanned at intake, weighed on the calibrated " - "scale, and matched against the manifest before any label is printed or any pallet leaves the dock.", - "Agents handling {city} tickets confirm the tracking number, the declared value, and the delivery window " - "with the sender before promising a reroute, and record the promise in the ticket with a timestamp.", - "If a parcel bound for {city} misses its scheduled scan by more than {hours} hours, the agent opens a " - "trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift.", - "Refunds for {city} shipments above {amount} dollars need a supervisor code, a photo of the damage when " - "one exists, and the original invoice number, and are paid to the original payment method within ten days.", -) -_SECTION_COUNT: Final = 32 - -SYSTEM_PROMPT: Final = "\n\n".join( - ( - "You are the support assistant for Northwind Logistics. Follow the playbook below exactly.", - *( - " ".join( - sentence.format(n=n, city=_CITIES[n % len(_CITIES)], hours=4 + (n % 5), amount=100 * (1 + (n % 7))) - for sentence in _SECTION_SENTENCES - ) - for n in range(1, _SECTION_COUNT + 1) - ), - f"When the user asks about the weather, call the {WEATHER_TOOL_NAME} tool with the city name. After a " - "tool result arrives, answer in one short sentence that repeats the temperature exactly as the tool " - "reported it.", - ) -) +TEST_MODEL_REQUESTS_DIR: Final = Path(__file__).resolve().parent / "test_model_requests" @dataclass(frozen=True, slots=True) @@ -232,6 +191,18 @@ _JSON_DOCUMENT: Final = TypeAdapter(object) _EMPTY_ASSISTANT: Final[Mapping[str, object]] = MappingProxyType({"role": "assistant", "content": None}) +def _request_file(name: str) -> Callable[[], Mapping[str, object]]: + """Turn 1 of each script is a plain JSON request body on disk, replayable with curl, parsed fresh per call so + the provider code never touches a shared copy.""" + text: Final = (TEST_MODEL_REQUESTS_DIR / f"{name}.json").read_text(encoding="utf-8") + return lambda: _PLAIN_OBJECT.validate_json(text) + + +_CHAT_TURN_ONE: Final = _request_file("chat_completions") +_RESPONSES_TURN_ONE: Final = _request_file("responses") +_MESSAGES_TURN_ONE: Final = _request_file("messages") + + def _plain(response: object) -> Mapping[str, object] | None: if isinstance(response, BaseModel): return _PLAIN_OBJECT.validate_python(response.model_dump(exclude_none=True)) @@ -254,29 +225,8 @@ def _arguments(raw: str) -> Mapping[str, object] | None: return MappingProxyType(_PLAIN_OBJECT.validate_python(parsed)) if isinstance(parsed, dict) else None -def _chat_turn_one() -> Mapping[str, object]: - return { - "messages": [ - {"role": "system", "content": SYSTEM_PROMPT}, - {"role": "user", "content": WEATHER_QUESTION}, - ], - "tools": [ - { - "type": "function", - "function": { - "name": WEATHER_TOOL_NAME, - "description": _WEATHER_DESCRIPTION, - "parameters": dict(_WEATHER_PARAMETERS), - }, - } - ], - "reasoning_effort": "low", - "max_tokens": MAX_OUTPUT_TOKENS, - } - - def _chat_turn_two(turn_one: ParsedTurn) -> Mapping[str, object]: - first: Final = _chat_turn_one() + first: Final = _CHAT_TURN_ONE() tool_call_id: Final = turn_one.tool_call.id if turn_one.tool_call is not None else "" return { **first, @@ -332,25 +282,8 @@ def _parse_chat(response: object) -> ParsedTurn | str: ) -def _responses_turn_one() -> Mapping[str, object]: - return { - "instructions": SYSTEM_PROMPT, - "input": [{"role": "user", "content": WEATHER_QUESTION}], - "tools": [ - { - "type": "function", - "name": WEATHER_TOOL_NAME, - "description": _WEATHER_DESCRIPTION, - "parameters": dict(_WEATHER_PARAMETERS), - } - ], - "reasoning": {"effort": "low"}, - "max_output_tokens": MAX_OUTPUT_TOKENS, - } - - def _responses_turn_two(turn_one: ParsedTurn) -> Mapping[str, object]: - first: Final = _responses_turn_one() + first: Final = _RESPONSES_TURN_ONE() call_id: Final = turn_one.tool_call.id if turn_one.tool_call is not None else "" return { **first, @@ -398,24 +331,8 @@ def _parse_responses(response: object) -> ParsedTurn | str: ) -def _messages_turn_one() -> Mapping[str, object]: - return { - "system": SYSTEM_PROMPT, - "messages": [{"role": "user", "content": WEATHER_QUESTION}], - "tools": [ - { - "name": WEATHER_TOOL_NAME, - "description": _WEATHER_DESCRIPTION, - "input_schema": dict(_WEATHER_PARAMETERS), - } - ], - "thinking": {"type": "enabled", "budget_tokens": THINKING_BUDGET_TOKENS}, - "max_tokens": MAX_OUTPUT_TOKENS, - } - - def _messages_turn_two(turn_one: ParsedTurn) -> Mapping[str, object]: - first: Final = _messages_turn_one() + first: Final = _MESSAGES_TURN_ONE() tool_use_id: Final = turn_one.tool_call.id if turn_one.tool_call is not None else "" return { **first, @@ -463,7 +380,7 @@ CHAT_COMPLETIONS_SCRIPT: Final = EndpointScript( endpoint="/v1/chat/completions", route_type="acompletion", metadata_key="metadata", - turn_one=_chat_turn_one, + turn_one=_CHAT_TURN_ONE, turn_two=_chat_turn_two, parse=_parse_chat, ) @@ -471,7 +388,7 @@ RESPONSES_SCRIPT: Final = EndpointScript( endpoint="/v1/responses", route_type="aresponses", metadata_key="litellm_metadata", - turn_one=_responses_turn_one, + turn_one=_RESPONSES_TURN_ONE, turn_two=_responses_turn_two, parse=_parse_responses, ) @@ -479,7 +396,7 @@ MESSAGES_SCRIPT: Final = EndpointScript( endpoint="/v1/messages", route_type="anthropic_messages", metadata_key="litellm_metadata", - turn_one=_messages_turn_one, + turn_one=_MESSAGES_TURN_ONE, turn_two=_messages_turn_two, parse=_parse_messages, ) diff --git a/litellm/proxy/health_endpoints/test_model_requests/chat_completions.json b/litellm/proxy/health_endpoints/test_model_requests/chat_completions.json new file mode 100644 index 00000000000..9b03f3758ab --- /dev/null +++ b/litellm/proxy/health_endpoints/test_model_requests/chat_completions.json @@ -0,0 +1,42 @@ +{ + "messages": [ + { + "role": "system", + "content": "You are the support assistant for Northwind Logistics. Follow the playbook below exactly.\n\nSection 1: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 2: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 3: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 4: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 5: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 6: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 7: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 8: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 9: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 10: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 11: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 12: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 13: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 14: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 15: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 16: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 17: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 18: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 19: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 20: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 21: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 22: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 23: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 24: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 25: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 26: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 27: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 28: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 29: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 30: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 31: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 32: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nWhen the user asks about the weather, call the get_weather tool with the city name. After a tool result arrives, answer in one short sentence that repeats the temperature exactly as the tool reported it." + }, + { + "role": "user", + "content": "What is the weather in Boston right now? Use the get_weather tool to find out." + } + ], + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather for a city", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "City name, for example Boston" + }, + "unit": { + "type": "string", + "enum": [ + "celsius", + "fahrenheit" + ] + } + }, + "required": [ + "location" + ] + } + } + } + ], + "reasoning_effort": "low", + "max_tokens": 2048 +} diff --git a/litellm/proxy/health_endpoints/test_model_requests/messages.json b/litellm/proxy/health_endpoints/test_model_requests/messages.json new file mode 100644 index 00000000000..514a8226caf --- /dev/null +++ b/litellm/proxy/health_endpoints/test_model_requests/messages.json @@ -0,0 +1,39 @@ +{ + "system": "You are the support assistant for Northwind Logistics. Follow the playbook below exactly.\n\nSection 1: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 2: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 3: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 4: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 5: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 6: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 7: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 8: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 9: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 10: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 11: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 12: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 13: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 14: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 15: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 16: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 17: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 18: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 19: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 20: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 21: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 22: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 23: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 24: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 25: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 26: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 27: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 28: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 29: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 30: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 31: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 32: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nWhen the user asks about the weather, call the get_weather tool with the city name. After a tool result arrives, answer in one short sentence that repeats the temperature exactly as the tool reported it.", + "messages": [ + { + "role": "user", + "content": "What is the weather in Boston right now? Use the get_weather tool to find out." + } + ], + "tools": [ + { + "name": "get_weather", + "description": "Get the current weather for a city", + "input_schema": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "City name, for example Boston" + }, + "unit": { + "type": "string", + "enum": [ + "celsius", + "fahrenheit" + ] + } + }, + "required": [ + "location" + ] + } + } + ], + "thinking": { + "type": "enabled", + "budget_tokens": 1024 + }, + "max_tokens": 2048 +} diff --git a/litellm/proxy/health_endpoints/test_model_requests/responses.json b/litellm/proxy/health_endpoints/test_model_requests/responses.json new file mode 100644 index 00000000000..f394f7a224d --- /dev/null +++ b/litellm/proxy/health_endpoints/test_model_requests/responses.json @@ -0,0 +1,39 @@ +{ + "instructions": "You are the support assistant for Northwind Logistics. Follow the playbook below exactly.\n\nSection 1: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 2: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 3: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 4: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 5: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 6: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 7: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 8: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 9: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 10: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 11: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 12: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 13: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 14: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 15: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 16: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 17: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 18: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 19: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 20: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 21: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 22: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 23: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 24: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 25: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 26: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 27: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 28: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 29: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 30: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 31: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 32: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nWhen the user asks about the weather, call the get_weather tool with the city name. After a tool result arrives, answer in one short sentence that repeats the temperature exactly as the tool reported it.", + "input": [ + { + "role": "user", + "content": "What is the weather in Boston right now? Use the get_weather tool to find out." + } + ], + "tools": [ + { + "type": "function", + "name": "get_weather", + "description": "Get the current weather for a city", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "City name, for example Boston" + }, + "unit": { + "type": "string", + "enum": [ + "celsius", + "fahrenheit" + ] + } + }, + "required": [ + "location" + ] + } + } + ], + "reasoning": { + "effort": "low" + }, + "max_output_tokens": 2048 +} diff --git a/pyproject.toml b/pyproject.toml index 6e9e75a5b7f..ef78006a002 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -332,6 +332,7 @@ include = [ "litellm/router_strategy/complexity_router/artifacts/*.json", "litellm/router_strategy/complexity_router/fuse_presets.json", "litellm/proxy/model_insights_tasks.json", + "litellm/proxy/health_endpoints/test_model_requests/*.json", "litellm/proxy/public_endpoints/whats_new.json", "litellm/proxy/common_utils/codex_base_instructions.md", "litellm/proxy/common_utils/codex_bundled_models_0.159.3.json", diff --git a/tests/integration/management/test_model_health_check.py b/tests/integration/management/test_model_health_check.py index 36572d9d5b8..8cb2e3016da 100644 --- a/tests/integration/management/test_model_health_check.py +++ b/tests/integration/management/test_model_health_check.py @@ -27,6 +27,8 @@ from integration._support.wire import Reply, Request, Wire, wire_server from integration.cost_calculation.cost_tracking_case import JsonResponse from pydantic import JsonValue +from litellm.proxy.health_endpoints.model_checks import TEST_MODEL_REQUESTS_DIR + _DECISIONS_PROBE_REPLY: Final = JsonResponse( content_type="application/json", body={ @@ -1438,6 +1440,9 @@ def test_health_check_of_gpt_transcribe_probes_without_verbose_json_and_reports_ _TEST_MODEL_TOOL_CALL_ID: Final = "call_boston_wire" +_TEST_MODEL_CHAT_REQUEST: Final = JSON_OBJECT.validate_json( + (TEST_MODEL_REQUESTS_DIR / "chat_completions.json").read_text(encoding="utf-8") +) def _test_model_chat_reply(request: Request, *, cached_tokens: int) -> Reply: @@ -1446,14 +1451,14 @@ def _test_model_chat_reply(request: Request, *, cached_tokens: int) -> Reply: assert "cache_control" not in request.body.decode(), request.body.decode() messages: Final = body["messages"] assert isinstance(messages, list) and isinstance(messages[0], dict), body - assert messages[0]["role"] == "system", messages[0] tool_turn: Final = any(isinstance(message, dict) and message.get("role") == "tool" for message in messages) + assert messages[: len(_TEST_MODEL_CHAT_REQUEST["messages"])] == _TEST_MODEL_CHAT_REQUEST["messages"], messages + assert body["tools"] == _TEST_MODEL_CHAT_REQUEST["tools"], body["tools"] if tool_turn: assert isinstance(messages[-1], dict), messages assert messages[-1]["tool_call_id"] == _TEST_MODEL_TOOL_CALL_ID, messages[-1] message: dict[str, JsonValue] = {"role": "assistant", "content": "Boston is 72F and sunny."} else: - assert body["tools"], body message = { "role": "assistant", "content": None, diff --git a/tests/unit/proxy/health_endpoints/test_model_checks.py b/tests/unit/proxy/health_endpoints/test_model_checks.py index 35094b8f32e..9ae29833f87 100644 --- a/tests/unit/proxy/health_endpoints/test_model_checks.py +++ b/tests/unit/proxy/health_endpoints/test_model_checks.py @@ -9,6 +9,7 @@ from litellm.proxy.health_endpoints.model_checks import ( CHAT_COMPLETIONS_SCRIPT, MESSAGES_SCRIPT, RESPONSES_SCRIPT, + TEST_MODEL_REQUESTS_DIR, WEATHER_TOOL_RESULT, CalledTurn, EndpointScript, @@ -371,3 +372,18 @@ def test_payload_lists_the_checks_in_order_with_their_details(): assert all(check["status"] == "pass" for check in payload["checks"]) assert payload["cache_read_percent"] == 96.0 assert isinstance(payload["duration_ms"], int) + + +def test_the_three_request_files_send_one_system_prompt_and_one_tool_so_endpoint_verdicts_are_comparable(): + bodies: Final = { + path.stem: json.loads(path.read_text(encoding="utf-8")) for path in TEST_MODEL_REQUESTS_DIR.glob("*.json") + } + chat, messages, responses = bodies["chat_completions"], bodies["messages"], bodies["responses"] + + assert set(bodies) == {"chat_completions", "messages", "responses"}, bodies.keys() + assert chat["messages"][0]["content"] == responses["instructions"] == messages["system"] + assert chat["tools"][0]["function"]["parameters"] == responses["tools"][0]["parameters"] + assert chat["tools"][0]["function"]["parameters"] == messages["tools"][0]["input_schema"] + assert CHAT_COMPLETIONS_SCRIPT.turn_one() == chat + assert RESPONSES_SCRIPT.turn_one() == responses + assert MESSAGES_SCRIPT.turn_one() == messages From a5e4bf9972f8a25e3f42754812b4a916b65d1350 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 31 Oct 2026 16:17:13 +0000 Subject: [PATCH 03/15] refactor(proxy): put both test_model turns in each request file --- .../proxy/health_endpoints/model_checks.py | 127 +++++++++--------- .../test_model_requests/chat_completions.json | 82 ++++++----- .../test_model_requests/messages.json | 86 +++++++----- .../test_model_requests/responses.json | 80 ++++++----- .../management/test_model_health_check.py | 4 +- .../health_endpoints/test_model_checks.py | 12 +- 6 files changed, 221 insertions(+), 170 deletions(-) diff --git a/litellm/proxy/health_endpoints/model_checks.py b/litellm/proxy/health_endpoints/model_checks.py index 2568a625e53..d8dc347302b 100644 --- a/litellm/proxy/health_endpoints/model_checks.py +++ b/litellm/proxy/health_endpoints/model_checks.py @@ -1,10 +1,12 @@ """Live checks behind ``POST /health/test_model``. -One two-turn tool-call conversation per selected endpoint. Turn 1 is the JSON body in ``test_model_requests/`` -for that endpoint, in its native request shape with no prompt-cache markers, so the second turn can only report -cached tokens when the provider caches the repeated prefix on its own. Turn 2 is built from turn 1's reply, since -it has to carry the tool call id the model returned. Three verdicts come out of the two turns: the conversation -itself (both turns parse, the tool call lands, the tool result is used), prompt caching, and cost tracking. +One two-turn tool-call conversation per selected endpoint, read from that endpoint's file in +``test_model_requests/``. ``turn_1`` is the full first request body in the endpoint's native shape with no +prompt-cache markers, so the second turn can only report cached tokens when the provider caches the repeated prefix +on its own. ``turn_2_appended_messages`` lists what the second request adds after turn 1's messages, with +```` and ```` filled in from the model's reply. +Three verdicts come out of the two turns: the conversation itself (both turns parse, the tool call lands, the tool +result is used), prompt caching, and cost tracking. """ from __future__ import annotations @@ -13,6 +15,7 @@ import json import time from collections.abc import Awaitable, Callable, Mapping, Sequence from dataclasses import dataclass +from itertools import chain from pathlib import Path from types import MappingProxyType from typing import Final, Literal, TypeAlias @@ -30,6 +33,8 @@ TEST_MODEL_ENDPOINTS: Final[tuple[TestModelEndpoint, ...]] = ("/v1/chat/completi WEATHER_TOOL_NAME: Final = "get_weather" WEATHER_TOOL_RESULT: Final = "72F and sunny" TEST_MODEL_REQUESTS_DIR: Final = Path(__file__).resolve().parent / "test_model_requests" +TURN_ONE_ASSISTANT_OUTPUT: Final = "" +TURN_ONE_TOOL_CALL_ID: Final = "" @dataclass(frozen=True, slots=True) @@ -191,16 +196,60 @@ _JSON_DOCUMENT: Final = TypeAdapter(object) _EMPTY_ASSISTANT: Final[Mapping[str, object]] = MappingProxyType({"role": "assistant", "content": None}) -def _request_file(name: str) -> Callable[[], Mapping[str, object]]: - """Turn 1 of each script is a plain JSON request body on disk, replayable with curl, parsed fresh per call so - the provider code never touches a shared copy.""" +class RequestFile(BaseModel): + """One two-turn conversation on disk: ``turn_1`` is the full first request body, ``turn_2_appended_messages`` + are the messages the second request adds after it, with the two placeholders filled from turn 1's response.""" + + model_config = ConfigDict(frozen=True) + + turn_1: Mapping[str, object] + turn_2_appended_messages: Sequence[object] + + +def _request_file(name: str) -> Callable[[], RequestFile]: text: Final = (TEST_MODEL_REQUESTS_DIR / f"{name}.json").read_text(encoding="utf-8") - return lambda: _PLAIN_OBJECT.validate_json(text) + return lambda: RequestFile.model_validate_json(text) -_CHAT_TURN_ONE: Final = _request_file("chat_completions") -_RESPONSES_TURN_ONE: Final = _request_file("responses") -_MESSAGES_TURN_ONE: Final = _request_file("messages") +_CHAT_FILE: Final = _request_file("chat_completions") +_RESPONSES_FILE: Final = _request_file("responses") +_MESSAGES_FILE: Final = _request_file("messages") + + +def _fill(value: object, turn_one: ParsedTurn) -> object: + if value == TURN_ONE_TOOL_CALL_ID: + return turn_one.tool_call.id if turn_one.tool_call is not None else "" + if value == TURN_ONE_ASSISTANT_OUTPUT: + return turn_one.assistant_echo + if isinstance(value, dict): + return {key: _fill(item, turn_one) for key, item in _PLAIN_OBJECT.validate_python(value).items()} + if isinstance(value, list): + return [_fill(item, turn_one) for item in _OBJECT_LIST.validate_python(value)] + return value + + +def _appended_messages(item: object, turn_one: ParsedTurn) -> Sequence[object]: + echo: Final = turn_one.assistant_echo + if item == TURN_ONE_ASSISTANT_OUTPUT and isinstance(echo, list): + return _OBJECT_LIST.validate_python(echo) + return (_fill(item, turn_one),) + + +def _turn_one(file: Callable[[], RequestFile]) -> Callable[[], Mapping[str, object]]: + return lambda: file().turn_1 + + +def _turn_two( + file: Callable[[], RequestFile], conversation_key: Literal["messages", "input"] +) -> Callable[[ParsedTurn], Mapping[str, object]]: + def build(turn_one: ParsedTurn) -> Mapping[str, object]: + first: Final = file() + appended: Final = chain.from_iterable( + _appended_messages(item, turn_one) for item in first.turn_2_appended_messages + ) + return {**first.turn_1, conversation_key: [*_message_list(first.turn_1.get(conversation_key)), *appended]} + + return build def _plain(response: object) -> Mapping[str, object] | None: @@ -225,19 +274,6 @@ def _arguments(raw: str) -> Mapping[str, object] | None: return MappingProxyType(_PLAIN_OBJECT.validate_python(parsed)) if isinstance(parsed, dict) else None -def _chat_turn_two(turn_one: ParsedTurn) -> Mapping[str, object]: - first: Final = _CHAT_TURN_ONE() - tool_call_id: Final = turn_one.tool_call.id if turn_one.tool_call is not None else "" - return { - **first, - "messages": [ - *_message_list(first["messages"]), - turn_one.assistant_echo, - {"role": "tool", "tool_call_id": tool_call_id, "content": WEATHER_TOOL_RESULT}, - ], - } - - def _message_list(messages: object) -> Sequence[object]: return _OBJECT_LIST.validate_python(messages) if isinstance(messages, list) else [] @@ -282,19 +318,6 @@ def _parse_chat(response: object) -> ParsedTurn | str: ) -def _responses_turn_two(turn_one: ParsedTurn) -> Mapping[str, object]: - first: Final = _RESPONSES_TURN_ONE() - call_id: Final = turn_one.tool_call.id if turn_one.tool_call is not None else "" - return { - **first, - "input": [ - *_message_list(first["input"]), - *_message_list(turn_one.assistant_echo), - {"type": "function_call_output", "call_id": call_id, "output": WEATHER_TOOL_RESULT}, - ], - } - - def _parse_responses(response: object) -> ParsedTurn | str: plain: Final = _plain(response) if plain is None: @@ -331,22 +354,6 @@ def _parse_responses(response: object) -> ParsedTurn | str: ) -def _messages_turn_two(turn_one: ParsedTurn) -> Mapping[str, object]: - first: Final = _MESSAGES_TURN_ONE() - tool_use_id: Final = turn_one.tool_call.id if turn_one.tool_call is not None else "" - return { - **first, - "messages": [ - *_message_list(first["messages"]), - {"role": "assistant", "content": turn_one.assistant_echo}, - { - "role": "user", - "content": [{"type": "tool_result", "tool_use_id": tool_use_id, "content": WEATHER_TOOL_RESULT}], - }, - ], - } - - def _parse_messages(response: object) -> ParsedTurn | str: plain: Final = _plain(response) if plain is None: @@ -380,24 +387,24 @@ CHAT_COMPLETIONS_SCRIPT: Final = EndpointScript( endpoint="/v1/chat/completions", route_type="acompletion", metadata_key="metadata", - turn_one=_CHAT_TURN_ONE, - turn_two=_chat_turn_two, + turn_one=_turn_one(_CHAT_FILE), + turn_two=_turn_two(_CHAT_FILE, "messages"), parse=_parse_chat, ) RESPONSES_SCRIPT: Final = EndpointScript( endpoint="/v1/responses", route_type="aresponses", metadata_key="litellm_metadata", - turn_one=_RESPONSES_TURN_ONE, - turn_two=_responses_turn_two, + turn_one=_turn_one(_RESPONSES_FILE), + turn_two=_turn_two(_RESPONSES_FILE, "input"), parse=_parse_responses, ) MESSAGES_SCRIPT: Final = EndpointScript( endpoint="/v1/messages", route_type="anthropic_messages", metadata_key="litellm_metadata", - turn_one=_MESSAGES_TURN_ONE, - turn_two=_messages_turn_two, + turn_one=_turn_one(_MESSAGES_FILE), + turn_two=_turn_two(_MESSAGES_FILE, "messages"), parse=_parse_messages, ) SCRIPTS: Final[Mapping[TestModelEndpoint, EndpointScript]] = MappingProxyType( diff --git a/litellm/proxy/health_endpoints/test_model_requests/chat_completions.json b/litellm/proxy/health_endpoints/test_model_requests/chat_completions.json index 9b03f3758ab..a46f54e2162 100644 --- a/litellm/proxy/health_endpoints/test_model_requests/chat_completions.json +++ b/litellm/proxy/health_endpoints/test_model_requests/chat_completions.json @@ -1,42 +1,52 @@ { - "messages": [ - { - "role": "system", - "content": "You are the support assistant for Northwind Logistics. Follow the playbook below exactly.\n\nSection 1: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 2: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 3: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 4: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 5: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 6: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 7: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 8: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 9: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 10: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 11: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 12: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 13: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 14: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 15: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 16: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 17: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 18: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 19: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 20: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 21: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 22: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 23: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 24: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 25: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 26: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 27: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 28: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 29: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 30: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 31: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 32: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nWhen the user asks about the weather, call the get_weather tool with the city name. After a tool result arrives, answer in one short sentence that repeats the temperature exactly as the tool reported it." - }, - { - "role": "user", - "content": "What is the weather in Boston right now? Use the get_weather tool to find out." - } - ], - "tools": [ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the current weather for a city", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City name, for example Boston" + "turn_1": { + "messages": [ + { + "role": "system", + "content": "You are the support assistant for Northwind Logistics. Follow the playbook below exactly.\n\nSection 1: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 2: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 3: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 4: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 5: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 6: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 7: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 8: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 9: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 10: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 11: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 12: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 13: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 14: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 15: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 16: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 17: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 18: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 19: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 20: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 21: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 22: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 23: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 24: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 25: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 26: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 27: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 28: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 29: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 30: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 31: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 32: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nWhen the user asks about the weather, call the get_weather tool with the city name. After a tool result arrives, answer in one short sentence that repeats the temperature exactly as the tool reported it." + }, + { + "role": "user", + "content": "What is the weather in Boston right now? Use the get_weather tool to find out." + } + ], + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather for a city", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "City name, for example Boston" + }, + "unit": { + "type": "string", + "enum": [ + "celsius", + "fahrenheit" + ] + } }, - "unit": { - "type": "string", - "enum": [ - "celsius", - "fahrenheit" - ] - } - }, - "required": [ - "location" - ] + "required": [ + "location" + ] + } } } + ], + "reasoning_effort": "low", + "max_tokens": 2048 + }, + "turn_2_appended_messages": [ + "", + { + "role": "tool", + "tool_call_id": "", + "content": "72F and sunny" } - ], - "reasoning_effort": "low", - "max_tokens": 2048 + ] } diff --git a/litellm/proxy/health_endpoints/test_model_requests/messages.json b/litellm/proxy/health_endpoints/test_model_requests/messages.json index 514a8226caf..5fe23fb3b45 100644 --- a/litellm/proxy/health_endpoints/test_model_requests/messages.json +++ b/litellm/proxy/health_endpoints/test_model_requests/messages.json @@ -1,39 +1,57 @@ { - "system": "You are the support assistant for Northwind Logistics. Follow the playbook below exactly.\n\nSection 1: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 2: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 3: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 4: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 5: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 6: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 7: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 8: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 9: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 10: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 11: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 12: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 13: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 14: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 15: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 16: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 17: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 18: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 19: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 20: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 21: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 22: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 23: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 24: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 25: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 26: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 27: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 28: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 29: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 30: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 31: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 32: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nWhen the user asks about the weather, call the get_weather tool with the city name. After a tool result arrives, answer in one short sentence that repeats the temperature exactly as the tool reported it.", - "messages": [ + "turn_1": { + "system": "You are the support assistant for Northwind Logistics. Follow the playbook below exactly.\n\nSection 1: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 2: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 3: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 4: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 5: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 6: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 7: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 8: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 9: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 10: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 11: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 12: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 13: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 14: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 15: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 16: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 17: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 18: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 19: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 20: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 21: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 22: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 23: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 24: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 25: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 26: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 27: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 28: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 29: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 30: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 31: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 32: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nWhen the user asks about the weather, call the get_weather tool with the city name. After a tool result arrives, answer in one short sentence that repeats the temperature exactly as the tool reported it.", + "messages": [ + { + "role": "user", + "content": "What is the weather in Boston right now? Use the get_weather tool to find out." + } + ], + "tools": [ + { + "name": "get_weather", + "description": "Get the current weather for a city", + "input_schema": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "City name, for example Boston" + }, + "unit": { + "type": "string", + "enum": [ + "celsius", + "fahrenheit" + ] + } + }, + "required": [ + "location" + ] + } + } + ], + "thinking": { + "type": "enabled", + "budget_tokens": 1024 + }, + "max_tokens": 2048 + }, + "turn_2_appended_messages": [ + { + "role": "assistant", + "content": "" + }, { "role": "user", - "content": "What is the weather in Boston right now? Use the get_weather tool to find out." + "content": [ + { + "type": "tool_result", + "tool_use_id": "", + "content": "72F and sunny" + } + ] } - ], - "tools": [ - { - "name": "get_weather", - "description": "Get the current weather for a city", - "input_schema": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City name, for example Boston" - }, - "unit": { - "type": "string", - "enum": [ - "celsius", - "fahrenheit" - ] - } - }, - "required": [ - "location" - ] - } - } - ], - "thinking": { - "type": "enabled", - "budget_tokens": 1024 - }, - "max_tokens": 2048 + ] } diff --git a/litellm/proxy/health_endpoints/test_model_requests/responses.json b/litellm/proxy/health_endpoints/test_model_requests/responses.json index f394f7a224d..4847990bb9b 100644 --- a/litellm/proxy/health_endpoints/test_model_requests/responses.json +++ b/litellm/proxy/health_endpoints/test_model_requests/responses.json @@ -1,39 +1,49 @@ { - "instructions": "You are the support assistant for Northwind Logistics. Follow the playbook below exactly.\n\nSection 1: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 2: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 3: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 4: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 5: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 6: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 7: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 8: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 9: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 10: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 11: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 12: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 13: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 14: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 15: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 16: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 17: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 18: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 19: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 20: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 21: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 22: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 23: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 24: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 25: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 26: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 27: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 28: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 29: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 30: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 31: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 32: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nWhen the user asks about the weather, call the get_weather tool with the city name. After a tool result arrives, answer in one short sentence that repeats the temperature exactly as the tool reported it.", - "input": [ - { - "role": "user", - "content": "What is the weather in Boston right now? Use the get_weather tool to find out." - } - ], - "tools": [ - { - "type": "function", - "name": "get_weather", - "description": "Get the current weather for a city", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City name, for example Boston" - }, - "unit": { - "type": "string", - "enum": [ - "celsius", - "fahrenheit" - ] - } - }, - "required": [ - "location" - ] + "turn_1": { + "instructions": "You are the support assistant for Northwind Logistics. Follow the playbook below exactly.\n\nSection 1: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 2: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 3: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 4: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 5: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 6: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 7: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 8: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 9: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 10: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 11: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 12: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 13: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 14: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 15: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 16: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 17: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 18: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 19: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 20: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 21: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 22: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 23: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 24: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 25: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 26: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 27: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 28: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 29: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 30: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 31: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 32: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nWhen the user asks about the weather, call the get_weather tool with the city name. After a tool result arrives, answer in one short sentence that repeats the temperature exactly as the tool reported it.", + "input": [ + { + "role": "user", + "content": "What is the weather in Boston right now? Use the get_weather tool to find out." } - } - ], - "reasoning": { - "effort": "low" + ], + "tools": [ + { + "type": "function", + "name": "get_weather", + "description": "Get the current weather for a city", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "City name, for example Boston" + }, + "unit": { + "type": "string", + "enum": [ + "celsius", + "fahrenheit" + ] + } + }, + "required": [ + "location" + ] + } + } + ], + "reasoning": { + "effort": "low" + }, + "max_output_tokens": 2048 }, - "max_output_tokens": 2048 + "turn_2_appended_messages": [ + "", + { + "type": "function_call_output", + "call_id": "", + "output": "72F and sunny" + } + ] } diff --git a/tests/integration/management/test_model_health_check.py b/tests/integration/management/test_model_health_check.py index 8cb2e3016da..d80dd1eaa15 100644 --- a/tests/integration/management/test_model_health_check.py +++ b/tests/integration/management/test_model_health_check.py @@ -1440,8 +1440,8 @@ def test_health_check_of_gpt_transcribe_probes_without_verbose_json_and_reports_ _TEST_MODEL_TOOL_CALL_ID: Final = "call_boston_wire" -_TEST_MODEL_CHAT_REQUEST: Final = JSON_OBJECT.validate_json( - (TEST_MODEL_REQUESTS_DIR / "chat_completions.json").read_text(encoding="utf-8") +_TEST_MODEL_CHAT_REQUEST: Final = JSON_OBJECT.validate_python( + JSON_OBJECT.validate_json((TEST_MODEL_REQUESTS_DIR / "chat_completions.json").read_text(encoding="utf-8"))["turn_1"] ) diff --git a/tests/unit/proxy/health_endpoints/test_model_checks.py b/tests/unit/proxy/health_endpoints/test_model_checks.py index 9ae29833f87..8f79a82d441 100644 --- a/tests/unit/proxy/health_endpoints/test_model_checks.py +++ b/tests/unit/proxy/health_endpoints/test_model_checks.py @@ -210,6 +210,7 @@ def test_every_endpoint_passes_all_three_checks_and_feeds_turn_one_back_in_its_o sent_json: Final = json.dumps(turn_two) assert _TOOL_CALL_ID in sent_json and WEATHER_TOOL_RESULT in sent_json, turn_two assert "cache_control" not in sent_json and "cachePoint" not in sent_json, turn_two + assert "" in appended and "" in appended, name assert chat["messages"][0]["content"] == responses["instructions"] == messages["system"] assert chat["tools"][0]["function"]["parameters"] == responses["tools"][0]["parameters"] assert chat["tools"][0]["function"]["parameters"] == messages["tools"][0]["input_schema"] From 46b3deadf98c982a340bd2bdd33a4d24ec2dbd97 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 31 Oct 2026 16:38:40 +0000 Subject: [PATCH 04/15] refactor(proxy): fill test_model placeholders without recursion --- litellm/proxy/health_endpoints/model_checks.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/litellm/proxy/health_endpoints/model_checks.py b/litellm/proxy/health_endpoints/model_checks.py index d8dc347302b..c05fb12d7cd 100644 --- a/litellm/proxy/health_endpoints/model_checks.py +++ b/litellm/proxy/health_endpoints/model_checks.py @@ -216,16 +216,14 @@ _RESPONSES_FILE: Final = _request_file("responses") _MESSAGES_FILE: Final = _request_file("messages") -def _fill(value: object, turn_one: ParsedTurn) -> object: - if value == TURN_ONE_TOOL_CALL_ID: - return turn_one.tool_call.id if turn_one.tool_call is not None else "" - if value == TURN_ONE_ASSISTANT_OUTPUT: - return turn_one.assistant_echo - if isinstance(value, dict): - return {key: _fill(item, turn_one) for key, item in _PLAIN_OBJECT.validate_python(value).items()} - if isinstance(value, list): - return [_fill(item, turn_one) for item in _OBJECT_LIST.validate_python(value)] - return value +def _fill(item: object, turn_one: ParsedTurn) -> object: + tool_call_id: Final = turn_one.tool_call.id if turn_one.tool_call is not None else "" + text: Final = ( + json.dumps(item) + .replace(json.dumps(TURN_ONE_TOOL_CALL_ID), json.dumps(tool_call_id)) + .replace(json.dumps(TURN_ONE_ASSISTANT_OUTPUT), json.dumps(turn_one.assistant_echo)) + ) + return _JSON_DOCUMENT.validate_json(text) def _appended_messages(item: object, turn_one: ParsedTurn) -> Sequence[object]: From e0cadc06236a54e7099875d86d1895655a5a768c Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 31 Oct 2026 17:16:52 +0000 Subject: [PATCH 05/15] feat(proxy): add a third user follow-up turn to test_model conversations --- .../health_endpoints/_health_endpoints.py | 9 +- .../proxy/health_endpoints/model_checks.py | 263 ++++++++++++------ .../test_model_requests/chat_completions.json | 7 + .../test_model_requests/messages.json | 10 + .../test_model_requests/responses.json | 7 + .../management/test_model_health_check.py | 21 +- .../health_endpoints/test_model_checks.py | 180 +++++++++--- 7 files changed, 351 insertions(+), 146 deletions(-) diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index 82709d64307..fdc41caf6d4 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -2780,11 +2780,12 @@ async def test_model( """ Run the live checks on a model, the way a client would use it, and report what works. - Each selected endpoint gets one two-turn conversation with a long system prompt and a tool. Turn 1 asks - a question the tool answers, turn 2 sends the tool result back. Three checks come out of it: + Each selected endpoint gets one three-turn conversation with a long system prompt and a tool. Turn 1 asks + a question the tool answers, turn 2 sends the tool result back, turn 3 asks a follow-up that needs the + answer from turn 2. Three checks come out of it: - - `conversation`: both turns parse, turn 1 calls the tool, turn 2 uses the tool result - - `prompt_caching`: turn 2 reports cached input tokens, with no cache markers in the request + - `conversation`: every turn parses, turn 1 calls the tool, turn 2 uses the tool result, turn 3 carries it on + - `prompt_caching`: turns 2 and 3 report cached input tokens, with no cache markers in the request - `cost_tracking`: every turn carries a non-zero response cost, and the spend log row lands when a database is connected diff --git a/litellm/proxy/health_endpoints/model_checks.py b/litellm/proxy/health_endpoints/model_checks.py index c05fb12d7cd..c85c81fc29f 100644 --- a/litellm/proxy/health_endpoints/model_checks.py +++ b/litellm/proxy/health_endpoints/model_checks.py @@ -1,12 +1,14 @@ """Live checks behind ``POST /health/test_model``. -One two-turn tool-call conversation per selected endpoint, read from that endpoint's file in +One three-turn tool-call conversation per selected endpoint, read from that endpoint's file in ``test_model_requests/``. ``turn_1`` is the full first request body in the endpoint's native shape with no -prompt-cache markers, so the second turn can only report cached tokens when the provider caches the repeated prefix -on its own. ``turn_2_appended_messages`` lists what the second request adds after turn 1's messages, with -```` and ```` filled in from the model's reply. -Three verdicts come out of the two turns: the conversation itself (both turns parse, the tool call lands, the tool -result is used), prompt caching, and cost tracking. +prompt-cache markers, so later turns can only report cached tokens when the provider caches the repeated prefix on +its own. ``turn_2_appended_messages`` lists what the second request adds after turn 1's messages (the model's tool +call echoed back plus the tool result), ``turn_3_appended_messages`` what the third adds after that (the model's +answer echoed back plus a user follow-up), with ````, +```` and ```` filled in from the model's replies. +Three verdicts come out of the three turns: the conversation itself (every turn parses, the tool call lands, the +tool result is used, the follow-up is answered from the earlier turn), prompt caching, and cost tracking. """ from __future__ import annotations @@ -15,6 +17,7 @@ import json import time from collections.abc import Awaitable, Callable, Mapping, Sequence from dataclasses import dataclass +from functools import partial, reduce from itertools import chain from pathlib import Path from types import MappingProxyType @@ -35,6 +38,7 @@ WEATHER_TOOL_RESULT: Final = "72F and sunny" TEST_MODEL_REQUESTS_DIR: Final = Path(__file__).resolve().parent / "test_model_requests" TURN_ONE_ASSISTANT_OUTPUT: Final = "" TURN_ONE_TOOL_CALL_ID: Final = "" +TURN_TWO_ASSISTANT_OUTPUT: Final = "" @dataclass(frozen=True, slots=True) @@ -61,6 +65,7 @@ class EndpointScript: metadata_key: Literal["metadata", "litellm_metadata"] turn_one: Callable[[], Mapping[str, object]] turn_two: Callable[[ParsedTurn], Mapping[str, object]] + turn_three: Callable[[ParsedTurn, ParsedTurn], Mapping[str, object]] parse: Callable[[object], ParsedTurn | str] @@ -98,6 +103,16 @@ class TurnOutcome: call_id: str +@dataclass(frozen=True, slots=True) +class RepeatedTurn: + """A turn whose prompt repeats the earlier turns, sent once more when the provider reports no cache hit.""" + + number: Literal[2, 3] + outcome: TurnOutcome | None + parsed: ParsedTurn | str | None + attempts: int + + TurnCaller: TypeAlias = Callable[[EndpointScript, Mapping[str, object], str], Awaitable[CalledTurn]] SpendLookup: TypeAlias = Callable[[Sequence[str]], Awaitable[Mapping[str, float]]] CallIdFactory: TypeAlias = Callable[[], str] @@ -197,13 +212,15 @@ _EMPTY_ASSISTANT: Final[Mapping[str, object]] = MappingProxyType({"role": "assis class RequestFile(BaseModel): - """One two-turn conversation on disk: ``turn_1`` is the full first request body, ``turn_2_appended_messages`` - are the messages the second request adds after it, with the two placeholders filled from turn 1's response.""" + """One three-turn conversation on disk: ``turn_1`` is the full first request body, ``turn_2_appended_messages`` + the messages the second request adds after it, ``turn_3_appended_messages`` the ones the third adds after those, + with the placeholders filled from the earlier responses.""" model_config = ConfigDict(frozen=True) turn_1: Mapping[str, object] turn_2_appended_messages: Sequence[object] + turn_3_appended_messages: Sequence[object] def _request_file(name: str) -> Callable[[], RequestFile]: @@ -216,36 +233,65 @@ _RESPONSES_FILE: Final = _request_file("responses") _MESSAGES_FILE: Final = _request_file("messages") -def _fill(item: object, turn_one: ParsedTurn) -> object: - tool_call_id: Final = turn_one.tool_call.id if turn_one.tool_call is not None else "" - text: Final = ( - json.dumps(item) - .replace(json.dumps(TURN_ONE_TOOL_CALL_ID), json.dumps(tool_call_id)) - .replace(json.dumps(TURN_ONE_ASSISTANT_OUTPUT), json.dumps(turn_one.assistant_echo)) - ) - return _JSON_DOCUMENT.validate_json(text) +Placeholders: TypeAlias = Mapping[str, object] +ConversationKey: TypeAlias = Literal["messages", "input"] -def _appended_messages(item: object, turn_one: ParsedTurn) -> Sequence[object]: - echo: Final = turn_one.assistant_echo - if item == TURN_ONE_ASSISTANT_OUTPUT and isinstance(echo, list): +def _replace_placeholder(text: str, placeholder: tuple[str, object]) -> str: + return text.replace(json.dumps(placeholder[0]), json.dumps(placeholder[1])) + + +def _fill(item: object, placeholders: Placeholders) -> object: + return _JSON_DOCUMENT.validate_json(reduce(_replace_placeholder, placeholders.items(), json.dumps(item))) + + +def _appended_messages(item: object, placeholders: Placeholders) -> Sequence[object]: + echo: Final = placeholders.get(item) if isinstance(item, str) else None + if isinstance(echo, list): return _OBJECT_LIST.validate_python(echo) - return (_fill(item, turn_one),) + return (_fill(item, placeholders),) + + +def _extended( + body: Mapping[str, object], key: ConversationKey, items: Sequence[object], placeholders: Placeholders +) -> Mapping[str, object]: + appended: Final = chain.from_iterable(_appended_messages(item, placeholders) for item in items) + return {**body, key: [*_message_list(body.get(key)), *appended]} + + +def _turn_one_placeholders(turn_one: ParsedTurn) -> Placeholders: + return MappingProxyType( + { + TURN_ONE_TOOL_CALL_ID: turn_one.tool_call.id if turn_one.tool_call is not None else "", + TURN_ONE_ASSISTANT_OUTPUT: turn_one.assistant_echo, + } + ) def _turn_one(file: Callable[[], RequestFile]) -> Callable[[], Mapping[str, object]]: return lambda: file().turn_1 -def _turn_two( - file: Callable[[], RequestFile], conversation_key: Literal["messages", "input"] -) -> Callable[[ParsedTurn], Mapping[str, object]]: +def _turn_two(file: Callable[[], RequestFile], key: ConversationKey) -> Callable[[ParsedTurn], Mapping[str, object]]: def build(turn_one: ParsedTurn) -> Mapping[str, object]: first: Final = file() - appended: Final = chain.from_iterable( - _appended_messages(item, turn_one) for item in first.turn_2_appended_messages + return _extended(first.turn_1, key, first.turn_2_appended_messages, _turn_one_placeholders(turn_one)) + + return build + + +def _turn_three( + file: Callable[[], RequestFile], key: ConversationKey +) -> Callable[[ParsedTurn, ParsedTurn], Mapping[str, object]]: + def build(turn_one: ParsedTurn, turn_two: ParsedTurn) -> Mapping[str, object]: + first: Final = file() + second: Final = _turn_two(file, key)(turn_one) + return _extended( + second, + key, + first.turn_3_appended_messages, + MappingProxyType({TURN_TWO_ASSISTANT_OUTPUT: turn_two.assistant_echo}), ) - return {**first.turn_1, conversation_key: [*_message_list(first.turn_1.get(conversation_key)), *appended]} return build @@ -387,6 +433,7 @@ CHAT_COMPLETIONS_SCRIPT: Final = EndpointScript( metadata_key="metadata", turn_one=_turn_one(_CHAT_FILE), turn_two=_turn_two(_CHAT_FILE, "messages"), + turn_three=_turn_three(_CHAT_FILE, "messages"), parse=_parse_chat, ) RESPONSES_SCRIPT: Final = EndpointScript( @@ -395,6 +442,7 @@ RESPONSES_SCRIPT: Final = EndpointScript( metadata_key="litellm_metadata", turn_one=_turn_one(_RESPONSES_FILE), turn_two=_turn_two(_RESPONSES_FILE, "input"), + turn_three=_turn_three(_RESPONSES_FILE, "input"), parse=_parse_responses, ) MESSAGES_SCRIPT: Final = EndpointScript( @@ -403,6 +451,7 @@ MESSAGES_SCRIPT: Final = EndpointScript( metadata_key="litellm_metadata", turn_one=_turn_one(_MESSAGES_FILE), turn_two=_turn_two(_MESSAGES_FILE, "messages"), + turn_three=_turn_three(_MESSAGES_FILE, "messages"), parse=_parse_messages, ) SCRIPTS: Final[Mapping[TestModelEndpoint, EndpointScript]] = MappingProxyType( @@ -426,11 +475,23 @@ async def _turn(script: EndpointScript, body: Mapping[str, object], call: TurnCa return TurnOutcome(response=called.response, cost=called.cost, error=None, call_id=call_id) +def _answer_failure(turn: RepeatedTurn, purpose: str, expectation: str) -> CheckResult | None: + if turn.outcome is None or turn.outcome.error is not None: + reason: Final = turn.outcome.error if turn.outcome is not None else "not sent" + return CheckResult("conversation", "fail", f"turn {turn.number} request with {purpose} failed: {reason}") + if isinstance(turn.parsed, str): + return CheckResult("conversation", "fail", f"turn {turn.number} {turn.parsed}") + if turn.parsed is None or "72" not in turn.parsed.text: + answer: Final = turn.parsed.text[:160] if turn.parsed is not None else "" + return CheckResult("conversation", "fail", f"turn {turn.number} {expectation}: {answer!r}") + return None + + def _conversation_check( turn_one: TurnOutcome, parsed_one: ParsedTurn | str | None, - turn_two: TurnOutcome | None, - parsed_two: ParsedTurn | str | None, + second: RepeatedTurn, + third: RepeatedTurn, ) -> CheckResult: if turn_one.error is not None: return CheckResult("conversation", "fail", f"turn 1 request failed: {turn_one.error}") @@ -448,27 +509,28 @@ def _conversation_check( ) if "location" not in call.arguments: return CheckResult("conversation", "fail", f"turn 1 tool arguments lack 'location': {call.raw_arguments!r}") - if turn_two is None or turn_two.error is not None: - reason: Final = turn_two.error if turn_two is not None else "not sent" - return CheckResult("conversation", "fail", f"turn 2 request with the tool result failed: {reason}") - if isinstance(parsed_two, str): - return CheckResult("conversation", "fail", f"turn 2 {parsed_two}") - if parsed_two is None or "72" not in parsed_two.text: - answer: Final = parsed_two.text[:160] if parsed_two is not None else "" - return CheckResult( - "conversation", "fail", f"turn 2 did not use the tool result {WEATHER_TOOL_RESULT!r}: {answer!r}" - ) - return CheckResult( - "conversation", - "pass", - f"turn 1 called {WEATHER_TOOL_NAME}({call.raw_arguments}), turn 2 answered {parsed_two.text[:120]!r}", + second_failure: Final = _answer_failure( + second, "the tool result", f"did not use the tool result {WEATHER_TOOL_RESULT!r}" ) + if second_failure is not None: + return second_failure + third_failure: Final = _answer_failure( + third, "the follow-up question", "did not carry the temperature from turn 2 into the follow-up answer" + ) + if third_failure is not None: + return third_failure + answers: Final = ", ".join( + f"turn {turn.number} answered {turn.parsed.text[:120]!r}" + for turn in (second, third) + if isinstance(turn.parsed, ParsedTurn) + ) + return CheckResult("conversation", "pass", f"turn 1 called {WEATHER_TOOL_NAME}({call.raw_arguments}), {answers}") -def _cache_percent(parsed_two: ParsedTurn | None) -> float | None: - if parsed_two is None or parsed_two.input_tokens <= 0: +def _cache_percent(parsed: ParsedTurn | None) -> float | None: + if parsed is None or parsed.input_tokens <= 0: return None - return round(100.0 * parsed_two.cache_read_tokens / parsed_two.input_tokens, 1) + return round(100.0 * parsed.cache_read_tokens / parsed.input_tokens, 1) def _missed_cache(parsed: ParsedTurn | str | None) -> bool: @@ -479,24 +541,32 @@ def _hit_cache(parsed: ParsedTurn | str | None) -> bool: return isinstance(parsed, ParsedTurn) and parsed.cache_read_tokens > 0 -def _prompt_caching_check(parsed_two: ParsedTurn | None, attempts: int) -> CheckResult: - if parsed_two is None: +def _parsed(turn: RepeatedTurn) -> ParsedTurn | None: + return turn.parsed if isinstance(turn.parsed, ParsedTurn) else None + + +def _cache_read_note(turn: RepeatedTurn, parsed: ParsedTurn) -> str: + attempt_note: Final = " on the second attempt" if turn.attempts > 1 else "" + return ( + f"turn {turn.number} read {parsed.cache_read_tokens} of {parsed.input_tokens} input tokens from cache " + f"({_cache_percent(parsed)}%){attempt_note}" + ) + + +def _prompt_caching_check(second: RepeatedTurn, third: RepeatedTurn) -> CheckResult: + reached: Final = tuple((turn, parsed) for turn in (second, third) if (parsed := _parsed(turn)) is not None) + if not reached: return CheckResult("prompt_caching", "fail", "not checked, the conversation did not reach turn 2") - percent: Final = _cache_percent(parsed_two) - attempt_note: Final = " on the second attempt" if attempts > 1 else "" - if parsed_two.cache_read_tokens > 0: + missed: Final = next(((turn, parsed) for turn, parsed in reached if parsed.cache_read_tokens == 0), None) + if missed is not None: + turn, parsed = missed return CheckResult( "prompt_caching", - "pass", - f"turn 2 read {parsed_two.cache_read_tokens} of {parsed_two.input_tokens} input tokens from cache " - f"({percent}%){attempt_note}", + "fail", + f"turn {turn.number} reported 0 cached tokens on {parsed.input_tokens} input tokens in {turn.attempts} " + "attempts, the provider did not reuse the repeated system prompt", ) - return CheckResult( - "prompt_caching", - "fail", - f"turn 2 reported 0 cached tokens on {parsed_two.input_tokens} input tokens in {attempts} attempts, the " - "provider did not reuse the repeated system prompt", - ) + return CheckResult("prompt_caching", "pass", ", ".join(_cache_read_note(turn, parsed) for turn, parsed in reached)) def _cost_verdict(turn_number: int, cost: float | None) -> CheckResult | None: @@ -514,10 +584,7 @@ def _cost_verdict(turn_number: int, cost: float | None) -> CheckResult | None: return None -def _cost_tracking_check( - turn_one: TurnOutcome, turn_two: TurnOutcome | None, spend_rows: Mapping[str, float] | None -) -> CheckResult: - turns: Final = tuple(turn for turn in (turn_one, turn_two) if turn is not None and turn.response is not None) +def _cost_tracking_check(turns: Sequence[TurnOutcome], spend_rows: Mapping[str, float] | None) -> CheckResult: if not turns: return CheckResult("cost_tracking", "fail", "not checked, no turn returned a response") costs: Final = tuple(turn.cost for turn in turns) @@ -555,6 +622,26 @@ def overall_status(results: Sequence[EndpointResult]) -> EndpointStatus: return "warning" if "warning" in statuses else "success" +async def _repeated_turn( + script: EndpointScript, + number: Literal[2, 3], + build: Callable[[], Mapping[str, object]] | None, + call: TurnCaller, + new_call_id: CallIdFactory, +) -> RepeatedTurn: + if build is None: + return RepeatedTurn(number=number, outcome=None, parsed=None, attempts=0) + first: Final = await _turn(script, build(), call, new_call_id()) + first_parsed: Final = script.parse(first.response) if first.response is not None else None + if not _missed_cache(first_parsed): + return RepeatedTurn(number=number, outcome=first, parsed=first_parsed, attempts=1) + retry: Final = await _turn(script, build(), call, new_call_id()) + retry_parsed: Final = script.parse(retry.response) if retry.response is not None else None + if _hit_cache(retry_parsed): + return RepeatedTurn(number=number, outcome=retry, parsed=retry_parsed, attempts=2) + return RepeatedTurn(number=number, outcome=first, parsed=first_parsed, attempts=2) + + async def run_endpoint_check( script: EndpointScript, call: TurnCaller, @@ -565,46 +652,42 @@ async def run_endpoint_check( started: Final = time.monotonic() turn_one: Final = await _turn(script, script.turn_one(), call, new_call_id()) parsed_one: Final = script.parse(turn_one.response) if turn_one.response is not None else None - ready_for_turn_two: Final = isinstance(parsed_one, ParsedTurn) and parsed_one.tool_call is not None - turn_two: Final = ( - await _turn(script, script.turn_two(parsed_one), call, new_call_id()) - if ready_for_turn_two and isinstance(parsed_one, ParsedTurn) - else None + second: Final = await _repeated_turn( + script, + 2, + partial(script.turn_two, parsed_one) + if isinstance(parsed_one, ParsedTurn) and parsed_one.tool_call is not None + else None, + call, + new_call_id, ) - first_parsed_two: Final = ( - script.parse(turn_two.response) if turn_two is not None and turn_two.response is not None else None + parsed_two: Final = _parsed(second) + third: Final = await _repeated_turn( + script, + 3, + partial(script.turn_three, parsed_one, parsed_two) + if isinstance(parsed_one, ParsedTurn) and parsed_two is not None + else None, + call, + new_call_id, ) - retried_turn_two: Final = ( - await _turn(script, script.turn_two(parsed_one), call, new_call_id()) - if isinstance(parsed_one, ParsedTurn) and _missed_cache(first_parsed_two) - else None - ) - retried_parsed_two: Final = ( - script.parse(retried_turn_two.response) - if retried_turn_two is not None and retried_turn_two.response is not None - else None - ) - turn_two_used: Final = retried_turn_two if _hit_cache(retried_parsed_two) else turn_two - parsed_two: Final = retried_parsed_two if _hit_cache(retried_parsed_two) else first_parsed_two - cache_attempts: Final = 1 if retried_turn_two is None else 2 answered: Final = tuple( - turn for turn in (turn_one, turn_two_used) if turn is not None and turn.response is not None + turn for turn in (turn_one, second.outcome, third.outcome) if turn is not None and turn.response is not None ) spend_rows: Final = ( await spend_lookup(tuple(turn.call_id for turn in answered)) if spend_lookup is not None and answered else None ) - second: Final = parsed_two if isinstance(parsed_two, ParsedTurn) else None checks: Final = ( - _conversation_check(turn_one, parsed_one, turn_two_used, parsed_two), - _prompt_caching_check(second, cache_attempts), - _cost_tracking_check(turn_one, turn_two_used, spend_rows), + _conversation_check(turn_one, parsed_one, second, third), + _prompt_caching_check(second, third), + _cost_tracking_check(answered, spend_rows), ) return EndpointResult( endpoint=script.endpoint, status=_status(checks), checks=checks, - cache_read_tokens=second.cache_read_tokens if second is not None else 0, - cache_read_percent=_cache_percent(second), + cache_read_tokens=parsed_two.cache_read_tokens if parsed_two is not None else 0, + cache_read_percent=_cache_percent(parsed_two), spend_usd=sum(turn.cost for turn in answered if turn.cost is not None), duration_ms=int((time.monotonic() - started) * 1000), ) diff --git a/litellm/proxy/health_endpoints/test_model_requests/chat_completions.json b/litellm/proxy/health_endpoints/test_model_requests/chat_completions.json index a46f54e2162..c103c60daaf 100644 --- a/litellm/proxy/health_endpoints/test_model_requests/chat_completions.json +++ b/litellm/proxy/health_endpoints/test_model_requests/chat_completions.json @@ -48,5 +48,12 @@ "tool_call_id": "", "content": "72F and sunny" } + ], + "turn_3_appended_messages": [ + "", + { + "role": "user", + "content": "Thanks. Is that warm enough for a short walk? Repeat the temperature you found in your answer." + } ] } diff --git a/litellm/proxy/health_endpoints/test_model_requests/messages.json b/litellm/proxy/health_endpoints/test_model_requests/messages.json index 5fe23fb3b45..96d381c7162 100644 --- a/litellm/proxy/health_endpoints/test_model_requests/messages.json +++ b/litellm/proxy/health_endpoints/test_model_requests/messages.json @@ -53,5 +53,15 @@ } ] } + ], + "turn_3_appended_messages": [ + { + "role": "assistant", + "content": "" + }, + { + "role": "user", + "content": "Thanks. Is that warm enough for a short walk? Repeat the temperature you found in your answer." + } ] } diff --git a/litellm/proxy/health_endpoints/test_model_requests/responses.json b/litellm/proxy/health_endpoints/test_model_requests/responses.json index 4847990bb9b..b8f8e03fd68 100644 --- a/litellm/proxy/health_endpoints/test_model_requests/responses.json +++ b/litellm/proxy/health_endpoints/test_model_requests/responses.json @@ -45,5 +45,12 @@ "call_id": "", "output": "72F and sunny" } + ], + "turn_3_appended_messages": [ + "", + { + "role": "user", + "content": "Thanks. Is that warm enough for a short walk? Repeat the temperature you found in your answer." + } ] } diff --git a/tests/integration/management/test_model_health_check.py b/tests/integration/management/test_model_health_check.py index d80dd1eaa15..3ab16ed82f0 100644 --- a/tests/integration/management/test_model_health_check.py +++ b/tests/integration/management/test_model_health_check.py @@ -1454,10 +1454,13 @@ def _test_model_chat_reply(request: Request, *, cached_tokens: int) -> Reply: tool_turn: Final = any(isinstance(message, dict) and message.get("role") == "tool" for message in messages) assert messages[: len(_TEST_MODEL_CHAT_REQUEST["messages"])] == _TEST_MODEL_CHAT_REQUEST["messages"], messages assert body["tools"] == _TEST_MODEL_CHAT_REQUEST["tools"], body["tools"] - if tool_turn: - assert isinstance(messages[-1], dict), messages + assert isinstance(messages[-1], dict), messages + if tool_turn and messages[-1]["role"] == "user": + assert messages[-2] == {"role": "assistant", "content": "Boston is 72F and sunny."}, messages[-2] + message: dict[str, JsonValue] = {"role": "assistant", "content": "Yes, 72F is warm enough for a walk."} + elif tool_turn: assert messages[-1]["tool_call_id"] == _TEST_MODEL_TOOL_CALL_ID, messages[-1] - message: dict[str, JsonValue] = {"role": "assistant", "content": "Boston is 72F and sunny."} + message = {"role": "assistant", "content": "Boston is 72F and sunny."} else: message = { "role": "assistant", @@ -1526,21 +1529,23 @@ def test_health_test_model_runs_the_tool_conversation_against_a_saved_deployment assert endpoint_result["endpoint"] == "/v1/chat/completions", response.text checks: Final = _check_statuses(endpoint_result) assert checks["conversation"][0] == "pass", response.text - assert checks["prompt_caching"] == ("pass", "turn 2 read 4800 of 5000 input tokens from cache (96.0%)"), ( - response.text - ) + assert checks["prompt_caching"] == ( + "pass", + "turn 2 read 4800 of 5000 input tokens from cache (96.0%), " + "turn 3 read 4800 of 5000 input tokens from cache (96.0%)", + ), response.text assert checks["cost_tracking"][0] == "pass", response.text assert "spend log rows written" in checks["cost_tracking"][1], response.text assert endpoint_result["cache_read_tokens"] == 4800, response.text requests: Final = wire.drain() - assert [request.headers["authorization"] for request in requests] == ["Bearer synthetic-test-model-key"] * 2, ( + assert [request.headers["authorization"] for request in requests] == ["Bearer synthetic-test-model-key"] * 3, ( response.text ) rows: Final = gateway.rows( 'SELECT "spend", "request_tags"::text AS tags FROM "LiteLLM_SpendLogs" WHERE "model_id" = %s', (model_id,), ) - assert len(rows) == 2, rows + assert len(rows) == 3, rows assert all(isinstance(row["spend"], int | float) and row["spend"] > 0 for row in rows), rows assert all("litellm_test_model:" in string_value(row["tags"]) for row in rows), rows diff --git a/tests/unit/proxy/health_endpoints/test_model_checks.py b/tests/unit/proxy/health_endpoints/test_model_checks.py index 8f79a82d441..99154dee851 100644 --- a/tests/unit/proxy/health_endpoints/test_model_checks.py +++ b/tests/unit/proxy/health_endpoints/test_model_checks.py @@ -21,6 +21,8 @@ from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.utils import ModelResponse _TOOL_CALL_ID: Final = "call_boston" +_TURN_TWO_ANSWER: Final = "Boston is 72F and sunny." +_TURN_THREE_ANSWER: Final = "Yes, 72F is warm enough for a walk." def _chat_reply(*, tool_call: bool, text: str = "", cached_tokens: int = 0, prompt_tokens: int = 5000) -> ModelResponse: @@ -123,29 +125,29 @@ def _messages_reply(*, tool_call: bool, text: str = "", cached_tokens: int = 0) class _ScriptedProvider: - """Answers turn 1 with a tool call and every later turn from ``second_turns``, recording what was sent.""" + """Answers turn 1 from ``first`` and every later request from ``later_turns`` in order, recording what was sent.""" def __init__( self, first: CalledTurn, - second_turns: Sequence[CalledTurn | Exception], + later_turns: Sequence[CalledTurn | Exception], ) -> None: self.first = first - self.second_turns = list(second_turns) + self.later_turns = list(later_turns) self.sent: list[Mapping[str, object]] = [] async def __call__(self, script: EndpointScript, body: Mapping[str, object], call_id: str) -> CalledTurn: self.sent.append(body) if len(self.sent) == 1: return self.first - outcome: Final = self.second_turns.pop(0) + outcome: Final = self.later_turns.pop(0) if isinstance(outcome, Exception): raise outcome return outcome def _ids() -> Sequence[str]: - return ("turn-1", "turn-2", "turn-2-retry") + return ("call-1", "call-2", "call-3", "call-4", "call-5") def _run(script: EndpointScript, provider: _ScriptedProvider, spend_rows: Mapping[str, float] | None = None): @@ -169,52 +171,61 @@ def _checks(result) -> Mapping[str, tuple[str, str]]: @pytest.mark.parametrize( - ("script", "first", "second"), + ("script", "reply", "key"), [ - ( - CHAT_COMPLETIONS_SCRIPT, - _chat_reply(tool_call=True), - _chat_reply(tool_call=False, text="Boston is 72F and sunny.", cached_tokens=4800), - ), - ( - RESPONSES_SCRIPT, - _responses_reply(tool_call=True), - _responses_reply(tool_call=False, text="Boston is 72F and sunny.", cached_tokens=4800), - ), - ( - MESSAGES_SCRIPT, - _messages_reply(tool_call=True), - _messages_reply(tool_call=False, text="Boston is 72F and sunny.", cached_tokens=4800), - ), + (CHAT_COMPLETIONS_SCRIPT, _chat_reply, "messages"), + (RESPONSES_SCRIPT, _responses_reply, "input"), + (MESSAGES_SCRIPT, _messages_reply, "messages"), ], ids=["chat", "responses", "messages"], ) -def test_every_endpoint_passes_all_three_checks_and_feeds_turn_one_back_in_its_own_shape(script, first, second): - provider: Final = _ScriptedProvider(CalledTurn(first, 0.004), [CalledTurn(second, 0.0005)]) +def test_every_endpoint_passes_all_three_checks_and_feeds_each_reply_back_in_its_own_shape(script, reply, key): + provider: Final = _ScriptedProvider( + CalledTurn(reply(tool_call=True), 0.004), + [ + CalledTurn(reply(tool_call=False, text=_TURN_TWO_ANSWER, cached_tokens=4800), 0.0005), + CalledTurn(reply(tool_call=False, text=_TURN_THREE_ANSWER, cached_tokens=4900), 0.0004), + ], + ) - result: Final = _run(script, provider, spend_rows={"turn-1": 0.004, "turn-2": 0.0005}) + result: Final = _run(script, provider, spend_rows={"call-1": 0.004, "call-2": 0.0005, "call-3": 0.0004}) assert result.status == "success", result checks: Final = _checks(result) - assert checks["conversation"][0] == "pass", checks - assert checks["prompt_caching"] == ("pass", "turn 2 read 4800 of 5000 input tokens from cache (96.0%)"), checks + assert checks["conversation"] == ( + "pass", + f'turn 1 called get_weather({{"location": "Boston"}}), turn 2 answered {_TURN_TWO_ANSWER!r}, ' + f"turn 3 answered {_TURN_THREE_ANSWER!r}", + ), checks + assert checks["prompt_caching"] == ( + "pass", + "turn 2 read 4800 of 5000 input tokens from cache (96.0%), " + "turn 3 read 4900 of 5000 input tokens from cache (98.0%)", + ), checks assert checks["cost_tracking"] == ( "pass", - "turn 1 $0.004000, turn 2 $0.000500, spend log rows written ($0.004000, $0.000500)", + "turn 1 $0.004000, turn 2 $0.000500, turn 3 $0.000400, " + "spend log rows written ($0.004000, $0.000500, $0.000400)", ), checks assert result.cache_read_tokens == 4800 assert result.cache_read_percent == 96.0 - assert result.spend_usd == pytest.approx(0.0045) - assert len(provider.sent) == 2 + assert result.spend_usd == pytest.approx(0.0049) + assert len(provider.sent) == 3 turn_two: Final = provider.sent[1] - sent_json: Final = json.dumps(turn_two) - assert _TOOL_CALL_ID in sent_json and WEATHER_TOOL_RESULT in sent_json, turn_two - assert "cache_control" not in sent_json and "cachePoint" not in sent_json, turn_two - assert "" in appended and "" in appended, name + third: Final = json.dumps(file["turn_3_appended_messages"]) + assert "" in third, (name, third) + assert file["turn_3_appended_messages"][-1]["role"] == "user", (name, third) assert chat["messages"][0]["content"] == responses["instructions"] == messages["system"] assert chat["tools"][0]["function"]["parameters"] == responses["tools"][0]["parameters"] assert chat["tools"][0]["function"]["parameters"] == messages["tools"][0]["input_schema"] From 4379d2651b1df8d8925b13ec1fe36fb903459067 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 31 Oct 2026 17:22:10 +0000 Subject: [PATCH 06/15] fix(proxy): name the unmapped-model case in the test_model zero-cost warning --- .../proxy/health_endpoints/model_checks.py | 24 ++++++++++++------- .../health_endpoints/test_model_checks.py | 3 ++- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/litellm/proxy/health_endpoints/model_checks.py b/litellm/proxy/health_endpoints/model_checks.py index c85c81fc29f..62c47ad7ea8 100644 --- a/litellm/proxy/health_endpoints/model_checks.py +++ b/litellm/proxy/health_endpoints/model_checks.py @@ -553,19 +553,22 @@ def _cache_read_note(turn: RepeatedTurn, parsed: ParsedTurn) -> str: ) +def _cache_miss(turn: RepeatedTurn, parsed: ParsedTurn) -> CheckResult: + return CheckResult( + "prompt_caching", + "fail", + f"turn {turn.number} reported 0 cached tokens on {parsed.input_tokens} input tokens in {turn.attempts} " + "attempts, the provider did not reuse the repeated system prompt", + ) + + def _prompt_caching_check(second: RepeatedTurn, third: RepeatedTurn) -> CheckResult: reached: Final = tuple((turn, parsed) for turn in (second, third) if (parsed := _parsed(turn)) is not None) if not reached: return CheckResult("prompt_caching", "fail", "not checked, the conversation did not reach turn 2") - missed: Final = next(((turn, parsed) for turn, parsed in reached if parsed.cache_read_tokens == 0), None) + missed: Final = next((_cache_miss(turn, parsed) for turn, parsed in reached if parsed.cache_read_tokens == 0), None) if missed is not None: - turn, parsed = missed - return CheckResult( - "prompt_caching", - "fail", - f"turn {turn.number} reported 0 cached tokens on {parsed.input_tokens} input tokens in {turn.attempts} " - "attempts, the provider did not reuse the repeated system prompt", - ) + return missed return CheckResult("prompt_caching", "pass", ", ".join(_cache_read_note(turn, parsed) for turn, parsed in reached)) @@ -579,7 +582,10 @@ def _cost_verdict(turn_number: int, cost: float | None) -> CheckResult | None: ) if cost == 0: return CheckResult( - "cost_tracking", "warn", f"turn {turn_number} response_cost is $0, this model is priced at zero" + "cost_tracking", + "warn", + f"turn {turn_number} response_cost is $0, either this model is priced at zero or no price resolved for " + "it (not in the cost map and no custom pricing on the deployment)", ) return None diff --git a/tests/unit/proxy/health_endpoints/test_model_checks.py b/tests/unit/proxy/health_endpoints/test_model_checks.py index 99154dee851..3befe4f71be 100644 --- a/tests/unit/proxy/health_endpoints/test_model_checks.py +++ b/tests/unit/proxy/health_endpoints/test_model_checks.py @@ -412,7 +412,8 @@ def test_a_model_without_a_price_fails_cost_tracking_and_a_zero_priced_model_onl assert unpriced_result.status == "error" assert _checks(free_result)["cost_tracking"] == ( "warn", - "turn 1 response_cost is $0, this model is priced at zero", + "turn 1 response_cost is $0, either this model is priced at zero or no price resolved for it (not in the " + "cost map and no custom pricing on the deployment)", ) assert free_result.status == "warning" assert overall_status([unpriced_result, free_result]) == "error" From ba9bd2fbc8aa4a043024bd264d1f319c3854e0c0 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 31 Oct 2026 17:27:04 +0000 Subject: [PATCH 07/15] chore(ui): regenerate schema.d.ts for the three-turn test_model docstring --- ui/litellm-dashboard/src/lib/http/schema.d.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 22cdf793eeb..59fcbf6644f 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -7980,11 +7980,12 @@ export interface paths { * Test Model * @description Run the live checks on a model, the way a client would use it, and report what works. * - * Each selected endpoint gets one two-turn conversation with a long system prompt and a tool. Turn 1 asks - * a question the tool answers, turn 2 sends the tool result back. Three checks come out of it: + * Each selected endpoint gets one three-turn conversation with a long system prompt and a tool. Turn 1 asks + * a question the tool answers, turn 2 sends the tool result back, turn 3 asks a follow-up that needs the + * answer from turn 2. Three checks come out of it: * - * - `conversation`: both turns parse, turn 1 calls the tool, turn 2 uses the tool result - * - `prompt_caching`: turn 2 reports cached input tokens, with no cache markers in the request + * - `conversation`: every turn parses, turn 1 calls the tool, turn 2 uses the tool result, turn 3 carries it on + * - `prompt_caching`: turns 2 and 3 report cached input tokens, with no cache markers in the request * - `cost_tracking`: every turn carries a non-zero response cost, and the spend log row lands when a * database is connected * From 5a06fec631b4af7c657fe8f67b7ce8da20130b01 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 31 Oct 2026 17:51:55 +0000 Subject: [PATCH 08/15] feat(proxy): ask for a one-sentence preamble before the test_model tool call --- .../health_endpoints/test_model_requests/chat_completions.json | 2 +- .../proxy/health_endpoints/test_model_requests/messages.json | 2 +- .../proxy/health_endpoints/test_model_requests/responses.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/health_endpoints/test_model_requests/chat_completions.json b/litellm/proxy/health_endpoints/test_model_requests/chat_completions.json index c103c60daaf..e6db17be541 100644 --- a/litellm/proxy/health_endpoints/test_model_requests/chat_completions.json +++ b/litellm/proxy/health_endpoints/test_model_requests/chat_completions.json @@ -7,7 +7,7 @@ }, { "role": "user", - "content": "What is the weather in Boston right now? Use the get_weather tool to find out." + "content": "What is the weather in Boston right now? First tell me in one sentence what you are about to do, then use the get_weather tool to find out." } ], "tools": [ diff --git a/litellm/proxy/health_endpoints/test_model_requests/messages.json b/litellm/proxy/health_endpoints/test_model_requests/messages.json index 96d381c7162..117880a2304 100644 --- a/litellm/proxy/health_endpoints/test_model_requests/messages.json +++ b/litellm/proxy/health_endpoints/test_model_requests/messages.json @@ -4,7 +4,7 @@ "messages": [ { "role": "user", - "content": "What is the weather in Boston right now? Use the get_weather tool to find out." + "content": "What is the weather in Boston right now? First tell me in one sentence what you are about to do, then use the get_weather tool to find out." } ], "tools": [ diff --git a/litellm/proxy/health_endpoints/test_model_requests/responses.json b/litellm/proxy/health_endpoints/test_model_requests/responses.json index b8f8e03fd68..a4d69035fd4 100644 --- a/litellm/proxy/health_endpoints/test_model_requests/responses.json +++ b/litellm/proxy/health_endpoints/test_model_requests/responses.json @@ -4,7 +4,7 @@ "input": [ { "role": "user", - "content": "What is the weather in Boston right now? Use the get_weather tool to find out." + "content": "What is the weather in Boston right now? First tell me in one sentence what you are about to do, then use the get_weather tool to find out." } ], "tools": [ From 9d57ff037684087d0f751327bdc3c9ba1413164b Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 31 Oct 2026 18:02:31 +0000 Subject: [PATCH 09/15] feat(proxy): send a mid-conversation system reminder on test_model turn 3 --- .../test_model_requests/chat_completions.json | 4 ++++ .../test_model_requests/messages.json | 11 ++++++++++- .../test_model_requests/responses.json | 4 ++++ .../management/test_model_health_check.py | 3 ++- .../proxy/health_endpoints/test_model_checks.py | 13 ++++++++----- 5 files changed, 28 insertions(+), 7 deletions(-) diff --git a/litellm/proxy/health_endpoints/test_model_requests/chat_completions.json b/litellm/proxy/health_endpoints/test_model_requests/chat_completions.json index e6db17be541..997e67685da 100644 --- a/litellm/proxy/health_endpoints/test_model_requests/chat_completions.json +++ b/litellm/proxy/health_endpoints/test_model_requests/chat_completions.json @@ -51,6 +51,10 @@ ], "turn_3_appended_messages": [ "", + { + "role": "system", + "content": "The user prefers short answers. Keep the temperature in the reply." + }, { "role": "user", "content": "Thanks. Is that warm enough for a short walk? Repeat the temperature you found in your answer." diff --git a/litellm/proxy/health_endpoints/test_model_requests/messages.json b/litellm/proxy/health_endpoints/test_model_requests/messages.json index 117880a2304..845beb7f9ff 100644 --- a/litellm/proxy/health_endpoints/test_model_requests/messages.json +++ b/litellm/proxy/health_endpoints/test_model_requests/messages.json @@ -61,7 +61,16 @@ }, { "role": "user", - "content": "Thanks. Is that warm enough for a short walk? Repeat the temperature you found in your answer." + "content": [ + { + "type": "text", + "text": "The user prefers short answers. Keep the temperature in the reply." + }, + { + "type": "text", + "text": "Thanks. Is that warm enough for a short walk? Repeat the temperature you found in your answer." + } + ] } ] } diff --git a/litellm/proxy/health_endpoints/test_model_requests/responses.json b/litellm/proxy/health_endpoints/test_model_requests/responses.json index a4d69035fd4..79587f044bb 100644 --- a/litellm/proxy/health_endpoints/test_model_requests/responses.json +++ b/litellm/proxy/health_endpoints/test_model_requests/responses.json @@ -48,6 +48,10 @@ ], "turn_3_appended_messages": [ "", + { + "role": "system", + "content": "The user prefers short answers. Keep the temperature in the reply." + }, { "role": "user", "content": "Thanks. Is that warm enough for a short walk? Repeat the temperature you found in your answer." diff --git a/tests/integration/management/test_model_health_check.py b/tests/integration/management/test_model_health_check.py index 3ab16ed82f0..583cb48cf48 100644 --- a/tests/integration/management/test_model_health_check.py +++ b/tests/integration/management/test_model_health_check.py @@ -1456,7 +1456,8 @@ def _test_model_chat_reply(request: Request, *, cached_tokens: int) -> Reply: assert body["tools"] == _TEST_MODEL_CHAT_REQUEST["tools"], body["tools"] assert isinstance(messages[-1], dict), messages if tool_turn and messages[-1]["role"] == "user": - assert messages[-2] == {"role": "assistant", "content": "Boston is 72F and sunny."}, messages[-2] + assert messages[-3] == {"role": "assistant", "content": "Boston is 72F and sunny."}, messages[-3] + assert isinstance(messages[-2], dict) and messages[-2]["role"] == "system", messages[-2] message: dict[str, JsonValue] = {"role": "assistant", "content": "Yes, 72F is warm enough for a walk."} elif tool_turn: assert messages[-1]["tool_call_id"] == _TEST_MODEL_TOOL_CALL_ID, messages[-1] diff --git a/tests/unit/proxy/health_endpoints/test_model_checks.py b/tests/unit/proxy/health_endpoints/test_model_checks.py index 3befe4f71be..3a2f3b03684 100644 --- a/tests/unit/proxy/health_endpoints/test_model_checks.py +++ b/tests/unit/proxy/health_endpoints/test_model_checks.py @@ -219,13 +219,15 @@ def test_every_endpoint_passes_all_three_checks_and_feeds_each_reply_back_in_its assert "" in json.dumps(turn_three[key][-2:]), turn_three[key][-2:] if script is CHAT_COMPLETIONS_SCRIPT: tool_message: Final = turn_two["messages"][-1] assert tool_message == {"role": "tool", "tool_call_id": _TOOL_CALL_ID, "content": WEATHER_TOOL_RESULT} assert turn_two["messages"][-2]["tool_calls"][0]["id"] == _TOOL_CALL_ID - assert turn_three["messages"][-2]["role"] == "assistant", turn_three - assert turn_three["messages"][-2]["content"] == _TURN_TWO_ANSWER, turn_three + assert turn_three["messages"][-2]["role"] == "system", turn_three + assert turn_three["messages"][-3]["role"] == "assistant", turn_three + assert turn_three["messages"][-3]["content"] == _TURN_TWO_ANSWER, turn_three elif script is RESPONSES_SCRIPT: assert turn_two["input"][-1] == { "type": "function_call_output", @@ -233,8 +235,9 @@ def test_every_endpoint_passes_all_three_checks_and_feeds_each_reply_back_in_its "output": WEATHER_TOOL_RESULT, } assert [item["type"] for item in turn_two["input"][1:-1]] == ["reasoning", "function_call"] - assert turn_three["input"][-2]["type"] == "message", turn_three - assert turn_three["input"][-2]["content"][0]["text"] == _TURN_TWO_ANSWER, turn_three + assert turn_three["input"][-2]["role"] == "system", turn_three + assert turn_three["input"][-3]["type"] == "message", turn_three + assert turn_three["input"][-3]["content"][0]["text"] == _TURN_TWO_ANSWER, turn_three else: assert turn_two["messages"][-1]["content"] == [ {"type": "tool_result", "tool_use_id": _TOOL_CALL_ID, "content": WEATHER_TOOL_RESULT} From cadd10067753ef9666095a02b4a5f30b9673d20b Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 31 Oct 2026 18:16:14 +0000 Subject: [PATCH 10/15] feat(proxy): give the test_model messages fixture the Claude Code cache_control and system reminder shape --- .../proxy/health_endpoints/model_checks.py | 10 +++++--- .../test_model_requests/messages.json | 25 +++++++++++-------- .../health_endpoints/test_model_checks.py | 19 +++++++------- 3 files changed, 30 insertions(+), 24 deletions(-) diff --git a/litellm/proxy/health_endpoints/model_checks.py b/litellm/proxy/health_endpoints/model_checks.py index 62c47ad7ea8..21dbe1502d3 100644 --- a/litellm/proxy/health_endpoints/model_checks.py +++ b/litellm/proxy/health_endpoints/model_checks.py @@ -1,11 +1,13 @@ """Live checks behind ``POST /health/test_model``. One three-turn tool-call conversation per selected endpoint, read from that endpoint's file in -``test_model_requests/``. ``turn_1`` is the full first request body in the endpoint's native shape with no -prompt-cache markers, so later turns can only report cached tokens when the provider caches the repeated prefix on -its own. ``turn_2_appended_messages`` lists what the second request adds after turn 1's messages (the model's tool +``test_model_requests/``. ``turn_1`` is the full first request body in the endpoint's native shape. Chat Completions +and Responses carry no prompt-cache markers, so later turns can only report cached tokens when the provider caches +the repeated prefix on its own. Messages mirrors Claude Code: ``cache_control`` on the system prompt and a +mid-conversation ``role: system`` reminder, so a proxy that hoists that reminder into the system prompt shows up as a +cache miss. ``turn_2_appended_messages`` lists what the second request adds after turn 1's messages (the model's tool call echoed back plus the tool result), ``turn_3_appended_messages`` what the third adds after that (the model's -answer echoed back plus a user follow-up), with ````, +answer echoed back, the reminder, a user follow-up), with ````, ```` and ```` filled in from the model's replies. Three verdicts come out of the three turns: the conversation itself (every turn parses, the tool call lands, the tool result is used, the follow-up is answered from the earlier turn), prompt caching, and cost tracking. diff --git a/litellm/proxy/health_endpoints/test_model_requests/messages.json b/litellm/proxy/health_endpoints/test_model_requests/messages.json index 845beb7f9ff..84ff9821268 100644 --- a/litellm/proxy/health_endpoints/test_model_requests/messages.json +++ b/litellm/proxy/health_endpoints/test_model_requests/messages.json @@ -1,6 +1,14 @@ { "turn_1": { - "system": "You are the support assistant for Northwind Logistics. Follow the playbook below exactly.\n\nSection 1: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 2: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 3: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 4: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 5: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 6: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 7: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 8: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 9: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 10: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 11: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 12: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 13: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 14: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 15: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 16: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 17: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 18: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 19: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 20: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 21: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 22: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 23: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 24: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 25: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 26: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 27: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 28: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 29: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 30: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 31: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 32: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nWhen the user asks about the weather, call the get_weather tool with the city name. After a tool result arrives, answer in one short sentence that repeats the temperature exactly as the tool reported it.", + "system": [ + { + "type": "text", + "text": "You are the support assistant for Northwind Logistics. Follow the playbook below exactly.\n\nSection 1: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 2: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 3: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 4: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 5: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 6: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 7: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 8: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 9: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 10: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 11: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 12: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 13: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 14: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 15: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 16: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 17: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 18: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 19: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 20: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 21: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 22: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 23: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 24: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 25: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 26: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 27: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 28: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 29: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 30: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 31: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 32: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nWhen the user asks about the weather, call the get_weather tool with the city name. After a tool result arrives, answer in one short sentence that repeats the temperature exactly as the tool reported it.", + "cache_control": { + "type": "ephemeral" + } + } + ], "messages": [ { "role": "user", @@ -61,16 +69,11 @@ }, { "role": "user", - "content": [ - { - "type": "text", - "text": "The user prefers short answers. Keep the temperature in the reply." - }, - { - "type": "text", - "text": "Thanks. Is that warm enough for a short walk? Repeat the temperature you found in your answer." - } - ] + "content": "Thanks. Is that warm enough for a short walk? Repeat the temperature you found in your answer." + }, + { + "role": "system", + "content": "The user prefers short answers. Keep the temperature in the reply." } ] } diff --git a/tests/unit/proxy/health_endpoints/test_model_checks.py b/tests/unit/proxy/health_endpoints/test_model_checks.py index 3a2f3b03684..7b2815e6e4a 100644 --- a/tests/unit/proxy/health_endpoints/test_model_checks.py +++ b/tests/unit/proxy/health_endpoints/test_model_checks.py @@ -215,11 +215,13 @@ def test_every_endpoint_passes_all_three_checks_and_feeds_each_reply_back_in_its turn_three: Final = provider.sent[2] sent_json: Final = json.dumps(provider.sent[1:]) assert _TOOL_CALL_ID in sent_json and WEATHER_TOOL_RESULT in sent_json, sent_json - assert "cache_control" not in sent_json and "cachePoint" not in sent_json, sent_json + if script is MESSAGES_SCRIPT: + assert turn_two["system"][0]["cache_control"] == {"type": "ephemeral"}, turn_two["system"] + else: + assert "cache_control" not in sent_json and "cachePoint" not in sent_json, sent_json assert "" in json.dumps(turn_three[key][-2:]), turn_three[key][-2:] if script is CHAT_COMPLETIONS_SCRIPT: tool_message: Final = turn_two["messages"][-1] @@ -243,10 +245,8 @@ def test_every_endpoint_passes_all_three_checks_and_feeds_each_reply_back_in_its {"type": "tool_result", "tool_use_id": _TOOL_CALL_ID, "content": WEATHER_TOOL_RESULT} ] assert turn_two["messages"][-2]["content"][0]["type"] == "thinking" - assert turn_three["messages"][-2] == { - "role": "assistant", - "content": [{"type": "text", "text": _TURN_TWO_ANSWER}], - }, turn_three + assert [m["role"] for m in turn_three["messages"][-3:]] == ["assistant", "user", "system"], turn_three + assert turn_three["messages"][-3]["content"] == [{"type": "text", "text": _TURN_TWO_ANSWER}], turn_three def test_a_model_that_answers_in_text_instead_of_calling_the_tool_fails_the_conversation_and_stops(): @@ -482,8 +482,9 @@ def test_the_three_request_files_send_one_system_prompt_and_one_tool_so_endpoint assert "" in appended and "" in appended, name third: Final = json.dumps(file["turn_3_appended_messages"]) assert "" in third, (name, third) - assert file["turn_3_appended_messages"][-1]["role"] == "user", (name, third) - assert chat["messages"][0]["content"] == responses["instructions"] == messages["system"] + assert "user" in {m["role"] for m in file["turn_3_appended_messages"][-2:]}, (name, third) + assert chat["messages"][0]["content"] == responses["instructions"] == messages["system"][0]["text"] + assert messages["system"][0]["cache_control"] == {"type": "ephemeral"}, messages["system"] assert chat["tools"][0]["function"]["parameters"] == responses["tools"][0]["parameters"] assert chat["tools"][0]["function"]["parameters"] == messages["tools"][0]["input_schema"] assert CHAT_COMPLETIONS_SCRIPT.turn_one() == chat From ee4cc1c3be12587c101b3c5fbfa930af2c5be95b Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 31 Oct 2026 18:52:17 +0000 Subject: [PATCH 11/15] feat(proxy): fail test_model prompt caching when turn 3 reads fewer cached tokens than turn 2 --- .../proxy/health_endpoints/model_checks.py | 22 +++++++-- .../test_model_requests/messages.json | 35 +++++++++----- .../health_endpoints/test_model_checks.py | 48 ++++++++++++++----- 3 files changed, 78 insertions(+), 27 deletions(-) diff --git a/litellm/proxy/health_endpoints/model_checks.py b/litellm/proxy/health_endpoints/model_checks.py index 21dbe1502d3..89537b5454b 100644 --- a/litellm/proxy/health_endpoints/model_checks.py +++ b/litellm/proxy/health_endpoints/model_checks.py @@ -3,9 +3,9 @@ One three-turn tool-call conversation per selected endpoint, read from that endpoint's file in ``test_model_requests/``. ``turn_1`` is the full first request body in the endpoint's native shape. Chat Completions and Responses carry no prompt-cache markers, so later turns can only report cached tokens when the provider caches -the repeated prefix on its own. Messages mirrors Claude Code: ``cache_control`` on the system prompt and a -mid-conversation ``role: system`` reminder, so a proxy that hoists that reminder into the system prompt shows up as a -cache miss. ``turn_2_appended_messages`` lists what the second request adds after turn 1's messages (the model's tool +the repeated prefix on its own. Messages mirrors Claude Code: ``cache_control`` on the latest user message and a +mid-conversation ``role: system`` reminder, so a proxy that hoists that reminder into the system prompt changes the +cached prefix and shows up as a cache miss. ``turn_2_appended_messages`` lists what the second request adds after turn 1's messages (the model's tool call echoed back plus the tool result), ``turn_3_appended_messages`` what the third adds after that (the model's answer echoed back, the reminder, a user follow-up), with ````, ```` and ```` filled in from the model's replies. @@ -564,6 +564,19 @@ def _cache_miss(turn: RepeatedTurn, parsed: ParsedTurn) -> CheckResult: ) +def _cache_shrank(second: ParsedTurn | None, third: ParsedTurn | None) -> CheckResult | None: + if second is None or third is None or third.cache_read_tokens >= second.cache_read_tokens: + return None + return CheckResult( + "prompt_caching", + "fail", + f"turn 3 read {third.cache_read_tokens} cached tokens, fewer than turn 2's {second.cache_read_tokens}, so the " + "prompt the provider saw on turn 3 no longer starts with the one it cached on turn 2 (something between the " + "client and the provider rewrote the earlier messages, for example a mid-conversation system message moved " + "into the system prompt)", + ) + + def _prompt_caching_check(second: RepeatedTurn, third: RepeatedTurn) -> CheckResult: reached: Final = tuple((turn, parsed) for turn in (second, third) if (parsed := _parsed(turn)) is not None) if not reached: @@ -571,6 +584,9 @@ def _prompt_caching_check(second: RepeatedTurn, third: RepeatedTurn) -> CheckRes missed: Final = next((_cache_miss(turn, parsed) for turn, parsed in reached if parsed.cache_read_tokens == 0), None) if missed is not None: return missed + shrank: Final = _cache_shrank(_parsed(second), _parsed(third)) + if shrank is not None: + return shrank return CheckResult("prompt_caching", "pass", ", ".join(_cache_read_note(turn, parsed) for turn, parsed in reached)) diff --git a/litellm/proxy/health_endpoints/test_model_requests/messages.json b/litellm/proxy/health_endpoints/test_model_requests/messages.json index 84ff9821268..b55e89507f0 100644 --- a/litellm/proxy/health_endpoints/test_model_requests/messages.json +++ b/litellm/proxy/health_endpoints/test_model_requests/messages.json @@ -1,18 +1,18 @@ { "turn_1": { - "system": [ - { - "type": "text", - "text": "You are the support assistant for Northwind Logistics. Follow the playbook below exactly.\n\nSection 1: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 2: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 3: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 4: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 5: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 6: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 7: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 8: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 9: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 10: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 11: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 12: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 13: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 14: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 15: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 16: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 17: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 18: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 19: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 20: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 21: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 22: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 23: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 24: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 25: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 26: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 27: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 28: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 29: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 30: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 31: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 32: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nWhen the user asks about the weather, call the get_weather tool with the city name. After a tool result arrives, answer in one short sentence that repeats the temperature exactly as the tool reported it.", - "cache_control": { - "type": "ephemeral" - } - } - ], + "system": "You are the support assistant for Northwind Logistics. Follow the playbook below exactly.\n\nSection 1: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 2: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 3: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 4: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 5: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 6: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 7: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 8: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 9: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 10: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 11: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 12: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 13: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 14: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 15: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 16: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 17: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 18: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 19: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 20: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 21: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 22: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 23: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 24: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 25: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 26: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 27: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 28: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 29: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 30: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 31: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 32: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nWhen the user asks about the weather, call the get_weather tool with the city name. After a tool result arrives, answer in one short sentence that repeats the temperature exactly as the tool reported it.", "messages": [ { "role": "user", - "content": "What is the weather in Boston right now? First tell me in one sentence what you are about to do, then use the get_weather tool to find out." + "content": [ + { + "type": "text", + "text": "What is the weather in Boston right now? First tell me in one sentence what you are about to do, then use the get_weather tool to find out.", + "cache_control": { + "type": "ephemeral" + } + } + ] } ], "tools": [ @@ -57,7 +57,10 @@ { "type": "tool_result", "tool_use_id": "", - "content": "72F and sunny" + "content": "72F and sunny", + "cache_control": { + "type": "ephemeral" + } } ] } @@ -69,7 +72,15 @@ }, { "role": "user", - "content": "Thanks. Is that warm enough for a short walk? Repeat the temperature you found in your answer." + "content": [ + { + "type": "text", + "text": "Thanks. Is that warm enough for a short walk? Repeat the temperature you found in your answer.", + "cache_control": { + "type": "ephemeral" + } + } + ] }, { "role": "system", diff --git a/tests/unit/proxy/health_endpoints/test_model_checks.py b/tests/unit/proxy/health_endpoints/test_model_checks.py index 7b2815e6e4a..8787e33b299 100644 --- a/tests/unit/proxy/health_endpoints/test_model_checks.py +++ b/tests/unit/proxy/health_endpoints/test_model_checks.py @@ -184,7 +184,7 @@ def test_every_endpoint_passes_all_three_checks_and_feeds_each_reply_back_in_its CalledTurn(reply(tool_call=True), 0.004), [ CalledTurn(reply(tool_call=False, text=_TURN_TWO_ANSWER, cached_tokens=4800), 0.0005), - CalledTurn(reply(tool_call=False, text=_TURN_THREE_ANSWER, cached_tokens=4900), 0.0004), + CalledTurn(reply(tool_call=False, text=_TURN_THREE_ANSWER, cached_tokens=4800), 0.0004), ], ) @@ -200,7 +200,7 @@ def test_every_endpoint_passes_all_three_checks_and_feeds_each_reply_back_in_its assert checks["prompt_caching"] == ( "pass", "turn 2 read 4800 of 5000 input tokens from cache (96.0%), " - "turn 3 read 4900 of 5000 input tokens from cache (98.0%)", + "turn 3 read 4800 of 5000 input tokens from cache (96.0%)", ), checks assert checks["cost_tracking"] == ( "pass", @@ -216,7 +216,9 @@ def test_every_endpoint_passes_all_three_checks_and_feeds_each_reply_back_in_its sent_json: Final = json.dumps(provider.sent[1:]) assert _TOOL_CALL_ID in sent_json and WEATHER_TOOL_RESULT in sent_json, sent_json if script is MESSAGES_SCRIPT: - assert turn_two["system"][0]["cache_control"] == {"type": "ephemeral"}, turn_two["system"] + assert turn_two["messages"][-1]["content"][0]["cache_control"] == {"type": "ephemeral"}, turn_two["messages"][ + -1 + ] else: assert "cache_control" not in sent_json and "cachePoint" not in sent_json, sent_json assert "" in third, (name, third) assert "user" in {m["role"] for m in file["turn_3_appended_messages"][-2:]}, (name, third) - assert chat["messages"][0]["content"] == responses["instructions"] == messages["system"][0]["text"] - assert messages["system"][0]["cache_control"] == {"type": "ephemeral"}, messages["system"] + assert chat["messages"][0]["content"] == responses["instructions"] == messages["system"] + assert messages["messages"][-1]["content"][0]["cache_control"] == {"type": "ephemeral"}, messages["messages"][-1] assert chat["tools"][0]["function"]["parameters"] == responses["tools"][0]["parameters"] assert chat["tools"][0]["function"]["parameters"] == messages["tools"][0]["input_schema"] assert CHAT_COMPLETIONS_SCRIPT.turn_one() == chat From 1d270a31b7e125b5456e7684cd1532e64a35efb8 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 31 Oct 2026 23:00:30 +0000 Subject: [PATCH 12/15] refactor(proxy): keep the test_model system prompt in system_prompt.txt so the request files stay readable --- .../proxy/health_endpoints/model_checks.py | 17 ++++- .../test_model_requests/chat_completions.json | 2 +- .../test_model_requests/messages.json | 2 +- .../test_model_requests/responses.json | 2 +- .../test_model_requests/system_prompt.txt | 67 +++++++++++++++++++ .../management/test_model_health_check.py | 4 +- .../health_endpoints/test_model_checks.py | 11 +-- 7 files changed, 93 insertions(+), 12 deletions(-) create mode 100644 litellm/proxy/health_endpoints/test_model_requests/system_prompt.txt diff --git a/litellm/proxy/health_endpoints/model_checks.py b/litellm/proxy/health_endpoints/model_checks.py index 89537b5454b..a01842ddac7 100644 --- a/litellm/proxy/health_endpoints/model_checks.py +++ b/litellm/proxy/health_endpoints/model_checks.py @@ -1,7 +1,8 @@ """Live checks behind ``POST /health/test_model``. One three-turn tool-call conversation per selected endpoint, read from that endpoint's file in -``test_model_requests/``. ``turn_1`` is the full first request body in the endpoint's native shape. Chat Completions +``test_model_requests/``. ``turn_1`` is the full first request body in the endpoint's native shape, with +```` standing for the long shared prompt in ``system_prompt.txt``. Chat Completions and Responses carry no prompt-cache markers, so later turns can only report cached tokens when the provider caches the repeated prefix on its own. Messages mirrors Claude Code: ``cache_control`` on the latest user message and a mid-conversation ``role: system`` reminder, so a proxy that hoists that reminder into the system prompt changes the @@ -225,9 +226,19 @@ class RequestFile(BaseModel): turn_3_appended_messages: Sequence[object] -def _request_file(name: str) -> Callable[[], RequestFile]: +SYSTEM_PROMPT_PLACEHOLDER: Final = "" + + +def load_request_file(name: str) -> RequestFile: + """The named request file with ```` filled from ``system_prompt.txt`` beside it.""" + prompt: Final = (TEST_MODEL_REQUESTS_DIR / "system_prompt.txt").read_text(encoding="utf-8") text: Final = (TEST_MODEL_REQUESTS_DIR / f"{name}.json").read_text(encoding="utf-8") - return lambda: RequestFile.model_validate_json(text) + return RequestFile.model_validate_json(text.replace(json.dumps(SYSTEM_PROMPT_PLACEHOLDER), json.dumps(prompt))) + + +def _request_file(name: str) -> Callable[[], RequestFile]: + file: Final = load_request_file(name) + return lambda: file _CHAT_FILE: Final = _request_file("chat_completions") diff --git a/litellm/proxy/health_endpoints/test_model_requests/chat_completions.json b/litellm/proxy/health_endpoints/test_model_requests/chat_completions.json index 997e67685da..143b472c641 100644 --- a/litellm/proxy/health_endpoints/test_model_requests/chat_completions.json +++ b/litellm/proxy/health_endpoints/test_model_requests/chat_completions.json @@ -3,7 +3,7 @@ "messages": [ { "role": "system", - "content": "You are the support assistant for Northwind Logistics. Follow the playbook below exactly.\n\nSection 1: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 2: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 3: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 4: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 5: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 6: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 7: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 8: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 9: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 10: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 11: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 12: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 13: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 14: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 15: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 16: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 17: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 18: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 19: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 20: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 21: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 22: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 23: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 24: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 25: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 26: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 27: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 28: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 29: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 30: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 31: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 32: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nWhen the user asks about the weather, call the get_weather tool with the city name. After a tool result arrives, answer in one short sentence that repeats the temperature exactly as the tool reported it." + "content": "" }, { "role": "user", diff --git a/litellm/proxy/health_endpoints/test_model_requests/messages.json b/litellm/proxy/health_endpoints/test_model_requests/messages.json index b55e89507f0..bb3b4993550 100644 --- a/litellm/proxy/health_endpoints/test_model_requests/messages.json +++ b/litellm/proxy/health_endpoints/test_model_requests/messages.json @@ -1,6 +1,6 @@ { "turn_1": { - "system": "You are the support assistant for Northwind Logistics. Follow the playbook below exactly.\n\nSection 1: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 2: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 3: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 4: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 5: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 6: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 7: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 8: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 9: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 10: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 11: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 12: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 13: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 14: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 15: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 16: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 17: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 18: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 19: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 20: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 21: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 22: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 23: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 24: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 25: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 26: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 27: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 28: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 29: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 30: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 31: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 32: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nWhen the user asks about the weather, call the get_weather tool with the city name. After a tool result arrives, answer in one short sentence that repeats the temperature exactly as the tool reported it.", + "system": "", "messages": [ { "role": "user", diff --git a/litellm/proxy/health_endpoints/test_model_requests/responses.json b/litellm/proxy/health_endpoints/test_model_requests/responses.json index 79587f044bb..68c1162b7ab 100644 --- a/litellm/proxy/health_endpoints/test_model_requests/responses.json +++ b/litellm/proxy/health_endpoints/test_model_requests/responses.json @@ -1,6 +1,6 @@ { "turn_1": { - "instructions": "You are the support assistant for Northwind Logistics. Follow the playbook below exactly.\n\nSection 1: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 2: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 3: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 4: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 5: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 6: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 7: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 8: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 9: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 10: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 11: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 12: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 13: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 14: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 15: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 16: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 17: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 18: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 19: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 20: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 21: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 22: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 23: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 24: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 25: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 26: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 27: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 28: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 29: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 30: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 31: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 32: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nWhen the user asks about the weather, call the get_weather tool with the city name. After a tool result arrives, answer in one short sentence that repeats the temperature exactly as the tool reported it.", + "instructions": "", "input": [ { "role": "user", diff --git a/litellm/proxy/health_endpoints/test_model_requests/system_prompt.txt b/litellm/proxy/health_endpoints/test_model_requests/system_prompt.txt new file mode 100644 index 00000000000..27e8f1f93dd --- /dev/null +++ b/litellm/proxy/health_endpoints/test_model_requests/system_prompt.txt @@ -0,0 +1,67 @@ +You are the support assistant for Northwind Logistics. Follow the playbook below exactly. + +Section 1: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days. + +Section 2: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days. + +Section 3: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days. + +Section 4: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days. + +Section 5: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days. + +Section 6: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days. + +Section 7: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days. + +Section 8: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days. + +Section 9: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days. + +Section 10: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days. + +Section 11: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days. + +Section 12: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days. + +Section 13: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days. + +Section 14: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days. + +Section 15: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days. + +Section 16: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days. + +Section 17: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days. + +Section 18: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days. + +Section 19: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days. + +Section 20: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days. + +Section 21: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days. + +Section 22: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days. + +Section 23: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days. + +Section 24: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days. + +Section 25: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days. + +Section 26: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days. + +Section 27: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days. + +Section 28: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days. + +Section 29: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days. + +Section 30: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days. + +Section 31: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days. + +Section 32: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days. + +When the user asks about the weather, call the get_weather tool with the city name. After a tool result arrives, answer in one short sentence that repeats the temperature exactly as the tool reported it. \ No newline at end of file diff --git a/tests/integration/management/test_model_health_check.py b/tests/integration/management/test_model_health_check.py index 583cb48cf48..dfe6d5d86ab 100644 --- a/tests/integration/management/test_model_health_check.py +++ b/tests/integration/management/test_model_health_check.py @@ -27,7 +27,7 @@ from integration._support.wire import Reply, Request, Wire, wire_server from integration.cost_calculation.cost_tracking_case import JsonResponse from pydantic import JsonValue -from litellm.proxy.health_endpoints.model_checks import TEST_MODEL_REQUESTS_DIR +from litellm.proxy.health_endpoints.model_checks import load_request_file _DECISIONS_PROBE_REPLY: Final = JsonResponse( content_type="application/json", @@ -1441,7 +1441,7 @@ def test_health_check_of_gpt_transcribe_probes_without_verbose_json_and_reports_ _TEST_MODEL_TOOL_CALL_ID: Final = "call_boston_wire" _TEST_MODEL_CHAT_REQUEST: Final = JSON_OBJECT.validate_python( - JSON_OBJECT.validate_json((TEST_MODEL_REQUESTS_DIR / "chat_completions.json").read_text(encoding="utf-8"))["turn_1"] + JSON_OBJECT.validate_python(dict(load_request_file("chat_completions").turn_1)) ) diff --git a/tests/unit/proxy/health_endpoints/test_model_checks.py b/tests/unit/proxy/health_endpoints/test_model_checks.py index 8787e33b299..673e22729d7 100644 --- a/tests/unit/proxy/health_endpoints/test_model_checks.py +++ b/tests/unit/proxy/health_endpoints/test_model_checks.py @@ -14,6 +14,7 @@ from litellm.proxy.health_endpoints.model_checks import ( CalledTurn, EndpointScript, endpoint_result_payload, + load_request_file, overall_status, run_endpoint_check, ) @@ -507,10 +508,12 @@ def test_the_three_request_files_send_one_system_prompt_and_one_tool_so_endpoint third: Final = json.dumps(file["turn_3_appended_messages"]) assert "" in third, (name, third) assert "user" in {m["role"] for m in file["turn_3_appended_messages"][-2:]}, (name, third) - assert chat["messages"][0]["content"] == responses["instructions"] == messages["system"] + assert chat["messages"][0]["content"] == responses["instructions"] == messages["system"] == "" + assert len((TEST_MODEL_REQUESTS_DIR / "system_prompt.txt").read_text(encoding="utf-8")) > 20000 + assert load_request_file("messages").turn_1["system"] != "" assert messages["messages"][-1]["content"][0]["cache_control"] == {"type": "ephemeral"}, messages["messages"][-1] assert chat["tools"][0]["function"]["parameters"] == responses["tools"][0]["parameters"] assert chat["tools"][0]["function"]["parameters"] == messages["tools"][0]["input_schema"] - assert CHAT_COMPLETIONS_SCRIPT.turn_one() == chat - assert RESPONSES_SCRIPT.turn_one() == responses - assert MESSAGES_SCRIPT.turn_one() == messages + assert CHAT_COMPLETIONS_SCRIPT.turn_one() == load_request_file("chat_completions").turn_1 + assert RESPONSES_SCRIPT.turn_one() == load_request_file("responses").turn_1 + assert MESSAGES_SCRIPT.turn_one() == load_request_file("messages").turn_1 From d8db1a41a1839665040577e060f80ade69660181 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 31 Oct 2026 23:08:02 +0000 Subject: [PATCH 13/15] refactor(proxy): define the test_model conversations in Python instead of JSON files --- .../proxy/health_endpoints/model_checks.py | 52 ++---- .../health_endpoints/test_model_requests.py | 162 ++++++++++++++++++ .../test_model_requests/chat_completions.json | 63 ------- .../test_model_requests/messages.json | 90 ---------- .../test_model_requests/responses.json | 60 ------- pyproject.toml | 2 +- .../management/test_model_health_check.py | 4 +- .../health_endpoints/test_model_checks.py | 38 ++-- 8 files changed, 194 insertions(+), 277 deletions(-) create mode 100644 litellm/proxy/health_endpoints/test_model_requests.py delete mode 100644 litellm/proxy/health_endpoints/test_model_requests/chat_completions.json delete mode 100644 litellm/proxy/health_endpoints/test_model_requests/messages.json delete mode 100644 litellm/proxy/health_endpoints/test_model_requests/responses.json diff --git a/litellm/proxy/health_endpoints/model_checks.py b/litellm/proxy/health_endpoints/model_checks.py index a01842ddac7..56814c1e598 100644 --- a/litellm/proxy/health_endpoints/model_checks.py +++ b/litellm/proxy/health_endpoints/model_checks.py @@ -1,7 +1,6 @@ """Live checks behind ``POST /health/test_model``. -One three-turn tool-call conversation per selected endpoint, read from that endpoint's file in -``test_model_requests/``. ``turn_1`` is the full first request body in the endpoint's native shape, with +One three-turn tool-call conversation per selected endpoint, defined in ``test_model_requests.py``. ``turn_1`` is the full first request body in the endpoint's native shape, with ```` standing for the long shared prompt in ``system_prompt.txt``. Chat Completions and Responses carry no prompt-cache markers, so later turns can only report cached tokens when the provider caches the repeated prefix on its own. Messages mirrors Claude Code: ``cache_control`` on the latest user message and a @@ -22,13 +21,22 @@ from collections.abc import Awaitable, Callable, Mapping, Sequence from dataclasses import dataclass from functools import partial, reduce from itertools import chain -from pathlib import Path from types import MappingProxyType from typing import Final, Literal, TypeAlias from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError from litellm.litellm_core_utils.add_retry_fallback_headers import get_hidden_params_dict +from litellm.proxy.health_endpoints.test_model_requests import ( + CHAT_COMPLETIONS, + MESSAGES, + RESPONSES, + TURN_ONE_ASSISTANT_OUTPUT, + TURN_ONE_TOOL_CALL_ID, + TURN_TWO_ASSISTANT_OUTPUT, + WEATHER_TOOL_RESULT, + RequestFile, +) TestModelEndpoint: TypeAlias = Literal["/v1/chat/completions", "/v1/responses", "/v1/messages"] TestModelRouteType: TypeAlias = Literal["acompletion", "aresponses", "anthropic_messages"] @@ -37,11 +45,6 @@ EndpointStatus: TypeAlias = Literal["success", "warning", "error"] TEST_MODEL_ENDPOINTS: Final[tuple[TestModelEndpoint, ...]] = ("/v1/chat/completions", "/v1/responses", "/v1/messages") WEATHER_TOOL_NAME: Final = "get_weather" -WEATHER_TOOL_RESULT: Final = "72F and sunny" -TEST_MODEL_REQUESTS_DIR: Final = Path(__file__).resolve().parent / "test_model_requests" -TURN_ONE_ASSISTANT_OUTPUT: Final = "" -TURN_ONE_TOOL_CALL_ID: Final = "" -TURN_TWO_ASSISTANT_OUTPUT: Final = "" @dataclass(frozen=True, slots=True) @@ -214,36 +217,9 @@ _JSON_DOCUMENT: Final = TypeAdapter(object) _EMPTY_ASSISTANT: Final[Mapping[str, object]] = MappingProxyType({"role": "assistant", "content": None}) -class RequestFile(BaseModel): - """One three-turn conversation on disk: ``turn_1`` is the full first request body, ``turn_2_appended_messages`` - the messages the second request adds after it, ``turn_3_appended_messages`` the ones the third adds after those, - with the placeholders filled from the earlier responses.""" - - model_config = ConfigDict(frozen=True) - - turn_1: Mapping[str, object] - turn_2_appended_messages: Sequence[object] - turn_3_appended_messages: Sequence[object] - - -SYSTEM_PROMPT_PLACEHOLDER: Final = "" - - -def load_request_file(name: str) -> RequestFile: - """The named request file with ```` filled from ``system_prompt.txt`` beside it.""" - prompt: Final = (TEST_MODEL_REQUESTS_DIR / "system_prompt.txt").read_text(encoding="utf-8") - text: Final = (TEST_MODEL_REQUESTS_DIR / f"{name}.json").read_text(encoding="utf-8") - return RequestFile.model_validate_json(text.replace(json.dumps(SYSTEM_PROMPT_PLACEHOLDER), json.dumps(prompt))) - - -def _request_file(name: str) -> Callable[[], RequestFile]: - file: Final = load_request_file(name) - return lambda: file - - -_CHAT_FILE: Final = _request_file("chat_completions") -_RESPONSES_FILE: Final = _request_file("responses") -_MESSAGES_FILE: Final = _request_file("messages") +_CHAT_FILE: Final = lambda: CHAT_COMPLETIONS # noqa: E731 # the scripts take a zero-arg loader +_RESPONSES_FILE: Final = lambda: RESPONSES # noqa: E731 # the scripts take a zero-arg loader +_MESSAGES_FILE: Final = lambda: MESSAGES # noqa: E731 # the scripts take a zero-arg loader Placeholders: TypeAlias = Mapping[str, object] diff --git a/litellm/proxy/health_endpoints/test_model_requests.py b/litellm/proxy/health_endpoints/test_model_requests.py new file mode 100644 index 00000000000..79547c2fb4a --- /dev/null +++ b/litellm/proxy/health_endpoints/test_model_requests.py @@ -0,0 +1,162 @@ +"""The three-turn conversations ``POST /health/test_model`` sends, one per endpoint, in that endpoint's native shape. + +Every conversation asks the same weather question in front of the same long system prompt (``test_model_requests/ +system_prompt.txt``), sends the same tool result back, then asks the same follow-up. Chat Completions and Responses +carry no prompt-cache markers, so cached tokens on turns 2 and 3 can only come from the provider caching the repeated +prefix on its own. Messages mirrors Claude Code: ``CACHE_CONTROL`` on the latest user message and a mid-conversation +``SYSTEM_REMINDER``, so a proxy that hoists the reminder into the system prompt changes the cached prefix and shows up +as a cache miss. The ``TURN_*`` placeholders are filled from the model's earlier replies. +""" + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from types import MappingProxyType +from typing import Final + +SYSTEM_PROMPT: Final = (Path(__file__).resolve().parent / "test_model_requests" / "system_prompt.txt").read_text( + encoding="utf-8" +) + +TURN_ONE_ASSISTANT_OUTPUT: Final = "" +TURN_ONE_TOOL_CALL_ID: Final = "" +TURN_TWO_ASSISTANT_OUTPUT: Final = "" + +WEATHER_QUESTION: Final = ( + "What is the weather in Boston right now? First tell me in one sentence what you are about to do, then use the " + "get_weather tool to find out." +) +WEATHER_TOOL_RESULT: Final = "72F and sunny" +FOLLOW_UP_QUESTION: Final = ( + "Thanks. Is that warm enough for a short walk? Repeat the temperature you found in your answer." +) +SYSTEM_REMINDER: Final = ( + "The user prefers short answers. Keep the temperature in the reply." +) +CACHE_CONTROL: Final = MappingProxyType({"type": "ephemeral"}) + +WEATHER_TOOL_PARAMETERS: Final = MappingProxyType( + { + "type": "object", + "properties": { + "location": {"type": "string", "description": "City name, for example Boston"}, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, + }, + "required": ["location"], + } +) +WEATHER_TOOL_DESCRIPTION: Final = "Get the current weather for a city" + + +@dataclass(frozen=True, slots=True) +class RequestFile: + """One three-turn conversation: ``turn_1`` is the full first request body, ``turn_2_appended_messages`` the + messages the second request adds after it, ``turn_3_appended_messages`` the ones the third adds after those.""" + + turn_1: Mapping[str, object] + turn_2_appended_messages: Sequence[object] + turn_3_appended_messages: Sequence[object] + + +CHAT_COMPLETIONS: Final = RequestFile( + turn_1={ + "messages": [ + {"role": "system", "content": SYSTEM_PROMPT}, + {"role": "user", "content": WEATHER_QUESTION}, + ], + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": WEATHER_TOOL_DESCRIPTION, + "parameters": dict(WEATHER_TOOL_PARAMETERS), + }, + } + ], + "reasoning_effort": "low", + "max_tokens": 2048, + }, + turn_2_appended_messages=( + TURN_ONE_ASSISTANT_OUTPUT, + {"role": "tool", "tool_call_id": TURN_ONE_TOOL_CALL_ID, "content": WEATHER_TOOL_RESULT}, + ), + turn_3_appended_messages=( + TURN_TWO_ASSISTANT_OUTPUT, + {"role": "system", "content": SYSTEM_REMINDER}, + {"role": "user", "content": FOLLOW_UP_QUESTION}, + ), +) + +RESPONSES: Final = RequestFile( + turn_1={ + "instructions": SYSTEM_PROMPT, + "input": [{"role": "user", "content": WEATHER_QUESTION}], + "tools": [ + { + "type": "function", + "name": "get_weather", + "description": WEATHER_TOOL_DESCRIPTION, + "parameters": dict(WEATHER_TOOL_PARAMETERS), + } + ], + "reasoning": {"effort": "low"}, + "max_output_tokens": 2048, + }, + turn_2_appended_messages=( + TURN_ONE_ASSISTANT_OUTPUT, + {"type": "function_call_output", "call_id": TURN_ONE_TOOL_CALL_ID, "output": WEATHER_TOOL_RESULT}, + ), + turn_3_appended_messages=( + TURN_TWO_ASSISTANT_OUTPUT, + {"role": "system", "content": SYSTEM_REMINDER}, + {"role": "user", "content": FOLLOW_UP_QUESTION}, + ), +) + +MESSAGES: Final = RequestFile( + turn_1={ + "system": SYSTEM_PROMPT, + "messages": [ + { + "role": "user", + "content": [{"type": "text", "text": WEATHER_QUESTION, "cache_control": dict(CACHE_CONTROL)}], + } + ], + "tools": [ + { + "name": "get_weather", + "description": WEATHER_TOOL_DESCRIPTION, + "input_schema": dict(WEATHER_TOOL_PARAMETERS), + } + ], + "thinking": {"type": "enabled", "budget_tokens": 1024}, + "max_tokens": 2048, + }, + turn_2_appended_messages=( + {"role": "assistant", "content": TURN_ONE_ASSISTANT_OUTPUT}, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": TURN_ONE_TOOL_CALL_ID, + "content": WEATHER_TOOL_RESULT, + "cache_control": dict(CACHE_CONTROL), + } + ], + }, + ), + turn_3_appended_messages=( + {"role": "assistant", "content": TURN_TWO_ASSISTANT_OUTPUT}, + { + "role": "user", + "content": [{"type": "text", "text": FOLLOW_UP_QUESTION, "cache_control": dict(CACHE_CONTROL)}], + }, + {"role": "system", "content": SYSTEM_REMINDER}, + ), +) + +REQUEST_FILES: Final = MappingProxyType( + {"chat_completions": CHAT_COMPLETIONS, "responses": RESPONSES, "messages": MESSAGES} +) diff --git a/litellm/proxy/health_endpoints/test_model_requests/chat_completions.json b/litellm/proxy/health_endpoints/test_model_requests/chat_completions.json deleted file mode 100644 index 143b472c641..00000000000 --- a/litellm/proxy/health_endpoints/test_model_requests/chat_completions.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "turn_1": { - "messages": [ - { - "role": "system", - "content": "" - }, - { - "role": "user", - "content": "What is the weather in Boston right now? First tell me in one sentence what you are about to do, then use the get_weather tool to find out." - } - ], - "tools": [ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the current weather for a city", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City name, for example Boston" - }, - "unit": { - "type": "string", - "enum": [ - "celsius", - "fahrenheit" - ] - } - }, - "required": [ - "location" - ] - } - } - } - ], - "reasoning_effort": "low", - "max_tokens": 2048 - }, - "turn_2_appended_messages": [ - "", - { - "role": "tool", - "tool_call_id": "", - "content": "72F and sunny" - } - ], - "turn_3_appended_messages": [ - "", - { - "role": "system", - "content": "The user prefers short answers. Keep the temperature in the reply." - }, - { - "role": "user", - "content": "Thanks. Is that warm enough for a short walk? Repeat the temperature you found in your answer." - } - ] -} diff --git a/litellm/proxy/health_endpoints/test_model_requests/messages.json b/litellm/proxy/health_endpoints/test_model_requests/messages.json deleted file mode 100644 index bb3b4993550..00000000000 --- a/litellm/proxy/health_endpoints/test_model_requests/messages.json +++ /dev/null @@ -1,90 +0,0 @@ -{ - "turn_1": { - "system": "", - "messages": [ - { - "role": "user", - "content": [ - { - "type": "text", - "text": "What is the weather in Boston right now? First tell me in one sentence what you are about to do, then use the get_weather tool to find out.", - "cache_control": { - "type": "ephemeral" - } - } - ] - } - ], - "tools": [ - { - "name": "get_weather", - "description": "Get the current weather for a city", - "input_schema": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City name, for example Boston" - }, - "unit": { - "type": "string", - "enum": [ - "celsius", - "fahrenheit" - ] - } - }, - "required": [ - "location" - ] - } - } - ], - "thinking": { - "type": "enabled", - "budget_tokens": 1024 - }, - "max_tokens": 2048 - }, - "turn_2_appended_messages": [ - { - "role": "assistant", - "content": "" - }, - { - "role": "user", - "content": [ - { - "type": "tool_result", - "tool_use_id": "", - "content": "72F and sunny", - "cache_control": { - "type": "ephemeral" - } - } - ] - } - ], - "turn_3_appended_messages": [ - { - "role": "assistant", - "content": "" - }, - { - "role": "user", - "content": [ - { - "type": "text", - "text": "Thanks. Is that warm enough for a short walk? Repeat the temperature you found in your answer.", - "cache_control": { - "type": "ephemeral" - } - } - ] - }, - { - "role": "system", - "content": "The user prefers short answers. Keep the temperature in the reply." - } - ] -} diff --git a/litellm/proxy/health_endpoints/test_model_requests/responses.json b/litellm/proxy/health_endpoints/test_model_requests/responses.json deleted file mode 100644 index 68c1162b7ab..00000000000 --- a/litellm/proxy/health_endpoints/test_model_requests/responses.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "turn_1": { - "instructions": "", - "input": [ - { - "role": "user", - "content": "What is the weather in Boston right now? First tell me in one sentence what you are about to do, then use the get_weather tool to find out." - } - ], - "tools": [ - { - "type": "function", - "name": "get_weather", - "description": "Get the current weather for a city", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City name, for example Boston" - }, - "unit": { - "type": "string", - "enum": [ - "celsius", - "fahrenheit" - ] - } - }, - "required": [ - "location" - ] - } - } - ], - "reasoning": { - "effort": "low" - }, - "max_output_tokens": 2048 - }, - "turn_2_appended_messages": [ - "", - { - "type": "function_call_output", - "call_id": "", - "output": "72F and sunny" - } - ], - "turn_3_appended_messages": [ - "", - { - "role": "system", - "content": "The user prefers short answers. Keep the temperature in the reply." - }, - { - "role": "user", - "content": "Thanks. Is that warm enough for a short walk? Repeat the temperature you found in your answer." - } - ] -} diff --git a/pyproject.toml b/pyproject.toml index ef78006a002..fa01c2f3c31 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -332,7 +332,7 @@ include = [ "litellm/router_strategy/complexity_router/artifacts/*.json", "litellm/router_strategy/complexity_router/fuse_presets.json", "litellm/proxy/model_insights_tasks.json", - "litellm/proxy/health_endpoints/test_model_requests/*.json", + "litellm/proxy/health_endpoints/test_model_requests/system_prompt.txt", "litellm/proxy/public_endpoints/whats_new.json", "litellm/proxy/common_utils/codex_base_instructions.md", "litellm/proxy/common_utils/codex_bundled_models_0.159.3.json", diff --git a/tests/integration/management/test_model_health_check.py b/tests/integration/management/test_model_health_check.py index dfe6d5d86ab..516c5e2c24e 100644 --- a/tests/integration/management/test_model_health_check.py +++ b/tests/integration/management/test_model_health_check.py @@ -27,7 +27,7 @@ from integration._support.wire import Reply, Request, Wire, wire_server from integration.cost_calculation.cost_tracking_case import JsonResponse from pydantic import JsonValue -from litellm.proxy.health_endpoints.model_checks import load_request_file +from litellm.proxy.health_endpoints.test_model_requests import CHAT_COMPLETIONS _DECISIONS_PROBE_REPLY: Final = JsonResponse( content_type="application/json", @@ -1441,7 +1441,7 @@ def test_health_check_of_gpt_transcribe_probes_without_verbose_json_and_reports_ _TEST_MODEL_TOOL_CALL_ID: Final = "call_boston_wire" _TEST_MODEL_CHAT_REQUEST: Final = JSON_OBJECT.validate_python( - JSON_OBJECT.validate_python(dict(load_request_file("chat_completions").turn_1)) + JSON_OBJECT.validate_python(dict(CHAT_COMPLETIONS.turn_1)) ) diff --git a/tests/unit/proxy/health_endpoints/test_model_checks.py b/tests/unit/proxy/health_endpoints/test_model_checks.py index 673e22729d7..57af7c75a57 100644 --- a/tests/unit/proxy/health_endpoints/test_model_checks.py +++ b/tests/unit/proxy/health_endpoints/test_model_checks.py @@ -9,15 +9,14 @@ from litellm.proxy.health_endpoints.model_checks import ( CHAT_COMPLETIONS_SCRIPT, MESSAGES_SCRIPT, RESPONSES_SCRIPT, - TEST_MODEL_REQUESTS_DIR, WEATHER_TOOL_RESULT, CalledTurn, EndpointScript, endpoint_result_payload, - load_request_file, overall_status, run_endpoint_check, ) +from litellm.proxy.health_endpoints.test_model_requests import REQUEST_FILES from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.utils import ModelResponse @@ -492,28 +491,21 @@ def test_payload_lists_the_checks_in_order_with_their_details(): def test_the_three_request_files_send_one_system_prompt_and_one_tool_so_endpoint_verdicts_are_comparable(): - files: Final = { - path.stem: json.loads(path.read_text(encoding="utf-8")) for path in TEST_MODEL_REQUESTS_DIR.glob("*.json") - } - chat, messages, responses = (files[name]["turn_1"] for name in ("chat_completions", "messages", "responses")) + chat, responses, messages = (REQUEST_FILES[name] for name in ("chat_completions", "responses", "messages")) - assert set(files) == {"chat_completions", "messages", "responses"}, files.keys() - assert all( - set(file) == {"turn_1", "turn_2_appended_messages", "turn_3_appended_messages"} for file in files.values() - ), files - for name, file in files.items(): - appended: Final = json.dumps(file["turn_2_appended_messages"]) + assert set(REQUEST_FILES) == {"chat_completions", "messages", "responses"} + for name, file in REQUEST_FILES.items(): + appended: Final = json.dumps(file.turn_2_appended_messages) assert WEATHER_TOOL_RESULT in appended, (name, appended) assert "" in appended and "" in appended, name - third: Final = json.dumps(file["turn_3_appended_messages"]) + third: Final = json.dumps(file.turn_3_appended_messages) assert "" in third, (name, third) - assert "user" in {m["role"] for m in file["turn_3_appended_messages"][-2:]}, (name, third) - assert chat["messages"][0]["content"] == responses["instructions"] == messages["system"] == "" - assert len((TEST_MODEL_REQUESTS_DIR / "system_prompt.txt").read_text(encoding="utf-8")) > 20000 - assert load_request_file("messages").turn_1["system"] != "" - assert messages["messages"][-1]["content"][0]["cache_control"] == {"type": "ephemeral"}, messages["messages"][-1] - assert chat["tools"][0]["function"]["parameters"] == responses["tools"][0]["parameters"] - assert chat["tools"][0]["function"]["parameters"] == messages["tools"][0]["input_schema"] - assert CHAT_COMPLETIONS_SCRIPT.turn_one() == load_request_file("chat_completions").turn_1 - assert RESPONSES_SCRIPT.turn_one() == load_request_file("responses").turn_1 - assert MESSAGES_SCRIPT.turn_one() == load_request_file("messages").turn_1 + assert "" in third, (name, third) + assert chat.turn_1["messages"][0]["content"] == responses.turn_1["instructions"] == messages.turn_1["system"] + assert len(messages.turn_1["system"]) > 20000 + assert messages.turn_1["messages"][-1]["content"][0]["cache_control"] == {"type": "ephemeral"} + assert chat.turn_1["tools"][0]["function"]["parameters"] == responses.turn_1["tools"][0]["parameters"] + assert chat.turn_1["tools"][0]["function"]["parameters"] == messages.turn_1["tools"][0]["input_schema"] + assert CHAT_COMPLETIONS_SCRIPT.turn_one() == chat.turn_1 + assert RESPONSES_SCRIPT.turn_one() == responses.turn_1 + assert MESSAGES_SCRIPT.turn_one() == messages.turn_1 From 65fef5f1725704fb94802557e6bc81fd7187398d Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 4 Nov 2026 00:27:34 +0000 Subject: [PATCH 14/15] fix(proxy): test_model skips the response cache, bounds the spend lookup, bills unsaved custom prices and checks spend rows against response_cost --- .../health_endpoints/_health_endpoints.py | 66 +++++++++++++++---- .../proxy/health_endpoints/model_checks.py | 20 +++++- .../management/test_model_health_check.py | 65 +++++++++++++++++- .../health_endpoints/test_model_checks.py | 34 ++++++++-- 4 files changed, 165 insertions(+), 20 deletions(-) diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index fdc41caf6d4..10b6ac5c76c 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -122,7 +122,11 @@ from litellm.router_api import deployment_usable_by_team from litellm.secret_managers.main import get_secret_bool from litellm.types.proxy.litellm_pre_call_utils import RedactedDict, SecretFields from litellm.types.router import Deployment, reject_server_owned_wif_fields -from litellm.types.utils import AdminApiKeySecretReference, admin_api_key_secret_reference +from litellm.types.utils import ( + AdminApiKeySecretReference, + CustomPricingLiteLLMParams, + admin_api_key_secret_reference, +) #### Health ENDPOINTS #### @@ -2641,7 +2645,8 @@ _TEST_MODEL_TAG_PREFIX: Final = "litellm_test_model:" _SPEND_LOG_BY_TAG_SQL: Final = """ SELECT tag, "spend" FROM "LiteLLM_SpendLogs" sl, jsonb_array_elements_text(sl."request_tags"::jsonb) AS tag - WHERE tag IN (SELECT jsonb_array_elements_text($1::jsonb)) + WHERE sl."startTime" >= ($2::timestamptz AT TIME ZONE 'UTC') + AND tag IN (SELECT jsonb_array_elements_text($1::jsonb)) """ @@ -2649,11 +2654,26 @@ def _test_model_tag(call_id: str) -> str: return f"{_TEST_MODEL_TAG_PREFIX}{call_id}" -async def _spend_rows_once(run_query: _RawQuery, request_ids: frozenset[str]) -> Mapping[str, float]: +def _custom_pricing_fields(model_info: Mapping[str, object] | None) -> Mapping[str, object]: + """The custom prices an unsaved model's ``model_info`` carries, sent on the call the way a saved deployment's + prices apply, so the check bills it as the proxy will once it is saved.""" + return { + key: value + for key, value in (model_info or {}).items() + if key in CustomPricingLiteLLMParams.model_fields and value is not None + } + + +async def _spend_rows_once(run_query: _RawQuery, request_ids: frozenset[str], since: datetime) -> Mapping[str, float]: """Spend per request id. The row's request_id is the provider's response id, so each turn is found by the - tag the check put on it, which the proxy writes to request_tags like any client tag.""" + tag the check put on it, which the proxy writes to request_tags like any client tag. Only rows started at or + after ``since`` are scanned, through the ``startTime`` index, so the check never reads the whole table.""" rows: Final = _SPEND_LOG_ROWS.validate_python( - await run_query(_SPEND_LOG_BY_TAG_SQL, json.dumps(sorted(_test_model_tag(call_id) for call_id in request_ids))) + await run_query( + _SPEND_LOG_BY_TAG_SQL, + json.dumps(sorted(_test_model_tag(call_id) for call_id in request_ids)), + since.isoformat(), + ) ) return MappingProxyType({row.tag.removeprefix(_TEST_MODEL_TAG_PREFIX): row.spend for row in rows}) @@ -2662,6 +2682,7 @@ async def _spend_log_rows( run_query: _RawQuery, request_ids: Sequence[str], *, + since: datetime, wait_seconds: float = TEST_MODEL_SPEND_LOG_WAIT_SECONDS, poll_seconds: float = 1.0, ) -> Mapping[str, float]: @@ -2672,7 +2693,9 @@ async def _spend_log_rows( """ deadline: Final = time.monotonic() + wait_seconds wanted: Final = frozenset(request_ids) - while not wanted <= (found := await _spend_rows_once(run_query, wanted)).keys() and time.monotonic() < deadline: + while ( + not wanted <= (found := await _spend_rows_once(run_query, wanted, since)).keys() and time.monotonic() < deadline + ): await asyncio.sleep(poll_seconds) return found @@ -2695,10 +2718,15 @@ def _test_model_endpoints( async def _logged_cost(logging_obj: LiteLLMLoggingObj, fallback: float | None) -> float | None: - """The cost the success handler logged for the call, which lands a moment after the call returns.""" + """The cost the success handler logged for the call, which lands a moment after the call returns. The wait + ends as soon as the handler has built the call's standard logging payload, so an unpriced call does not hold + every turn for the full budget.""" deadline: Final = time.monotonic() + TEST_MODEL_COST_WAIT_SECONDS - while (logged := _OBJECT_MAPPING.validate_python(logging_obj.model_call_details).get("response_cost")) is None and ( - time.monotonic() < deadline + while ( + (logged := (details := _OBJECT_MAPPING.validate_python(logging_obj.model_call_details)).get("response_cost")) + is None + and "standard_logging_object" not in details + and time.monotonic() < deadline ): await asyncio.sleep(0.05) return float(logged) if isinstance(logged, int | float) else fallback @@ -2714,7 +2742,8 @@ def _turn_caller( """The two ways a turn reaches the provider: pinned to a saved deployment through the router, so the deployment's own credentials and pricing apply, or straight through litellm with the submitted params. Either way the call carries the proxy's own logging object, the way every proxied request does, so the - cost read back is the one the spend log row gets.""" + cost read back is the one the spend log row gets. Every turn skips LiteLLM's response cache, so the reply, + and the cache-miss retry of the same prompt, come from the provider and not from a stored answer.""" async def call(script: EndpointScript, body: Mapping[str, object], call_id: str) -> CalledTurn: target_params: Final = {"model": pinned_model_id} if pinned_model_id is not None else call_params @@ -2728,6 +2757,7 @@ def _turn_caller( script.metadata_key: {**proxy_metadata, "tags": [_test_model_tag(call_id)]}, "litellm_call_id": call_id, "drop_params": True, + "cache": {"no-cache": True}, }, ) prepared["litellm_logging_obj"] = logging_obj @@ -2785,7 +2815,9 @@ async def test_model( answer from turn 2. Three checks come out of it: - `conversation`: every turn parses, turn 1 calls the tool, turn 2 uses the tool result, turn 3 carries it on - - `prompt_caching`: turns 2 and 3 report cached input tokens, with no cache markers in the request + - `prompt_caching`: turns 2 and 3 report cached input tokens and turn 3 reads at least as many as turn 2. + Chat Completions and Responses send no cache markers, so the provider has to cache the repeated prefix on + its own. Messages sends `cache_control` on the latest user message the way Claude Code does - `cost_tracking`: every turn carries a non-zero response cost, and the spend log row lands when a database is connected @@ -2852,12 +2884,17 @@ async def test_model( status_code=status.HTTP_403_FORBIDDEN, detail={"error": "Only proxy admins can test models."} ) - call_params: Final = {key: value for key, value in resolved.litellm_params.items() if key != "mode"} + call_params: Final = { + **_custom_pricing_fields(resolved.resolved_model_info), + **{key: value for key, value in resolved.litellm_params.items() if key != "mode"}, + } mode: Final = _string_mode_or_bad_request(resolved.litellm_params.get("mode")) or resolve_health_check_mode( resolved.probe_model_info, _OBJECT_MAPPING.validate_python(call_params) ) selected: Final = _test_model_endpoints(endpoints, mode) - overrides_saved_deployment: Final = any(key != "model" for key in resolved.request_litellm_params) + overrides_saved_deployment: Final = any( + key != "model" or value != resolved.stored_model for key, value in resolved.request_litellm_params.items() + ) pinned_model_id: Final = ( resolved.deployment_by_id.model_info.id if resolved.deployment_by_id is not None and not overrides_saved_deployment @@ -2880,9 +2917,10 @@ async def test_model( proxy_metadata=proxy_metadata, ) run_query: Final[_RawQuery | None] = prisma_client.db.query_raw if prisma_client is not None else None + started_at: Final = datetime.now(timezone.utc) - timedelta(minutes=1) async def spend_lookup(request_ids: Sequence[str]) -> Mapping[str, float]: - return await _spend_log_rows(run_query, request_ids) if run_query is not None else {} + return await _spend_log_rows(run_query, request_ids, since=started_at) if run_query is not None else {} results: Final[Sequence[EndpointResult]] = await asyncio.gather( *( diff --git a/litellm/proxy/health_endpoints/model_checks.py b/litellm/proxy/health_endpoints/model_checks.py index 56814c1e598..d920179dfd0 100644 --- a/litellm/proxy/health_endpoints/model_checks.py +++ b/litellm/proxy/health_endpoints/model_checks.py @@ -16,6 +16,7 @@ tool result is used, the follow-up is answered from the earlier turn), prompt ca from __future__ import annotations import json +import math import time from collections.abc import Awaitable, Callable, Mapping, Sequence from dataclasses import dataclass @@ -316,7 +317,10 @@ def _first_choice_message(plain: Mapping[str, object]) -> object: choices: Final = _CHOICES.validate_python(plain.get("choices")) except ValidationError: return dict(_EMPTY_ASSISTANT) - return choices[0].get("message", dict(_EMPTY_ASSISTANT)) if choices else dict(_EMPTY_ASSISTANT) + message: Final = choices[0].get("message") if choices else None + if not isinstance(message, dict): + return dict(_EMPTY_ASSISTANT) + return {**_EMPTY_ASSISTANT, **_PLAIN_OBJECT.validate_python(message)} def _parse_chat(response: object) -> ParsedTurn | str: @@ -595,6 +599,12 @@ def _cost_verdict(turn_number: int, cost: float | None) -> CheckResult | None: return None +def _spend_disagreement(turn: TurnOutcome, logged: float) -> str: + if turn.cost is None or math.isclose(logged, turn.cost, rel_tol=1e-6, abs_tol=1e-12): + return "" + return f"request_id {turn.call_id} row ${logged:.6f} vs response_cost ${turn.cost:.6f}" + + def _cost_tracking_check(turns: Sequence[TurnOutcome], spend_rows: Mapping[str, float] | None) -> CheckResult: if not turns: return CheckResult("cost_tracking", "fail", "not checked, no turn returned a response") @@ -615,6 +625,14 @@ def _cost_tracking_check(turns: Sequence[TurnOutcome], spend_rows: Mapping[str, "fail", f"{priced}, but no LiteLLM_SpendLogs row landed for request_id {', '.join(missing)}", ) + disagreeing: Final = tuple(_spend_disagreement(turn, spend_rows[turn.call_id]) for turn in turns) + if any(disagreeing): + return CheckResult( + "cost_tracking", + "fail", + f"{priced}, but the spend log row disagrees with the response cost (" + f"{', '.join(note for note in disagreeing if note)})", + ) logged: Final = ", ".join(f"${spend_rows[turn.call_id]:.6f}" for turn in turns) return CheckResult("cost_tracking", "pass", f"{priced}, spend log rows written ({logged})") diff --git a/tests/integration/management/test_model_health_check.py b/tests/integration/management/test_model_health_check.py index 516c5e2c24e..38f8fc126a0 100644 --- a/tests/integration/management/test_model_health_check.py +++ b/tests/integration/management/test_model_health_check.py @@ -1581,4 +1581,67 @@ def test_health_test_model_names_the_check_that_failed_for_an_unsaved_model_that ), response.text assert checks["cost_tracking"][0] == "fail", response.text assert "no price resolved for this model" in checks["cost_tracking"][1], response.text - assert len(wire.drain()) >= 2, response.text + assert len(wire.drain()) == 5, response.text + + +@pytest.mark.timeout(240) +def test_health_test_model_bills_an_unsaved_model_at_the_custom_prices_in_its_model_info(gateway: Gateway) -> None: + with wire_server(lambda request: _test_model_chat_reply(request, cached_tokens=4800)) as wire: + response: Final = gateway.request( + "POST", + "/health/test_model", + { + "litellm_params": { + "model": f"openai/priced-{uuid.uuid4().hex}", + "api_base": f"{wire.url}/v1", + "api_key": "synthetic-priced-key", + }, + "model_info": {"input_cost_per_token": 1e-6, "output_cost_per_token": 2e-6}, + "endpoints": ["/v1/chat/completions"], + }, + ) + + assert response.status_code == 200, response.text + payload: Final = JSON_OBJECT.validate_json(response.content) + assert payload["status"] == "success", response.text + checks: Final = _check_statuses(object_value(list_value(payload["endpoints"])[0])) + assert checks["cost_tracking"][0] == "pass", response.text + assert checks["cost_tracking"][1].startswith("turn 1 $0.005024, "), response.text + assert "spend log rows written" in checks["cost_tracking"][1], response.text + assert len(wire.drain()) == 3, response.text + + +@pytest.mark.timeout(240) +def test_health_test_model_with_a_different_model_than_the_saved_deployment_calls_that_model(gateway: Gateway) -> None: + with ( + gateway.scenario() as scenario, + wire_server(lambda request: _test_model_chat_reply(request, cached_tokens=4800)) as wire, + ): + model_name: Final = scenario.model(api_base=f"{wire.url}/v1", api_key="synthetic-override-key") + model_id: Final = next( + string_value(object_value(info["model_info"])["id"]) + for info in (object_value(entry) for entry in list_value(gateway.get("/v1/model/info")["data"])) + if info["model_name"] == model_name + ) + override: Final = f"openai/override-{uuid.uuid4().hex}" + + response: Final = gateway.request( + "POST", + "/health/test_model", + {"model_id": model_id, "litellm_params": {"model": override}, "endpoints": ["/v1/chat/completions"]}, + ) + + assert response.status_code == 200, response.text + payload: Final = JSON_OBJECT.validate_json(response.content) + assert payload["model"] == override, response.text + assert payload["model_id"] is None, response.text + assert _check_statuses(object_value(list_value(payload["endpoints"])[0]))["conversation"][0] == "pass", ( + response.text + ) + requests: Final = wire.drain() + assert [JSON_OBJECT.validate_json(request.body)["model"] for request in requests] == [ + override.removeprefix("openai/") + ] * 3, response.text + assert [request.headers["authorization"] for request in requests] == ["Bearer synthetic-override-key"] * 3, ( + response.text + ) diff --git a/tests/unit/proxy/health_endpoints/test_model_checks.py b/tests/unit/proxy/health_endpoints/test_model_checks.py index 57af7c75a57..27527ae1f1d 100644 --- a/tests/unit/proxy/health_endpoints/test_model_checks.py +++ b/tests/unit/proxy/health_endpoints/test_model_checks.py @@ -1,6 +1,6 @@ import asyncio import json -from collections.abc import Mapping, Sequence +from collections.abc import Callable, Mapping, Sequence from typing import Final import pytest @@ -11,6 +11,7 @@ from litellm.proxy.health_endpoints.model_checks import ( RESPONSES_SCRIPT, WEATHER_TOOL_RESULT, CalledTurn, + EndpointResult, EndpointScript, endpoint_result_payload, overall_status, @@ -150,7 +151,9 @@ def _ids() -> Sequence[str]: return ("call-1", "call-2", "call-3", "call-4", "call-5") -def _run(script: EndpointScript, provider: _ScriptedProvider, spend_rows: Mapping[str, float] | None = None): +def _run( + script: EndpointScript, provider: _ScriptedProvider, spend_rows: Mapping[str, float] | None = None +) -> EndpointResult: ids: Final = iter(_ids()) async def spend_lookup(request_ids: Sequence[str]) -> Mapping[str, float]: @@ -166,7 +169,7 @@ def _run(script: EndpointScript, provider: _ScriptedProvider, spend_rows: Mappin ) -def _checks(result) -> Mapping[str, tuple[str, str]]: +def _checks(result: EndpointResult) -> Mapping[str, tuple[str, str]]: return {check.name: (check.status, check.detail) for check in result.checks} @@ -179,7 +182,9 @@ def _checks(result) -> Mapping[str, tuple[str, str]]: ], ids=["chat", "responses", "messages"], ) -def test_every_endpoint_passes_all_three_checks_and_feeds_each_reply_back_in_its_own_shape(script, reply, key): +def test_every_endpoint_passes_all_three_checks_and_feeds_each_reply_back_in_its_own_shape( + script: EndpointScript, reply: Callable[..., object], key: str +) -> None: provider: Final = _ScriptedProvider( CalledTurn(reply(tool_call=True), 0.004), [ @@ -229,6 +234,7 @@ def test_every_endpoint_passes_all_three_checks_and_feeds_each_reply_back_in_its tool_message: Final = turn_two["messages"][-1] assert tool_message == {"role": "tool", "tool_call_id": _TOOL_CALL_ID, "content": WEATHER_TOOL_RESULT} assert turn_two["messages"][-2]["tool_calls"][0]["id"] == _TOOL_CALL_ID + assert "content" in turn_two["messages"][-2] and turn_two["messages"][-2]["content"] is None, turn_two assert turn_three["messages"][-2]["role"] == "system", turn_three assert turn_three["messages"][-3]["role"] == "assistant", turn_three assert turn_three["messages"][-3]["content"] == _TURN_TWO_ANSWER, turn_three @@ -465,6 +471,26 @@ def test_a_missing_spend_log_row_fails_cost_tracking_even_when_the_response_was_ ) +def test_a_spend_log_row_that_disagrees_with_the_response_cost_fails_cost_tracking() -> None: + provider: Final = _ScriptedProvider( + CalledTurn(_chat_reply(tool_call=True), 0.004), + [ + CalledTurn(_chat_reply(tool_call=False, text="72F", cached_tokens=4800), 0.0005), + CalledTurn(_chat_reply(tool_call=False, text="72F", cached_tokens=4800), 0.0004), + ], + ) + + checks: Final = _checks( + _run(CHAT_COMPLETIONS_SCRIPT, provider, spend_rows={"call-1": 0.004, "call-2": 0.0, "call-3": 0.0004}) + ) + + assert checks["cost_tracking"] == ( + "fail", + "turn 1 $0.004000, turn 2 $0.000500, turn 3 $0.000400, but the spend log row disagrees with the response " + "cost (request_id call-2 row $0.000000 vs response_cost $0.000500)", + ) + + def test_a_response_that_is_not_the_endpoint_shape_fails_the_conversation_with_the_parse_error(): provider: Final = _ScriptedProvider(CalledTurn({"id": "x", "object": "nonsense"}, 0.001), []) From 4389f753543f52b95e843761e6745b869d3526bd Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 4 Nov 2026 00:35:16 +0000 Subject: [PATCH 15/15] chore(ui): regenerate API types for the test_model docstring --- ui/litellm-dashboard/src/lib/http/schema.d.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index ae7b8120289..546200c2c82 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -8029,7 +8029,9 @@ export interface paths { * answer from turn 2. Three checks come out of it: * * - `conversation`: every turn parses, turn 1 calls the tool, turn 2 uses the tool result, turn 3 carries it on - * - `prompt_caching`: turns 2 and 3 report cached input tokens, with no cache markers in the request + * - `prompt_caching`: turns 2 and 3 report cached input tokens and turn 3 reads at least as many as turn 2. + * Chat Completions and Responses send no cache markers, so the provider has to cache the repeated prefix on + * its own. Messages sends `cache_control` on the latest user message the way Claude Code does * - `cost_tracking`: every turn carries a non-zero response cost, and the spend log row lands when a * database is connected *