scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-01 07:03:02 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 129

test: update integration tests left stale by recent product changes (#49143)

Browse source 
* test: update integration tests left stale by recent product changes

* test: expect proxy Messages calls to stay on Python with Rust enabled
This commit is contained in:
yuneng-jiang 2026-10-29 18:06:28 -07:00 • committed by GitHub
parent 24afa12253
commit d76db470f9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
24 changed files with 415 additions and 259 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
tests/integration/management/test_auto_router_databricks_classifier_writes.py
View file

@ -3,6 +3,7 @@ import json
import threading
import uuid
from collections.abc import Callable, Mapping, Sequence
from pathlib import Path
from queue import SimpleQueue
from typing import Final
@ -64,6 +65,21 @@ _ROWS_QUERY: Final = (
)
def _number(value: JsonValue) -> float:
assert isinstance(value, (int, float)) and not isinstance(value, bool), value
return float(value)
def _endpoint_cost() -> float:
prices: Final = object_value(
json.loads(Path("model_prices_and_context_window.json").read_text())[f"databricks/{_ENDPOINT}"]
)
usage: Final = object_value(_ENDPOINT_ANSWER["usage"])
return _number(usage["input_tokens"]) * _number(prices["input_cost_per_token"]) + _number(
usage["output_tokens"]
) * _number(prices["output_cost_per_token"])
def _classifier(request: Request) -> Reply:
answer: Final = _ENDPOINT_ANSWER if request.target.endswith("/invocations") else _AI_DECIDE_ANSWER
return Reply(body=json.dumps(answer).encode())
@ -221,11 +237,14 @@ def test_validate_names_each_databricks_classifier_misconfiguration(gateway: Gat
@pytest.mark.parametrize(
("model", "target", "cost"),
[(_MODEL, _ROUTE, {}), (_ENDPOINT, f"/{_ENDPOINT}/invocations", {"classifier_cost": 0.0})],
[
(_MODEL, _ROUTE, {}),
(_ENDPOINT, f"/{_ENDPOINT}/invocations", {"classifier_cost": pytest.approx(_endpoint_cost())}),
],
ids=["ai_decide", "serving_endpoint"],
)
def test_test_routing_classifies_through_databricks_without_calling_the_tier_it_picked(
gateway: Gateway, model: str, target: str, cost: dict[str, JsonValue]
gateway: Gateway, model: str, target: str, cost: dict[str, object]
) -> None:
prompt: Final = f"design a distributed cache {uuid.uuid4().hex}"
with (

31
tests/integration/mcp/test_mcp_access_matrix.py
View file

@ -181,7 +181,6 @@ def test_same_tool_name_on_two_servers_routes_by_prefix(gateway: Gateway) -> Non
_PROBE: Final = "catalog-probe"
_ECHO: Final = "catalog-echo"
_UNLISTED: Final = ""
_GUARDRAIL_CODE: Final = (
"def apply_guardrail(inputs, request_data, input_type):\n"
f' if "{_PROBE}" not in list(inputs.get("texts") or []):\n'
@ -217,13 +216,26 @@ def _echoed_description(outcome: Outcome) -> str:
return found.group(1)
def _add_described_anew_per_fetch(described: str) -> ScriptedTool:
return ScriptedTool("add", lambda _: text_result("3"), description=lambda _: f"{described} {uuid.uuid4().hex}")
def _fetched(described: str) -> re.Pattern[str]:
return re.compile(rf"{re.escape(described)} [0-9a-f]{{32}}")
def _listed_description(listing: Outcome, described: str) -> str:
found: Final = _fetched(described).search(listing.raw)
assert found is not None, listing.raw
return found.group(0)
@pytest.mark.parametrize("subject", SUBJECTS)
def test_each_subjects_call_is_evaluated_only_against_the_catalog_its_own_listing_served(
echo_rig: Gateway, subject: Subject
) -> None:
described: Final = "Adds under grant " + uuid.uuid4().hex[:8]
tool: Final = ScriptedTool("add", lambda _: text_result("3"), description=described)
with scripted_peer(tool) as peer, echo_rig.scenario() as scenario:
with scripted_peer(_add_described_anew_per_fetch(described)) as peer, echo_rig.scenario() as scenario:
group: Final = "grp" + uuid.uuid4().hex[:8]
alias: Final = "cat" + uuid.uuid4().hex[:8]
identity: Final = register_mcp(scenario, peer, alias, mcp_access_groups=[group])
@ -235,8 +247,10 @@ def test_each_subjects_call_is_evaluated_only_against_the_catalog_its_own_listin
cold: Final = _echoed_description(reach.call(f"{alias}-add", {"probe": _PROBE}))
listed: Final = reach.list_tools()
assert listed.ok and f"{alias}-add" in listed.tools, listed.raw
served: Final = _listed_description(listed, described)
warm: Final = _echoed_description(reach.call(f"{alias}-add", {"probe": _PROBE}))
assert (cold, warm) == (_UNLISTED, described), (cold, warm)
assert _fetched(described).fullmatch(cold) and cold != served, (cold, served)
assert warm == served, (cold, served, warm)
assert tool_calls(peer.drain()) == (), "a blocked probe reached the peer"
@ -244,8 +258,7 @@ def test_end_users_of_one_key_share_its_catalog_slot_because_the_identity_exclud
echo_rig: Gateway,
) -> None:
described: Final = "Adds for end users " + uuid.uuid4().hex[:8]
tool: Final = ScriptedTool("add", lambda _: text_result("3"), description=described)
with scripted_peer(tool) as peer, echo_rig.scenario() as scenario:
with scripted_peer(_add_described_anew_per_fetch(described)) as peer, echo_rig.scenario() as scenario:
alias: Final = "eu" + uuid.uuid4().hex[:8]
identity: Final = register_mcp(scenario, peer, alias)
granted: Final = grant(scenario, "end_user", (identity,), (identity,))
@ -254,10 +267,12 @@ def test_end_users_of_one_key_share_its_catalog_slot_because_the_identity_exclud
echo_rig, granted.key, "mcp", alias, {"x-litellm-end-user-id": "integration-" + uuid.uuid4().hex[:10]}
)
assert second.initialize().ok
assert _echoed_description(second.call(f"{alias}-add", {"probe": _PROBE})) == _UNLISTED
cold: Final = _echoed_description(second.call(f"{alias}-add", {"probe": _PROBE}))
listed: Final = first.list_tools()
assert listed.ok and f"{alias}-add" in listed.tools, listed.raw
assert _echoed_description(second.call(f"{alias}-add", {"probe": _PROBE})) == described, (
served: Final = _listed_description(listed, described)
assert _fetched(described).fullmatch(cold) and cold != served, (cold, served)
assert _echoed_description(second.call(f"{alias}-add", {"probe": _PROBE})) == served, (
"the end-user header is intentionally not part of the catalog identity: one key, one slot"
)
assert tool_calls(peer.drain()) == (), "a blocked probe reached the peer"

96
tests/integration/mcp/test_mcp_credentials.py
View file

@ -448,7 +448,6 @@ def test_deprecated_string_x_mcp_auth_lists_a_byok_server_for_a_key_without_a_us
_PROBE: Final = "catalog-probe"
_ECHO: Final = "catalog-echo"
_UNLISTED: Final = ""
_GUARDRAIL_CODE: Final = (
"def apply_guardrail(inputs, request_data, input_type):\n"
f' if "{_PROBE}" not in list(inputs.get("texts") or []):\n'
@ -488,6 +487,28 @@ _PROBE_ARGUMENTS: Final = {"probe": _PROBE}
_HEADERS: Final = TypeAdapter(dict[str, str])
def _add_described_anew_per_fetch_for(header: str) -> ScriptedTool:
return ScriptedTool(
"add",
lambda _: text_result("3"),
description=lambda headers: f"Adds for {headers.get(header, 'nobody')} at {uuid.uuid4().hex}",
)
def _fetched_for(credential: str) -> re.Pattern[str]:
return re.compile(rf"Adds for {re.escape(credential)} at [0-9a-f]{{32}}")
def _listed_description(listing: Outcome, credential: str) -> str:
found: Final = _fetched_for(credential).search(listing.raw)
assert found is not None, listing.raw
return found.group(0)
def _sent_header(request: dict[str, object], name: str) -> str | None:
return _HEADERS.validate_python(request["headers"]).get(name)
def _store_byok_credential(scenario: Scenario, identity: str, key: str, secret: str) -> None:
stored: Final = scenario.gateway.client.post(
f"/v1/mcp/server/{identity}/user-credential", json={"credential": secret}, headers={"x-litellm-api-key": key}
@ -499,7 +520,7 @@ def _store_byok_credential(scenario: Scenario, identity: str, key: str, secret:
def test_rotating_the_credential_drops_the_callers_listing_until_it_lists_again(echo_rig: Gateway) -> None:
with mcp_peer() as peer, echo_rig.scenario() as scenario:
with scripted_peer(_add_described_anew_per_fetch_for("authorization")) as peer, echo_rig.scenario() as scenario:
first: Final = "cred-" + uuid.uuid4().hex
second: Final = "cred-" + uuid.uuid4().hex
alias: Final = "rot" + uuid.uuid4().hex[:8]
@ -508,53 +529,62 @@ def test_rotating_the_credential_drops_the_callers_listing_until_it_lists_again(
)
key: Final = scenario.key(object_permission={"mcp_servers": [identity]})
caller: Final = McpCaller(echo_rig, key, "mcp", alias)
assert caller.list_tools().ok
assert _echoed_description(caller.call(f"{alias}-add", _PROBE_ARGUMENTS)) == "Add two integers"
listed: Final = caller.list_tools()
assert listed.ok, listed.raw
served: Final = _listed_description(listed, f"Bearer {first}")
assert _echoed_description(caller.call(f"{alias}-add", _PROBE_ARGUMENTS)) == served
rotated: Final = echo_rig.request(
"PUT", "/v1/mcp/server", {"server_id": identity, "credentials": {"auth_value": second}}
)
assert rotated.status_code == 202, rotated.text
eventually(
lambda: _echoed_description(caller.call(f"{alias}-add", _PROBE_ARGUMENTS)), lambda seen: seen == _UNLISTED
lambda: _echoed_description(caller.call(f"{alias}-add", _PROBE_ARGUMENTS)),
lambda seen: _fetched_for(f"Bearer {second}").fullmatch(seen) is not None,
)
peer.drain()
assert caller.list_tools().ok
assert _echoed_description(caller.call(f"{alias}-add", _PROBE_ARGUMENTS)) == "Add two integers"
relisting: Final = caller.list_tools()
assert relisting.ok, relisting.raw
served_again: Final = _listed_description(relisting, f"Bearer {second}")
assert _echoed_description(caller.call(f"{alias}-add", _PROBE_ARGUMENTS)) == served_again
relisted: Final = _listings(peer)
assert len(relisted) == 1, relisted
assert _header(relisted[0], b"authorization") == f"Bearer {second}".encode(), relisted[0]["headers"]
assert _sent_header(relisted[0], "authorization") == f"Bearer {second}", relisted[0]["headers"]
def test_byok_callers_are_evaluated_against_their_own_listing_and_the_stored_secret_never_keys_the_slot(
echo_rig: Gateway,
) -> None:
with mcp_peer() as peer, echo_rig.scenario() as scenario:
with scripted_peer(_add_described_anew_per_fetch_for("x-api-key")) as peer, echo_rig.scenario() as scenario:
alias: Final = "byok" + uuid.uuid4().hex[:8]
identity: Final = register_mcp(scenario, peer, alias, auth_type="api_key", is_byok=True)
owner_key: Final = scenario.key(user_id=scenario.user(), object_permission={"mcp_servers": [identity]})
stranger_key: Final = scenario.key(user_id=scenario.user(), object_permission={"mcp_servers": [identity]})
owner_secret: Final = "byok-" + uuid.uuid4().hex
stranger_secret: Final = "byok-" + uuid.uuid4().hex
replacement: Final = "byok-" + uuid.uuid4().hex
_store_byok_credential(scenario, identity, owner_key, owner_secret)
_store_byok_credential(scenario, identity, stranger_key, "byok-" + uuid.uuid4().hex)
_store_byok_credential(scenario, identity, stranger_key, stranger_secret)
owner: Final = McpCaller(echo_rig, owner_key, "mcp", alias)
stranger: Final = McpCaller(echo_rig, stranger_key, "mcp", alias)
peer.drain()
assert owner.list_tools().ok
listed: Final = owner.list_tools()
assert listed.ok, listed.raw
listings: Final = _listings(peer)
assert [_header(item, b"x-api-key") for item in listings] == [owner_secret.encode()], listings
assert [_sent_header(item, "x-api-key") for item in listings] == [owner_secret], listings
served: Final = _listed_description(listed, owner_secret)
own: Final = _echoed_description(owner.call(f"{alias}-add", _PROBE_ARGUMENTS))
other: Final = _echoed_description(stranger.call(f"{alias}-add", _PROBE_ARGUMENTS))
assert (own, other) == ("Add two integers", _UNLISTED), (own, other)
assert own == served, (own, served)
assert _fetched_for(stranger_secret).fullmatch(other), (other, served)
_store_byok_credential(scenario, identity, owner_key, replacement)
assert _echoed_description(owner.call(f"{alias}-add", _PROBE_ARGUMENTS)) == "Add two integers", (
"the slot is keyed by the client-supplied header, never by the stored credential"
)
sent: Final = eventually(
lambda: (owner.call(f"{alias}-add", ADD).ok, tool_calls(peer.drain())),
lambda value: any(_header(call, b"x-api-key") == replacement.encode() for call in value[1]),
lambda value: any(_sent_header(call, "x-api-key") == replacement for call in value[1]),
)
assert sent[0], sent
assert _echoed_description(owner.call(f"{alias}-add", _PROBE_ARGUMENTS)) == served, (
"the slot is keyed by the client-supplied header, never by the stored credential"
)
def test_callers_with_different_server_scoped_auth_headers_are_evaluated_against_their_own_listings(
@ -585,12 +615,7 @@ def test_callers_with_different_server_scoped_auth_headers_are_evaluated_against
def test_deprecated_string_x_mcp_auth_callers_on_a_user_less_key_own_separate_listings(echo_rig: Gateway) -> None:
tool: Final = ScriptedTool(
"add",
lambda _: text_result("3"),
description=lambda headers: "Adds for " + headers.get("authorization", "nobody"),
)
with scripted_peer(tool) as peer, echo_rig.scenario() as scenario:
with scripted_peer(_add_described_anew_per_fetch_for("authorization")) as peer, echo_rig.scenario() as scenario:
alias: Final = "legacy" + uuid.uuid4().hex[:8]
identity: Final = register_mcp(scenario, peer, alias, auth_type="bearer_token")
key: Final = scenario.key(object_permission={"mcp_servers": [identity]})
@ -598,28 +623,33 @@ def test_deprecated_string_x_mcp_auth_callers_on_a_user_less_key_own_separate_li
second_token: Final = "second-" + uuid.uuid4().hex
first: Final = McpCaller(echo_rig, key, "mcp", alias, {"x-mcp-auth": f"Bearer {first_token}"})
second: Final = McpCaller(echo_rig, key, "mcp", alias, {"x-mcp-auth": f"Bearer {second_token}"})
assert _echoed_description(first.call(f"{alias}-add", _PROBE_ARGUMENTS)) == _UNLISTED
first_cold: Final = _echoed_description(first.call(f"{alias}-add", _PROBE_ARGUMENTS))
assert _fetched_for(f"Bearer {first_token}").fullmatch(first_cold), first_cold
peer.drain()
assert first.list_tools().ok
first_listing: Final = first.list_tools()
assert first_listing.ok, first_listing.raw
listings: Final = _listings(peer)
assert len(listings) == 1, listings
listed_with: Final = _HEADERS.validate_python(listings[0]["headers"])
assert listed_with.get("authorization") == f"Bearer {first_token}", listed_with
assert _sent_header(listings[0], "authorization") == f"Bearer {first_token}", listings[0]["headers"]
first_served: Final = _listed_description(first_listing, f"Bearer {first_token}")
warm: Final = eventually(
lambda: _echoed_description(first.call(f"{alias}-add", _PROBE_ARGUMENTS)),
lambda seen: seen != _UNLISTED,
lambda seen: seen == first_served,
)
assert warm == f"Adds for Bearer {first_token}", warm
assert _echoed_description(second.call(f"{alias}-add", _PROBE_ARGUMENTS)) == _UNLISTED
assert second.list_tools().ok
assert warm == first_served != first_cold, (first_cold, first_served, warm)
second_cold: Final = _echoed_description(second.call(f"{alias}-add", _PROBE_ARGUMENTS))
assert _fetched_for(f"Bearer {second_token}").fullmatch(second_cold), (second_cold, first_served)
second_listing: Final = second.list_tools()
assert second_listing.ok, second_listing.raw
second_served: Final = _listed_description(second_listing, f"Bearer {second_token}")
seen: Final = eventually(
lambda: (
_echoed_description(first.call(f"{alias}-add", _PROBE_ARGUMENTS)),
_echoed_description(second.call(f"{alias}-add", _PROBE_ARGUMENTS)),
),
lambda pair: _UNLISTED not in pair,
lambda pair: pair == (first_served, second_served),
)
assert seen == (f"Adds for Bearer {first_token}", f"Adds for Bearer {second_token}"), seen
assert seen == (first_served, second_served), seen
assert tool_calls(peer.drain()) == (), "a blocked probe reached the peer"

35
tests/integration/mcp/test_mcp_llm_endpoints.py
View file

@ -1,5 +1,6 @@
import asyncio
import json
import re
import uuid
from collections.abc import Callable, Generator, Iterator, Mapping, Sequence
from concurrent.futures import ThreadPoolExecutor
@ -1408,10 +1409,6 @@ REPORT: Final[tuple[str, dict[str, JsonValue]]] = (
"Write one report",
{"type": "object", "properties": {"query": {"type": "string"}, "format": {}}},
)
COLD: Final[tuple[str, dict[str, JsonValue]]] = (
"",
{"type": "object", "properties": {}, "additionalProperties": False},
)
RELOAD_FAST: Final = {"PROXY_CONFIG_RELOAD_INTERVAL_SECONDS": "5"}
CALL_ID: Final = "x-litellm-call-id"
T = TypeVar("T")
@ -1577,12 +1574,14 @@ def _echoed(value: JsonValue) -> Echo:
@contextmanager
def _bridge_rig(hooked: Hooked, bridge: Bridge) -> Generator[BridgeRig, None, None]:
def _bridge_rig(
hooked: Hooked, bridge: Bridge, lookup_description: str | Callable[[Mapping[str, str]], str] = LOOKUP[0]
) -> Generator[BridgeRig, None, None]:
alias: Final = "brg" + uuid.uuid4().hex[:8]
lookup: Final = ScriptedTool(
"lookup",
lambda params: text_result("found:" + json.dumps(JSON_OBJECT.validate_python(params)["arguments"])),
description=LOOKUP[0],
description=lookup_description,
input_schema=LOOKUP[1],
)
report: Final = ScriptedTool(
@ -1753,23 +1752,35 @@ def test_bridge_hook_sees_the_definition_of_the_tool_filtered_for_that_request(
assert _spend_row(key, found.call_id)["status"] == "success"
def _fetched_lookup(description: JsonValue) -> bool:
return isinstance(description, str) and re.fullmatch(rf"{LOOKUP[0]} [0-9a-f]{{32}}", description) is not None
@pytest.mark.parametrize("bridge", BRIDGES)
def test_direct_call_after_bridge_only_discovery_stays_cold(hooked: Hooked, bridge: Bridge) -> None:
with _bridge_rig(hooked, bridge) as rig:
with _bridge_rig(hooked, bridge, lambda _: f"{LOOKUP[0]} {uuid.uuid4().hex}") as rig:
key: Final = _bridge_key(rig)
workers: Final = _both_workers(rig.hooked.proxy)
bridged: Final = _ask(rig, key, HOOK_PROBE, [rig.mcp("lookup")], False, "sync")
assert _echoed(bridged.text) == _served(LOOKUP), bridged
discovered: Final = _echoed(bridged.text)
offered: Final = rig.upstream_by_prompt()
assert _fetched_lookup(discovered[0]) and discovered[1] == _served(LOOKUP)[1], bridged
assert set(offered) == {HOOK_PROBE} and all(
definitions == ((rig.tool("lookup"), *discovered),) for definitions in offered[HOOK_PROBE]
), (discovered, offered)
direct: Final = eventually(
lambda: tuple(
_on_worker(rig.hooked.proxy, lambda client: _direct_probe(rig, key, "lookup", client)) for _ in range(6)
),
lambda seen: frozenset(pid for pid, _ in seen) == workers,
)
assert all(echo == COLD for _, echo in direct) and frozenset(pid for pid, _ in direct) == workers, (
direct,
workers,
)
assert (
all(
_fetched_lookup(description) and description != discovered[0] and parameters == discovered[1]
for _, (description, parameters) in direct
)
and frozenset(pid for pid, _ in direct) == workers
), (direct, discovered, workers)
assert rig.peer_calls() == ()

56
tests/integration/mcp/test_mcp_oauth_flows.py
View file

@ -19,7 +19,7 @@ import jwt
import pytest
import yaml
from cryptography.hazmat.primitives.asymmetric.rsa import RSAPrivateKey, generate_private_key
from integration._support.client import Gateway, eventually, gateway_from_environment
from integration._support.client import Gateway, Scenario, eventually, gateway_from_environment
from integration._support.database import read_rows
from integration._support.mcp import (
ENTRY_POINTS,
@ -786,13 +786,20 @@ def test_dcr_bridge_relays_client_registration_and_advertises_gateway_endpoints(
assert "S256" in issuer.json()["code_challenge_methods_supported"]
def _ui_session_cookie(gateway: Gateway, user_id: str) -> dict[str, str]:
claims: Final = {"user_id": user_id, "login_method": "username_password", "exp": int(time.time()) + 600}
return {"token": jwt.encode(claims, gateway.key, algorithm="HS256")}
def _ui_session_cookie(scenario: Scenario, user_id: str) -> dict[str, str]:
session_key: Final = scenario.key(user_id=user_id, duration="10m")
claims: Final = {
"user_id": user_id,
"key": session_key,
"login_method": "username_password",
"exp": int(time.time()) + 600,
}
return {"token": jwt.encode(claims, scenario.gateway.key, algorithm="HS256")}
def _gateway_session_sign_in(
gateway: Gateway,
scenario: Scenario,
user_id: str,
resource: str | None = None,
*,
@ -806,7 +813,7 @@ def _gateway_session_sign_in(
assert registered.status_code in (200, 201), registered.text
client_id: Final = str(_response_object(registered)["client_id"])
pkce: Final = _Pkce(secrets.token_urlsafe(48))
cookies: Final = _ui_session_cookie(gateway, user_id)
cookies: Final = _ui_session_cookie(scenario, user_id)
started: Final = gateway.client.get(
authorization_endpoint,
params={
@ -843,8 +850,8 @@ def _gateway_session_sign_in(
return _response_object(issued), client_id
def _gateway_session_bearer(gateway: Gateway, user_id: str, resource: str | None = None) -> str:
issued, _ = _gateway_session_sign_in(gateway, user_id, resource)
def _gateway_session_bearer(gateway: Gateway, scenario: Scenario, user_id: str, resource: str | None = None) -> str:
issued, _ = _gateway_session_sign_in(gateway, scenario, user_id, resource)
return _issued_token(issued)
@ -887,7 +894,7 @@ def test_spec_client_discovers_the_aggregate_authorization_server_and_signs_in_t
assert registered.status_code in (200, 201), registered.text
client_id: Final = str(_response_object(registered)["client_id"])
pkce: Final = _Pkce(secrets.token_urlsafe(48))
cookies: Final = _ui_session_cookie(gateway, member)
cookies: Final = _ui_session_cookie(scenario, member)
started: Final = gateway.client.get(
authorization_endpoint,
params={
@ -997,7 +1004,7 @@ def test_per_server_resource_scopes_the_session_bearer_to_the_named_server(gatew
assert registered.status_code in (200, 201), registered.text
client_id: Final = str(_response_object(registered)["client_id"])
pkce: Final = _Pkce(secrets.token_urlsafe(48))
cookies: Final = _ui_session_cookie(gateway, member)
cookies: Final = _ui_session_cookie(scenario, member)
started: Final = gateway.client.get(
"/authorize/mcp-session",
params={
@ -1110,7 +1117,7 @@ def test_rotated_session_refresh_token_cannot_be_replayed_after_revocation(gatew
alias: Final = "rt6" + uuid.uuid4().hex[:10]
identity: Final = register_mcp(scenario, peer, alias)
member: Final = scenario.member(scenario.team(object_permission={"mcp_servers": [identity]}))
issued, client_id = _gateway_session_sign_in(gateway, member)
issued, client_id = _gateway_session_sign_in(gateway, scenario, member)
r1: Final = str(issued["refresh_token"])
a1: Final = str(issued["access_token"])
assert r1.startswith("llm_srefresh_"), r1
@ -1207,7 +1214,7 @@ def test_lite_logout_revokes_the_cli_refresh_token_sent_the_way_the_cli_sends_it
assert registered.status_code in (200, 201), registered.text
client_id: Final = str(_response_object(registered)["client_id"])
pkce: Final = _Pkce(secrets.token_urlsafe(64))
cookies: Final = _ui_session_cookie(gateway, member)
cookies: Final = _ui_session_cookie(scenario, member)
consent: Final = gateway.client.get(
str(contract["authorization_endpoint"]),
params={
@ -1362,6 +1369,7 @@ def test_gateway_session_bearer_of_a_team_member_is_served_the_team_toolset_on_i
token_endpoint: Final = str(authorization_body["token_endpoint"])
issued, client_id = _gateway_session_sign_in(
gateway,
scenario,
member,
resource,
registration_endpoint=str(authorization_body["registration_endpoint"]),
@ -1413,7 +1421,7 @@ def test_resource_scoped_session_bearer_opens_a_team_toolset_inside_its_server_a
inside_id: Final = create_toolset(scenario, ((inside_server, "add"),), toolset_name=inside_name)
outside_id: Final = create_toolset(scenario, ((outside_server, "add"),), toolset_name=outside_name)
member: Final = scenario.member(scenario.team(object_permission={"mcp_toolsets": [inside_id, outside_id]}))
bearer: Final = _gateway_session_bearer(gateway, member, resource=f"{_base(gateway)}/{inside}/mcp")
bearer: Final = _gateway_session_bearer(gateway, scenario, member, resource=f"{_base(gateway)}/{inside}/mcp")
assert bearer.startswith("llm_session_"), bearer[:16]
listed: Final = _toolset_rpc(gateway, bearer, inside_name, "tools/list", {})
assert listed.tools == (f"{inside}-add",), listed.raw
@ -1430,7 +1438,6 @@ def test_resource_scoped_session_bearer_opens_a_team_toolset_inside_its_server_a
_PROBE: Final = "catalog-probe"
_ECHO: Final = "catalog-echo"
_UNLISTED: Final = ""
_GUARDRAIL_CODE: Final = (
"def apply_guardrail(inputs, request_data, input_type):\n"
f' if "{_PROBE}" not in list(inputs.get("texts") or []):\n'
@ -1466,8 +1473,22 @@ def _echoed_description(outcome: Outcome) -> str:
return found.group(1)
def _add_described_for_the_exchanged_subject(auth: AuthorizationServer) -> ScriptedTool:
def described(headers: Mapping[str, str]) -> str:
token: Final = headers.get("authorization", "").removeprefix("Bearer ")
with auth.lock:
grant: Final = auth.access_tokens.get(token, {})
return "Add for " + grant.get("subject", "nobody")
return ScriptedTool("add", lambda _: text_result("must not run"), description=described)
def test_token_exchange_callers_with_different_subject_tokens_own_separate_listings(echo_rig: Gateway) -> None:
with mcp_peer() as peer, oauth_server() as auth, echo_rig.scenario() as scenario:
with (
oauth_server() as auth,
scripted_peer(_add_described_for_the_exchanged_subject(auth)) as peer,
echo_rig.scenario() as scenario,
):
alias: Final = "te" + uuid.uuid4().hex[:8]
identity: Final = register_mcp(
scenario,
@ -1479,19 +1500,20 @@ def test_token_exchange_callers_with_different_subject_tokens_own_separate_listi
)
key: Final = scenario.key(object_permission={"mcp_servers": [identity]})
first_subject: Final = "subject-" + uuid.uuid4().hex
second_subject: Final = "subject-" + uuid.uuid4().hex
first: Final = McpCaller(echo_rig, key, "mcp", alias, {"Authorization": f"Bearer {first_subject}"})
second: Final = McpCaller(echo_rig, key, "mcp", alias, {"Authorization": "Bearer subject-" + uuid.uuid4().hex})
second: Final = McpCaller(echo_rig, key, "mcp", alias, {"Authorization": f"Bearer {second_subject}"})
auth.drain()
assert first.list_tools().ok
assert [request["subject_token"] for request in auth.token_requests()] == [first_subject]
probe: Final = {"probe": _PROBE}
own: Final = _echoed_description(first.call(f"{alias}-add", probe))
other: Final = _echoed_description(second.call(f"{alias}-add", probe))
assert (own, other) == ("Add two integers", _UNLISTED), (
assert (own, other) == (f"Add for exchanged:{first_subject}", f"Add for exchanged:{second_subject}"), (
"the caller bearer is part of the identity on a token-exchange server: one subject, one slot"
)
assert second.list_tools().ok
assert _echoed_description(second.call(f"{alias}-add", probe)) == "Add two integers"
assert _echoed_description(second.call(f"{alias}-add", probe)) == f"Add for exchanged:{second_subject}"
assert tool_calls(peer.drain()) == (), "a blocked probe reached the peer"

66
tests/integration/mcp/test_mcp_resilience.py
View file

@ -1,4 +1,5 @@
import itertools
import re
import uuid
from collections.abc import Mapping
from concurrent.futures import ThreadPoolExecutor
@ -37,6 +38,7 @@ pytestmark: Final = pytest.mark.shares_worker_state
_ECHO: Final = "catalog-echo:"
_PROBE: Final = "catalog-probe"
_DESCRIPTION: Final = "Look up one record"
_FETCHED: Final = re.compile(rf"{_DESCRIPTION} [0-9a-f]{{32}}")
_GUARDRAIL_CODE: Final = (
"def apply_guardrail(inputs, request_data, input_type):\n"
' texts = list(inputs.get("texts") or [])\n'
@ -69,6 +71,20 @@ class _RpcReply(BaseModel):
result: _Result
class _ListedTool(BaseModel):
name: str
description: str | None = None
class _ListedTools(BaseModel):
tools: tuple[_ListedTool, ...]
class _ListReply(BaseModel):
id: int
result: _ListedTools
class _SessionsReport(BaseModel):
worker_pid: int
@ -207,9 +223,13 @@ def test_peer_restart_on_the_same_url_is_picked_up_without_gateway_restart(gatew
assert len(tool_calls(replacement.drain())) >= 1
def _rpc_reply(raw: str) -> _RpcReply:
def _rpc_payload(raw: str) -> str:
data: Final = tuple(line[5:].strip() for line in raw.splitlines() if line.startswith("data:"))
return _RpcReply.model_validate_json(data[-1] if data else raw)
return data[-1] if data else raw
def _rpc_reply(raw: str) -> _RpcReply:
return _RpcReply.model_validate_json(_rpc_payload(raw))
def _call_params(call: Mapping[str, object]) -> Mapping[str, object]:
@ -220,9 +240,18 @@ def _call_nonce(call: Mapping[str, object]) -> str:
return _STRINGS.validate_python(_call_params(call)["arguments"])["nonce"]
def _listed(caller: McpCaller, name: str) -> None:
def _listed(caller: McpCaller, name: str) -> Outcome:
listing: Final = eventually(caller.list_tools, lambda outcome: name in outcome.tools, seconds=45)
assert listing.error is None, (caller.gateway.client.base_url, listing.raw)
return listing
def _listed_description(caller: McpCaller, name: str) -> str:
listing: Final = _listed(caller, name)
tools: Final = _ListReply.model_validate_json(_rpc_payload(listing.raw)).result.tools
described: Final = next(tool.description for tool in tools if tool.name == name)
assert described is not None and _FETCHED.fullmatch(described), listing.raw
return described
@dataclass(frozen=True, slots=True)
@ -251,15 +280,18 @@ def _probed_description(worker: _Worker, name: str) -> str | None:
return _ECHOED.validate_json(carrier.split(_ECHO, 1)[1])["description"]
def _catalog_is_cold(worker: _Worker, name: str) -> bool:
return not _probed_description(worker, name)
def _catalog_is_cold(worker: _Worker, name: str, listings: tuple[str, ...]) -> bool:
probed: Final = _probed_description(worker, name)
return probed is not None and _FETCHED.fullmatch(probed) is not None and probed not in listings
@pytest.mark.timeout(600)
def test_worker_restart_cools_its_listed_catalog_while_the_sibling_worker_keeps_serving(
gateway: Gateway, echo_config: Path, tmp_path: Path
) -> None:
tool: Final = ScriptedTool("lookup", lambda _: text_result("found"), description=_DESCRIPTION)
tool: Final = ScriptedTool(
"lookup", lambda _: text_result("found"), description=lambda _: f"{_DESCRIPTION} {uuid.uuid4().hex}"
)
with scripted_peer(tool) as peer, gateway.scenario() as scenario:
alias: Final = "cold" + uuid.uuid4().hex[:8]
identity: Final = register_mcp(scenario, peer, alias)
@ -272,21 +304,23 @@ def test_worker_restart_cools_its_listed_catalog_while_the_sibling_worker_keeps_
assert first.pid != sibling.pid
_served(first, name, "first-unlisted")
_served(sibling, name, "sibling-unlisted")
assert _catalog_is_cold(first, name) and _catalog_is_cold(sibling, name)
_listed(first.caller, name)
assert _probed_description(first, name) == _DESCRIPTION
assert _catalog_is_cold(sibling, name), "a listing on one worker warmed its sibling"
_listed(sibling.caller, name)
assert _probed_description(sibling, name) == _DESCRIPTION
assert _catalog_is_cold(first, name, ()) and _catalog_is_cold(sibling, name, ())
first_listing: Final = _listed_description(first.caller, name)
assert _probed_description(first, name) == first_listing
assert _catalog_is_cold(sibling, name, (first_listing,)), "a listing on one worker warmed its sibling"
sibling_listing: Final = _listed_description(sibling.caller, name)
assert _probed_description(sibling, name) == sibling_listing
_served(sibling, name, "sibling-alone")
with owned_proxy_process(gateway, tmp_path / "restarted", {}, config=echo_config) as restarted_proxy:
restarted: Final = _worker(restarted_proxy.gateway, key, alias)
assert restarted.pid not in (first.pid, sibling.pid)
assert _catalog_is_cold(restarted, name), "a restarted worker kept the old process's catalog"
assert _probed_description(sibling, name) == _DESCRIPTION
assert _catalog_is_cold(restarted, name, (first_listing, sibling_listing)), (
"a restarted worker kept the old process's catalog"
)
assert _probed_description(sibling, name) == sibling_listing
_served(restarted, name, "restarted-unlisted")
_listed(restarted.caller, name)
assert _probed_description(restarted, name) == _DESCRIPTION
restarted_listing: Final = _listed_description(restarted.caller, name)
assert _probed_description(restarted, name) == restarted_listing
calls: Final = tool_calls(peer.drain())
assert [_call_nonce(call) for call in calls] == [
"first-unlisted",

23
tests/integration/mcp/test_mcp_toolsets.py
View file

@ -538,7 +538,6 @@ def test_owned_proxy_loss_retains_exit_status_and_first_http_failure(gateway: Ga
_PROBE: Final = "catalog-probe"
_ECHO: Final = "catalog-echo"
_UNLISTED: Final = ""
_GUARDRAIL_CODE: Final = (
"def apply_guardrail(inputs, request_data, input_type):\n"
f' if "{_PROBE}" not in list(inputs.get("texts") or []):\n'
@ -574,9 +573,21 @@ def _echoed_description(outcome: Outcome) -> str:
return found.group(1)
def _fetched(described: str) -> re.Pattern[str]:
return re.compile(rf"{re.escape(described)} [0-9a-f]{{32}}")
def _listed_description(listing: Outcome, described: str) -> str:
found: Final = _fetched(described).search(listing.raw)
assert found is not None, listing.raw
return found.group(0)
def test_a_team_keys_toolset_route_listing_feeds_its_own_calls_but_not_a_team_mates(echo_rig: Gateway) -> None:
described: Final = "Adds for the team " + uuid.uuid4().hex[:8]
tool: Final = ScriptedTool("add", lambda _: text_result("9"), description=described)
tool: Final = ScriptedTool(
"add", lambda _: text_result("9"), description=lambda _: f"{described} {uuid.uuid4().hex}"
)
with scripted_peer(tool) as peer, echo_rig.scenario() as scenario:
alias: Final = "lit6029echo" + uuid.uuid4().hex[:6]
server_id: Final = register_mcp(scenario, peer, alias)
@ -587,10 +598,14 @@ def test_a_team_keys_toolset_route_listing_feeds_its_own_calls_but_not_a_team_ma
_assert_team_grants_only(echo_rig, team_id, key, granted_id)
listed: Final = _toolset_rpc(echo_rig, _bearer(key), granted_name, "tools/list", {})
assert listed.ok and listed.tools == (f"{alias}-add",), listed.raw
served: Final = _listed_description(listed, described)
probe: Final[dict[str, object]] = {"name": f"{alias}-add", "arguments": {"probe": _PROBE}}
own: Final = _echoed_description(_toolset_rpc(echo_rig, _bearer(key), granted_name, "tools/call", probe))
mate: Final = _echoed_description(_toolset_rpc(echo_rig, _bearer(team_mate), granted_name, "tools/call", probe))
assert (own, mate) == (described, _UNLISTED), (
"the slot is keyed by the hashed key, so a team-mate that never listed is handed nothing"
assert own == served, (own, served)
assert _fetched(described).fullmatch(mate) and mate != served, (
"the slot is keyed by the hashed key, so a team-mate that never listed is not handed the key's listing",
mate,
served,
)
assert tool_calls(peer.drain()) == (), "a blocked probe reached the peer"

9
tests/integration/observability/test_guardrail_effects.py
View file

@ -1085,7 +1085,8 @@ def test_request_selected_mcp_guardrail_blocks_direct_and_virtual_calls(gateway:
mcp_peer() as peer,
candidate.scenario() as scenario,
):
identity = register_mcp(scenario, peer, "guardrail" + uuid.uuid4().hex)
server = "guardrail" + uuid.uuid4().hex
identity = register_mcp(scenario, peer, server)
permission = {"mcp_servers": [identity], "mcp_tool_search_enabled": True}
key = scenario.key(object_permission=permission)
key_selected = scenario.key(object_permission=permission, guardrails=[guardrail])
@ -1116,8 +1117,10 @@ def test_request_selected_mcp_guardrail_blocks_direct_and_virtual_calls(gateway:
)
calls = tuple(item for item in peer.drain() if item["body"].get("method") == "tools/call")
if expected is None:
assert response.status_code == 400, response.text
assert "integration resolved add denied" in response.text, response.text
assert response.status_code == 403, response.text
assert response.json() == {
"detail": {"error": f"Tool add is unavailable after MCP guardrail checks for server {server}."}
}, response.text
assert calls == (), "pre-call denial must prevent upstream execution"
else:
assert response.status_code == 200, response.text

40
tests/integration/providers/test_databricks_decisions_wire.py
View file

@ -1,3 +1,4 @@
import json
import threading
import time
import uuid
@ -67,7 +68,12 @@ _NON_BARE_NAMES: Final[tuple[tuple[str, str], ...]] = (
)
_DATABRICKS_ERRORS: Final[tuple[tuple[str, str, int, dict[str, JsonValue]], ...]] = (
(_MODEL, _ROUTE, 400, {"error_code": "BAD_REQUEST", "message": "Invalid input: questions must not be empty"}),
(_MODEL, _ROUTE, 403, {"error_code": "PERMISSION_DENIED", "message": "ai_decide is not enabled for this workspace"}),
(
_MODEL,
_ROUTE,
403,
{"error_code": "PERMISSION_DENIED", "message": "ai_decide is not enabled for this workspace"},
),
(
_ENDPOINT,
f"/{_ENDPOINT}/invocations",
@ -89,6 +95,15 @@ def _number(value: JsonValue) -> float:
return float(value)
def _endpoint_spend() -> float:
prices: Final = object_value(
json.loads(Path("model_prices_and_context_window.json").read_text())[f"databricks/{_ENDPOINT}"]
)
return _number(_USAGE["input_tokens"]) * _number(prices["input_cost_per_token"]) + _number(
_USAGE["output_tokens"]
) * _number(prices["output_cost_per_token"])
def _register(scenario: Scenario, body: dict[str, JsonValue], *, status: int = 200) -> ScenarioHandle:
handle: Final = register_scenario(
f"databricks-{uuid.uuid4().hex[:12]}", JsonResponse(content_type="application/json", body=body, status=status)
@ -300,9 +315,14 @@ def test_an_openai_format_request_at_v1_decisions_reaches_the_serving_endpoint_a
row["model_group"],
row["api_base"],
_number(row["spend"]),
) == ("success", "adecisions", "databricks", deployment, f"{handle.api_base()}/{_ENDPOINT}/invocations", 0.0), (
row
)
) == (
"success",
"adecisions",
"databricks",
deployment,
f"{handle.api_base()}/{_ENDPOINT}/invocations",
pytest.approx(_endpoint_spend(), rel=1e-9),
), row
def test_an_openai_format_request_at_v1_decisions_reaches_the_ai_decide_route_without_a_model_field(
@ -381,6 +401,7 @@ def test_a_noul_probability_reaches_the_caller_as_the_system_one_noul_answer(gat
def test_a_wildcard_deployment_sends_each_request_to_the_named_endpoint_and_bills_each_once(gateway: Gateway) -> None:
expected_spend: Final = _endpoint_spend()
with gateway.scenario() as scenario:
handle: Final = _register(scenario, _ANSWER_BODY)
prefix: Final = _wildcard_deployment(scenario, api_base=handle.api_base(), api_key=_API_KEY)
@ -408,9 +429,14 @@ def test_a_wildcard_deployment_sends_each_request_to_the_named_endpoint_and_bill
row["model_group"],
row["api_base"],
_number(row["spend"]),
) == ("success", "asystemone", "databricks", model, f"{handle.api_base()}/{_ENDPOINT}/invocations", 0.0), (
row
)
) == (
"success",
"asystemone",
"databricks",
model,
f"{handle.api_base()}/{_ENDPOINT}/invocations",
pytest.approx(expected_spend, rel=1e-9),
), row
def test_a_wildcard_deployment_sends_each_ai_decide_request_to_the_route_and_logs_each_once(gateway: Gateway) -> None:

13
tests/integration/providers/test_decisions_openai_format_wire.py
View file

@ -187,7 +187,16 @@ _PROVIDERS: Final = (
"cloudflare/@cf/cloudflare/clef",
takes_images=True,
),
_Provider("hosted_vllm", "hosted_vllm/Qwen/Qwen3-0.6B", "/v1/systemone", "Qwen/Qwen3-0.6B", None, False, None),
_Provider(
"hosted_vllm",
"hosted_vllm/Qwen/Qwen3-0.6B",
"/v1/systemone",
"Qwen/Qwen3-0.6B",
None,
False,
None,
takes_images=True,
),
_Provider(
"databricks",
"databricks/ai_decide",
@ -205,7 +214,7 @@ _PROVIDERS: Final = (
"databricks-openjev-qwen35-4b",
_API_KEY,
False,
None,
"databricks/databricks-openjev-qwen35-4b",
),
_Provider(
"azure_ai", "azure_ai/decision-1", "/providers/microsoft/v1/systemone", "decision-1", _API_KEY, False, None

5
tests/integration/providers/test_idle_upstream_keepalive_edges_wire.py
View file

@ -33,15 +33,14 @@ pytestmark: Final = pytest.mark.timeout(int(2 * graceful_stop_seconds() + 120 +
REUSE_ROWS: Final = (
Row("reuse-chat-openai-sdk-sync", openai_deployment, chat_sdk),
Row("reuse-messages-rust-anthropic-sdk-sync", anthropic_deployment, messages_sdk, rust=True),
Row("reuse-messages-anthropic-sdk-sync", anthropic_deployment, messages_sdk),
)
QUIET_ROWS: Final = (
Row("quiet-chat-openai-sdk-async-stream", openai_deployment, chat_sdk_stream, respond=quiet_stream_answer),
Row(
"quiet-messages-rust-anthropic-sdk-async-stream",
"quiet-messages-anthropic-sdk-async-stream",
anthropic_deployment,
messages_sdk_stream,
rust=True,
respond=quiet_stream_answer,
),
)

2
tests/integration/providers/test_idle_upstream_keepalive_override_wire.py
View file

@ -27,7 +27,7 @@ pytestmark: Final = pytest.mark.timeout(int(2 * graceful_stop_seconds() + 120 +
ROWS: Final = (
Row("override-chat-openai-sdk-sync", openai_deployment, chat_sdk),
Row("override-messages-rust-anthropic-sdk-sync", anthropic_deployment, messages_sdk, rust=True),
Row("override-messages-anthropic-sdk-sync", anthropic_deployment, messages_sdk),
Row("override-ocr-mistral-rust-bridge", ocr_deployment, ocr, rust=True),
)

56
tests/integration/providers/test_idle_upstream_reconnect_rust_messages_wire.py
View file

@ -1,56 +0,0 @@
from collections.abc import Iterator
from typing import Final
import pytest
from integration._support.client import Gateway
from integration._support.idle_gateway import Drop
from integration._support.process import graceful_stop_seconds
from integration.providers._idle_upstream_support import (
IDLE_SECONDS,
Row,
StalePools,
answered,
anthropic_deployment,
expected_answers,
landed,
landed_once_undropped,
messages_sdk,
messages_sdk_stream,
served_by_rust,
spend_statuses,
stale_pools,
)
pytestmark: Final = pytest.mark.timeout(int(2 * graceful_stop_seconds() + 120 + IDLE_SECONDS))
ROWS: Final = (
Row("messages-rust-anthropic-sdk-sync", anthropic_deployment, messages_sdk, rust=True),
Row("messages-rust-anthropic-sdk-async-stream", anthropic_deployment, messages_sdk_stream, rust=True),
Row(
"messages-rust-upstream-resets-stale-connection",
anthropic_deployment,
messages_sdk,
drop=Drop.RESET,
rust=True,
),
)
@pytest.fixture(scope="module")
def pools(session_gateway: Gateway, tmp_path_factory: pytest.TempPathFactory) -> Iterator[StalePools]:
with stale_pools(
session_gateway, tmp_path_factory.mktemp("idle-upstream-rust"), ROWS, {"LITELLM_RUST": "1"}
) as value:
yield value
@pytest.mark.parametrize("name", [row.name for row in ROWS])
def test_rust_messages_after_upstream_idle_past_keepalive_open_a_fresh_connection_on_each_worker(
pools: StalePools, name: str
) -> None:
row: Final = pools.rows[name]
sents, arrivals = pools.probe(name)
assert answered(sents) == expected_answers(row, sents), sents
assert landed(arrivals) == landed_once_undropped(sents), arrivals
assert served_by_rust(sents) == (row.rust,) * len(sents), sents
assert spend_statuses(sents) == ("success",) * len(sents), sents

10
tests/integration/providers/test_inferenceapis_wire.py
View file

@ -30,6 +30,7 @@ _WEATHER_TOOL: Final[JsonValue] = {
},
}
_COST_MAP_PATH: Final = Path(__file__).resolve().parents[3] / "model_prices_and_context_window.json"
_ENDPOINT_SUPPORT_PATH: Final = Path(__file__).resolve().parents[3] / "litellm/provider_endpoints_support_backup.json"
_COST_MAP: Final = TypeAdapter(dict[str, dict[str, JsonValue]])
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
_JSON_LIST: Final = TypeAdapter(list[JsonValue])
@ -512,6 +513,12 @@ def _lists_inferenceapis(endpoint: Mapping[str, JsonValue]) -> bool:
return any(object_value(provider)["slug"] == "inferenceapis" for provider in providers)
def _bundled_inferenceapis_endpoints() -> frozenset[str]:
providers: Final = object_value(_JSON_OBJECT.validate_json(_ENDPOINT_SUPPORT_PATH.read_bytes())["providers"])
flags: Final = object_value(object_value(providers["inferenceapis"])["endpoints"])
return frozenset(key for key, supported in flags.items() if supported)
def test_public_routes_list_the_provider_with_its_endpoints_and_fields(gateway: Gateway) -> None:
providers: Final = gateway.request("GET", "/public/providers")
assert providers.status_code == 200, providers.text
@ -538,7 +545,8 @@ def test_public_routes_list_the_provider_with_its_endpoints_and_fields(gateway:
for endpoint in _JSON_LIST.validate_python(_JSON_OBJECT.validate_json(endpoints_response.content)["endpoints"])
)
listed: Final = frozenset(str(endpoint["key"]) for endpoint in endpoints if _lists_inferenceapis(endpoint))
assert listed == {"chat_completions", "messages", "responses"}
assert {"chat_completions", "messages", "responses"} <= listed, listed
assert listed == _bundled_inferenceapis_endpoints()
def test_openai_and_anthropic_sdk_clients_reach_the_vendor(gateway: Gateway) -> None:

33
tests/integration/providers/test_responses_bridge_tool_search_undeferred_wire.py
View file

@ -113,10 +113,8 @@ def _namespace_tool() -> dict[str, JsonValue]:
}
def _search_tools(route: Route, search: Search) -> tuple[dict[str, JsonValue], ...]:
if search == "absent":
return ()
return (_tool_search(execution="client"),) if route == "anthropic" else (_tool_search(),)
def _search_tools(search: Search) -> tuple[dict[str, JsonValue], ...]:
return () if search == "absent" else (_tool_search(),)
def _gemini_frame(marker: str) -> dict[str, JsonValue]:
@ -406,7 +404,6 @@ _UNDEFERRING: Final[tuple[tuple[Route, Search], ...]] = (
_KEEPING: Final[tuple[tuple[Route, Search], ...]] = (
("unlisted_claude", "absent"),
("anthropic", "absent"),
("anthropic", "dropped"),
("azure_ai", "absent"),
)
@ -428,7 +425,7 @@ def test_without_a_search_tool_on_the_wire_every_tool_arrives_without_defer_load
call: Final = _Call(route, stream, uuid.uuid4().hex)
with wire_server(_vendor(route)) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, route, wire.url)
tools: Final = (*_search_tools(route, search), _function_tool(), _namespace_tool())
tools: Final = (*_search_tools(search), _function_tool(), _namespace_tool())
response: Final = _raw(gateway, _responses_body(model, call.marker, tools, stream))
_assert_served(response, call.marker, stream)
request, body = _only_request(wire)
@ -442,7 +439,7 @@ def test_a_claude_route_sent_no_server_search_tool_keeps_each_tools_own_defer_lo
call: Final = _Call(route, stream, uuid.uuid4().hex)
with wire_server(_vendor(route)) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, route, wire.url)
tools: Final = (*_search_tools(route, search), _function_tool(), _namespace_tool(), _loaded_tool())
tools: Final = (*_search_tools(search), _function_tool(), _namespace_tool(), _loaded_tool())
response: Final = _raw(gateway, _responses_body(model, call.marker, tools, stream))
_assert_served(response, call.marker, stream)
request, body = _only_request(wire)
@ -456,6 +453,26 @@ def test_a_claude_route_sent_no_server_search_tool_keeps_each_tools_own_defer_lo
], sent
@pytest.mark.parametrize("stream", [False, True], ids=["sync", "stream"])
def test_a_client_search_tool_on_anthropic_goes_out_as_a_function_and_holds_back_the_deferred_tools(
gateway: Gateway, stream: bool
) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(_claude) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, "anthropic", wire.url)
tools: Final = (_tool_search(execution="client"), _function_tool(), _namespace_tool(), _loaded_tool())
response: Final = _raw(gateway, _responses_body(model, marker, tools, stream))
_assert_served(response, marker, stream)
request, body = _only_request(wire)
_assert_target("anthropic", stream, request, body)
sent: Final = _ITEMS.validate_python(body["tools"])
assert [(tool["name"], tool.get("defer_loading")) for tool in sent] == [
("tool_search", None),
("send_email", None),
], sent
assert "defer_loading" not in request.body.decode(), body
def test_a_false_defer_loading_never_reaches_an_openai_compatible_wire_as_a_key(gateway: Gateway) -> None:
call: Final = _Call("mistral", False, uuid.uuid4().hex)
with wire_server(_mistral) as wire, gateway.scenario() as scenario:
@ -682,7 +699,7 @@ def test_a_streamed_answer_closes_its_message_item_with_every_streamed_delta(gat
async def _send(client: httpx.AsyncClient, key: str, model: str, call: _Call) -> _Served:
tools: Final = (*_search_tools(call.route, "dropped"), _function_tool(), _namespace_tool())
tools: Final = (*_search_tools("dropped"), _function_tool(), _namespace_tool())
body: Final = _responses_body(model, call.marker, tools, call.stream)
async with client.stream(
"POST", "/v1/responses", json=body, headers={"Authorization": f"Bearer {key}"}

8
tests/integration/providers/test_responses_bridge_tool_search_wire.py
View file

@ -835,15 +835,17 @@ def test_a_claude_the_cost_map_does_not_list_gets_the_tool_dropped_on_the_anthro
_assert_dropped_on_wire(body, _DEFERRED_NAMES)
def test_client_execution_drops_the_tool_search_tool(gateway: Gateway) -> None:
def test_client_execution_offers_tool_search_as_a_function_and_holds_back_the_deferred_tools(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(_Vendor().respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, "anthropic", wire.url, gateway.upstream_url)
tools: Final = (_tool_search(execution="client"), _function_tool(defer_loading=False), _namespace_tool())
answer: Final = _raw_answer(gateway, _responses_body(model, marker, tools, False))
assert _output_types(answer.payload) == ["message", "function_call"], answer
_, body = _only_request(wire)
_assert_dropped_on_wire(body, _DEFERRED_NAMES)
request, body = _only_request(wire)
assert request.target == "/v1/messages", request.target
_assert_dropped_on_wire(body, ("tool_search", "get_weather"))
assert "defer_loading" not in request.body.decode(), body
@pytest.mark.parametrize("drop_params", [False, True], ids=["refused", "dropped"])

2
tests/integration/providers/test_rust_anthropic_safeguards_beta_chaos.py
View file

@ -45,7 +45,7 @@ def _clients(stack: ExitStack, base_url: str, count: int) -> tuple[httpx.Client,
def _post(client: httpx.Client, key: str, marker: str) -> int:
body: Final = safeguarded(messages_body(ANTHROPIC_DEPLOYMENT, marker), list(SAFEGUARDS))
response: Final = client.post("/v1/messages", json=body, headers={"Authorization": f"Bearer {key}"})
assert response.status_code != 200 or rust_routed(response.headers), dict(response.headers)
assert response.status_code != 200 or not rust_routed(response.headers), dict(response.headers)
return response.status_code

18
tests/integration/providers/test_rust_anthropic_safeguards_beta_wire.py
View file

@ -155,7 +155,7 @@ def test_sdk_messages_with_safeguards_go_upstream_with_the_dangerous_tool_use_be
extra_body={"safeguards": list(SAFEGUARDS), "tools": [dict(BASH_TOOL)]},
)
message: Final = raw.parse()
assert rust_routed(raw.headers), dict(raw.headers)
assert not rust_routed(raw.headers), dict(raw.headers)
received: Final = json_object(message.model_dump_json())
assert message.id == f"msg_{marker}", received
assert received.get("safeguard_results") == list(SAFEGUARD_RESULTS), received
@ -192,7 +192,7 @@ async def test_sdk_streamed_messages_with_safeguards_go_upstream_with_the_beta(
) as stream:
text: Final = "".join([piece async for piece in stream.text_stream])
final: Final = await stream.get_final_message()
assert rust_routed(stream.response.headers), dict(stream.response.headers)
assert not rust_routed(stream.response.headers), dict(stream.response.headers)
assert (final.id, text) == (f"msg_{marker}", REPLY_TEXT), (final, text)
sent: Final = one_delivery(rig.wire, marker)
assert (sent.target, sent.beta, sent.safeguards) == (
@ -209,7 +209,7 @@ def test_raw_messages_merge_the_safeguards_beta_into_the_client_betas(rig: _Rig,
body: Final = safeguarded(messages_body(ANTHROPIC_DEPLOYMENT, marker, stream=stream), list(SAFEGUARDS))
response: Final = _post(rig.gateway, body, beta=CLIENT_BETAS)
assert response.status_code == 200, response.text
assert rust_routed(response.headers), dict(response.headers)
assert not rust_routed(response.headers), dict(response.headers)
assert f"msg_{marker}" in response.text and "safeguard_results" in response.text, response.text
sent: Final = one_delivery(rig.wire, marker)
assert (sent.target, sent.beta) == (ANTHROPIC_TARGET, MERGED_BETAS), sent
@ -221,7 +221,7 @@ def test_null_or_absent_safeguards_add_no_beta(rig: _Rig, extra: Mapping[str, Js
marker: Final = new_marker()
response: Final = _post(rig.gateway, {**messages_body(ANTHROPIC_DEPLOYMENT, marker), **extra})
assert response.status_code == 200, response.text
assert rust_routed(response.headers), dict(response.headers)
assert not rust_routed(response.headers), dict(response.headers)
assert f"msg_{marker}" in response.text and "safeguard_results" not in response.text, response.text
sent: Final = one_delivery(rig.wire, marker)
assert (sent.beta, sent.safeguards) == (None, None), sent
@ -253,7 +253,7 @@ def test_malformed_safeguards_still_add_the_beta_and_relay_the_vendor_verdict(
control: Final = new_marker()
after: Final = _post(rig.gateway, messages_body(ANTHROPIC_DEPLOYMENT, control))
assert after.status_code == 200, after.text
assert rust_routed(after.headers), dict(after.headers)
assert not rust_routed(after.headers), dict(after.headers)
assert one_delivery(rig.wire, control).beta is None
assert rig.gateway.request("GET", "/health/liveliness").status_code == 200
@ -276,7 +276,7 @@ def test_duplicated_safeguards_keys_resolve_to_the_last_value(rig: _Rig, last: s
},
)
assert response.status_code == 200, response.text
assert rust_routed(response.headers), dict(response.headers)
assert not rust_routed(response.headers), dict(response.headers)
assert ("safeguard_results" in response.text) == (last == "list"), response.text
sent: Final = one_delivery(rig.wire, marker)
assert sent.beta == (DANGEROUS_TOOL_USE if last == "list" else None), sent
@ -304,7 +304,7 @@ def test_vertex_messages_with_safeguards_keep_the_beta_from_the_shared_helper(
body: Final = safeguarded(messages_body(VERTEX_DEPLOYMENT, marker, stream=stream), list(SAFEGUARDS))
response: Final = _post(rig.gateway, {**body, "tools": tools})
assert response.status_code == 200, response.text
assert rust_routed(response.headers), dict(response.headers)
assert not rust_routed(response.headers), dict(response.headers)
assert f"msg_{marker}" in response.text and "safeguard_results" in response.text, response.text
sent: Final = one_delivery(rig.wire, marker)
assert sent.target == (VERTEX_STREAM_TARGET if stream else VERTEX_TARGET), sent
@ -326,7 +326,7 @@ def test_deepseek_messages_forward_safeguards_without_adding_a_beta(rig: _Rig, b
beta=beta,
)
assert response.status_code == 200, response.text
assert rust_routed(response.headers), dict(response.headers)
assert not rust_routed(response.headers), dict(response.headers)
assert f"msg_{marker}" in response.text, response.text
sent: Final = one_delivery(rig.wire, marker)
assert (sent.target, sent.api_key, sent.safeguards) == (DEEPSEEK_TARGET, DEEPSEEK_KEY, list(SAFEGUARDS)), sent
@ -370,7 +370,7 @@ def _wave(pool: ThreadPoolExecutor, gateway: Gateway, markers: tuple[str, ...],
def _assert_served(answer: _Answer) -> None:
assert answer.status == 200, answer
assert REPLY_TEXT in answer.text, answer
assert answer.rust == (answer.path == "/v1/messages"), answer
assert not answer.rust, answer
if answer.path == "/v1/messages":
assert f"msg_{answer.marker}" in answer.text and "safeguard_results" in answer.text, answer

4
tests/integration/providers/test_vertex_credentials_source_wire.py
View file

@ -416,7 +416,7 @@ def vertex_claude_peer(request: Request) -> Reply:
@pytest.mark.timeout(2 * graceful_stop_seconds() + 120)
def test_rust_messages_route_loads_a_claude_key_file_and_names_unloadable_credentials(
def test_rust_enabled_messages_route_loads_a_claude_key_file_and_names_unloadable_credentials(
gateway: Gateway, tmp_path: Path
) -> None:
token_url: Final = gateway.upstream_url.rstrip("/")
@ -452,7 +452,7 @@ def test_rust_messages_route_loads_a_claude_key_file_and_names_unloadable_creden
candidate,
wire,
ask(str(tmp_path / "missing.json"), "rust refuses a missing file"),
"failed to open application credentials file",
FILE_MESSAGE,
str(tmp_path),
)
refused(

29
tests/integration/spend/golden/daily_activity_team_paginated.json
View file

@ -177,35 +177,6 @@
"endpoints": {
"/v1/chat/completions": {
"api_key_breakdown": {
"__ptu_flat_cost__": {
"metadata": {
"key_alias": null,
"key_exists": false,
"team_id": null,
"user_email": null,
"user_id": null
},
"metrics": {
"api_requests": 0,
"autorouter_savings_spend": 0.0,
"cache_creation_input_tokens": 0,
"cache_read_input_tokens": 0,
"completion_tokens": 0,
"compression_saved_tokens": 0,
"compression_savings_spend": 0.0,
"failed_requests": 0,
"flat_cost": 0.0,
"ptu_hours": 0.0,
"gateway_injected_caching_savings_spend": 0.0,
"prompt_caching_savings_spend": 0.0,
"prompt_tokens": 0,
"spend": 1000.0,
"successful_requests": 0,
"timed_requests": 0,
"total_response_time_ms": 0,
"total_tokens": 0
}
},
"key-a": {
"metadata": {
"key_alias": "alias-a",

29
tests/integration/spend/golden/daily_activity_user_paginated.json
View file

@ -177,35 +177,6 @@
"endpoints": {
"/v1/chat/completions": {
"api_key_breakdown": {
"__ptu_flat_cost__": {
"metadata": {
"key_alias": null,
"key_exists": false,
"team_id": null,
"user_email": null,
"user_id": null
},
"metrics": {
"api_requests": 0,
"autorouter_savings_spend": 0.0,
"cache_creation_input_tokens": 0,
"cache_read_input_tokens": 0,
"completion_tokens": 0,
"compression_saved_tokens": 0,
"compression_savings_spend": 0.0,
"failed_requests": 0,
"flat_cost": 0.0,
"ptu_hours": 0.0,
"gateway_injected_caching_savings_spend": 0.0,
"prompt_caching_savings_spend": 0.0,
"prompt_tokens": 0,
"spend": 1000.0,
"successful_requests": 0,
"timed_requests": 0,
"total_response_time_ms": 0,
"total_tokens": 0
}
},
"key-a": {
"metadata": {
"key_alias": "alias-a",

16
tests/integration/spend/test_shutdown_flush.py
View file

@ -16,7 +16,7 @@ import httpx
import psycopg
import pytest
import yaml
from integration._support.client import Gateway, delete_key_if_present, eventually, string_value
from integration._support.client import Gateway, delete_key_if_present, eventually, object_value, string_value
from integration._support.database import read_rows
from integration._support.process import OwnedProxy, owned_proxy_process
from integration._support.wire import Reply, Request, wire_server
@ -25,7 +25,6 @@ from psycopg import sql
REQUESTS_WHILE_BLOCKED: Final = 6
CANCEL_LOG_LINE: Final = "in-flight scheduled job(s) for shutdown"
BATCH_DRAINED_LOG_LINE: Final = f"flushed {REQUESTS_WHILE_BLOCKED} daily spend update items from in-memory queue"
MODEL_INSERT_ARRIVED_LOG_LINE: Final = "path=/model/new"
COMMIT_DELAY_SECONDS: Final = 15
BURST_REQUESTS: Final = 30
@ -356,8 +355,15 @@ class _Shutdown:
for outcome in pool.map(lambda _: self.chat(), range(requests)):
assert outcome is None
def logged(self, line: str, times: int = 1) -> bool:
return self.owned.log.read_text(errors="replace").count(line) >= times
def logged(self, line: str) -> bool:
return line in self.owned.log.read_text(errors="replace")
def other_requests_in_flight(self) -> int:
response: Final = self.owned.gateway.request("GET", "/health/backlog")
assert response.status_code == 200, response.text
in_flight: Final = object_value(response.json())["in_flight_requests"]
assert isinstance(in_flight, int)
return in_flight - 1
def chat_while_spend_update_is_blocked(self, blocker: psycopg.Connection, table: str) -> None:
blocker.execute(f'LOCK TABLE "{table}" IN EXCLUSIVE MODE')
@ -463,7 +469,7 @@ def test_daily_spend_batch_cancelled_while_waiting_for_a_pool_connection_is_writ
eventually(lambda: _waiting_on("LiteLLM_ProxyModelTable"), lambda waiting: waiting == 1, seconds=30)
shutdown.chat_while_spend_update_is_blocked(memberships, "LiteLLM_TeamMembership")
second: Final = shutdown.start_blocked_model_insert()
eventually(lambda: shutdown.logged(MODEL_INSERT_ARRIVED_LOG_LINE, times=2), lambda seen: seen, seconds=30)
eventually(shutdown.other_requests_in_flight, lambda in_flight: in_flight == 2, seconds=30)
memberships.rollback()
eventually(lambda: _waiting_on("LiteLLM_ProxyModelTable"), lambda waiting: waiting == 2, seconds=30)
shutdown.terminate_once(lambda: shutdown.logged(BATCH_DRAINED_LOG_LINE), models.rollback)

7
tests/integration/translation/chat_completions/bases/openai.py
View file

@ -1,4 +1,5 @@
from typing import Final
from unittest.mock import ANY
from integration._support.client import object_value
from integration.translation.case import TranslationTestCase
@ -420,6 +421,7 @@ GPT_6_LUNA_TEST_CASE: Final = TranslationTestCase(
"summary": [],
}
],
"provider_specific_fields": {"phase": "commentary", "openai_responses_replay": ANY},
},
}
],
@ -630,6 +632,7 @@ GPT_5_4_MINI_TEST_CASE: Final = TranslationTestCase(
"type": "function",
},
],
"provider_specific_fields": {"phase": "commentary", "openai_responses_replay": ANY},
},
}
],
@ -729,6 +732,7 @@ GPT_6_SOL_TEST_CASE: Final = TranslationTestCase(
},
],
"content": None,
"provider_specific_fields": {"openai_responses_replay": ANY},
},
}
],
@ -886,6 +890,7 @@ GPT_6_1_SOL_TEST_CASE: Final = TranslationTestCase(
"summary": [],
}
],
"provider_specific_fields": {"phase": "commentary", "openai_responses_replay": ANY},
},
}
],
@ -1042,6 +1047,7 @@ GPT_6_ASTRA_TEST_CASE: Final = TranslationTestCase(
"summary": [],
}
],
"provider_specific_fields": {"phase": "commentary", "openai_responses_replay": ANY},
},
}
],
@ -1201,6 +1207,7 @@ GPT_5_6_TERRA_TEST_CASE: Final = TranslationTestCase(
"summary": [],
}
],
"provider_specific_fields": {"phase": "commentary", "openai_responses_replay": ANY},
},
}
],

63
tests/integration/translation/chat_completions/cross_model/openai.py
View file

@ -1,5 +1,7 @@
from collections.abc import Mapping
from dataclasses import replace
from typing import Final
from unittest.mock import ANY
from integration._support.client import list_value, object_value
from integration.translation.case import (
@ -29,6 +31,30 @@ from integration.translation.templates.openai import (
OPENAI_RESPONSES_AGENT_REQUEST,
OPENAI_RESPONSES_AGENT_RESPONSE,
)
from pydantic import JsonValue
def _without_replay_record(response: Mapping[str, JsonValue]) -> dict[str, JsonValue]:
choices: Final = response["choices"]
assert isinstance(choices, list) and isinstance(choices[0], dict), choices
message: Final = choices[0]["message"]
assert isinstance(message, dict), message
fields: Final = message["provider_specific_fields"]
assert isinstance(fields, dict), fields
kept: Final = {key: value for key, value in fields.items() if key != "openai_responses_replay"}
return {
**response,
"choices": [
{
**choices[0],
"message": {
**{key: value for key, value in message.items() if key != "provider_specific_fields"},
**({"provider_specific_fields": kept} if kept else {}),
},
}
],
}
GPT_5_MINI_AGENT_FOLLOWUP_TEST_CASE: Final = replace(
GPT_5_MINI_TEST_CASE,
@ -559,7 +585,7 @@ GPT_5_4_MINI_AGENT_FOLLOWUP_TEST_CASE: Final = replace(
GPT_5_4_MINI_TEST_CASE,
scenario="agent_followup",
litellm_request=followup_request(
GPT_5_4_MINI_TEST_CASE.litellm_request, GPT_5_4_MINI_TEST_CASE.expected_litellm_response
GPT_5_4_MINI_TEST_CASE.litellm_request, _without_replay_record(GPT_5_4_MINI_TEST_CASE.expected_litellm_response)
),
expected_provider_endpoint="/responses",
expected_provider_request={
@ -570,6 +596,7 @@ GPT_5_4_MINI_AGENT_FOLLOWUP_TEST_CASE: Final = replace(
{
"type": "message",
"role": "assistant",
"phase": "commentary",
"content": [
{
"type": "output_text",
@ -679,6 +706,7 @@ GPT_5_4_MINI_AGENT_FOLLOWUP_TEST_CASE: Final = replace(
"caller in `src/worker.py` (`remaining = retries_remaining(3, attempts)`), and "
"`tests/test_retry.py` failed with `assert 4 == 2` for `retries_remaining(3, 1)`.",
"role": "assistant",
"provider_specific_fields": {"phase": "final_answer", "openai_responses_replay": ANY},
},
}
],
@ -1310,6 +1338,7 @@ GPT_5_6_LUNA_AGENT_TEST_CASE: Final = replace(
}
],
"content": None,
"provider_specific_fields": {"openai_responses_replay": ANY},
},
}
],
@ -1343,7 +1372,8 @@ GPT_5_6_LUNA_AGENT_FOLLOWUP_TEST_CASE: Final = replace(
GPT_5_6_LUNA_AGENT_TEST_CASE,
scenario="agent_followup",
litellm_request=followup_request(
GPT_5_6_LUNA_AGENT_TEST_CASE.litellm_request, GPT_5_6_LUNA_AGENT_TEST_CASE.expected_litellm_response
GPT_5_6_LUNA_AGENT_TEST_CASE.litellm_request,
_without_replay_record(GPT_5_6_LUNA_AGENT_TEST_CASE.expected_litellm_response),
),
expected_provider_endpoint="/responses",
expected_provider_request={
@ -1456,6 +1486,7 @@ GPT_5_6_LUNA_AGENT_FOLLOWUP_TEST_CASE: Final = replace(
"of 2 and potentially negative values are not clamped. It is called by "
"`src/worker.py:12`, and `tests/test_retry.py::test_retries_remaining` failed.",
"role": "assistant",
"provider_specific_fields": {"phase": "final_answer", "openai_responses_replay": ANY},
},
}
],
@ -1819,6 +1850,7 @@ GPT_5_6_SOL_AGENT_TEST_CASE: Final = replace(
},
],
"content": None,
"provider_specific_fields": {"openai_responses_replay": ANY},
},
}
],
@ -1852,7 +1884,8 @@ GPT_5_6_SOL_AGENT_FOLLOWUP_TEST_CASE: Final = replace(
GPT_5_6_SOL_AGENT_TEST_CASE,
scenario="agent_followup",
litellm_request=followup_request(
GPT_5_6_SOL_AGENT_TEST_CASE.litellm_request, GPT_5_6_SOL_AGENT_TEST_CASE.expected_litellm_response
GPT_5_6_SOL_AGENT_TEST_CASE.litellm_request,
_without_replay_record(GPT_5_6_SOL_AGENT_TEST_CASE.expected_litellm_response),
),
expected_provider_endpoint="/responses",
expected_provider_request={
@ -1961,6 +1994,7 @@ GPT_5_6_SOL_AGENT_FOLLOWUP_TEST_CASE: Final = replace(
"`tests/test_retry.py` failed because `retries_remaining(3, 1)` returned `4` "
"instead of `2`.",
"role": "assistant",
"provider_specific_fields": {"phase": "final_answer", "openai_responses_replay": ANY},
},
}
],
@ -2278,6 +2312,7 @@ GPT_5_6_TERRA_AGENT_TEST_CASE: Final = replace(
},
],
"content": None,
"provider_specific_fields": {"openai_responses_replay": ANY},
},
}
],
@ -2311,7 +2346,8 @@ GPT_5_6_TERRA_AGENT_FOLLOWUP_TEST_CASE: Final = replace(
GPT_5_6_TERRA_AGENT_TEST_CASE,
scenario="agent_followup",
litellm_request=followup_request(
GPT_5_6_TERRA_AGENT_TEST_CASE.litellm_request, GPT_5_6_TERRA_AGENT_TEST_CASE.expected_litellm_response
GPT_5_6_TERRA_AGENT_TEST_CASE.litellm_request,
_without_replay_record(GPT_5_6_TERRA_AGENT_TEST_CASE.expected_litellm_response),
),
expected_provider_endpoint="/responses",
expected_provider_request={
@ -2420,6 +2456,7 @@ GPT_5_6_TERRA_AGENT_FOLLOWUP_TEST_CASE: Final = replace(
"`tests/test_retry.py` fails because `retries_remaining(3, 1)` returns `4` instead "
"of `2`.",
"role": "assistant",
"provider_specific_fields": {"phase": "final_answer", "openai_responses_replay": ANY},
},
}
],
@ -2737,6 +2774,7 @@ GPT_6_ASTRA_AGENT_TEST_CASE: Final = replace(
},
],
"content": None,
"provider_specific_fields": {"openai_responses_replay": ANY},
},
}
],
@ -2769,7 +2807,8 @@ GPT_6_ASTRA_AGENT_FOLLOWUP_TEST_CASE: Final = replace(
GPT_6_ASTRA_AGENT_TEST_CASE,
scenario="agent_followup",
litellm_request=followup_request(
GPT_6_ASTRA_AGENT_TEST_CASE.litellm_request, GPT_6_ASTRA_AGENT_TEST_CASE.expected_litellm_response
GPT_6_ASTRA_AGENT_TEST_CASE.litellm_request,
_without_replay_record(GPT_6_ASTRA_AGENT_TEST_CASE.expected_litellm_response),
),
expected_provider_endpoint="/responses",
expected_provider_request={
@ -2878,6 +2917,7 @@ GPT_6_ASTRA_AGENT_FOLLOWUP_TEST_CASE: Final = replace(
"and `tests/test_retry.py::test_retries_remaining` failed because it returned 4 "
"instead of 2.",
"role": "assistant",
"provider_specific_fields": {"phase": "final_answer", "openai_responses_replay": ANY},
},
}
],
@ -3194,6 +3234,7 @@ GPT_6_LUNA_AGENT_TEST_CASE: Final = replace(
},
],
"content": None,
"provider_specific_fields": {"openai_responses_replay": ANY},
},
}
],
@ -3227,7 +3268,8 @@ GPT_6_LUNA_AGENT_FOLLOWUP_TEST_CASE: Final = replace(
GPT_6_LUNA_AGENT_TEST_CASE,
scenario="agent_followup",
litellm_request=followup_request(
GPT_6_LUNA_AGENT_TEST_CASE.litellm_request, GPT_6_LUNA_AGENT_TEST_CASE.expected_litellm_response
GPT_6_LUNA_AGENT_TEST_CASE.litellm_request,
_without_replay_record(GPT_6_LUNA_AGENT_TEST_CASE.expected_litellm_response),
),
expected_provider_endpoint="/responses",
expected_provider_request={
@ -3336,6 +3378,7 @@ GPT_6_LUNA_AGENT_FOLLOWUP_TEST_CASE: Final = replace(
"`tests/test_retry.py` failed because `retries_remaining(3, 1)` returned 4 instead "
"of 2.",
"role": "assistant",
"provider_specific_fields": {"phase": "final_answer", "openai_responses_replay": ANY},
},
}
],
@ -3587,7 +3630,7 @@ GPT_6_SOL_AGENT_FOLLOWUP_TEST_CASE: Final = replace(
GPT_6_SOL_TEST_CASE,
scenario="agent_followup",
litellm_request=followup_request(
GPT_6_SOL_TEST_CASE.litellm_request, GPT_6_SOL_TEST_CASE.expected_litellm_response
GPT_6_SOL_TEST_CASE.litellm_request, _without_replay_record(GPT_6_SOL_TEST_CASE.expected_litellm_response)
),
expected_provider_endpoint="/responses",
expected_provider_request={
@ -3694,6 +3737,7 @@ GPT_6_SOL_AGENT_FOLLOWUP_TEST_CASE: Final = replace(
"clamping the result at zero; `src/worker.py` calls it. `tests/test_retry.py` "
"failed because `retries_remaining(3, 1)` returned 4 instead of 2.",
"role": "assistant",
"provider_specific_fields": {"phase": "final_answer", "openai_responses_replay": ANY},
},
}
],
@ -4007,6 +4051,7 @@ GPT_6_1_SOL_AGENT_TEST_CASE: Final = replace(
},
],
"content": None,
"provider_specific_fields": {"openai_responses_replay": ANY},
},
}
],
@ -4039,7 +4084,8 @@ GPT_6_1_SOL_AGENT_FOLLOWUP_TEST_CASE: Final = replace(
GPT_6_1_SOL_AGENT_TEST_CASE,
scenario="agent_followup",
litellm_request=followup_request(
GPT_6_1_SOL_AGENT_TEST_CASE.litellm_request, GPT_6_1_SOL_AGENT_TEST_CASE.expected_litellm_response
GPT_6_1_SOL_AGENT_TEST_CASE.litellm_request,
_without_replay_record(GPT_6_1_SOL_AGENT_TEST_CASE.expected_litellm_response),
),
expected_provider_endpoint="/responses",
expected_provider_request={
@ -4148,6 +4194,7 @@ GPT_6_1_SOL_AGENT_FOLLOWUP_TEST_CASE: Final = replace(
"`src/worker.py:12`. `tests/test_retry.py::test_retries_remaining` failed because "
"it returned 4 instead of 2; no files were changed.",
"role": "assistant",
"provider_specific_fields": {"phase": "final_answer", "openai_responses_replay": ANY},
},
}
],