scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-29 06:33:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 200

test: update stale integration and guardrail e2e tests to intended behavior (#48665)

Browse source 
* test(integration): update stale expectations to intended provider behavior

Each expectation moved because a merged PR changed behavior on purpose.
- chatgpt string input is wrapped as one user message (#41522)
- Gemini 2.5 tool media sits beside function_response (#48428)
- sampling knobs move to Gemini 3.5 Flash, which keeps them (#47645)
- a body base_model is stripped without allow_client_pricing_override (#47639)
- ai_decide answers without usage return usage null (#48261)
- azure_ai forwards mapped client anthropic-beta values (#48033)
- the OpenCode scripted upstream mints a fresh id per request

* test(e2e-ui): type the guardrail name before deleting it

GuardrailDeleteModal requires the exact name before Delete enables since
#45893. All three guardrail delete flows now go through one helper.

* test(observability): accept any sampled traceresponse flags and expect a 400 for a non-object body

OpenTelemetry 1.45 (#48055) sets the W3C random trace id flag on new traces,
so the gateway's traceresponse now ends in -03. The producer copies the server
span's own trace flags, so the tests now accept any two-digit flags with the
sampled bit set instead of pinning -01

#48438 made a non-object JSON body a 400 "Invalid JSON payload: expected an
object" on purpose, so the [1] body case is renamed non_object_body and
expects that 400

* test(observability): stop waiting on OpenTelemetry's export log line in the langtrace outage test

The outage test waited for "Exception while exporting Span batch" in the proxy
log, which is OpenTelemetry's wording and no longer appears with 1.45. The test
keeps asserting what the gateway guarantees, callers unaffected during the
outage, a live gateway, and every span after recovery delivered once with no
outage span duplicated

* test(guardrails): count the Responses disconnect spend rows #47611 added

Since #47611 a streamed /v1/responses disconnect writes its own spend row, so
the concurrent disconnect test saw 20 rows where it counted 10 by model group.
It now reads each row's call type and expects ten acompletion rows with the
outage verdict and ten aresponses rows with success. /v1/messages disconnects
still write no row on main, so they are left out of the count rather than
pinned as absent

The responses-httpx case of the single disconnect test no longer needs its
LIT-8603 skip and passes

* test(observability): size K4's timeout from the harness boot budget

K4 boots three two-worker proxies in a row. On CircleCI each boot takes 43 to
58 seconds, so a passing run took 179.97s against its fixed 180s cap and 6 of
30 runs hit pytest-timeout. The cap and the restart gate wait now come from
readiness_seconds and graceful_stop_seconds, like other multi-boot tests
This commit is contained in:
yuneng-jiang 2026-10-27 18:11:25 -07:00 • committed by GitHub
parent 15eca24fc2
commit 21ab2cb87b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
18 changed files with 161 additions and 376 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
tests/e2e/ui/helpers/guardrails.ts Normal file
View file

@ -0,0 +1,20 @@
import { Page as PlaywrightPage, expect } from "@playwright/test";
import { Page } from "../fixtures/pages";
import { dismissFeedbackPopup, navigateToPage } from "./navigation";
export async function deleteGuardrailFromTable(page: PlaywrightPage, guardrailName: string): Promise<void> {
await navigateToPage(page, Page.Guardrails);
await dismissFeedbackPopup(page);
const row = page.getByRole("row").filter({ hasText: guardrailName });
await expect(row).toHaveCount(1, { timeout: 15_000 });
await row.getByRole("button", { name: "Open guardrail actions" }).click();
await page.getByRole("menuitem", { name: "Delete" }).click();
const deleteModal = page.getByRole("dialog", { name: "Delete Guardrail" });
await expect(deleteModal).toBeVisible({ timeout: 5_000 });
await deleteModal.getByPlaceholder(guardrailName, { exact: true }).fill(guardrailName);
await deleteModal.getByRole("button", { name: "Delete", exact: true }).click();
await expect(page.getByText(`Guardrail "${guardrailName}" deleted successfully`)).toBeVisible({ timeout: 10_000 });
await expect(row).toHaveCount(0, { timeout: 15_000 });
}

31
tests/e2e/ui/tests/guardrails/guardrails.spec.ts
View file

@ -2,6 +2,7 @@ import { test, expect, type Page as PlaywrightPage } from "@playwright/test";
import { ADMIN_STORAGE_PATH, E2E_TEAM_NO_ADMIN_ID } from "../../constants";
import { Page } from "../../fixtures/pages";
import { navigateToPage, dismissFeedbackPopup, clickTeamId } from "../../helpers/navigation";
import { deleteGuardrailFromTable } from "../../helpers/guardrails";
import { CHAT_MODEL_A, MOCK_RESPONSE_TEXT, masterKey } from "../../helpers/traffic";
interface StoredGuardrail {
@ -183,21 +184,9 @@ test.describe("Guardrails", () => {
test("Delete a guardrail", async ({ page }) => {
const stamp = Date.now();
const guardrailName = `e2e-guardrail-delete-${stamp}`;
const guardrailId = await createKeywordGuardrailViaApi(page, guardrailName, `e2edelete${stamp}`);
await createKeywordGuardrailViaApi(page, guardrailName, `e2edelete${stamp}`);
await navigateToPage(page, Page.Guardrails);
await dismissFeedbackPopup(page);
await expect(page.getByRole("row").filter({ hasText: guardrailName })).toBeVisible({ timeout: 15_000 });
await page.getByTestId(`guardrail-actions-${guardrailId}`).click();
await page.getByTestId("guardrail-action-delete").click();
const modal = page.getByRole("dialog");
await expect(modal).toBeVisible({ timeout: 5_000 });
await modal.getByRole("button", { name: "Delete", exact: true }).click();
await expect(page.getByRole("row").filter({ hasText: guardrailName })).toHaveCount(0, { timeout: 15_000 });
await deleteGuardrailFromTable(page, guardrailName);
// The RC checklist deletes then reloads, because a row vanishing from the table has
// fooled us before; assert against the route the reload would read.
@ -274,19 +263,7 @@ test.describe("Guardrails", () => {
await expect(page.getByRole("option", { name: guardrailName })).toBeVisible({ timeout: 10_000 });
await page.keyboard.press("Escape");
await navigateToPage(page, Page.Guardrails);
await expect(row).toHaveCount(1, { timeout: 15_000 });
await row.getByRole("button", { name: "Open guardrail actions" }).click();
await page.getByRole("menuitem", { name: "Delete" }).click();
const deleteModal = page.getByRole("dialog", { name: "Delete Guardrail" });
await expect(deleteModal).toBeVisible({ timeout: 5_000 });
await deleteModal.getByRole("button", { name: "Delete", exact: true }).click();
await expect(page.getByText(`Guardrail "${guardrailName}" deleted successfully`)).toBeVisible({
timeout: 10_000,
});
await expect(row).toHaveCount(0, { timeout: 15_000 });
await deleteGuardrailFromTable(page, guardrailName);
await page.reload();
await expect(page.getByRole("button", { name: /Add New Guardrail/i })).toBeVisible({ timeout: 20_000 });

16
tests/e2e/ui/tests/guardrails/presidioUserStory.spec.ts
View file

@ -1,6 +1,7 @@
import { test as base, expect, type Page as PlaywrightPage } from "@playwright/test";
import { ADMIN_STORAGE_PATH, MOCK_PRESIDIO_URL, PROPAGATION_TIMEOUT_MS } from "../../constants";
import { navigateToPage, dismissFeedbackPopup } from "../../helpers/navigation";
import { deleteGuardrailFromTable } from "../../helpers/guardrails";
import { Page } from "../../fixtures/pages";
import { CHAT_MODEL_A, masterKey, rootPath, uniqueSuffix, waitForSpendLog } from "../../helpers/traffic";
import { composer, openPlayground, selectModel, sendButton, onlyVisible } from "../../helpers/playground";
@ -49,19 +50,6 @@ async function createPresidioGuardrail(page: PlaywrightPage, guardrailName: stri
await expect(page.getByRole("row").filter({ hasText: guardrailName })).toHaveCount(1, { timeout: 15_000 });
}
async function deleteGuardrail(page: PlaywrightPage, guardrailName: string): Promise<void> {
await navigateToPage(page, Page.Guardrails);
await dismissFeedbackPopup(page);
const row = page.getByRole("row").filter({ hasText: guardrailName });
await expect(row).toHaveCount(1, { timeout: 15_000 });
await row.getByRole("button", { name: "Open guardrail actions" }).click();
await page.getByRole("menuitem", { name: "Delete" }).click();
const deleteModal = page.getByRole("dialog", { name: "Delete Guardrail" });
await expect(deleteModal).toBeVisible({ timeout: 5_000 });
await deleteModal.getByRole("button", { name: "Delete", exact: true }).click();
await expect(page.getByText(`Guardrail "${guardrailName}" deleted successfully`)).toBeVisible({ timeout: 10_000 });
}
const test = base.extend<{ guardrailName: string }>({
guardrailName: async ({ page }, use) => {
const name = `e2e-presidio-story-${uniqueSuffix()}`;
@ -69,7 +57,7 @@ const test = base.extend<{ guardrailName: string }>({
try {
await use(name);
} finally {
await deleteGuardrail(page, name);
await deleteGuardrailFromTable(page, name);
}
},
});

19
tests/integration/messages_endpoint/providers/anthropic/test_anthropic_safeguards_beta_wire.py
View file

@ -13,7 +13,7 @@ import anthropic
import httpx
import openai
import pytest
from integration._support.client import Gateway, eventually
from integration._support.client import Gateway, eventually, object_value
from integration._support.database import read_rows
from integration._support.process import owned_proxy
from integration._support.vertex import service_account_json
@ -38,6 +38,9 @@ _CLIENT_BETAS: Final = ",".join(_CLAUDE_CODE_BETAS)
_CLIENT_BETAS_WITH_DANGEROUS: Final = ",".join((*_CLAUDE_CODE_BETAS, BETA))
_MERGED_BETAS: Final = ",".join(sorted((*_CLAUDE_CODE_BETAS, BETA)))
_FOUNDRY_MODEL: Final = "azure_ai/claude-sonnet-4-6"
_FOUNDRY_BETA_COLUMN: Final = object_value(
object_value(json.loads(Path("litellm/anthropic_beta_headers_config.json").read_text()))["azure_ai"]
)
_FOUNDRY_KEY: Final = "synthetic-foundry-key"
_FOUNDRY_TARGET: Final = "/anthropic/v1/messages"
_FOUNDRY_ARRAY_ERROR: Final = "safeguards: Input should be a valid array"
@ -284,15 +287,23 @@ _ANTHROPIC_SHAPES: Final = MappingProxyType(
"D6_missing_safeguards": _ShapeCase(OMITTED, OMITTED, None, None),
}
)
def _foundry_betas(*client_betas: str) -> str:
return ",".join(
sorted({mapped for beta in client_betas if isinstance(mapped := _FOUNDRY_BETA_COLUMN.get(beta), str)})
)
_FOUNDRY_SHAPES: Final = MappingProxyType(
{
"D9_streamed_safeguards_with_claude_code_header": _ShapeCase(
SAFEGUARDS, SAFEGUARDS, _CLIENT_BETAS_WITH_DANGEROUS, BETA, stream=True
SAFEGUARDS, SAFEGUARDS, _CLIENT_BETAS_WITH_DANGEROUS, _foundry_betas(*_CLAUDE_CODE_BETAS, BETA), stream=True
),
"D11_empty_safeguards": _ShapeCase([], [], None, BETA),
"D12_null_safeguards": _ShapeCase(None, OMITTED, None, None),
"D13_missing_safeguards_with_claude_code_header_lacking_the_beta": _ShapeCase(
OMITTED, OMITTED, _CLIENT_BETAS, None
OMITTED, OMITTED, _CLIENT_BETAS, _foundry_betas(*_CLAUDE_CODE_BETAS)
),
}
)
@ -407,7 +418,7 @@ def test_a_deployment_dropping_context_management_sends_neither_the_field_nor_it
@pytest.mark.parametrize("row", tuple(_FOUNDRY_SHAPES))
def test_azure_ai_messages_derive_the_beta_instead_of_forwarding_the_client_header(gateway: Gateway, row: str) -> None:
def test_azure_ai_messages_forward_the_mapped_client_betas_beside_the_derived_one(gateway: Gateway, row: str) -> None:
case: Final = _FOUNDRY_SHAPES[row]
marker: Final = uuid.uuid4().hex
with wire_server(anthropic_peer) as wire, gateway.scenario() as scenario:

26
tests/integration/observability/test_guardrail_effects.py
View file

@ -2578,9 +2578,15 @@ def _scanned_while_upstream_is_held(
rig.gate.set()
def _post_call_statuses(rig: _DisconnectRig, model: str, rows: int = 1) -> tuple[tuple[str, ...], ...]:
def _post_call_rows(
rig: _DisconnectRig, model: str, rows: int = 1, call_types: frozenset[str] | None = None
) -> tuple[tuple[str, tuple[str, ...]], ...]:
found: Final = eventually(
lambda: read_rows('SELECT metadata FROM "LiteLLM_SpendLogs" WHERE model_group=%s', (model,)),
lambda: tuple(
row
for row in read_rows('SELECT call_type, metadata FROM "LiteLLM_SpendLogs" WHERE model_group=%s', (model,))
if call_types is None or row["call_type"] in call_types
),
lambda values: len(values) == rows,
seconds=70,
)
@ -2595,7 +2601,11 @@ def _post_call_statuses(rig: _DisconnectRig, model: str, rows: int = 1) -> tuple
)
return tuple(str(entry["guardrail_status"]) for entry in post_call)
return tuple(statuses(row["metadata"]) for row in found)
return tuple((str(row["call_type"]), statuses(row["metadata"])) for row in found)
def _post_call_statuses(rig: _DisconnectRig, model: str, rows: int = 1) -> tuple[tuple[str, ...], ...]:
return tuple(statuses for _, statuses in _post_call_rows(rig, model, rows))
_ENDPOINT_CLIENTS: Final = (
@ -2628,7 +2638,7 @@ def test_client_disconnect_mid_stream_still_scans_the_content_it_already_receive
(
pytest.param(_chat_httpx, id="chat-httpx"),
pytest.param(_chat_async_openai_sdk, id="chat-async-openai-sdk"),
pytest.param(_responses_httpx, id="responses-httpx", marks=_NO_DISCONNECT_ROW_LIT_8603),
pytest.param(_responses_httpx, id="responses-httpx"),
pytest.param(_messages_anthropic_sdk, id="messages-anthropic-sdk", marks=_NO_DISCONNECT_ROW_LIT_8603),
pytest.param(
_gemini_httpx,
@ -3184,10 +3194,10 @@ def test_concurrent_disconnects_during_a_guardrail_outage_each_record_exactly_on
seconds=20,
)
assert scanned == (1,) * 30, scanned
chat_rows: Final = _post_call_statuses(rig, rig.model, rows=10)
assert sorted(chat_rows) == sorted(
("guardrail_failed_to_respond",) if index % 3 == 0 else ("success",) for index in range(0, 30, 3)
), chat_rows
rows: Final = _post_call_rows(rig, rig.model, rows=20, call_types=frozenset({"acompletion", "aresponses"}))
assert sorted(rows) == sorted(
(("acompletion", ("guardrail_failed_to_respond",)),) * 10 + (("aresponses", ("success",)),) * 10
), rows
def test_disconnect_scans_keep_recording_after_a_worker_is_killed(gateway: Gateway, tmp_path: Path) -> None:

7
tests/integration/observability/test_langtrace_delivery.py
View file

@ -610,14 +610,7 @@ def test_langtrace_sink_outage_mid_burst_recovers_on_the_same_port(gateway: Gate
with wire_server(_accepted, port=port) as sink:
rig: Final = _Rig(owned.gateway, model, provider, _Sink(sink, key))
_assert_each_once(rig.sink, _burst(rig, 6))
log: Final = owned.log
failures_before: Final = log.read_text().count("Exception while exporting Span batch")
outage: Final = _burst(rig, 12)
eventually(
lambda: log.read_text().count("Exception while exporting Span batch"),
lambda value: value > failures_before,
seconds=20,
)
assert owned.gateway.request("GET", "/health/liveliness").status_code == 200
with wire_server(_accepted, port=port) as revived:
recovered: Final = _Rig(owned.gateway, model, provider, _Sink(revived, key))

7
tests/integration/observability/test_logging_only_scope_chaos.py
View file

@ -42,9 +42,10 @@ from _logging_only_scope_support import (
_record_audit_properties as _record_audit_properties,
)
from anthropic import APIConnectionError as AnthropicAPIConnectionError
from integration._support.budget import readiness_seconds
from integration._support.client import Gateway, eventually
from integration._support.ports import allocate_port
from integration._support.process import owned_proxy, owned_proxy_process
from integration._support.process import graceful_stop_seconds, owned_proxy, owned_proxy_process
from integration._support.timing import Timed, assert_not_delayed_by, timed
from integration._support.wire import Reply, Request
from openai import APIConnectionError as OpenAIAPIConnectionError
@ -574,7 +575,7 @@ def test_K3_two_worker_sigkill_checks_post_kill_spend_rows(
)
@pytest.mark.timeout(180)
@pytest.mark.timeout(3 * readiness_seconds(2) + 3 * graceful_stop_seconds() + 180)
def test_K4_proxy_restart_after_fifteen_responses_records_lost_ids(
gateway: Gateway,
tmp_path: Path,
@ -627,7 +628,7 @@ def test_K4_proxy_restart_after_fifteen_responses_records_lost_ids(
if call.index < 15:
return first_gateway
second_wave_barrier.wait(timeout=90)
assert restart_gate.wait(timeout=90)
assert restart_gate.wait(timeout=readiness_seconds(2) + 90)
return restarted_gateways.get()
def run(call: ChaosCall, first_gateway: Gateway) -> tuple[int, CallerResult]:

16
tests/integration/observability/test_otel_trace_response_header.py
View file

@ -26,7 +26,8 @@ from pydantic import JsonValue, TypeAdapter
pytestmark: Final = pytest.mark.shares_worker_state
MARKER: Final = re.compile(rb"traceresp-[0-9a-f]{32}")
TRACERESPONSE: Final = re.compile(r"00-([0-9a-f]{32})-([0-9a-f]{16})-01")
TRACERESPONSE: Final = re.compile(r"00-([0-9a-f]{32})-([0-9a-f]{16})-([0-9a-f]{2})")
SAMPLED: Final = 0x01
SERVER: Final = 2
CALL_ID: Final = "litellm.call_id"
ANSWER: Final = "trace ok"
@ -136,8 +137,13 @@ def _query(request: Request) -> Mapping[str, str]:
return MappingProxyType({name: values[0] for name, values in parse_qs(urlsplit(request.target).query).items()})
def _sampled(value: str) -> re.Match[str] | None:
match: Final = TRACERESPONSE.fullmatch(value)
return match if match is not None and int(match.group(3), 16) & SAMPLED else None
def _continued_traceresponse(request: Request) -> Reply:
incoming: Final = TRACERESPONSE.fullmatch(request.headers.get("traceparent", ""))
incoming: Final = _sampled(request.headers.get("traceparent", ""))
if incoming is None:
return Reply(status=400, body=b'{"error":"no sampled traceparent reached the upstream"}')
return Reply(body=b'{"case":"continued"}', headers={"traceresponse": f"00-{incoming.group(1)}-{_span_id()}-01"})
@ -374,7 +380,7 @@ def control(factory: Callable[[bool], RigFactory]) -> Iterator[Rig]:
def _named(headers: httpx.Headers) -> tuple[str, str]:
values: Final = headers.get_list("traceresponse")
assert len(values) == 1, values
match: Final = TRACERESPONSE.fullmatch(values[0])
match: Final = _sampled(values[0])
assert match is not None, values
return match.group(1), match.group(2)
@ -543,7 +549,7 @@ def _error_invalid_body(rig: Rig) -> httpx.Response:
return rig.post("/key/generate", {"models": "not-a-list"})
def _error_unhandled(rig: Rig) -> httpx.Response:
def _error_non_object_body(rig: Rig) -> httpx.Response:
return rig.proxy.client.post(
"/v1/chat/completions",
content=b"[1]",
@ -556,7 +562,7 @@ ERRORS: Final[Mapping[str, tuple[int, Callable[[Rig], httpx.Response]]]] = Mappi
"bad_key": (401, _error_bad_key),
"unknown_model": (400, _error_unknown_model),
"invalid_body": (422, _error_invalid_body),
"unhandled": (500, _error_unhandled),
"non_object_body": (400, _error_non_object_body),
}
)

42
tests/integration/pricing/test_opencode_go_reported_cost_wire.py
View file

@ -1,4 +1,5 @@
import json
import uuid
from collections.abc import Mapping
from typing import Final
@ -44,10 +45,10 @@ def _event_frame(event: Mapping[str, JsonValue]) -> bytes:
return f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode()
def _chat_completion() -> bytes:
def _chat_completion(tag: str) -> bytes:
return json.dumps(
{
"id": "chatcmpl-go",
"id": f"chatcmpl-go-{tag}",
"object": "chat.completion",
"created": 1,
"model": "glm-5.3",
@ -57,9 +58,9 @@ def _chat_completion() -> bytes:
).encode()
def _chat_stream() -> tuple[bytes, ...]:
def _chat_stream(tag: str) -> tuple[bytes, ...]:
text: Final = {
"id": "chatcmpl-go",
"id": f"chatcmpl-go-{tag}",
"object": "chat.completion.chunk",
"created": 1,
"model": "glm-5.3",
@ -70,16 +71,16 @@ def _chat_stream() -> tuple[bytes, ...]:
return (*(_data_frame(event) for event in (text, finish, final_usage)), b"data: [DONE]\n\n")
def _responses_response() -> Mapping[str, JsonValue]:
def _responses_response(tag: str) -> Mapping[str, JsonValue]:
return {
"id": "resp_go",
"id": f"resp_go_{tag}",
"object": "response",
"created_at": 1,
"status": "completed",
"model": "gpt-6-luna",
"output": [
{
"id": "msg_go",
"id": f"msg_go_{tag}",
"type": "message",
"status": "completed",
"role": "assistant",
@ -90,8 +91,8 @@ def _responses_response() -> Mapping[str, JsonValue]:
}
def _responses_stream() -> tuple[bytes, ...]:
response: Final = _responses_response()
def _responses_stream(tag: str) -> tuple[bytes, ...]:
response: Final = _responses_response(tag)
return (
_event_frame(
{"type": "response.created", "sequence_number": 0, "response": {**response, "status": "in_progress"}}
@ -100,7 +101,7 @@ def _responses_stream() -> tuple[bytes, ...]:
{
"type": "response.output_text.delta",
"sequence_number": 1,
"item_id": "msg_go",
"item_id": f"msg_go_{tag}",
"output_index": 0,
"content_index": 0,
"delta": "ok",
@ -110,9 +111,9 @@ def _responses_stream() -> tuple[bytes, ...]:
)
def _anthropic_message() -> Mapping[str, JsonValue]:
def _anthropic_message(tag: str) -> Mapping[str, JsonValue]:
return {
"id": "msg_go",
"id": f"msg_go_{tag}",
"type": "message",
"role": "assistant",
"model": "claude-haiku-5-5",
@ -123,8 +124,8 @@ def _anthropic_message() -> Mapping[str, JsonValue]:
}
def _anthropic_stream() -> tuple[bytes, ...]:
start: Final = {**_anthropic_message(), "content": [], "stop_reason": None}
def _anthropic_stream(tag: str) -> tuple[bytes, ...]:
start: Final = {**_anthropic_message(tag), "content": [], "stop_reason": None}
return (
_event_frame(
{"type": "message_start", "message": {**start, "usage": {"input_tokens": 1200, "output_tokens": 0}}}
@ -146,20 +147,21 @@ def _anthropic_stream() -> tuple[bytes, ...]:
def _respond(request: Request) -> Reply:
assert request.method == "POST", request.method
streaming: Final = _JSON_OBJECT.validate_json(request.body).get("stream") is True
tag: Final = uuid.uuid4().hex
if request.target.endswith("/chat/completions"):
return (
Reply(content_type="text/event-stream", chunks=_chat_stream())
Reply(content_type="text/event-stream", chunks=_chat_stream(tag))
if streaming
else Reply(body=_chat_completion())
else Reply(body=_chat_completion(tag))
)
if request.target.endswith("/responses"):
if streaming:
return Reply(content_type="text/event-stream", chunks=_responses_stream())
return Reply(body=json.dumps(_responses_response()).encode())
return Reply(content_type="text/event-stream", chunks=_responses_stream(tag))
return Reply(body=json.dumps(_responses_response(tag)).encode())
assert request.target.endswith("/v1/messages"), request.target
if streaming:
return Reply(content_type="text/event-stream", chunks=_anthropic_stream())
return Reply(body=json.dumps(_anthropic_message()).encode())
return Reply(content_type="text/event-stream", chunks=_anthropic_stream(tag))
return Reply(body=json.dumps(_anthropic_message(tag)).encode())
def _logged_spend(call_id: str) -> float:

6
tests/integration/providers/test_bedrock_converse_lookaround_regex_wire.py
View file

@ -577,10 +577,10 @@ def test_a_deployment_flag_never_reaches_its_sibling_on_the_same_model(
@pytest.mark.parametrize(
("model_id", "body_base_model", "expected"),
((_NOVA, f"bedrock/{_KIMI}", _WIRE_LOOKAROUND_FREE), (_KIMI, f"bedrock/{_NOVA}", _WIRE_LOOKAROUND_FREE)),
ids=("client-base-model-can-loosen-an-unflagged-deployment", "client-base-model-cannot-restore-a-flagged-one"),
((_NOVA, f"bedrock/{_KIMI}", _WIRE_AS_SENT), (_KIMI, f"bedrock/{_NOVA}", _WIRE_LOOKAROUND_FREE)),
ids=("client-base-model-cannot-flag-an-unflagged-deployment", "client-base-model-cannot-restore-a-flagged-one"),
)
def test_a_base_model_in_the_request_body_only_ever_loosens(
def test_a_base_model_in_the_request_body_leaves_the_deployment_decision(
gateway: Gateway, model_id: str, body_base_model: str, expected: Mapping[str, JsonValue]
) -> None:
with wire_server(_bedrock_peer) as wire, gateway.scenario() as scenario:

9
tests/integration/providers/test_chatgpt_responses_caller_stream_wire.py
View file

@ -392,12 +392,13 @@ def test_extra_body_stream_false_gets_json(rig: _Rig, model: str) -> None:
_landed(model, identity)
def test_string_input_is_refused_by_the_vendor_as_a_400(rig: _Rig, model: str) -> None:
def test_string_input_reaches_the_vendor_as_one_user_message(rig: _Rig, model: str) -> None:
marker: Final = uuid.uuid4().hex
response: Final = _post(rig, "/v1/responses", {"model": model, "input": _prompt(marker), **_NO_CACHE})
_assert_error(response, status=400, message=cv.INPUT_MUST_BE_A_LIST)
(request,) = rig.wire.drain()
assert rv.JSON_OBJECT.validate_json(request.body)["input"] == _prompt(marker), request.body
identity: Final = _assert_json_answer(response, marker)
forwarded: Final = _only_forwarded(rig, marker)
assert forwarded["input"] == [{"role": "user", "content": _prompt(marker)}], forwarded
_landed(model, identity)
def test_vendor_401_reaches_the_caller_as_401(rig: _Rig, model: str) -> None:

2
tests/integration/providers/test_databricks_decisions_wire.py
View file

@ -48,7 +48,7 @@ _AI_DECIDE_BODY: Final[dict[str, JsonValue]] = {"response": {"answers": _ANSWERS
_SYSTEM_ONE_BODY: Final[dict[str, JsonValue]] = {
"model": None,
"answers": _ANSWERS,
"usage": {"input_tokens": 0, "output_tokens": 0},
"usage": None,
"metadata": _METADATA,
}
_MODEL_RULE: Final = (

11
tests/integration/providers/test_gemini_tool_result_part_reference_wire.py
View file

@ -536,13 +536,8 @@ def _call_part(call_id: str | None = None, index: int = 0) -> Mapping[str, JsonV
return {"function_call": call, **({"thoughtSignature": _SKIP_SIGNATURE} if call_id else {})}
def _result_part(response: JsonValue, call_id: str | None = None, parts: JsonValue = None) -> Mapping[str, JsonValue]:
result: Final = {
"name": _TOOL,
"response": response,
**({"id": call_id} if call_id else {}),
**({"parts": parts} if parts is not None else {}),
}
def _result_part(response: JsonValue, call_id: str | None = None) -> Mapping[str, JsonValue]:
result: Final = {"name": _TOOL, "response": response, **({"id": call_id} if call_id else {})}
return {"function_response": result}
@ -680,7 +675,7 @@ def test_a_multimodal_tool_result_with_a_ref_keeps_its_image_parts_beside_the_ra
**expected,
"contents": [
*_items(expected["contents"])[:2],
{"role": "user", "parts": [_result_part({"content": _CHART_TEXT}, parts=_INLINE_PNG)]},
{"role": "user", "parts": [_result_part({"content": _CHART_TEXT}), *_INLINE_PNG]},
],
}
assert _spend_row(response_id)["status"] == "success"

2
tests/integration/providers/test_vertex_gemini_fragmented_stream_wire.py
View file

@ -102,7 +102,6 @@ def _scripted_vertex(fragments: tuple[bytes, ...]) -> Callable[[Request], Reply]
assert request.headers["authorization"] == "Bearer scripted-token"
body: Final = _JSON_OBJECT.validate_json(request.body)
assert body["contents"] == [{"role": "user", "parts": [{"text": _PROMPT}]}]
assert body["generationConfig"] == {"temperature": 0.0}
return Reply(content_type="text/event-stream", chunks=fragments)
return respond
@ -127,7 +126,6 @@ def _streamed_lines(gateway: Gateway, model: str) -> tuple[str, ...]:
"model": model,
"messages": [{"role": "user", "content": _PROMPT}],
"stream": True,
"temperature": 0.0,
},
headers={"Authorization": f"Bearer {gateway.key}"},
timeout=_STREAM_BUDGET_SECONDS,

4
tests/integration/providers/test_vertex_gemini_function_call_wire.py
View file

@ -197,7 +197,7 @@ def test_vertex_gemini_kwargs_extra_param_reaches_generation_config(gateway: Gat
def respond(request: Request) -> Reply:
assert request.target == f"{_MODEL_PATH}:generateContent"
body: Final = _JSON_OBJECT.validate_json(request.body)
assert body["generationConfig"]["top_k"] == 3, body
assert body["generationConfig"]["candidate_count"] == 3, body
return Reply(
body=json.dumps(
{
@ -219,7 +219,7 @@ def test_vertex_gemini_kwargs_extra_param_reaches_generation_config(gateway: Gat
"/v1/chat/completions",
json={
"model": model,
"top_k": 3,
"candidate_count": 3,
"messages": [{"role": "user", "content": "hi"}],
},
headers={"Authorization": f"Bearer {gateway.key}"},

289
tests/integration/translation/chat_completions/sampling/test_chat_completions_sampling_gemini.py
View file

@ -5,38 +5,42 @@ import pytest
from integration._support.client import Gateway, object_value
from integration._support.provider import SharedProvider
from integration.translation.case import TranslationTestCase
from integration.translation.chat_completions.bases.gemini import GEMINI_3_8_FLASH_TEST_CASE
from integration.translation.chat_completions.bases.vertex_ai import (
GEMINI_3_8_FLASH_TEST_CASE as VERTEX_AI_GEMINI_3_8_FLASH_TEST_CASE,
)
from integration.translation.chat_completions.templates import CHAT_COMPLETIONS_RESPONSE
from integration.translation.runner import assert_translation
from integration.translation.templates.vertex_ai import (
VERTEX_AI_GEMINI_GENERATE_CONTENT_RESPONSE,
)
GEMINI_GENERATION_CONFIG: Final = object_value(GEMINI_3_8_FLASH_TEST_CASE.expected_provider_request["generationConfig"])
VERTEX_AI_GENERATION_CONFIG: Final = object_value(
VERTEX_AI_GEMINI_3_8_FLASH_TEST_CASE.expected_provider_request["generationConfig"]
)
GEMINI_3_8_FLASH_TEMPERATURE_TEST_CASE: Final = replace(
from integration.translation.chat_completions.bases.gemini import (
GEMINI_3_5_FLASH_TEST_CASE,
GEMINI_3_8_FLASH_TEST_CASE,
)
from integration.translation.chat_completions.bases.vertex_ai import (
GEMINI_3_5_FLASH_TEST_CASE as VERTEX_AI_GEMINI_3_5_FLASH_TEST_CASE,
)
from integration.translation.runner import assert_translation
GEMINI_3_5_FLASH_GENERATION_CONFIG: Final = object_value(
GEMINI_3_5_FLASH_TEST_CASE.expected_provider_request["generationConfig"]
)
GEMINI_3_8_FLASH_GENERATION_CONFIG: Final = object_value(
GEMINI_3_8_FLASH_TEST_CASE.expected_provider_request["generationConfig"]
)
VERTEX_AI_GENERATION_CONFIG: Final = object_value(
VERTEX_AI_GEMINI_3_5_FLASH_TEST_CASE.expected_provider_request["generationConfig"]
)
GEMINI_3_5_FLASH_TEMPERATURE_TEST_CASE: Final = replace(
GEMINI_3_5_FLASH_TEST_CASE,
scenario="temperature",
litellm_request={**GEMINI_3_8_FLASH_TEST_CASE.litellm_request, "temperature": 0.4},
litellm_request={**GEMINI_3_5_FLASH_TEST_CASE.litellm_request, "temperature": 0.4},
expected_provider_request={
**GEMINI_3_8_FLASH_TEST_CASE.expected_provider_request,
"generationConfig": {**GEMINI_GENERATION_CONFIG, "temperature": 0.4},
**GEMINI_3_5_FLASH_TEST_CASE.expected_provider_request,
"generationConfig": {**GEMINI_3_5_FLASH_GENERATION_CONFIG, "temperature": 0.4},
},
)
GEMINI_3_8_FLASH_ZERO_TEMPERATURE_TEST_CASE: Final = replace(
GEMINI_3_8_FLASH_TEST_CASE,
GEMINI_3_5_FLASH_ZERO_TEMPERATURE_TEST_CASE: Final = replace(
GEMINI_3_5_FLASH_TEST_CASE,
scenario="zero_temperature",
litellm_request={**GEMINI_3_8_FLASH_TEST_CASE.litellm_request, "temperature": 0},
litellm_request={**GEMINI_3_5_FLASH_TEST_CASE.litellm_request, "temperature": 0},
expected_provider_request={
**GEMINI_3_8_FLASH_TEST_CASE.expected_provider_request,
"generationConfig": {**GEMINI_GENERATION_CONFIG, "temperature": 0},
**GEMINI_3_5_FLASH_TEST_CASE.expected_provider_request,
"generationConfig": {**GEMINI_3_5_FLASH_GENERATION_CONFIG, "temperature": 0},
},
)
@ -47,251 +51,30 @@ GEMINI_3_8_FLASH_REASONING_EFFORT_TEST_CASE: Final = replace(
expected_provider_request={
**GEMINI_3_8_FLASH_TEST_CASE.expected_provider_request,
"generationConfig": {
**GEMINI_GENERATION_CONFIG,
**GEMINI_3_8_FLASH_GENERATION_CONFIG,
"thinkingConfig": {"thinkingLevel": "medium", "includeThoughts": True},
},
},
)
VERTEX_AI_GEMINI_3_8_FLASH_TEMPERATURE_THOUGHT_SIGNATURE_0: Final = (
"AY89a18+3vC8Y6wtLQ8YLNoWj0pZ1qcMTAdhuIp6v+o7VjO8nN81l72KF3fiSKtgKQYvOKc4hz38BNeldTuj8NebwQ7BneePvMzc"
"hDx/5j5DXPVIIW7bRE2vYMhffaNpyjdltlQRDAxhiqjbSHIIGZQdxxAE7GfdGxlhstemN91VzQR4KnPKk6MXh0at/4JacXcOFRRX"
"DnYmUqj1a1tSIoSjeTsHMvSj5OpJyxxa8Fafd/yqwL8g4scNZgGzckn4O5quaQmJT9VPtoAsmh8qgXUr/qzNSZoFGzRN6m7OzSuT"
"yQ246hxKwp0+ZB9P+Nn5fvG96JVXsRq7dF00HBAbhebjUfbNTmIQauZHGSGbF2e+ck0aG5Lf8se2qRu6NCqiF+EnTFMUg7g+1cN1"
"nIXLw7X6cAPPrpXvVdkfAoHksgSFWY5uUYwW9yDTZYwOWqF6bxDapPFU5HmJj/Bf5YuVH7VhGB7fyO8SVW6AUe0/S/o0/e//wn6A"
"aFomUST2wXw0hsi/OSZLEhRAi16AeUI0Y4SenblpEYtDAyhzFi77mqsceuF7QIq672m5yWq2qoGhPh7S+5lMEvuD8YLITzrC81Xc"
"b4HN3rxOvi6skmP4WJqZO8zPIw65WQUhRdhadhxM81HE+3InBToo1IhAF+11qt6hK0EuUf1guaGrUEDZRPAgomcOLBSdK6gQ9pn5"
"z3SiRPpw9SV4LWMqBjulGRD6K8u7XHXLPAiGZTF1cr/N2sEQuyri57XytugNISVcxh0q/Lvz0v0DyUAZM8H2ymbusxnKdIHanI9Z"
"US/XIXdia3PwqiOCgXDQaceOmotUnl8i0IXvSKwMjv3wB7UaC4YKyDurZ+gJJOzbjVjZ/mEf9h4JW1ZetNWVtQl2s+oZXCHOtVJD"
"2qLDyuMPO7EuafP8w7mbpKBTmw4ZAyCfrTL73FnEYhp6M3IEG+c4fdUCHMvT6ecG42DLKhdmWfCqzTdXmXFLj/bePo7YzfYpLOLl"
"hEzVdO5ojXPd6squqM+YHQS640UkdQ4v20UQ9ghFRkW1xUl7uPQkIAI0KWiDmaoi0YGFBxWpTs1U+wN2K6hfqgtJy80rl5ceUT7f"
"6B3MnwiIoAjzVc3n3TH6GodPIO08vrhrf4oz+G3OkdCaMHXURfzYkhji50/ThfAxgD7XDuzUYlUWGotbx0FiYEDBbi2b7t7L/w3f"
"P3nXYoSgaaHumXEnOAtugy6immasBFawtZq+GrOIYHJbH2tZhMpUysSTSnGF3BmqczIctDCwC82AbXU5sKEKFShEGnJSw9B0OlWu"
"Pa6kC/Yfapk6NzDyNB2ttsLhy0hCkT0BmgDn2P4h5T4PCVgLWUbA7LGQoYO5zbsUFBwtPUsVFK0yLyf/Gg9h4ebosT6NVh7AkzS2"
"riaNfm//lGUKK/8YDY49PAu0jpHp6JoXYg/z1+BjPvITdIQgf/8a0X36uqO+Yp2HZPzZzl/Ed8sC3ZadKxulNV9kkXN2Gc3v/VyB"
"ruICqdhHSXJkusQJw+vtClqbP2uT18wQN4SRLjFTC3/ZruElXJFipevCQcC8KNVkXW39qZNWJGmKAR15mprXot6VKXkRHfyQJJz0"
"lAMTaACA5LJoA1dDn7+UaikqUsfrlLE6ObbCasA5/wIp5zFsGvslVq/vJqGWk2mdXbas2fpWTSUECEB3gGoIFo9rmHTURX4cAALa"
"oZnjSkaKy1IdC3XDQsqT2vmAX3XjvaLhkCwWg56pst5LvS0AlcRGo49QHATNsY4h0SFkUmMyAGgS7+Pa2VwmCgWUoFFHRbO8ryoy"
"W57qONicyCxA2/OL28OrKGC/e25UsXwKBAXx+9uamcdxFvOQr9WqEJK1g4Q="
)
VERTEX_AI_GEMINI_3_8_FLASH_TEMPERATURE_TEST_CASE: Final = replace(
VERTEX_AI_GEMINI_3_8_FLASH_TEST_CASE,
VERTEX_AI_GEMINI_3_5_FLASH_TEMPERATURE_TEST_CASE: Final = replace(
VERTEX_AI_GEMINI_3_5_FLASH_TEST_CASE,
scenario="temperature",
litellm_request={**VERTEX_AI_GEMINI_3_8_FLASH_TEST_CASE.litellm_request, "temperature": 0.4},
litellm_request={**VERTEX_AI_GEMINI_3_5_FLASH_TEST_CASE.litellm_request, "temperature": 0.4},
expected_provider_request={
**VERTEX_AI_GEMINI_3_8_FLASH_TEST_CASE.expected_provider_request,
**VERTEX_AI_GEMINI_3_5_FLASH_TEST_CASE.expected_provider_request,
"generationConfig": {**VERTEX_AI_GENERATION_CONFIG, "temperature": 0.4},
},
mock_provider_response={
**VERTEX_AI_GEMINI_GENERATE_CONTENT_RESPONSE,
"candidates": [
{
"content": {
"role": "model",
"parts": [
{
"text": (
"**My Thought Process on Handling Quantity Validation**\n\nAlright, to properly address the handling "
"of zero, negative, and potentially non-integer quantities, my immediate next step needs to be "
"understanding the existing validation logic. This is crucial before I can even begin to think about "
'suggesting any fixes or further code. My protocol dictates that I must "read the relevant files '
"before suggesting a fix,\" and that's exactly what I'm prioritizing.\n\nTherefore, I need to examine "
"the `shop/validation.py` file. This is where I anticipate finding the rules and checks that govern "
"how different types of quantities are processed and validated. My plan is to read a reasonable "
"initial segment of this file, say lines 1 through 50, to get a solid grasp of the current "
"implementation. After that, I'll be able to determine precisely *which* other files, if any, I'll "
"need to investigate next, keeping in mind the specific constraints of zero, negative, and "
'non-integer values.\n\nTo do this, I will now use the `read_file` tool:\n\n```json\n{\n "tool_code": '
"\"print(read_file(path='shop/validation.py', start_line=1, end_line=50))\"\n}\n```\n\n"
),
"thought": True,
},
{
"text": (
"To examine how zero, negative, and non-integer quantities are handled, I need to inspect "
"`shop/validation.py` next."
),
},
{
"functionCall": {
"name": "read_file",
"args": {"start_line": 1, "end_line": 50, "path": "shop/validation.py"},
"id": "call_1464376",
},
"thoughtSignature": VERTEX_AI_GEMINI_3_8_FLASH_TEMPERATURE_THOUGHT_SIGNATURE_0,
},
],
},
"finishReason": "STOP",
},
],
"usageMetadata": {
"promptTokenCount": 246,
"candidatesTokenCount": 61,
"totalTokenCount": 646,
"trafficType": "ON_DEMAND",
"promptTokensDetails": [{"modality": "TEXT", "tokenCount": 246}],
"candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 61}],
"thoughtsTokenCount": 339,
},
"modelVersion": "gemini-3.8-flash",
"createTime": "2026-10-15T01:22:48.202504Z",
"responseId": "6CrQaoiuDNKZq8YPjZCokAw",
},
expected_litellm_response={
**CHAT_COMPLETIONS_RESPONSE,
"model": "vertex_ai/gemini-3.8-flash",
"choices": [
{
"finish_reason": "tool_calls",
"index": 0,
"message": {
"content": (
"To examine how zero, negative, and non-integer quantities are handled, I need to inspect "
"`shop/validation.py` next."
),
"role": "assistant",
"tool_calls": [
{
"index": 0,
"provider_specific_fields": {
"thought_signature": (
"AY89a18+3vC8Y6wtLQ8YLNoWj0pZ1qcMTAdhuIp6v+o7VjO8nN81l72KF3fiSKtgKQYvOKc4hz38BNeldTuj8NebwQ7BneePvMzc"
"hDx/5j5DXPVIIW7bRE2vYMhffaNpyjdltlQRDAxhiqjbSHIIGZQdxxAE7GfdGxlhstemN91VzQR4KnPKk6MXh0at/4JacXcOFRRX"
"DnYmUqj1a1tSIoSjeTsHMvSj5OpJyxxa8Fafd/yqwL8g4scNZgGzckn4O5quaQmJT9VPtoAsmh8qgXUr/qzNSZoFGzRN6m7OzSuT"
"yQ246hxKwp0+ZB9P+Nn5fvG96JVXsRq7dF00HBAbhebjUfbNTmIQauZHGSGbF2e+ck0aG5Lf8se2qRu6NCqiF+EnTFMUg7g+1cN1"
"nIXLw7X6cAPPrpXvVdkfAoHksgSFWY5uUYwW9yDTZYwOWqF6bxDapPFU5HmJj/Bf5YuVH7VhGB7fyO8SVW6AUe0/S/o0/e//wn6A"
"aFomUST2wXw0hsi/OSZLEhRAi16AeUI0Y4SenblpEYtDAyhzFi77mqsceuF7QIq672m5yWq2qoGhPh7S+5lMEvuD8YLITzrC81Xc"
"b4HN3rxOvi6skmP4WJqZO8zPIw65WQUhRdhadhxM81HE+3InBToo1IhAF+11qt6hK0EuUf1guaGrUEDZRPAgomcOLBSdK6gQ9pn5"
"z3SiRPpw9SV4LWMqBjulGRD6K8u7XHXLPAiGZTF1cr/N2sEQuyri57XytugNISVcxh0q/Lvz0v0DyUAZM8H2ymbusxnKdIHanI9Z"
"US/XIXdia3PwqiOCgXDQaceOmotUnl8i0IXvSKwMjv3wB7UaC4YKyDurZ+gJJOzbjVjZ/mEf9h4JW1ZetNWVtQl2s+oZXCHOtVJD"
"2qLDyuMPO7EuafP8w7mbpKBTmw4ZAyCfrTL73FnEYhp6M3IEG+c4fdUCHMvT6ecG42DLKhdmWfCqzTdXmXFLj/bePo7YzfYpLOLl"
"hEzVdO5ojXPd6squqM+YHQS640UkdQ4v20UQ9ghFRkW1xUl7uPQkIAI0KWiDmaoi0YGFBxWpTs1U+wN2K6hfqgtJy80rl5ceUT7f"
"6B3MnwiIoAjzVc3n3TH6GodPIO08vrhrf4oz+G3OkdCaMHXURfzYkhji50/ThfAxgD7XDuzUYlUWGotbx0FiYEDBbi2b7t7L/w3f"
"P3nXYoSgaaHumXEnOAtugy6immasBFawtZq+GrOIYHJbH2tZhMpUysSTSnGF3BmqczIctDCwC82AbXU5sKEKFShEGnJSw9B0OlWu"
"Pa6kC/Yfapk6NzDyNB2ttsLhy0hCkT0BmgDn2P4h5T4PCVgLWUbA7LGQoYO5zbsUFBwtPUsVFK0yLyf/Gg9h4ebosT6NVh7AkzS2"
"riaNfm//lGUKK/8YDY49PAu0jpHp6JoXYg/z1+BjPvITdIQgf/8a0X36uqO+Yp2HZPzZzl/Ed8sC3ZadKxulNV9kkXN2Gc3v/VyB"
"ruICqdhHSXJkusQJw+vtClqbP2uT18wQN4SRLjFTC3/ZruElXJFipevCQcC8KNVkXW39qZNWJGmKAR15mprXot6VKXkRHfyQJJz0"
"lAMTaACA5LJoA1dDn7+UaikqUsfrlLE6ObbCasA5/wIp5zFsGvslVq/vJqGWk2mdXbas2fpWTSUECEB3gGoIFo9rmHTURX4cAALa"
"oZnjSkaKy1IdC3XDQsqT2vmAX3XjvaLhkCwWg56pst5LvS0AlcRGo49QHATNsY4h0SFkUmMyAGgS7+Pa2VwmCgWUoFFHRbO8ryoy"
"W57qONicyCxA2/OL28OrKGC/e25UsXwKBAXx+9uamcdxFvOQr9WqEJK1g4Q="
),
},
"function": {
"arguments": '{"start_line": 1, "end_line": 50, "path": "shop/validation.py"}',
"name": "read_file",
},
"id": (
"call_1464376__thought__AY89a18+3vC8Y6wtLQ8YLNoWj0pZ1qcMTAdhuIp6v+o7VjO8nN81l72KF3fiSKtgKQYvOKc4hz38B"
"NeldTuj8NebwQ7BneePvMzchDx/5j5DXPVIIW7bRE2vYMhffaNpyjdltlQRDAxhiqjbSHIIGZQdxxAE7GfdGxlhstemN91VzQR4K"
"nPKk6MXh0at/4JacXcOFRRXDnYmUqj1a1tSIoSjeTsHMvSj5OpJyxxa8Fafd/yqwL8g4scNZgGzckn4O5quaQmJT9VPtoAsmh8qg"
"XUr/qzNSZoFGzRN6m7OzSuTyQ246hxKwp0+ZB9P+Nn5fvG96JVXsRq7dF00HBAbhebjUfbNTmIQauZHGSGbF2e+ck0aG5Lf8se2q"
"Ru6NCqiF+EnTFMUg7g+1cN1nIXLw7X6cAPPrpXvVdkfAoHksgSFWY5uUYwW9yDTZYwOWqF6bxDapPFU5HmJj/Bf5YuVH7VhGB7fy"
"O8SVW6AUe0/S/o0/e//wn6AaFomUST2wXw0hsi/OSZLEhRAi16AeUI0Y4SenblpEYtDAyhzFi77mqsceuF7QIq672m5yWq2qoGhP"
"h7S+5lMEvuD8YLITzrC81Xcb4HN3rxOvi6skmP4WJqZO8zPIw65WQUhRdhadhxM81HE+3InBToo1IhAF+11qt6hK0EuUf1guaGrU"
"EDZRPAgomcOLBSdK6gQ9pn5z3SiRPpw9SV4LWMqBjulGRD6K8u7XHXLPAiGZTF1cr/N2sEQuyri57XytugNISVcxh0q/Lvz0v0Dy"
"UAZM8H2ymbusxnKdIHanI9ZUS/XIXdia3PwqiOCgXDQaceOmotUnl8i0IXvSKwMjv3wB7UaC4YKyDurZ+gJJOzbjVjZ/mEf9h4JW"
"1ZetNWVtQl2s+oZXCHOtVJD2qLDyuMPO7EuafP8w7mbpKBTmw4ZAyCfrTL73FnEYhp6M3IEG+c4fdUCHMvT6ecG42DLKhdmWfCqz"
"TdXmXFLj/bePo7YzfYpLOLlhEzVdO5ojXPd6squqM+YHQS640UkdQ4v20UQ9ghFRkW1xUl7uPQkIAI0KWiDmaoi0YGFBxWpTs1U+"
"wN2K6hfqgtJy80rl5ceUT7f6B3MnwiIoAjzVc3n3TH6GodPIO08vrhrf4oz+G3OkdCaMHXURfzYkhji50/ThfAxgD7XDuzUYlUWG"
"otbx0FiYEDBbi2b7t7L/w3fP3nXYoSgaaHumXEnOAtugy6immasBFawtZq+GrOIYHJbH2tZhMpUysSTSnGF3BmqczIctDCwC82Ab"
"XU5sKEKFShEGnJSw9B0OlWuPa6kC/Yfapk6NzDyNB2ttsLhy0hCkT0BmgDn2P4h5T4PCVgLWUbA7LGQoYO5zbsUFBwtPUsVFK0yL"
"yf/Gg9h4ebosT6NVh7AkzS2riaNfm//lGUKK/8YDY49PAu0jpHp6JoXYg/z1+BjPvITdIQgf/8a0X36uqO+Yp2HZPzZzl/Ed8sC3"
"ZadKxulNV9kkXN2Gc3v/VyBruICqdhHSXJkusQJw+vtClqbP2uT18wQN4SRLjFTC3/ZruElXJFipevCQcC8KNVkXW39qZNWJGmKA"
"R15mprXot6VKXkRHfyQJJz0lAMTaACA5LJoA1dDn7+UaikqUsfrlLE6ObbCasA5/wIp5zFsGvslVq/vJqGWk2mdXbas2fpWTSUEC"
"EB3gGoIFo9rmHTURX4cAALaoZnjSkaKy1IdC3XDQsqT2vmAX3XjvaLhkCwWg56pst5LvS0AlcRGo49QHATNsY4h0SFkUmMyAGgS7"
"+Pa2VwmCgWUoFFHRbO8ryoyW57qONicyCxA2/OL28OrKGC/e25UsXwKBAXx+9uamcdxFvOQr9WqEJK1g4Q="
),
"type": "function",
},
],
"images": [],
"reasoning_content": (
"**My Thought Process on Handling Quantity Validation**\n\nAlright, to properly address the handling "
"of zero, negative, and potentially non-integer quantities, my immediate next step needs to be "
"understanding the existing validation logic. This is crucial before I can even begin to think about "
'suggesting any fixes or further code. My protocol dictates that I must "read the relevant files '
"before suggesting a fix,\" and that's exactly what I'm prioritizing.\n\nTherefore, I need to examine "
"the `shop/validation.py` file. This is where I anticipate finding the rules and checks that govern "
"how different types of quantities are processed and validated. My plan is to read a reasonable "
"initial segment of this file, say lines 1 through 50, to get a solid grasp of the current "
"implementation. After that, I'll be able to determine precisely *which* other files, if any, I'll "
"need to investigate next, keeping in mind the specific constraints of zero, negative, and "
'non-integer values.\n\nTo do this, I will now use the `read_file` tool:\n\n```json\n{\n "tool_code": '
"\"print(read_file(path='shop/validation.py', start_line=1, end_line=50))\"\n}\n```\n\n"
),
"thinking_blocks": [
{
"type": "thinking",
"thinking": (
"**My Thought Process on Handling Quantity Validation**\n\nAlright, to properly address the handling "
"of zero, negative, and potentially non-integer quantities, my immediate next step needs to be "
"understanding the existing validation logic. This is crucial before I can even begin to think about "
'suggesting any fixes or further code. My protocol dictates that I must "read the relevant files '
"before suggesting a fix,\" and that's exactly what I'm prioritizing.\n\nTherefore, I need to examine "
"the `shop/validation.py` file. This is where I anticipate finding the rules and checks that govern "
"how different types of quantities are processed and validated. My plan is to read a reasonable "
"initial segment of this file, say lines 1 through 50, to get a solid grasp of the current "
"implementation. After that, I'll be able to determine precisely *which* other files, if any, I'll "
"need to investigate next, keeping in mind the specific constraints of zero, negative, and "
'non-integer values.\n\nTo do this, I will now use the `read_file` tool:\n\n```json\n{\n "tool_code": '
"\"print(read_file(path='shop/validation.py', start_line=1, end_line=50))\"\n}\n```\n\n"
),
},
],
"provider_specific_fields": {
"thought_signatures": [
(
"AY89a18+3vC8Y6wtLQ8YLNoWj0pZ1qcMTAdhuIp6v+o7VjO8nN81l72KF3fiSKtgKQYvOKc4hz38BNeldTuj8NebwQ7BneePvMzc"
"hDx/5j5DXPVIIW7bRE2vYMhffaNpyjdltlQRDAxhiqjbSHIIGZQdxxAE7GfdGxlhstemN91VzQR4KnPKk6MXh0at/4JacXcOFRRX"
"DnYmUqj1a1tSIoSjeTsHMvSj5OpJyxxa8Fafd/yqwL8g4scNZgGzckn4O5quaQmJT9VPtoAsmh8qgXUr/qzNSZoFGzRN6m7OzSuT"
"yQ246hxKwp0+ZB9P+Nn5fvG96JVXsRq7dF00HBAbhebjUfbNTmIQauZHGSGbF2e+ck0aG5Lf8se2qRu6NCqiF+EnTFMUg7g+1cN1"
"nIXLw7X6cAPPrpXvVdkfAoHksgSFWY5uUYwW9yDTZYwOWqF6bxDapPFU5HmJj/Bf5YuVH7VhGB7fyO8SVW6AUe0/S/o0/e//wn6A"
"aFomUST2wXw0hsi/OSZLEhRAi16AeUI0Y4SenblpEYtDAyhzFi77mqsceuF7QIq672m5yWq2qoGhPh7S+5lMEvuD8YLITzrC81Xc"
"b4HN3rxOvi6skmP4WJqZO8zPIw65WQUhRdhadhxM81HE+3InBToo1IhAF+11qt6hK0EuUf1guaGrUEDZRPAgomcOLBSdK6gQ9pn5"
"z3SiRPpw9SV4LWMqBjulGRD6K8u7XHXLPAiGZTF1cr/N2sEQuyri57XytugNISVcxh0q/Lvz0v0DyUAZM8H2ymbusxnKdIHanI9Z"
"US/XIXdia3PwqiOCgXDQaceOmotUnl8i0IXvSKwMjv3wB7UaC4YKyDurZ+gJJOzbjVjZ/mEf9h4JW1ZetNWVtQl2s+oZXCHOtVJD"
"2qLDyuMPO7EuafP8w7mbpKBTmw4ZAyCfrTL73FnEYhp6M3IEG+c4fdUCHMvT6ecG42DLKhdmWfCqzTdXmXFLj/bePo7YzfYpLOLl"
"hEzVdO5ojXPd6squqM+YHQS640UkdQ4v20UQ9ghFRkW1xUl7uPQkIAI0KWiDmaoi0YGFBxWpTs1U+wN2K6hfqgtJy80rl5ceUT7f"
"6B3MnwiIoAjzVc3n3TH6GodPIO08vrhrf4oz+G3OkdCaMHXURfzYkhji50/ThfAxgD7XDuzUYlUWGotbx0FiYEDBbi2b7t7L/w3f"
"P3nXYoSgaaHumXEnOAtugy6immasBFawtZq+GrOIYHJbH2tZhMpUysSTSnGF3BmqczIctDCwC82AbXU5sKEKFShEGnJSw9B0OlWu"
"Pa6kC/Yfapk6NzDyNB2ttsLhy0hCkT0BmgDn2P4h5T4PCVgLWUbA7LGQoYO5zbsUFBwtPUsVFK0yLyf/Gg9h4ebosT6NVh7AkzS2"
"riaNfm//lGUKK/8YDY49PAu0jpHp6JoXYg/z1+BjPvITdIQgf/8a0X36uqO+Yp2HZPzZzl/Ed8sC3ZadKxulNV9kkXN2Gc3v/VyB"
"ruICqdhHSXJkusQJw+vtClqbP2uT18wQN4SRLjFTC3/ZruElXJFipevCQcC8KNVkXW39qZNWJGmKAR15mprXot6VKXkRHfyQJJz0"
"lAMTaACA5LJoA1dDn7+UaikqUsfrlLE6ObbCasA5/wIp5zFsGvslVq/vJqGWk2mdXbas2fpWTSUECEB3gGoIFo9rmHTURX4cAALa"
"oZnjSkaKy1IdC3XDQsqT2vmAX3XjvaLhkCwWg56pst5LvS0AlcRGo49QHATNsY4h0SFkUmMyAGgS7+Pa2VwmCgWUoFFHRbO8ryoy"
"W57qONicyCxA2/OL28OrKGC/e25UsXwKBAXx+9uamcdxFvOQr9WqEJK1g4Q="
),
],
},
},
"provider_specific_fields": {"native_finish_reason": "STOP"},
},
],
"usage": {
"completion_tokens": 400,
"prompt_tokens": 246,
"total_tokens": 646,
"completion_tokens_details": {"reasoning_tokens": 339, "text_tokens": 61},
"prompt_tokens_details": {"text_tokens": 246},
},
"vertex_ai_grounding_metadata": [],
"vertex_ai_url_context_metadata": [],
"vertex_ai_safety_results": [],
"vertex_ai_citation_metadata": [],
},
)
@pytest.mark.parametrize(
"case",
[
GEMINI_3_8_FLASH_TEMPERATURE_TEST_CASE,
GEMINI_3_8_FLASH_ZERO_TEMPERATURE_TEST_CASE,
GEMINI_3_5_FLASH_TEMPERATURE_TEST_CASE,
GEMINI_3_5_FLASH_ZERO_TEMPERATURE_TEST_CASE,
GEMINI_3_8_FLASH_REASONING_EFFORT_TEST_CASE,
VERTEX_AI_GEMINI_3_8_FLASH_TEMPERATURE_TEST_CASE,
VERTEX_AI_GEMINI_3_5_FLASH_TEMPERATURE_TEST_CASE,
],
ids=lambda case: case.id,
)

14
tests/integration/translation/messages/sampling/test_messages_sampling_gemini.py
View file

@ -5,22 +5,22 @@ import pytest
from integration._support.client import Gateway, object_value
from integration._support.provider import SharedProvider
from integration.translation.case import TranslationTestCase
from integration.translation.messages.bases.gemini import GEMINI_3_8_FLASH_TEST_CASE
from integration.translation.messages.bases.gemini import GEMINI_3_5_FLASH_TEST_CASE
from integration.translation.runner import assert_translation
GENERATION_CONFIG: Final = object_value(GEMINI_3_8_FLASH_TEST_CASE.expected_provider_request["generationConfig"])
GENERATION_CONFIG: Final = object_value(GEMINI_3_5_FLASH_TEST_CASE.expected_provider_request["generationConfig"])
GEMINI_3_8_FLASH_TEMPERATURE_TEST_CASE: Final = replace(
GEMINI_3_8_FLASH_TEST_CASE,
GEMINI_3_5_FLASH_TEMPERATURE_TEST_CASE: Final = replace(
GEMINI_3_5_FLASH_TEST_CASE,
scenario="temperature",
litellm_request={**GEMINI_3_8_FLASH_TEST_CASE.litellm_request, "temperature": 0.4},
litellm_request={**GEMINI_3_5_FLASH_TEST_CASE.litellm_request, "temperature": 0.4},
expected_provider_request={
**GEMINI_3_8_FLASH_TEST_CASE.expected_provider_request,
**GEMINI_3_5_FLASH_TEST_CASE.expected_provider_request,
"generationConfig": {**GENERATION_CONFIG, "temperature": 0.4},
},
)
@pytest.mark.parametrize("case", [GEMINI_3_8_FLASH_TEMPERATURE_TEST_CASE], ids=lambda case: case.id)
@pytest.mark.parametrize("case", [GEMINI_3_5_FLASH_TEMPERATURE_TEST_CASE], ids=lambda case: case.id)
def test_messages_sampling_gemini(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None:
assert_translation(case, gateway, provider)

16
tests/integration/translation/responses/sampling/test_responses_sampling_gemini.py
View file

@ -5,23 +5,23 @@ import pytest
from integration._support.client import Gateway, object_value
from integration._support.provider import SharedProvider
from integration.translation.case import TranslationTestCase
from integration.translation.responses.bases.gemini import GEMINI_3_8_FLASH_TEST_CASE
from integration.translation.responses.bases.gemini import GEMINI_3_5_FLASH_TEST_CASE
from integration.translation.runner import assert_translation
GENERATION_CONFIG: Final = object_value(GEMINI_3_8_FLASH_TEST_CASE.expected_provider_request["generationConfig"])
GENERATION_CONFIG: Final = object_value(GEMINI_3_5_FLASH_TEST_CASE.expected_provider_request["generationConfig"])
GEMINI_3_8_FLASH_TEMPERATURE_TEST_CASE: Final = replace(
GEMINI_3_8_FLASH_TEST_CASE,
GEMINI_3_5_FLASH_TEMPERATURE_TEST_CASE: Final = replace(
GEMINI_3_5_FLASH_TEST_CASE,
scenario="temperature",
litellm_request={**GEMINI_3_8_FLASH_TEST_CASE.litellm_request, "temperature": 0.4},
litellm_request={**GEMINI_3_5_FLASH_TEST_CASE.litellm_request, "temperature": 0.4},
expected_provider_request={
**GEMINI_3_8_FLASH_TEST_CASE.expected_provider_request,
**GEMINI_3_5_FLASH_TEST_CASE.expected_provider_request,
"generationConfig": {**GENERATION_CONFIG, "temperature": 0.4},
},
expected_litellm_response={**GEMINI_3_8_FLASH_TEST_CASE.expected_litellm_response, "temperature": 0.4},
expected_litellm_response={**GEMINI_3_5_FLASH_TEST_CASE.expected_litellm_response, "temperature": 0.4},
)
@pytest.mark.parametrize("case", [GEMINI_3_8_FLASH_TEMPERATURE_TEST_CASE], ids=lambda case: case.id)
@pytest.mark.parametrize("case", [GEMINI_3_5_FLASH_TEMPERATURE_TEST_CASE], ids=lambda case: case.id)
def test_responses_sampling_gemini(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None:
assert_translation(case, gateway, provider)