From 21ab2cb87befb4994e569271d13e9ae100552c69 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 27 Oct 2026 18:11:25 -0700 Subject: [PATCH] test: update stale integration and guardrail e2e tests to intended behavior (#48665) * test(integration): update stale expectations to intended provider behavior Each expectation moved because a merged PR changed behavior on purpose. - chatgpt string input is wrapped as one user message (#41522) - Gemini 2.5 tool media sits beside function_response (#48428) - sampling knobs move to Gemini 3.5 Flash, which keeps them (#47645) - a body base_model is stripped without allow_client_pricing_override (#47639) - ai_decide answers without usage return usage null (#48261) - azure_ai forwards mapped client anthropic-beta values (#48033) - the OpenCode scripted upstream mints a fresh id per request * test(e2e-ui): type the guardrail name before deleting it GuardrailDeleteModal requires the exact name before Delete enables since #45893. All three guardrail delete flows now go through one helper. * test(observability): accept any sampled traceresponse flags and expect a 400 for a non-object body OpenTelemetry 1.45 (#48055) sets the W3C random trace id flag on new traces, so the gateway's traceresponse now ends in -03. The producer copies the server span's own trace flags, so the tests now accept any two-digit flags with the sampled bit set instead of pinning -01 #48438 made a non-object JSON body a 400 "Invalid JSON payload: expected an object" on purpose, so the [1] body case is renamed non_object_body and expects that 400 * test(observability): stop waiting on OpenTelemetry's export log line in the langtrace outage test The outage test waited for "Exception while exporting Span batch" in the proxy log, which is OpenTelemetry's wording and no longer appears with 1.45. The test keeps asserting what the gateway guarantees, callers unaffected during the outage, a live gateway, and every span after recovery delivered once with no outage span duplicated * test(guardrails): count the Responses disconnect spend rows #47611 added Since #47611 a streamed /v1/responses disconnect writes its own spend row, so the concurrent disconnect test saw 20 rows where it counted 10 by model group. It now reads each row's call type and expects ten acompletion rows with the outage verdict and ten aresponses rows with success. /v1/messages disconnects still write no row on main, so they are left out of the count rather than pinned as absent The responses-httpx case of the single disconnect test no longer needs its LIT-8603 skip and passes * test(observability): size K4's timeout from the harness boot budget K4 boots three two-worker proxies in a row. On CircleCI each boot takes 43 to 58 seconds, so a passing run took 179.97s against its fixed 180s cap and 6 of 30 runs hit pytest-timeout. The cap and the restart gate wait now come from readiness_seconds and graceful_stop_seconds, like other multi-boot tests --- tests/e2e/ui/helpers/guardrails.ts | 20 ++ .../ui/tests/guardrails/guardrails.spec.ts | 31 +- .../guardrails/presidioUserStory.spec.ts | 16 +- .../test_anthropic_safeguards_beta_wire.py | 19 +- .../observability/test_guardrail_effects.py | 26 +- .../observability/test_langtrace_delivery.py | 7 - .../test_logging_only_scope_chaos.py | 7 +- .../test_otel_trace_response_header.py | 16 +- .../test_opencode_go_reported_cost_wire.py | 42 +-- ..._bedrock_converse_lookaround_regex_wire.py | 6 +- ...st_chatgpt_responses_caller_stream_wire.py | 9 +- .../test_databricks_decisions_wire.py | 2 +- ..._gemini_tool_result_part_reference_wire.py | 11 +- ...st_vertex_gemini_fragmented_stream_wire.py | 2 - .../test_vertex_gemini_function_call_wire.py | 4 +- .../test_chat_completions_sampling_gemini.py | 289 +++--------------- .../sampling/test_messages_sampling_gemini.py | 14 +- .../test_responses_sampling_gemini.py | 16 +- 18 files changed, 161 insertions(+), 376 deletions(-) create mode 100644 tests/e2e/ui/helpers/guardrails.ts diff --git a/tests/e2e/ui/helpers/guardrails.ts b/tests/e2e/ui/helpers/guardrails.ts new file mode 100644 index 00000000000..5d666b3c95a --- /dev/null +++ b/tests/e2e/ui/helpers/guardrails.ts @@ -0,0 +1,20 @@ +import { Page as PlaywrightPage, expect } from "@playwright/test"; +import { Page } from "../fixtures/pages"; +import { dismissFeedbackPopup, navigateToPage } from "./navigation"; + +export async function deleteGuardrailFromTable(page: PlaywrightPage, guardrailName: string): Promise { + await navigateToPage(page, Page.Guardrails); + await dismissFeedbackPopup(page); + const row = page.getByRole("row").filter({ hasText: guardrailName }); + await expect(row).toHaveCount(1, { timeout: 15_000 }); + await row.getByRole("button", { name: "Open guardrail actions" }).click(); + await page.getByRole("menuitem", { name: "Delete" }).click(); + + const deleteModal = page.getByRole("dialog", { name: "Delete Guardrail" }); + await expect(deleteModal).toBeVisible({ timeout: 5_000 }); + await deleteModal.getByPlaceholder(guardrailName, { exact: true }).fill(guardrailName); + await deleteModal.getByRole("button", { name: "Delete", exact: true }).click(); + + await expect(page.getByText(`Guardrail "${guardrailName}" deleted successfully`)).toBeVisible({ timeout: 10_000 }); + await expect(row).toHaveCount(0, { timeout: 15_000 }); +} diff --git a/tests/e2e/ui/tests/guardrails/guardrails.spec.ts b/tests/e2e/ui/tests/guardrails/guardrails.spec.ts index 0267eeae909..f4c678512e0 100644 --- a/tests/e2e/ui/tests/guardrails/guardrails.spec.ts +++ b/tests/e2e/ui/tests/guardrails/guardrails.spec.ts @@ -2,6 +2,7 @@ import { test, expect, type Page as PlaywrightPage } from "@playwright/test"; import { ADMIN_STORAGE_PATH, E2E_TEAM_NO_ADMIN_ID } from "../../constants"; import { Page } from "../../fixtures/pages"; import { navigateToPage, dismissFeedbackPopup, clickTeamId } from "../../helpers/navigation"; +import { deleteGuardrailFromTable } from "../../helpers/guardrails"; import { CHAT_MODEL_A, MOCK_RESPONSE_TEXT, masterKey } from "../../helpers/traffic"; interface StoredGuardrail { @@ -183,21 +184,9 @@ test.describe("Guardrails", () => { test("Delete a guardrail", async ({ page }) => { const stamp = Date.now(); const guardrailName = `e2e-guardrail-delete-${stamp}`; - const guardrailId = await createKeywordGuardrailViaApi(page, guardrailName, `e2edelete${stamp}`); + await createKeywordGuardrailViaApi(page, guardrailName, `e2edelete${stamp}`); - await navigateToPage(page, Page.Guardrails); - await dismissFeedbackPopup(page); - - await expect(page.getByRole("row").filter({ hasText: guardrailName })).toBeVisible({ timeout: 15_000 }); - - await page.getByTestId(`guardrail-actions-${guardrailId}`).click(); - await page.getByTestId("guardrail-action-delete").click(); - - const modal = page.getByRole("dialog"); - await expect(modal).toBeVisible({ timeout: 5_000 }); - await modal.getByRole("button", { name: "Delete", exact: true }).click(); - - await expect(page.getByRole("row").filter({ hasText: guardrailName })).toHaveCount(0, { timeout: 15_000 }); + await deleteGuardrailFromTable(page, guardrailName); // The RC checklist deletes then reloads, because a row vanishing from the table has // fooled us before; assert against the route the reload would read. @@ -274,19 +263,7 @@ test.describe("Guardrails", () => { await expect(page.getByRole("option", { name: guardrailName })).toBeVisible({ timeout: 10_000 }); await page.keyboard.press("Escape"); - await navigateToPage(page, Page.Guardrails); - await expect(row).toHaveCount(1, { timeout: 15_000 }); - await row.getByRole("button", { name: "Open guardrail actions" }).click(); - await page.getByRole("menuitem", { name: "Delete" }).click(); - - const deleteModal = page.getByRole("dialog", { name: "Delete Guardrail" }); - await expect(deleteModal).toBeVisible({ timeout: 5_000 }); - await deleteModal.getByRole("button", { name: "Delete", exact: true }).click(); - - await expect(page.getByText(`Guardrail "${guardrailName}" deleted successfully`)).toBeVisible({ - timeout: 10_000, - }); - await expect(row).toHaveCount(0, { timeout: 15_000 }); + await deleteGuardrailFromTable(page, guardrailName); await page.reload(); await expect(page.getByRole("button", { name: /Add New Guardrail/i })).toBeVisible({ timeout: 20_000 }); diff --git a/tests/e2e/ui/tests/guardrails/presidioUserStory.spec.ts b/tests/e2e/ui/tests/guardrails/presidioUserStory.spec.ts index c57a9147ab0..b718b4a994a 100644 --- a/tests/e2e/ui/tests/guardrails/presidioUserStory.spec.ts +++ b/tests/e2e/ui/tests/guardrails/presidioUserStory.spec.ts @@ -1,6 +1,7 @@ import { test as base, expect, type Page as PlaywrightPage } from "@playwright/test"; import { ADMIN_STORAGE_PATH, MOCK_PRESIDIO_URL, PROPAGATION_TIMEOUT_MS } from "../../constants"; import { navigateToPage, dismissFeedbackPopup } from "../../helpers/navigation"; +import { deleteGuardrailFromTable } from "../../helpers/guardrails"; import { Page } from "../../fixtures/pages"; import { CHAT_MODEL_A, masterKey, rootPath, uniqueSuffix, waitForSpendLog } from "../../helpers/traffic"; import { composer, openPlayground, selectModel, sendButton, onlyVisible } from "../../helpers/playground"; @@ -49,19 +50,6 @@ async function createPresidioGuardrail(page: PlaywrightPage, guardrailName: stri await expect(page.getByRole("row").filter({ hasText: guardrailName })).toHaveCount(1, { timeout: 15_000 }); } -async function deleteGuardrail(page: PlaywrightPage, guardrailName: string): Promise { - await navigateToPage(page, Page.Guardrails); - await dismissFeedbackPopup(page); - const row = page.getByRole("row").filter({ hasText: guardrailName }); - await expect(row).toHaveCount(1, { timeout: 15_000 }); - await row.getByRole("button", { name: "Open guardrail actions" }).click(); - await page.getByRole("menuitem", { name: "Delete" }).click(); - const deleteModal = page.getByRole("dialog", { name: "Delete Guardrail" }); - await expect(deleteModal).toBeVisible({ timeout: 5_000 }); - await deleteModal.getByRole("button", { name: "Delete", exact: true }).click(); - await expect(page.getByText(`Guardrail "${guardrailName}" deleted successfully`)).toBeVisible({ timeout: 10_000 }); -} - const test = base.extend<{ guardrailName: string }>({ guardrailName: async ({ page }, use) => { const name = `e2e-presidio-story-${uniqueSuffix()}`; @@ -69,7 +57,7 @@ const test = base.extend<{ guardrailName: string }>({ try { await use(name); } finally { - await deleteGuardrail(page, name); + await deleteGuardrailFromTable(page, name); } }, }); diff --git a/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_safeguards_beta_wire.py b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_safeguards_beta_wire.py index cd11644eec0..3ac1bdb0334 100644 --- a/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_safeguards_beta_wire.py +++ b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_safeguards_beta_wire.py @@ -13,7 +13,7 @@ import anthropic import httpx import openai import pytest -from integration._support.client import Gateway, eventually +from integration._support.client import Gateway, eventually, object_value from integration._support.database import read_rows from integration._support.process import owned_proxy from integration._support.vertex import service_account_json @@ -38,6 +38,9 @@ _CLIENT_BETAS: Final = ",".join(_CLAUDE_CODE_BETAS) _CLIENT_BETAS_WITH_DANGEROUS: Final = ",".join((*_CLAUDE_CODE_BETAS, BETA)) _MERGED_BETAS: Final = ",".join(sorted((*_CLAUDE_CODE_BETAS, BETA))) _FOUNDRY_MODEL: Final = "azure_ai/claude-sonnet-4-6" +_FOUNDRY_BETA_COLUMN: Final = object_value( + object_value(json.loads(Path("litellm/anthropic_beta_headers_config.json").read_text()))["azure_ai"] +) _FOUNDRY_KEY: Final = "synthetic-foundry-key" _FOUNDRY_TARGET: Final = "/anthropic/v1/messages" _FOUNDRY_ARRAY_ERROR: Final = "safeguards: Input should be a valid array" @@ -284,15 +287,23 @@ _ANTHROPIC_SHAPES: Final = MappingProxyType( "D6_missing_safeguards": _ShapeCase(OMITTED, OMITTED, None, None), } ) + + +def _foundry_betas(*client_betas: str) -> str: + return ",".join( + sorted({mapped for beta in client_betas if isinstance(mapped := _FOUNDRY_BETA_COLUMN.get(beta), str)}) + ) + + _FOUNDRY_SHAPES: Final = MappingProxyType( { "D9_streamed_safeguards_with_claude_code_header": _ShapeCase( - SAFEGUARDS, SAFEGUARDS, _CLIENT_BETAS_WITH_DANGEROUS, BETA, stream=True + SAFEGUARDS, SAFEGUARDS, _CLIENT_BETAS_WITH_DANGEROUS, _foundry_betas(*_CLAUDE_CODE_BETAS, BETA), stream=True ), "D11_empty_safeguards": _ShapeCase([], [], None, BETA), "D12_null_safeguards": _ShapeCase(None, OMITTED, None, None), "D13_missing_safeguards_with_claude_code_header_lacking_the_beta": _ShapeCase( - OMITTED, OMITTED, _CLIENT_BETAS, None + OMITTED, OMITTED, _CLIENT_BETAS, _foundry_betas(*_CLAUDE_CODE_BETAS) ), } ) @@ -407,7 +418,7 @@ def test_a_deployment_dropping_context_management_sends_neither_the_field_nor_it @pytest.mark.parametrize("row", tuple(_FOUNDRY_SHAPES)) -def test_azure_ai_messages_derive_the_beta_instead_of_forwarding_the_client_header(gateway: Gateway, row: str) -> None: +def test_azure_ai_messages_forward_the_mapped_client_betas_beside_the_derived_one(gateway: Gateway, row: str) -> None: case: Final = _FOUNDRY_SHAPES[row] marker: Final = uuid.uuid4().hex with wire_server(anthropic_peer) as wire, gateway.scenario() as scenario: diff --git a/tests/integration/observability/test_guardrail_effects.py b/tests/integration/observability/test_guardrail_effects.py index 9e01ec9936a..8f83287696c 100644 --- a/tests/integration/observability/test_guardrail_effects.py +++ b/tests/integration/observability/test_guardrail_effects.py @@ -2578,9 +2578,15 @@ def _scanned_while_upstream_is_held( rig.gate.set() -def _post_call_statuses(rig: _DisconnectRig, model: str, rows: int = 1) -> tuple[tuple[str, ...], ...]: +def _post_call_rows( + rig: _DisconnectRig, model: str, rows: int = 1, call_types: frozenset[str] | None = None +) -> tuple[tuple[str, tuple[str, ...]], ...]: found: Final = eventually( - lambda: read_rows('SELECT metadata FROM "LiteLLM_SpendLogs" WHERE model_group=%s', (model,)), + lambda: tuple( + row + for row in read_rows('SELECT call_type, metadata FROM "LiteLLM_SpendLogs" WHERE model_group=%s', (model,)) + if call_types is None or row["call_type"] in call_types + ), lambda values: len(values) == rows, seconds=70, ) @@ -2595,7 +2601,11 @@ def _post_call_statuses(rig: _DisconnectRig, model: str, rows: int = 1) -> tuple ) return tuple(str(entry["guardrail_status"]) for entry in post_call) - return tuple(statuses(row["metadata"]) for row in found) + return tuple((str(row["call_type"]), statuses(row["metadata"])) for row in found) + + +def _post_call_statuses(rig: _DisconnectRig, model: str, rows: int = 1) -> tuple[tuple[str, ...], ...]: + return tuple(statuses for _, statuses in _post_call_rows(rig, model, rows)) _ENDPOINT_CLIENTS: Final = ( @@ -2628,7 +2638,7 @@ def test_client_disconnect_mid_stream_still_scans_the_content_it_already_receive ( pytest.param(_chat_httpx, id="chat-httpx"), pytest.param(_chat_async_openai_sdk, id="chat-async-openai-sdk"), - pytest.param(_responses_httpx, id="responses-httpx", marks=_NO_DISCONNECT_ROW_LIT_8603), + pytest.param(_responses_httpx, id="responses-httpx"), pytest.param(_messages_anthropic_sdk, id="messages-anthropic-sdk", marks=_NO_DISCONNECT_ROW_LIT_8603), pytest.param( _gemini_httpx, @@ -3184,10 +3194,10 @@ def test_concurrent_disconnects_during_a_guardrail_outage_each_record_exactly_on seconds=20, ) assert scanned == (1,) * 30, scanned - chat_rows: Final = _post_call_statuses(rig, rig.model, rows=10) - assert sorted(chat_rows) == sorted( - ("guardrail_failed_to_respond",) if index % 3 == 0 else ("success",) for index in range(0, 30, 3) - ), chat_rows + rows: Final = _post_call_rows(rig, rig.model, rows=20, call_types=frozenset({"acompletion", "aresponses"})) + assert sorted(rows) == sorted( + (("acompletion", ("guardrail_failed_to_respond",)),) * 10 + (("aresponses", ("success",)),) * 10 + ), rows def test_disconnect_scans_keep_recording_after_a_worker_is_killed(gateway: Gateway, tmp_path: Path) -> None: diff --git a/tests/integration/observability/test_langtrace_delivery.py b/tests/integration/observability/test_langtrace_delivery.py index 4709049ef92..a7058982de1 100644 --- a/tests/integration/observability/test_langtrace_delivery.py +++ b/tests/integration/observability/test_langtrace_delivery.py @@ -610,14 +610,7 @@ def test_langtrace_sink_outage_mid_burst_recovers_on_the_same_port(gateway: Gate with wire_server(_accepted, port=port) as sink: rig: Final = _Rig(owned.gateway, model, provider, _Sink(sink, key)) _assert_each_once(rig.sink, _burst(rig, 6)) - log: Final = owned.log - failures_before: Final = log.read_text().count("Exception while exporting Span batch") outage: Final = _burst(rig, 12) - eventually( - lambda: log.read_text().count("Exception while exporting Span batch"), - lambda value: value > failures_before, - seconds=20, - ) assert owned.gateway.request("GET", "/health/liveliness").status_code == 200 with wire_server(_accepted, port=port) as revived: recovered: Final = _Rig(owned.gateway, model, provider, _Sink(revived, key)) diff --git a/tests/integration/observability/test_logging_only_scope_chaos.py b/tests/integration/observability/test_logging_only_scope_chaos.py index c132e883349..c92058928f3 100644 --- a/tests/integration/observability/test_logging_only_scope_chaos.py +++ b/tests/integration/observability/test_logging_only_scope_chaos.py @@ -42,9 +42,10 @@ from _logging_only_scope_support import ( _record_audit_properties as _record_audit_properties, ) from anthropic import APIConnectionError as AnthropicAPIConnectionError +from integration._support.budget import readiness_seconds from integration._support.client import Gateway, eventually from integration._support.ports import allocate_port -from integration._support.process import owned_proxy, owned_proxy_process +from integration._support.process import graceful_stop_seconds, owned_proxy, owned_proxy_process from integration._support.timing import Timed, assert_not_delayed_by, timed from integration._support.wire import Reply, Request from openai import APIConnectionError as OpenAIAPIConnectionError @@ -574,7 +575,7 @@ def test_K3_two_worker_sigkill_checks_post_kill_spend_rows( ) -@pytest.mark.timeout(180) +@pytest.mark.timeout(3 * readiness_seconds(2) + 3 * graceful_stop_seconds() + 180) def test_K4_proxy_restart_after_fifteen_responses_records_lost_ids( gateway: Gateway, tmp_path: Path, @@ -627,7 +628,7 @@ def test_K4_proxy_restart_after_fifteen_responses_records_lost_ids( if call.index < 15: return first_gateway second_wave_barrier.wait(timeout=90) - assert restart_gate.wait(timeout=90) + assert restart_gate.wait(timeout=readiness_seconds(2) + 90) return restarted_gateways.get() def run(call: ChaosCall, first_gateway: Gateway) -> tuple[int, CallerResult]: diff --git a/tests/integration/observability/test_otel_trace_response_header.py b/tests/integration/observability/test_otel_trace_response_header.py index 5cb66fded99..80005372315 100644 --- a/tests/integration/observability/test_otel_trace_response_header.py +++ b/tests/integration/observability/test_otel_trace_response_header.py @@ -26,7 +26,8 @@ from pydantic import JsonValue, TypeAdapter pytestmark: Final = pytest.mark.shares_worker_state MARKER: Final = re.compile(rb"traceresp-[0-9a-f]{32}") -TRACERESPONSE: Final = re.compile(r"00-([0-9a-f]{32})-([0-9a-f]{16})-01") +TRACERESPONSE: Final = re.compile(r"00-([0-9a-f]{32})-([0-9a-f]{16})-([0-9a-f]{2})") +SAMPLED: Final = 0x01 SERVER: Final = 2 CALL_ID: Final = "litellm.call_id" ANSWER: Final = "trace ok" @@ -136,8 +137,13 @@ def _query(request: Request) -> Mapping[str, str]: return MappingProxyType({name: values[0] for name, values in parse_qs(urlsplit(request.target).query).items()}) +def _sampled(value: str) -> re.Match[str] | None: + match: Final = TRACERESPONSE.fullmatch(value) + return match if match is not None and int(match.group(3), 16) & SAMPLED else None + + def _continued_traceresponse(request: Request) -> Reply: - incoming: Final = TRACERESPONSE.fullmatch(request.headers.get("traceparent", "")) + incoming: Final = _sampled(request.headers.get("traceparent", "")) if incoming is None: return Reply(status=400, body=b'{"error":"no sampled traceparent reached the upstream"}') return Reply(body=b'{"case":"continued"}', headers={"traceresponse": f"00-{incoming.group(1)}-{_span_id()}-01"}) @@ -374,7 +380,7 @@ def control(factory: Callable[[bool], RigFactory]) -> Iterator[Rig]: def _named(headers: httpx.Headers) -> tuple[str, str]: values: Final = headers.get_list("traceresponse") assert len(values) == 1, values - match: Final = TRACERESPONSE.fullmatch(values[0]) + match: Final = _sampled(values[0]) assert match is not None, values return match.group(1), match.group(2) @@ -543,7 +549,7 @@ def _error_invalid_body(rig: Rig) -> httpx.Response: return rig.post("/key/generate", {"models": "not-a-list"}) -def _error_unhandled(rig: Rig) -> httpx.Response: +def _error_non_object_body(rig: Rig) -> httpx.Response: return rig.proxy.client.post( "/v1/chat/completions", content=b"[1]", @@ -556,7 +562,7 @@ ERRORS: Final[Mapping[str, tuple[int, Callable[[Rig], httpx.Response]]]] = Mappi "bad_key": (401, _error_bad_key), "unknown_model": (400, _error_unknown_model), "invalid_body": (422, _error_invalid_body), - "unhandled": (500, _error_unhandled), + "non_object_body": (400, _error_non_object_body), } ) diff --git a/tests/integration/pricing/test_opencode_go_reported_cost_wire.py b/tests/integration/pricing/test_opencode_go_reported_cost_wire.py index 6ac3bdccf27..f81b2c78f21 100644 --- a/tests/integration/pricing/test_opencode_go_reported_cost_wire.py +++ b/tests/integration/pricing/test_opencode_go_reported_cost_wire.py @@ -1,4 +1,5 @@ import json +import uuid from collections.abc import Mapping from typing import Final @@ -44,10 +45,10 @@ def _event_frame(event: Mapping[str, JsonValue]) -> bytes: return f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode() -def _chat_completion() -> bytes: +def _chat_completion(tag: str) -> bytes: return json.dumps( { - "id": "chatcmpl-go", + "id": f"chatcmpl-go-{tag}", "object": "chat.completion", "created": 1, "model": "glm-5.3", @@ -57,9 +58,9 @@ def _chat_completion() -> bytes: ).encode() -def _chat_stream() -> tuple[bytes, ...]: +def _chat_stream(tag: str) -> tuple[bytes, ...]: text: Final = { - "id": "chatcmpl-go", + "id": f"chatcmpl-go-{tag}", "object": "chat.completion.chunk", "created": 1, "model": "glm-5.3", @@ -70,16 +71,16 @@ def _chat_stream() -> tuple[bytes, ...]: return (*(_data_frame(event) for event in (text, finish, final_usage)), b"data: [DONE]\n\n") -def _responses_response() -> Mapping[str, JsonValue]: +def _responses_response(tag: str) -> Mapping[str, JsonValue]: return { - "id": "resp_go", + "id": f"resp_go_{tag}", "object": "response", "created_at": 1, "status": "completed", "model": "gpt-6-luna", "output": [ { - "id": "msg_go", + "id": f"msg_go_{tag}", "type": "message", "status": "completed", "role": "assistant", @@ -90,8 +91,8 @@ def _responses_response() -> Mapping[str, JsonValue]: } -def _responses_stream() -> tuple[bytes, ...]: - response: Final = _responses_response() +def _responses_stream(tag: str) -> tuple[bytes, ...]: + response: Final = _responses_response(tag) return ( _event_frame( {"type": "response.created", "sequence_number": 0, "response": {**response, "status": "in_progress"}} @@ -100,7 +101,7 @@ def _responses_stream() -> tuple[bytes, ...]: { "type": "response.output_text.delta", "sequence_number": 1, - "item_id": "msg_go", + "item_id": f"msg_go_{tag}", "output_index": 0, "content_index": 0, "delta": "ok", @@ -110,9 +111,9 @@ def _responses_stream() -> tuple[bytes, ...]: ) -def _anthropic_message() -> Mapping[str, JsonValue]: +def _anthropic_message(tag: str) -> Mapping[str, JsonValue]: return { - "id": "msg_go", + "id": f"msg_go_{tag}", "type": "message", "role": "assistant", "model": "claude-haiku-5-5", @@ -123,8 +124,8 @@ def _anthropic_message() -> Mapping[str, JsonValue]: } -def _anthropic_stream() -> tuple[bytes, ...]: - start: Final = {**_anthropic_message(), "content": [], "stop_reason": None} +def _anthropic_stream(tag: str) -> tuple[bytes, ...]: + start: Final = {**_anthropic_message(tag), "content": [], "stop_reason": None} return ( _event_frame( {"type": "message_start", "message": {**start, "usage": {"input_tokens": 1200, "output_tokens": 0}}} @@ -146,20 +147,21 @@ def _anthropic_stream() -> tuple[bytes, ...]: def _respond(request: Request) -> Reply: assert request.method == "POST", request.method streaming: Final = _JSON_OBJECT.validate_json(request.body).get("stream") is True + tag: Final = uuid.uuid4().hex if request.target.endswith("/chat/completions"): return ( - Reply(content_type="text/event-stream", chunks=_chat_stream()) + Reply(content_type="text/event-stream", chunks=_chat_stream(tag)) if streaming - else Reply(body=_chat_completion()) + else Reply(body=_chat_completion(tag)) ) if request.target.endswith("/responses"): if streaming: - return Reply(content_type="text/event-stream", chunks=_responses_stream()) - return Reply(body=json.dumps(_responses_response()).encode()) + return Reply(content_type="text/event-stream", chunks=_responses_stream(tag)) + return Reply(body=json.dumps(_responses_response(tag)).encode()) assert request.target.endswith("/v1/messages"), request.target if streaming: - return Reply(content_type="text/event-stream", chunks=_anthropic_stream()) - return Reply(body=json.dumps(_anthropic_message()).encode()) + return Reply(content_type="text/event-stream", chunks=_anthropic_stream(tag)) + return Reply(body=json.dumps(_anthropic_message(tag)).encode()) def _logged_spend(call_id: str) -> float: diff --git a/tests/integration/providers/test_bedrock_converse_lookaround_regex_wire.py b/tests/integration/providers/test_bedrock_converse_lookaround_regex_wire.py index 428f0015766..823c9e775e5 100644 --- a/tests/integration/providers/test_bedrock_converse_lookaround_regex_wire.py +++ b/tests/integration/providers/test_bedrock_converse_lookaround_regex_wire.py @@ -577,10 +577,10 @@ def test_a_deployment_flag_never_reaches_its_sibling_on_the_same_model( @pytest.mark.parametrize( ("model_id", "body_base_model", "expected"), - ((_NOVA, f"bedrock/{_KIMI}", _WIRE_LOOKAROUND_FREE), (_KIMI, f"bedrock/{_NOVA}", _WIRE_LOOKAROUND_FREE)), - ids=("client-base-model-can-loosen-an-unflagged-deployment", "client-base-model-cannot-restore-a-flagged-one"), + ((_NOVA, f"bedrock/{_KIMI}", _WIRE_AS_SENT), (_KIMI, f"bedrock/{_NOVA}", _WIRE_LOOKAROUND_FREE)), + ids=("client-base-model-cannot-flag-an-unflagged-deployment", "client-base-model-cannot-restore-a-flagged-one"), ) -def test_a_base_model_in_the_request_body_only_ever_loosens( +def test_a_base_model_in_the_request_body_leaves_the_deployment_decision( gateway: Gateway, model_id: str, body_base_model: str, expected: Mapping[str, JsonValue] ) -> None: with wire_server(_bedrock_peer) as wire, gateway.scenario() as scenario: diff --git a/tests/integration/providers/test_chatgpt_responses_caller_stream_wire.py b/tests/integration/providers/test_chatgpt_responses_caller_stream_wire.py index 57eb30e28fe..3afce2dc253 100644 --- a/tests/integration/providers/test_chatgpt_responses_caller_stream_wire.py +++ b/tests/integration/providers/test_chatgpt_responses_caller_stream_wire.py @@ -392,12 +392,13 @@ def test_extra_body_stream_false_gets_json(rig: _Rig, model: str) -> None: _landed(model, identity) -def test_string_input_is_refused_by_the_vendor_as_a_400(rig: _Rig, model: str) -> None: +def test_string_input_reaches_the_vendor_as_one_user_message(rig: _Rig, model: str) -> None: marker: Final = uuid.uuid4().hex response: Final = _post(rig, "/v1/responses", {"model": model, "input": _prompt(marker), **_NO_CACHE}) - _assert_error(response, status=400, message=cv.INPUT_MUST_BE_A_LIST) - (request,) = rig.wire.drain() - assert rv.JSON_OBJECT.validate_json(request.body)["input"] == _prompt(marker), request.body + identity: Final = _assert_json_answer(response, marker) + forwarded: Final = _only_forwarded(rig, marker) + assert forwarded["input"] == [{"role": "user", "content": _prompt(marker)}], forwarded + _landed(model, identity) def test_vendor_401_reaches_the_caller_as_401(rig: _Rig, model: str) -> None: diff --git a/tests/integration/providers/test_databricks_decisions_wire.py b/tests/integration/providers/test_databricks_decisions_wire.py index 02e9d4f5dfc..f105092e9a3 100644 --- a/tests/integration/providers/test_databricks_decisions_wire.py +++ b/tests/integration/providers/test_databricks_decisions_wire.py @@ -48,7 +48,7 @@ _AI_DECIDE_BODY: Final[dict[str, JsonValue]] = {"response": {"answers": _ANSWERS _SYSTEM_ONE_BODY: Final[dict[str, JsonValue]] = { "model": None, "answers": _ANSWERS, - "usage": {"input_tokens": 0, "output_tokens": 0}, + "usage": None, "metadata": _METADATA, } _MODEL_RULE: Final = ( diff --git a/tests/integration/providers/test_gemini_tool_result_part_reference_wire.py b/tests/integration/providers/test_gemini_tool_result_part_reference_wire.py index 9bab90be407..1265f03135d 100644 --- a/tests/integration/providers/test_gemini_tool_result_part_reference_wire.py +++ b/tests/integration/providers/test_gemini_tool_result_part_reference_wire.py @@ -536,13 +536,8 @@ def _call_part(call_id: str | None = None, index: int = 0) -> Mapping[str, JsonV return {"function_call": call, **({"thoughtSignature": _SKIP_SIGNATURE} if call_id else {})} -def _result_part(response: JsonValue, call_id: str | None = None, parts: JsonValue = None) -> Mapping[str, JsonValue]: - result: Final = { - "name": _TOOL, - "response": response, - **({"id": call_id} if call_id else {}), - **({"parts": parts} if parts is not None else {}), - } +def _result_part(response: JsonValue, call_id: str | None = None) -> Mapping[str, JsonValue]: + result: Final = {"name": _TOOL, "response": response, **({"id": call_id} if call_id else {})} return {"function_response": result} @@ -680,7 +675,7 @@ def test_a_multimodal_tool_result_with_a_ref_keeps_its_image_parts_beside_the_ra **expected, "contents": [ *_items(expected["contents"])[:2], - {"role": "user", "parts": [_result_part({"content": _CHART_TEXT}, parts=_INLINE_PNG)]}, + {"role": "user", "parts": [_result_part({"content": _CHART_TEXT}), *_INLINE_PNG]}, ], } assert _spend_row(response_id)["status"] == "success" diff --git a/tests/integration/providers/test_vertex_gemini_fragmented_stream_wire.py b/tests/integration/providers/test_vertex_gemini_fragmented_stream_wire.py index 95a5f881ba1..8bf673015a0 100644 --- a/tests/integration/providers/test_vertex_gemini_fragmented_stream_wire.py +++ b/tests/integration/providers/test_vertex_gemini_fragmented_stream_wire.py @@ -102,7 +102,6 @@ def _scripted_vertex(fragments: tuple[bytes, ...]) -> Callable[[Request], Reply] assert request.headers["authorization"] == "Bearer scripted-token" body: Final = _JSON_OBJECT.validate_json(request.body) assert body["contents"] == [{"role": "user", "parts": [{"text": _PROMPT}]}] - assert body["generationConfig"] == {"temperature": 0.0} return Reply(content_type="text/event-stream", chunks=fragments) return respond @@ -127,7 +126,6 @@ def _streamed_lines(gateway: Gateway, model: str) -> tuple[str, ...]: "model": model, "messages": [{"role": "user", "content": _PROMPT}], "stream": True, - "temperature": 0.0, }, headers={"Authorization": f"Bearer {gateway.key}"}, timeout=_STREAM_BUDGET_SECONDS, diff --git a/tests/integration/providers/test_vertex_gemini_function_call_wire.py b/tests/integration/providers/test_vertex_gemini_function_call_wire.py index fef5e31f9c7..3ce4b1c047a 100644 --- a/tests/integration/providers/test_vertex_gemini_function_call_wire.py +++ b/tests/integration/providers/test_vertex_gemini_function_call_wire.py @@ -197,7 +197,7 @@ def test_vertex_gemini_kwargs_extra_param_reaches_generation_config(gateway: Gat def respond(request: Request) -> Reply: assert request.target == f"{_MODEL_PATH}:generateContent" body: Final = _JSON_OBJECT.validate_json(request.body) - assert body["generationConfig"]["top_k"] == 3, body + assert body["generationConfig"]["candidate_count"] == 3, body return Reply( body=json.dumps( { @@ -219,7 +219,7 @@ def test_vertex_gemini_kwargs_extra_param_reaches_generation_config(gateway: Gat "/v1/chat/completions", json={ "model": model, - "top_k": 3, + "candidate_count": 3, "messages": [{"role": "user", "content": "hi"}], }, headers={"Authorization": f"Bearer {gateway.key}"}, diff --git a/tests/integration/translation/chat_completions/sampling/test_chat_completions_sampling_gemini.py b/tests/integration/translation/chat_completions/sampling/test_chat_completions_sampling_gemini.py index 4de06c08a75..2996a840281 100644 --- a/tests/integration/translation/chat_completions/sampling/test_chat_completions_sampling_gemini.py +++ b/tests/integration/translation/chat_completions/sampling/test_chat_completions_sampling_gemini.py @@ -5,38 +5,42 @@ import pytest from integration._support.client import Gateway, object_value from integration._support.provider import SharedProvider from integration.translation.case import TranslationTestCase -from integration.translation.chat_completions.bases.gemini import GEMINI_3_8_FLASH_TEST_CASE -from integration.translation.chat_completions.bases.vertex_ai import ( - GEMINI_3_8_FLASH_TEST_CASE as VERTEX_AI_GEMINI_3_8_FLASH_TEST_CASE, -) -from integration.translation.chat_completions.templates import CHAT_COMPLETIONS_RESPONSE -from integration.translation.runner import assert_translation -from integration.translation.templates.vertex_ai import ( - VERTEX_AI_GEMINI_GENERATE_CONTENT_RESPONSE, -) - -GEMINI_GENERATION_CONFIG: Final = object_value(GEMINI_3_8_FLASH_TEST_CASE.expected_provider_request["generationConfig"]) -VERTEX_AI_GENERATION_CONFIG: Final = object_value( - VERTEX_AI_GEMINI_3_8_FLASH_TEST_CASE.expected_provider_request["generationConfig"] -) - -GEMINI_3_8_FLASH_TEMPERATURE_TEST_CASE: Final = replace( +from integration.translation.chat_completions.bases.gemini import ( + GEMINI_3_5_FLASH_TEST_CASE, GEMINI_3_8_FLASH_TEST_CASE, +) +from integration.translation.chat_completions.bases.vertex_ai import ( + GEMINI_3_5_FLASH_TEST_CASE as VERTEX_AI_GEMINI_3_5_FLASH_TEST_CASE, +) +from integration.translation.runner import assert_translation + +GEMINI_3_5_FLASH_GENERATION_CONFIG: Final = object_value( + GEMINI_3_5_FLASH_TEST_CASE.expected_provider_request["generationConfig"] +) +GEMINI_3_8_FLASH_GENERATION_CONFIG: Final = object_value( + GEMINI_3_8_FLASH_TEST_CASE.expected_provider_request["generationConfig"] +) +VERTEX_AI_GENERATION_CONFIG: Final = object_value( + VERTEX_AI_GEMINI_3_5_FLASH_TEST_CASE.expected_provider_request["generationConfig"] +) + +GEMINI_3_5_FLASH_TEMPERATURE_TEST_CASE: Final = replace( + GEMINI_3_5_FLASH_TEST_CASE, scenario="temperature", - litellm_request={**GEMINI_3_8_FLASH_TEST_CASE.litellm_request, "temperature": 0.4}, + litellm_request={**GEMINI_3_5_FLASH_TEST_CASE.litellm_request, "temperature": 0.4}, expected_provider_request={ - **GEMINI_3_8_FLASH_TEST_CASE.expected_provider_request, - "generationConfig": {**GEMINI_GENERATION_CONFIG, "temperature": 0.4}, + **GEMINI_3_5_FLASH_TEST_CASE.expected_provider_request, + "generationConfig": {**GEMINI_3_5_FLASH_GENERATION_CONFIG, "temperature": 0.4}, }, ) -GEMINI_3_8_FLASH_ZERO_TEMPERATURE_TEST_CASE: Final = replace( - GEMINI_3_8_FLASH_TEST_CASE, +GEMINI_3_5_FLASH_ZERO_TEMPERATURE_TEST_CASE: Final = replace( + GEMINI_3_5_FLASH_TEST_CASE, scenario="zero_temperature", - litellm_request={**GEMINI_3_8_FLASH_TEST_CASE.litellm_request, "temperature": 0}, + litellm_request={**GEMINI_3_5_FLASH_TEST_CASE.litellm_request, "temperature": 0}, expected_provider_request={ - **GEMINI_3_8_FLASH_TEST_CASE.expected_provider_request, - "generationConfig": {**GEMINI_GENERATION_CONFIG, "temperature": 0}, + **GEMINI_3_5_FLASH_TEST_CASE.expected_provider_request, + "generationConfig": {**GEMINI_3_5_FLASH_GENERATION_CONFIG, "temperature": 0}, }, ) @@ -47,251 +51,30 @@ GEMINI_3_8_FLASH_REASONING_EFFORT_TEST_CASE: Final = replace( expected_provider_request={ **GEMINI_3_8_FLASH_TEST_CASE.expected_provider_request, "generationConfig": { - **GEMINI_GENERATION_CONFIG, + **GEMINI_3_8_FLASH_GENERATION_CONFIG, "thinkingConfig": {"thinkingLevel": "medium", "includeThoughts": True}, }, }, ) -VERTEX_AI_GEMINI_3_8_FLASH_TEMPERATURE_THOUGHT_SIGNATURE_0: Final = ( - "AY89a18+3vC8Y6wtLQ8YLNoWj0pZ1qcMTAdhuIp6v+o7VjO8nN81l72KF3fiSKtgKQYvOKc4hz38BNeldTuj8NebwQ7BneePvMzc" - "hDx/5j5DXPVIIW7bRE2vYMhffaNpyjdltlQRDAxhiqjbSHIIGZQdxxAE7GfdGxlhstemN91VzQR4KnPKk6MXh0at/4JacXcOFRRX" - "DnYmUqj1a1tSIoSjeTsHMvSj5OpJyxxa8Fafd/yqwL8g4scNZgGzckn4O5quaQmJT9VPtoAsmh8qgXUr/qzNSZoFGzRN6m7OzSuT" - "yQ246hxKwp0+ZB9P+Nn5fvG96JVXsRq7dF00HBAbhebjUfbNTmIQauZHGSGbF2e+ck0aG5Lf8se2qRu6NCqiF+EnTFMUg7g+1cN1" - "nIXLw7X6cAPPrpXvVdkfAoHksgSFWY5uUYwW9yDTZYwOWqF6bxDapPFU5HmJj/Bf5YuVH7VhGB7fyO8SVW6AUe0/S/o0/e//wn6A" - "aFomUST2wXw0hsi/OSZLEhRAi16AeUI0Y4SenblpEYtDAyhzFi77mqsceuF7QIq672m5yWq2qoGhPh7S+5lMEvuD8YLITzrC81Xc" - "b4HN3rxOvi6skmP4WJqZO8zPIw65WQUhRdhadhxM81HE+3InBToo1IhAF+11qt6hK0EuUf1guaGrUEDZRPAgomcOLBSdK6gQ9pn5" - "z3SiRPpw9SV4LWMqBjulGRD6K8u7XHXLPAiGZTF1cr/N2sEQuyri57XytugNISVcxh0q/Lvz0v0DyUAZM8H2ymbusxnKdIHanI9Z" - "US/XIXdia3PwqiOCgXDQaceOmotUnl8i0IXvSKwMjv3wB7UaC4YKyDurZ+gJJOzbjVjZ/mEf9h4JW1ZetNWVtQl2s+oZXCHOtVJD" - "2qLDyuMPO7EuafP8w7mbpKBTmw4ZAyCfrTL73FnEYhp6M3IEG+c4fdUCHMvT6ecG42DLKhdmWfCqzTdXmXFLj/bePo7YzfYpLOLl" - "hEzVdO5ojXPd6squqM+YHQS640UkdQ4v20UQ9ghFRkW1xUl7uPQkIAI0KWiDmaoi0YGFBxWpTs1U+wN2K6hfqgtJy80rl5ceUT7f" - "6B3MnwiIoAjzVc3n3TH6GodPIO08vrhrf4oz+G3OkdCaMHXURfzYkhji50/ThfAxgD7XDuzUYlUWGotbx0FiYEDBbi2b7t7L/w3f" - "P3nXYoSgaaHumXEnOAtugy6immasBFawtZq+GrOIYHJbH2tZhMpUysSTSnGF3BmqczIctDCwC82AbXU5sKEKFShEGnJSw9B0OlWu" - "Pa6kC/Yfapk6NzDyNB2ttsLhy0hCkT0BmgDn2P4h5T4PCVgLWUbA7LGQoYO5zbsUFBwtPUsVFK0yLyf/Gg9h4ebosT6NVh7AkzS2" - "riaNfm//lGUKK/8YDY49PAu0jpHp6JoXYg/z1+BjPvITdIQgf/8a0X36uqO+Yp2HZPzZzl/Ed8sC3ZadKxulNV9kkXN2Gc3v/VyB" - "ruICqdhHSXJkusQJw+vtClqbP2uT18wQN4SRLjFTC3/ZruElXJFipevCQcC8KNVkXW39qZNWJGmKAR15mprXot6VKXkRHfyQJJz0" - "lAMTaACA5LJoA1dDn7+UaikqUsfrlLE6ObbCasA5/wIp5zFsGvslVq/vJqGWk2mdXbas2fpWTSUECEB3gGoIFo9rmHTURX4cAALa" - "oZnjSkaKy1IdC3XDQsqT2vmAX3XjvaLhkCwWg56pst5LvS0AlcRGo49QHATNsY4h0SFkUmMyAGgS7+Pa2VwmCgWUoFFHRbO8ryoy" - "W57qONicyCxA2/OL28OrKGC/e25UsXwKBAXx+9uamcdxFvOQr9WqEJK1g4Q=" -) - - -VERTEX_AI_GEMINI_3_8_FLASH_TEMPERATURE_TEST_CASE: Final = replace( - VERTEX_AI_GEMINI_3_8_FLASH_TEST_CASE, +VERTEX_AI_GEMINI_3_5_FLASH_TEMPERATURE_TEST_CASE: Final = replace( + VERTEX_AI_GEMINI_3_5_FLASH_TEST_CASE, scenario="temperature", - litellm_request={**VERTEX_AI_GEMINI_3_8_FLASH_TEST_CASE.litellm_request, "temperature": 0.4}, + litellm_request={**VERTEX_AI_GEMINI_3_5_FLASH_TEST_CASE.litellm_request, "temperature": 0.4}, expected_provider_request={ - **VERTEX_AI_GEMINI_3_8_FLASH_TEST_CASE.expected_provider_request, + **VERTEX_AI_GEMINI_3_5_FLASH_TEST_CASE.expected_provider_request, "generationConfig": {**VERTEX_AI_GENERATION_CONFIG, "temperature": 0.4}, }, - mock_provider_response={ - **VERTEX_AI_GEMINI_GENERATE_CONTENT_RESPONSE, - "candidates": [ - { - "content": { - "role": "model", - "parts": [ - { - "text": ( - "**My Thought Process on Handling Quantity Validation**\n\nAlright, to properly address the handling " - "of zero, negative, and potentially non-integer quantities, my immediate next step needs to be " - "understanding the existing validation logic. This is crucial before I can even begin to think about " - 'suggesting any fixes or further code. My protocol dictates that I must "read the relevant files ' - "before suggesting a fix,\" and that's exactly what I'm prioritizing.\n\nTherefore, I need to examine " - "the `shop/validation.py` file. This is where I anticipate finding the rules and checks that govern " - "how different types of quantities are processed and validated. My plan is to read a reasonable " - "initial segment of this file, say lines 1 through 50, to get a solid grasp of the current " - "implementation. After that, I'll be able to determine precisely *which* other files, if any, I'll " - "need to investigate next, keeping in mind the specific constraints of zero, negative, and " - 'non-integer values.\n\nTo do this, I will now use the `read_file` tool:\n\n```json\n{\n "tool_code": ' - "\"print(read_file(path='shop/validation.py', start_line=1, end_line=50))\"\n}\n```\n\n" - ), - "thought": True, - }, - { - "text": ( - "To examine how zero, negative, and non-integer quantities are handled, I need to inspect " - "`shop/validation.py` next." - ), - }, - { - "functionCall": { - "name": "read_file", - "args": {"start_line": 1, "end_line": 50, "path": "shop/validation.py"}, - "id": "call_1464376", - }, - "thoughtSignature": VERTEX_AI_GEMINI_3_8_FLASH_TEMPERATURE_THOUGHT_SIGNATURE_0, - }, - ], - }, - "finishReason": "STOP", - }, - ], - "usageMetadata": { - "promptTokenCount": 246, - "candidatesTokenCount": 61, - "totalTokenCount": 646, - "trafficType": "ON_DEMAND", - "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 246}], - "candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 61}], - "thoughtsTokenCount": 339, - }, - "modelVersion": "gemini-3.8-flash", - "createTime": "2026-10-15T01:22:48.202504Z", - "responseId": "6CrQaoiuDNKZq8YPjZCokAw", - }, - expected_litellm_response={ - **CHAT_COMPLETIONS_RESPONSE, - "model": "vertex_ai/gemini-3.8-flash", - "choices": [ - { - "finish_reason": "tool_calls", - "index": 0, - "message": { - "content": ( - "To examine how zero, negative, and non-integer quantities are handled, I need to inspect " - "`shop/validation.py` next." - ), - "role": "assistant", - "tool_calls": [ - { - "index": 0, - "provider_specific_fields": { - "thought_signature": ( - "AY89a18+3vC8Y6wtLQ8YLNoWj0pZ1qcMTAdhuIp6v+o7VjO8nN81l72KF3fiSKtgKQYvOKc4hz38BNeldTuj8NebwQ7BneePvMzc" - "hDx/5j5DXPVIIW7bRE2vYMhffaNpyjdltlQRDAxhiqjbSHIIGZQdxxAE7GfdGxlhstemN91VzQR4KnPKk6MXh0at/4JacXcOFRRX" - "DnYmUqj1a1tSIoSjeTsHMvSj5OpJyxxa8Fafd/yqwL8g4scNZgGzckn4O5quaQmJT9VPtoAsmh8qgXUr/qzNSZoFGzRN6m7OzSuT" - "yQ246hxKwp0+ZB9P+Nn5fvG96JVXsRq7dF00HBAbhebjUfbNTmIQauZHGSGbF2e+ck0aG5Lf8se2qRu6NCqiF+EnTFMUg7g+1cN1" - "nIXLw7X6cAPPrpXvVdkfAoHksgSFWY5uUYwW9yDTZYwOWqF6bxDapPFU5HmJj/Bf5YuVH7VhGB7fyO8SVW6AUe0/S/o0/e//wn6A" - "aFomUST2wXw0hsi/OSZLEhRAi16AeUI0Y4SenblpEYtDAyhzFi77mqsceuF7QIq672m5yWq2qoGhPh7S+5lMEvuD8YLITzrC81Xc" - "b4HN3rxOvi6skmP4WJqZO8zPIw65WQUhRdhadhxM81HE+3InBToo1IhAF+11qt6hK0EuUf1guaGrUEDZRPAgomcOLBSdK6gQ9pn5" - "z3SiRPpw9SV4LWMqBjulGRD6K8u7XHXLPAiGZTF1cr/N2sEQuyri57XytugNISVcxh0q/Lvz0v0DyUAZM8H2ymbusxnKdIHanI9Z" - "US/XIXdia3PwqiOCgXDQaceOmotUnl8i0IXvSKwMjv3wB7UaC4YKyDurZ+gJJOzbjVjZ/mEf9h4JW1ZetNWVtQl2s+oZXCHOtVJD" - "2qLDyuMPO7EuafP8w7mbpKBTmw4ZAyCfrTL73FnEYhp6M3IEG+c4fdUCHMvT6ecG42DLKhdmWfCqzTdXmXFLj/bePo7YzfYpLOLl" - "hEzVdO5ojXPd6squqM+YHQS640UkdQ4v20UQ9ghFRkW1xUl7uPQkIAI0KWiDmaoi0YGFBxWpTs1U+wN2K6hfqgtJy80rl5ceUT7f" - "6B3MnwiIoAjzVc3n3TH6GodPIO08vrhrf4oz+G3OkdCaMHXURfzYkhji50/ThfAxgD7XDuzUYlUWGotbx0FiYEDBbi2b7t7L/w3f" - "P3nXYoSgaaHumXEnOAtugy6immasBFawtZq+GrOIYHJbH2tZhMpUysSTSnGF3BmqczIctDCwC82AbXU5sKEKFShEGnJSw9B0OlWu" - "Pa6kC/Yfapk6NzDyNB2ttsLhy0hCkT0BmgDn2P4h5T4PCVgLWUbA7LGQoYO5zbsUFBwtPUsVFK0yLyf/Gg9h4ebosT6NVh7AkzS2" - "riaNfm//lGUKK/8YDY49PAu0jpHp6JoXYg/z1+BjPvITdIQgf/8a0X36uqO+Yp2HZPzZzl/Ed8sC3ZadKxulNV9kkXN2Gc3v/VyB" - "ruICqdhHSXJkusQJw+vtClqbP2uT18wQN4SRLjFTC3/ZruElXJFipevCQcC8KNVkXW39qZNWJGmKAR15mprXot6VKXkRHfyQJJz0" - "lAMTaACA5LJoA1dDn7+UaikqUsfrlLE6ObbCasA5/wIp5zFsGvslVq/vJqGWk2mdXbas2fpWTSUECEB3gGoIFo9rmHTURX4cAALa" - "oZnjSkaKy1IdC3XDQsqT2vmAX3XjvaLhkCwWg56pst5LvS0AlcRGo49QHATNsY4h0SFkUmMyAGgS7+Pa2VwmCgWUoFFHRbO8ryoy" - "W57qONicyCxA2/OL28OrKGC/e25UsXwKBAXx+9uamcdxFvOQr9WqEJK1g4Q=" - ), - }, - "function": { - "arguments": '{"start_line": 1, "end_line": 50, "path": "shop/validation.py"}', - "name": "read_file", - }, - "id": ( - "call_1464376__thought__AY89a18+3vC8Y6wtLQ8YLNoWj0pZ1qcMTAdhuIp6v+o7VjO8nN81l72KF3fiSKtgKQYvOKc4hz38B" - "NeldTuj8NebwQ7BneePvMzchDx/5j5DXPVIIW7bRE2vYMhffaNpyjdltlQRDAxhiqjbSHIIGZQdxxAE7GfdGxlhstemN91VzQR4K" - "nPKk6MXh0at/4JacXcOFRRXDnYmUqj1a1tSIoSjeTsHMvSj5OpJyxxa8Fafd/yqwL8g4scNZgGzckn4O5quaQmJT9VPtoAsmh8qg" - "XUr/qzNSZoFGzRN6m7OzSuTyQ246hxKwp0+ZB9P+Nn5fvG96JVXsRq7dF00HBAbhebjUfbNTmIQauZHGSGbF2e+ck0aG5Lf8se2q" - "Ru6NCqiF+EnTFMUg7g+1cN1nIXLw7X6cAPPrpXvVdkfAoHksgSFWY5uUYwW9yDTZYwOWqF6bxDapPFU5HmJj/Bf5YuVH7VhGB7fy" - "O8SVW6AUe0/S/o0/e//wn6AaFomUST2wXw0hsi/OSZLEhRAi16AeUI0Y4SenblpEYtDAyhzFi77mqsceuF7QIq672m5yWq2qoGhP" - "h7S+5lMEvuD8YLITzrC81Xcb4HN3rxOvi6skmP4WJqZO8zPIw65WQUhRdhadhxM81HE+3InBToo1IhAF+11qt6hK0EuUf1guaGrU" - "EDZRPAgomcOLBSdK6gQ9pn5z3SiRPpw9SV4LWMqBjulGRD6K8u7XHXLPAiGZTF1cr/N2sEQuyri57XytugNISVcxh0q/Lvz0v0Dy" - "UAZM8H2ymbusxnKdIHanI9ZUS/XIXdia3PwqiOCgXDQaceOmotUnl8i0IXvSKwMjv3wB7UaC4YKyDurZ+gJJOzbjVjZ/mEf9h4JW" - "1ZetNWVtQl2s+oZXCHOtVJD2qLDyuMPO7EuafP8w7mbpKBTmw4ZAyCfrTL73FnEYhp6M3IEG+c4fdUCHMvT6ecG42DLKhdmWfCqz" - "TdXmXFLj/bePo7YzfYpLOLlhEzVdO5ojXPd6squqM+YHQS640UkdQ4v20UQ9ghFRkW1xUl7uPQkIAI0KWiDmaoi0YGFBxWpTs1U+" - "wN2K6hfqgtJy80rl5ceUT7f6B3MnwiIoAjzVc3n3TH6GodPIO08vrhrf4oz+G3OkdCaMHXURfzYkhji50/ThfAxgD7XDuzUYlUWG" - "otbx0FiYEDBbi2b7t7L/w3fP3nXYoSgaaHumXEnOAtugy6immasBFawtZq+GrOIYHJbH2tZhMpUysSTSnGF3BmqczIctDCwC82Ab" - "XU5sKEKFShEGnJSw9B0OlWuPa6kC/Yfapk6NzDyNB2ttsLhy0hCkT0BmgDn2P4h5T4PCVgLWUbA7LGQoYO5zbsUFBwtPUsVFK0yL" - "yf/Gg9h4ebosT6NVh7AkzS2riaNfm//lGUKK/8YDY49PAu0jpHp6JoXYg/z1+BjPvITdIQgf/8a0X36uqO+Yp2HZPzZzl/Ed8sC3" - "ZadKxulNV9kkXN2Gc3v/VyBruICqdhHSXJkusQJw+vtClqbP2uT18wQN4SRLjFTC3/ZruElXJFipevCQcC8KNVkXW39qZNWJGmKA" - "R15mprXot6VKXkRHfyQJJz0lAMTaACA5LJoA1dDn7+UaikqUsfrlLE6ObbCasA5/wIp5zFsGvslVq/vJqGWk2mdXbas2fpWTSUEC" - "EB3gGoIFo9rmHTURX4cAALaoZnjSkaKy1IdC3XDQsqT2vmAX3XjvaLhkCwWg56pst5LvS0AlcRGo49QHATNsY4h0SFkUmMyAGgS7" - "+Pa2VwmCgWUoFFHRbO8ryoyW57qONicyCxA2/OL28OrKGC/e25UsXwKBAXx+9uamcdxFvOQr9WqEJK1g4Q=" - ), - "type": "function", - }, - ], - "images": [], - "reasoning_content": ( - "**My Thought Process on Handling Quantity Validation**\n\nAlright, to properly address the handling " - "of zero, negative, and potentially non-integer quantities, my immediate next step needs to be " - "understanding the existing validation logic. This is crucial before I can even begin to think about " - 'suggesting any fixes or further code. My protocol dictates that I must "read the relevant files ' - "before suggesting a fix,\" and that's exactly what I'm prioritizing.\n\nTherefore, I need to examine " - "the `shop/validation.py` file. This is where I anticipate finding the rules and checks that govern " - "how different types of quantities are processed and validated. My plan is to read a reasonable " - "initial segment of this file, say lines 1 through 50, to get a solid grasp of the current " - "implementation. After that, I'll be able to determine precisely *which* other files, if any, I'll " - "need to investigate next, keeping in mind the specific constraints of zero, negative, and " - 'non-integer values.\n\nTo do this, I will now use the `read_file` tool:\n\n```json\n{\n "tool_code": ' - "\"print(read_file(path='shop/validation.py', start_line=1, end_line=50))\"\n}\n```\n\n" - ), - "thinking_blocks": [ - { - "type": "thinking", - "thinking": ( - "**My Thought Process on Handling Quantity Validation**\n\nAlright, to properly address the handling " - "of zero, negative, and potentially non-integer quantities, my immediate next step needs to be " - "understanding the existing validation logic. This is crucial before I can even begin to think about " - 'suggesting any fixes or further code. My protocol dictates that I must "read the relevant files ' - "before suggesting a fix,\" and that's exactly what I'm prioritizing.\n\nTherefore, I need to examine " - "the `shop/validation.py` file. This is where I anticipate finding the rules and checks that govern " - "how different types of quantities are processed and validated. My plan is to read a reasonable " - "initial segment of this file, say lines 1 through 50, to get a solid grasp of the current " - "implementation. After that, I'll be able to determine precisely *which* other files, if any, I'll " - "need to investigate next, keeping in mind the specific constraints of zero, negative, and " - 'non-integer values.\n\nTo do this, I will now use the `read_file` tool:\n\n```json\n{\n "tool_code": ' - "\"print(read_file(path='shop/validation.py', start_line=1, end_line=50))\"\n}\n```\n\n" - ), - }, - ], - "provider_specific_fields": { - "thought_signatures": [ - ( - "AY89a18+3vC8Y6wtLQ8YLNoWj0pZ1qcMTAdhuIp6v+o7VjO8nN81l72KF3fiSKtgKQYvOKc4hz38BNeldTuj8NebwQ7BneePvMzc" - "hDx/5j5DXPVIIW7bRE2vYMhffaNpyjdltlQRDAxhiqjbSHIIGZQdxxAE7GfdGxlhstemN91VzQR4KnPKk6MXh0at/4JacXcOFRRX" - "DnYmUqj1a1tSIoSjeTsHMvSj5OpJyxxa8Fafd/yqwL8g4scNZgGzckn4O5quaQmJT9VPtoAsmh8qgXUr/qzNSZoFGzRN6m7OzSuT" - "yQ246hxKwp0+ZB9P+Nn5fvG96JVXsRq7dF00HBAbhebjUfbNTmIQauZHGSGbF2e+ck0aG5Lf8se2qRu6NCqiF+EnTFMUg7g+1cN1" - "nIXLw7X6cAPPrpXvVdkfAoHksgSFWY5uUYwW9yDTZYwOWqF6bxDapPFU5HmJj/Bf5YuVH7VhGB7fyO8SVW6AUe0/S/o0/e//wn6A" - "aFomUST2wXw0hsi/OSZLEhRAi16AeUI0Y4SenblpEYtDAyhzFi77mqsceuF7QIq672m5yWq2qoGhPh7S+5lMEvuD8YLITzrC81Xc" - "b4HN3rxOvi6skmP4WJqZO8zPIw65WQUhRdhadhxM81HE+3InBToo1IhAF+11qt6hK0EuUf1guaGrUEDZRPAgomcOLBSdK6gQ9pn5" - "z3SiRPpw9SV4LWMqBjulGRD6K8u7XHXLPAiGZTF1cr/N2sEQuyri57XytugNISVcxh0q/Lvz0v0DyUAZM8H2ymbusxnKdIHanI9Z" - "US/XIXdia3PwqiOCgXDQaceOmotUnl8i0IXvSKwMjv3wB7UaC4YKyDurZ+gJJOzbjVjZ/mEf9h4JW1ZetNWVtQl2s+oZXCHOtVJD" - "2qLDyuMPO7EuafP8w7mbpKBTmw4ZAyCfrTL73FnEYhp6M3IEG+c4fdUCHMvT6ecG42DLKhdmWfCqzTdXmXFLj/bePo7YzfYpLOLl" - "hEzVdO5ojXPd6squqM+YHQS640UkdQ4v20UQ9ghFRkW1xUl7uPQkIAI0KWiDmaoi0YGFBxWpTs1U+wN2K6hfqgtJy80rl5ceUT7f" - "6B3MnwiIoAjzVc3n3TH6GodPIO08vrhrf4oz+G3OkdCaMHXURfzYkhji50/ThfAxgD7XDuzUYlUWGotbx0FiYEDBbi2b7t7L/w3f" - "P3nXYoSgaaHumXEnOAtugy6immasBFawtZq+GrOIYHJbH2tZhMpUysSTSnGF3BmqczIctDCwC82AbXU5sKEKFShEGnJSw9B0OlWu" - "Pa6kC/Yfapk6NzDyNB2ttsLhy0hCkT0BmgDn2P4h5T4PCVgLWUbA7LGQoYO5zbsUFBwtPUsVFK0yLyf/Gg9h4ebosT6NVh7AkzS2" - "riaNfm//lGUKK/8YDY49PAu0jpHp6JoXYg/z1+BjPvITdIQgf/8a0X36uqO+Yp2HZPzZzl/Ed8sC3ZadKxulNV9kkXN2Gc3v/VyB" - "ruICqdhHSXJkusQJw+vtClqbP2uT18wQN4SRLjFTC3/ZruElXJFipevCQcC8KNVkXW39qZNWJGmKAR15mprXot6VKXkRHfyQJJz0" - "lAMTaACA5LJoA1dDn7+UaikqUsfrlLE6ObbCasA5/wIp5zFsGvslVq/vJqGWk2mdXbas2fpWTSUECEB3gGoIFo9rmHTURX4cAALa" - "oZnjSkaKy1IdC3XDQsqT2vmAX3XjvaLhkCwWg56pst5LvS0AlcRGo49QHATNsY4h0SFkUmMyAGgS7+Pa2VwmCgWUoFFHRbO8ryoy" - "W57qONicyCxA2/OL28OrKGC/e25UsXwKBAXx+9uamcdxFvOQr9WqEJK1g4Q=" - ), - ], - }, - }, - "provider_specific_fields": {"native_finish_reason": "STOP"}, - }, - ], - "usage": { - "completion_tokens": 400, - "prompt_tokens": 246, - "total_tokens": 646, - "completion_tokens_details": {"reasoning_tokens": 339, "text_tokens": 61}, - "prompt_tokens_details": {"text_tokens": 246}, - }, - "vertex_ai_grounding_metadata": [], - "vertex_ai_url_context_metadata": [], - "vertex_ai_safety_results": [], - "vertex_ai_citation_metadata": [], - }, ) @pytest.mark.parametrize( "case", [ - GEMINI_3_8_FLASH_TEMPERATURE_TEST_CASE, - GEMINI_3_8_FLASH_ZERO_TEMPERATURE_TEST_CASE, + GEMINI_3_5_FLASH_TEMPERATURE_TEST_CASE, + GEMINI_3_5_FLASH_ZERO_TEMPERATURE_TEST_CASE, GEMINI_3_8_FLASH_REASONING_EFFORT_TEST_CASE, - VERTEX_AI_GEMINI_3_8_FLASH_TEMPERATURE_TEST_CASE, + VERTEX_AI_GEMINI_3_5_FLASH_TEMPERATURE_TEST_CASE, ], ids=lambda case: case.id, ) diff --git a/tests/integration/translation/messages/sampling/test_messages_sampling_gemini.py b/tests/integration/translation/messages/sampling/test_messages_sampling_gemini.py index e2b54774b1b..99d05872656 100644 --- a/tests/integration/translation/messages/sampling/test_messages_sampling_gemini.py +++ b/tests/integration/translation/messages/sampling/test_messages_sampling_gemini.py @@ -5,22 +5,22 @@ import pytest from integration._support.client import Gateway, object_value from integration._support.provider import SharedProvider from integration.translation.case import TranslationTestCase -from integration.translation.messages.bases.gemini import GEMINI_3_8_FLASH_TEST_CASE +from integration.translation.messages.bases.gemini import GEMINI_3_5_FLASH_TEST_CASE from integration.translation.runner import assert_translation -GENERATION_CONFIG: Final = object_value(GEMINI_3_8_FLASH_TEST_CASE.expected_provider_request["generationConfig"]) +GENERATION_CONFIG: Final = object_value(GEMINI_3_5_FLASH_TEST_CASE.expected_provider_request["generationConfig"]) -GEMINI_3_8_FLASH_TEMPERATURE_TEST_CASE: Final = replace( - GEMINI_3_8_FLASH_TEST_CASE, +GEMINI_3_5_FLASH_TEMPERATURE_TEST_CASE: Final = replace( + GEMINI_3_5_FLASH_TEST_CASE, scenario="temperature", - litellm_request={**GEMINI_3_8_FLASH_TEST_CASE.litellm_request, "temperature": 0.4}, + litellm_request={**GEMINI_3_5_FLASH_TEST_CASE.litellm_request, "temperature": 0.4}, expected_provider_request={ - **GEMINI_3_8_FLASH_TEST_CASE.expected_provider_request, + **GEMINI_3_5_FLASH_TEST_CASE.expected_provider_request, "generationConfig": {**GENERATION_CONFIG, "temperature": 0.4}, }, ) -@pytest.mark.parametrize("case", [GEMINI_3_8_FLASH_TEMPERATURE_TEST_CASE], ids=lambda case: case.id) +@pytest.mark.parametrize("case", [GEMINI_3_5_FLASH_TEMPERATURE_TEST_CASE], ids=lambda case: case.id) def test_messages_sampling_gemini(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: assert_translation(case, gateway, provider) diff --git a/tests/integration/translation/responses/sampling/test_responses_sampling_gemini.py b/tests/integration/translation/responses/sampling/test_responses_sampling_gemini.py index 6e82132aecb..1b07c9fda7c 100644 --- a/tests/integration/translation/responses/sampling/test_responses_sampling_gemini.py +++ b/tests/integration/translation/responses/sampling/test_responses_sampling_gemini.py @@ -5,23 +5,23 @@ import pytest from integration._support.client import Gateway, object_value from integration._support.provider import SharedProvider from integration.translation.case import TranslationTestCase -from integration.translation.responses.bases.gemini import GEMINI_3_8_FLASH_TEST_CASE +from integration.translation.responses.bases.gemini import GEMINI_3_5_FLASH_TEST_CASE from integration.translation.runner import assert_translation -GENERATION_CONFIG: Final = object_value(GEMINI_3_8_FLASH_TEST_CASE.expected_provider_request["generationConfig"]) +GENERATION_CONFIG: Final = object_value(GEMINI_3_5_FLASH_TEST_CASE.expected_provider_request["generationConfig"]) -GEMINI_3_8_FLASH_TEMPERATURE_TEST_CASE: Final = replace( - GEMINI_3_8_FLASH_TEST_CASE, +GEMINI_3_5_FLASH_TEMPERATURE_TEST_CASE: Final = replace( + GEMINI_3_5_FLASH_TEST_CASE, scenario="temperature", - litellm_request={**GEMINI_3_8_FLASH_TEST_CASE.litellm_request, "temperature": 0.4}, + litellm_request={**GEMINI_3_5_FLASH_TEST_CASE.litellm_request, "temperature": 0.4}, expected_provider_request={ - **GEMINI_3_8_FLASH_TEST_CASE.expected_provider_request, + **GEMINI_3_5_FLASH_TEST_CASE.expected_provider_request, "generationConfig": {**GENERATION_CONFIG, "temperature": 0.4}, }, - expected_litellm_response={**GEMINI_3_8_FLASH_TEST_CASE.expected_litellm_response, "temperature": 0.4}, + expected_litellm_response={**GEMINI_3_5_FLASH_TEST_CASE.expected_litellm_response, "temperature": 0.4}, ) -@pytest.mark.parametrize("case", [GEMINI_3_8_FLASH_TEMPERATURE_TEST_CASE], ids=lambda case: case.id) +@pytest.mark.parametrize("case", [GEMINI_3_5_FLASH_TEMPERATURE_TEST_CASE], ids=lambda case: case.id) def test_responses_sampling_gemini(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: assert_translation(case, gateway, provider)