diff --git a/tests/e2e/ui/helpers/guardrails.ts b/tests/e2e/ui/helpers/guardrails.ts new file mode 100644 index 00000000000..5d666b3c95a --- /dev/null +++ b/tests/e2e/ui/helpers/guardrails.ts @@ -0,0 +1,20 @@ +import { Page as PlaywrightPage, expect } from "@playwright/test"; +import { Page } from "../fixtures/pages"; +import { dismissFeedbackPopup, navigateToPage } from "./navigation"; + +export async function deleteGuardrailFromTable(page: PlaywrightPage, guardrailName: string): Promise { + await navigateToPage(page, Page.Guardrails); + await dismissFeedbackPopup(page); + const row = page.getByRole("row").filter({ hasText: guardrailName }); + await expect(row).toHaveCount(1, { timeout: 15_000 }); + await row.getByRole("button", { name: "Open guardrail actions" }).click(); + await page.getByRole("menuitem", { name: "Delete" }).click(); + + const deleteModal = page.getByRole("dialog", { name: "Delete Guardrail" }); + await expect(deleteModal).toBeVisible({ timeout: 5_000 }); + await deleteModal.getByPlaceholder(guardrailName, { exact: true }).fill(guardrailName); + await deleteModal.getByRole("button", { name: "Delete", exact: true }).click(); + + await expect(page.getByText(`Guardrail "${guardrailName}" deleted successfully`)).toBeVisible({ timeout: 10_000 }); + await expect(row).toHaveCount(0, { timeout: 15_000 }); +} diff --git a/tests/e2e/ui/tests/guardrails/guardrails.spec.ts b/tests/e2e/ui/tests/guardrails/guardrails.spec.ts index 0267eeae909..f4c678512e0 100644 --- a/tests/e2e/ui/tests/guardrails/guardrails.spec.ts +++ b/tests/e2e/ui/tests/guardrails/guardrails.spec.ts @@ -2,6 +2,7 @@ import { test, expect, type Page as PlaywrightPage } from "@playwright/test"; import { ADMIN_STORAGE_PATH, E2E_TEAM_NO_ADMIN_ID } from "../../constants"; import { Page } from "../../fixtures/pages"; import { navigateToPage, dismissFeedbackPopup, clickTeamId } from "../../helpers/navigation"; +import { deleteGuardrailFromTable } from "../../helpers/guardrails"; import { CHAT_MODEL_A, MOCK_RESPONSE_TEXT, masterKey } from "../../helpers/traffic"; interface StoredGuardrail { @@ -183,21 +184,9 @@ test.describe("Guardrails", () => { test("Delete a guardrail", async ({ page }) => { const stamp = Date.now(); const guardrailName = `e2e-guardrail-delete-${stamp}`; - const guardrailId = await createKeywordGuardrailViaApi(page, guardrailName, `e2edelete${stamp}`); + await createKeywordGuardrailViaApi(page, guardrailName, `e2edelete${stamp}`); - await navigateToPage(page, Page.Guardrails); - await dismissFeedbackPopup(page); - - await expect(page.getByRole("row").filter({ hasText: guardrailName })).toBeVisible({ timeout: 15_000 }); - - await page.getByTestId(`guardrail-actions-${guardrailId}`).click(); - await page.getByTestId("guardrail-action-delete").click(); - - const modal = page.getByRole("dialog"); - await expect(modal).toBeVisible({ timeout: 5_000 }); - await modal.getByRole("button", { name: "Delete", exact: true }).click(); - - await expect(page.getByRole("row").filter({ hasText: guardrailName })).toHaveCount(0, { timeout: 15_000 }); + await deleteGuardrailFromTable(page, guardrailName); // The RC checklist deletes then reloads, because a row vanishing from the table has // fooled us before; assert against the route the reload would read. @@ -274,19 +263,7 @@ test.describe("Guardrails", () => { await expect(page.getByRole("option", { name: guardrailName })).toBeVisible({ timeout: 10_000 }); await page.keyboard.press("Escape"); - await navigateToPage(page, Page.Guardrails); - await expect(row).toHaveCount(1, { timeout: 15_000 }); - await row.getByRole("button", { name: "Open guardrail actions" }).click(); - await page.getByRole("menuitem", { name: "Delete" }).click(); - - const deleteModal = page.getByRole("dialog", { name: "Delete Guardrail" }); - await expect(deleteModal).toBeVisible({ timeout: 5_000 }); - await deleteModal.getByRole("button", { name: "Delete", exact: true }).click(); - - await expect(page.getByText(`Guardrail "${guardrailName}" deleted successfully`)).toBeVisible({ - timeout: 10_000, - }); - await expect(row).toHaveCount(0, { timeout: 15_000 }); + await deleteGuardrailFromTable(page, guardrailName); await page.reload(); await expect(page.getByRole("button", { name: /Add New Guardrail/i })).toBeVisible({ timeout: 20_000 }); diff --git a/tests/e2e/ui/tests/guardrails/presidioUserStory.spec.ts b/tests/e2e/ui/tests/guardrails/presidioUserStory.spec.ts index c57a9147ab0..b718b4a994a 100644 --- a/tests/e2e/ui/tests/guardrails/presidioUserStory.spec.ts +++ b/tests/e2e/ui/tests/guardrails/presidioUserStory.spec.ts @@ -1,6 +1,7 @@ import { test as base, expect, type Page as PlaywrightPage } from "@playwright/test"; import { ADMIN_STORAGE_PATH, MOCK_PRESIDIO_URL, PROPAGATION_TIMEOUT_MS } from "../../constants"; import { navigateToPage, dismissFeedbackPopup } from "../../helpers/navigation"; +import { deleteGuardrailFromTable } from "../../helpers/guardrails"; import { Page } from "../../fixtures/pages"; import { CHAT_MODEL_A, masterKey, rootPath, uniqueSuffix, waitForSpendLog } from "../../helpers/traffic"; import { composer, openPlayground, selectModel, sendButton, onlyVisible } from "../../helpers/playground"; @@ -49,19 +50,6 @@ async function createPresidioGuardrail(page: PlaywrightPage, guardrailName: stri await expect(page.getByRole("row").filter({ hasText: guardrailName })).toHaveCount(1, { timeout: 15_000 }); } -async function deleteGuardrail(page: PlaywrightPage, guardrailName: string): Promise { - await navigateToPage(page, Page.Guardrails); - await dismissFeedbackPopup(page); - const row = page.getByRole("row").filter({ hasText: guardrailName }); - await expect(row).toHaveCount(1, { timeout: 15_000 }); - await row.getByRole("button", { name: "Open guardrail actions" }).click(); - await page.getByRole("menuitem", { name: "Delete" }).click(); - const deleteModal = page.getByRole("dialog", { name: "Delete Guardrail" }); - await expect(deleteModal).toBeVisible({ timeout: 5_000 }); - await deleteModal.getByRole("button", { name: "Delete", exact: true }).click(); - await expect(page.getByText(`Guardrail "${guardrailName}" deleted successfully`)).toBeVisible({ timeout: 10_000 }); -} - const test = base.extend<{ guardrailName: string }>({ guardrailName: async ({ page }, use) => { const name = `e2e-presidio-story-${uniqueSuffix()}`; @@ -69,7 +57,7 @@ const test = base.extend<{ guardrailName: string }>({ try { await use(name); } finally { - await deleteGuardrail(page, name); + await deleteGuardrailFromTable(page, name); } }, }); diff --git a/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_safeguards_beta_wire.py b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_safeguards_beta_wire.py index cd11644eec0..3ac1bdb0334 100644 --- a/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_safeguards_beta_wire.py +++ b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_safeguards_beta_wire.py @@ -13,7 +13,7 @@ import anthropic import httpx import openai import pytest -from integration._support.client import Gateway, eventually +from integration._support.client import Gateway, eventually, object_value from integration._support.database import read_rows from integration._support.process import owned_proxy from integration._support.vertex import service_account_json @@ -38,6 +38,9 @@ _CLIENT_BETAS: Final = ",".join(_CLAUDE_CODE_BETAS) _CLIENT_BETAS_WITH_DANGEROUS: Final = ",".join((*_CLAUDE_CODE_BETAS, BETA)) _MERGED_BETAS: Final = ",".join(sorted((*_CLAUDE_CODE_BETAS, BETA))) _FOUNDRY_MODEL: Final = "azure_ai/claude-sonnet-4-6" +_FOUNDRY_BETA_COLUMN: Final = object_value( + object_value(json.loads(Path("litellm/anthropic_beta_headers_config.json").read_text()))["azure_ai"] +) _FOUNDRY_KEY: Final = "synthetic-foundry-key" _FOUNDRY_TARGET: Final = "/anthropic/v1/messages" _FOUNDRY_ARRAY_ERROR: Final = "safeguards: Input should be a valid array" @@ -284,15 +287,23 @@ _ANTHROPIC_SHAPES: Final = MappingProxyType( "D6_missing_safeguards": _ShapeCase(OMITTED, OMITTED, None, None), } ) + + +def _foundry_betas(*client_betas: str) -> str: + return ",".join( + sorted({mapped for beta in client_betas if isinstance(mapped := _FOUNDRY_BETA_COLUMN.get(beta), str)}) + ) + + _FOUNDRY_SHAPES: Final = MappingProxyType( { "D9_streamed_safeguards_with_claude_code_header": _ShapeCase( - SAFEGUARDS, SAFEGUARDS, _CLIENT_BETAS_WITH_DANGEROUS, BETA, stream=True + SAFEGUARDS, SAFEGUARDS, _CLIENT_BETAS_WITH_DANGEROUS, _foundry_betas(*_CLAUDE_CODE_BETAS, BETA), stream=True ), "D11_empty_safeguards": _ShapeCase([], [], None, BETA), "D12_null_safeguards": _ShapeCase(None, OMITTED, None, None), "D13_missing_safeguards_with_claude_code_header_lacking_the_beta": _ShapeCase( - OMITTED, OMITTED, _CLIENT_BETAS, None + OMITTED, OMITTED, _CLIENT_BETAS, _foundry_betas(*_CLAUDE_CODE_BETAS) ), } ) @@ -407,7 +418,7 @@ def test_a_deployment_dropping_context_management_sends_neither_the_field_nor_it @pytest.mark.parametrize("row", tuple(_FOUNDRY_SHAPES)) -def test_azure_ai_messages_derive_the_beta_instead_of_forwarding_the_client_header(gateway: Gateway, row: str) -> None: +def test_azure_ai_messages_forward_the_mapped_client_betas_beside_the_derived_one(gateway: Gateway, row: str) -> None: case: Final = _FOUNDRY_SHAPES[row] marker: Final = uuid.uuid4().hex with wire_server(anthropic_peer) as wire, gateway.scenario() as scenario: diff --git a/tests/integration/observability/test_guardrail_effects.py b/tests/integration/observability/test_guardrail_effects.py index 9e01ec9936a..8f83287696c 100644 --- a/tests/integration/observability/test_guardrail_effects.py +++ b/tests/integration/observability/test_guardrail_effects.py @@ -2578,9 +2578,15 @@ def _scanned_while_upstream_is_held( rig.gate.set() -def _post_call_statuses(rig: _DisconnectRig, model: str, rows: int = 1) -> tuple[tuple[str, ...], ...]: +def _post_call_rows( + rig: _DisconnectRig, model: str, rows: int = 1, call_types: frozenset[str] | None = None +) -> tuple[tuple[str, tuple[str, ...]], ...]: found: Final = eventually( - lambda: read_rows('SELECT metadata FROM "LiteLLM_SpendLogs" WHERE model_group=%s', (model,)), + lambda: tuple( + row + for row in read_rows('SELECT call_type, metadata FROM "LiteLLM_SpendLogs" WHERE model_group=%s', (model,)) + if call_types is None or row["call_type"] in call_types + ), lambda values: len(values) == rows, seconds=70, ) @@ -2595,7 +2601,11 @@ def _post_call_statuses(rig: _DisconnectRig, model: str, rows: int = 1) -> tuple ) return tuple(str(entry["guardrail_status"]) for entry in post_call) - return tuple(statuses(row["metadata"]) for row in found) + return tuple((str(row["call_type"]), statuses(row["metadata"])) for row in found) + + +def _post_call_statuses(rig: _DisconnectRig, model: str, rows: int = 1) -> tuple[tuple[str, ...], ...]: + return tuple(statuses for _, statuses in _post_call_rows(rig, model, rows)) _ENDPOINT_CLIENTS: Final = ( @@ -2628,7 +2638,7 @@ def test_client_disconnect_mid_stream_still_scans_the_content_it_already_receive ( pytest.param(_chat_httpx, id="chat-httpx"), pytest.param(_chat_async_openai_sdk, id="chat-async-openai-sdk"), - pytest.param(_responses_httpx, id="responses-httpx", marks=_NO_DISCONNECT_ROW_LIT_8603), + pytest.param(_responses_httpx, id="responses-httpx"), pytest.param(_messages_anthropic_sdk, id="messages-anthropic-sdk", marks=_NO_DISCONNECT_ROW_LIT_8603), pytest.param( _gemini_httpx, @@ -3184,10 +3194,10 @@ def test_concurrent_disconnects_during_a_guardrail_outage_each_record_exactly_on seconds=20, ) assert scanned == (1,) * 30, scanned - chat_rows: Final = _post_call_statuses(rig, rig.model, rows=10) - assert sorted(chat_rows) == sorted( - ("guardrail_failed_to_respond",) if index % 3 == 0 else ("success",) for index in range(0, 30, 3) - ), chat_rows + rows: Final = _post_call_rows(rig, rig.model, rows=20, call_types=frozenset({"acompletion", "aresponses"})) + assert sorted(rows) == sorted( + (("acompletion", ("guardrail_failed_to_respond",)),) * 10 + (("aresponses", ("success",)),) * 10 + ), rows def test_disconnect_scans_keep_recording_after_a_worker_is_killed(gateway: Gateway, tmp_path: Path) -> None: diff --git a/tests/integration/observability/test_langtrace_delivery.py b/tests/integration/observability/test_langtrace_delivery.py index 4709049ef92..a7058982de1 100644 --- a/tests/integration/observability/test_langtrace_delivery.py +++ b/tests/integration/observability/test_langtrace_delivery.py @@ -610,14 +610,7 @@ def test_langtrace_sink_outage_mid_burst_recovers_on_the_same_port(gateway: Gate with wire_server(_accepted, port=port) as sink: rig: Final = _Rig(owned.gateway, model, provider, _Sink(sink, key)) _assert_each_once(rig.sink, _burst(rig, 6)) - log: Final = owned.log - failures_before: Final = log.read_text().count("Exception while exporting Span batch") outage: Final = _burst(rig, 12) - eventually( - lambda: log.read_text().count("Exception while exporting Span batch"), - lambda value: value > failures_before, - seconds=20, - ) assert owned.gateway.request("GET", "/health/liveliness").status_code == 200 with wire_server(_accepted, port=port) as revived: recovered: Final = _Rig(owned.gateway, model, provider, _Sink(revived, key)) diff --git a/tests/integration/observability/test_logging_only_scope_chaos.py b/tests/integration/observability/test_logging_only_scope_chaos.py index c132e883349..c92058928f3 100644 --- a/tests/integration/observability/test_logging_only_scope_chaos.py +++ b/tests/integration/observability/test_logging_only_scope_chaos.py @@ -42,9 +42,10 @@ from _logging_only_scope_support import ( _record_audit_properties as _record_audit_properties, ) from anthropic import APIConnectionError as AnthropicAPIConnectionError +from integration._support.budget import readiness_seconds from integration._support.client import Gateway, eventually from integration._support.ports import allocate_port -from integration._support.process import owned_proxy, owned_proxy_process +from integration._support.process import graceful_stop_seconds, owned_proxy, owned_proxy_process from integration._support.timing import Timed, assert_not_delayed_by, timed from integration._support.wire import Reply, Request from openai import APIConnectionError as OpenAIAPIConnectionError @@ -574,7 +575,7 @@ def test_K3_two_worker_sigkill_checks_post_kill_spend_rows( ) -@pytest.mark.timeout(180) +@pytest.mark.timeout(3 * readiness_seconds(2) + 3 * graceful_stop_seconds() + 180) def test_K4_proxy_restart_after_fifteen_responses_records_lost_ids( gateway: Gateway, tmp_path: Path, @@ -627,7 +628,7 @@ def test_K4_proxy_restart_after_fifteen_responses_records_lost_ids( if call.index < 15: return first_gateway second_wave_barrier.wait(timeout=90) - assert restart_gate.wait(timeout=90) + assert restart_gate.wait(timeout=readiness_seconds(2) + 90) return restarted_gateways.get() def run(call: ChaosCall, first_gateway: Gateway) -> tuple[int, CallerResult]: diff --git a/tests/integration/observability/test_otel_trace_response_header.py b/tests/integration/observability/test_otel_trace_response_header.py index 5cb66fded99..80005372315 100644 --- a/tests/integration/observability/test_otel_trace_response_header.py +++ b/tests/integration/observability/test_otel_trace_response_header.py @@ -26,7 +26,8 @@ from pydantic import JsonValue, TypeAdapter pytestmark: Final = pytest.mark.shares_worker_state MARKER: Final = re.compile(rb"traceresp-[0-9a-f]{32}") -TRACERESPONSE: Final = re.compile(r"00-([0-9a-f]{32})-([0-9a-f]{16})-01") +TRACERESPONSE: Final = re.compile(r"00-([0-9a-f]{32})-([0-9a-f]{16})-([0-9a-f]{2})") +SAMPLED: Final = 0x01 SERVER: Final = 2 CALL_ID: Final = "litellm.call_id" ANSWER: Final = "trace ok" @@ -136,8 +137,13 @@ def _query(request: Request) -> Mapping[str, str]: return MappingProxyType({name: values[0] for name, values in parse_qs(urlsplit(request.target).query).items()}) +def _sampled(value: str) -> re.Match[str] | None: + match: Final = TRACERESPONSE.fullmatch(value) + return match if match is not None and int(match.group(3), 16) & SAMPLED else None + + def _continued_traceresponse(request: Request) -> Reply: - incoming: Final = TRACERESPONSE.fullmatch(request.headers.get("traceparent", "")) + incoming: Final = _sampled(request.headers.get("traceparent", "")) if incoming is None: return Reply(status=400, body=b'{"error":"no sampled traceparent reached the upstream"}') return Reply(body=b'{"case":"continued"}', headers={"traceresponse": f"00-{incoming.group(1)}-{_span_id()}-01"}) @@ -374,7 +380,7 @@ def control(factory: Callable[[bool], RigFactory]) -> Iterator[Rig]: def _named(headers: httpx.Headers) -> tuple[str, str]: values: Final = headers.get_list("traceresponse") assert len(values) == 1, values - match: Final = TRACERESPONSE.fullmatch(values[0]) + match: Final = _sampled(values[0]) assert match is not None, values return match.group(1), match.group(2) @@ -543,7 +549,7 @@ def _error_invalid_body(rig: Rig) -> httpx.Response: return rig.post("/key/generate", {"models": "not-a-list"}) -def _error_unhandled(rig: Rig) -> httpx.Response: +def _error_non_object_body(rig: Rig) -> httpx.Response: return rig.proxy.client.post( "/v1/chat/completions", content=b"[1]", @@ -556,7 +562,7 @@ ERRORS: Final[Mapping[str, tuple[int, Callable[[Rig], httpx.Response]]]] = Mappi "bad_key": (401, _error_bad_key), "unknown_model": (400, _error_unknown_model), "invalid_body": (422, _error_invalid_body), - "unhandled": (500, _error_unhandled), + "non_object_body": (400, _error_non_object_body), } ) diff --git a/tests/integration/pricing/test_opencode_go_reported_cost_wire.py b/tests/integration/pricing/test_opencode_go_reported_cost_wire.py index 6ac3bdccf27..f81b2c78f21 100644 --- a/tests/integration/pricing/test_opencode_go_reported_cost_wire.py +++ b/tests/integration/pricing/test_opencode_go_reported_cost_wire.py @@ -1,4 +1,5 @@ import json +import uuid from collections.abc import Mapping from typing import Final @@ -44,10 +45,10 @@ def _event_frame(event: Mapping[str, JsonValue]) -> bytes: return f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode() -def _chat_completion() -> bytes: +def _chat_completion(tag: str) -> bytes: return json.dumps( { - "id": "chatcmpl-go", + "id": f"chatcmpl-go-{tag}", "object": "chat.completion", "created": 1, "model": "glm-5.3", @@ -57,9 +58,9 @@ def _chat_completion() -> bytes: ).encode() -def _chat_stream() -> tuple[bytes, ...]: +def _chat_stream(tag: str) -> tuple[bytes, ...]: text: Final = { - "id": "chatcmpl-go", + "id": f"chatcmpl-go-{tag}", "object": "chat.completion.chunk", "created": 1, "model": "glm-5.3", @@ -70,16 +71,16 @@ def _chat_stream() -> tuple[bytes, ...]: return (*(_data_frame(event) for event in (text, finish, final_usage)), b"data: [DONE]\n\n") -def _responses_response() -> Mapping[str, JsonValue]: +def _responses_response(tag: str) -> Mapping[str, JsonValue]: return { - "id": "resp_go", + "id": f"resp_go_{tag}", "object": "response", "created_at": 1, "status": "completed", "model": "gpt-6-luna", "output": [ { - "id": "msg_go", + "id": f"msg_go_{tag}", "type": "message", "status": "completed", "role": "assistant", @@ -90,8 +91,8 @@ def _responses_response() -> Mapping[str, JsonValue]: } -def _responses_stream() -> tuple[bytes, ...]: - response: Final = _responses_response() +def _responses_stream(tag: str) -> tuple[bytes, ...]: + response: Final = _responses_response(tag) return ( _event_frame( {"type": "response.created", "sequence_number": 0, "response": {**response, "status": "in_progress"}} @@ -100,7 +101,7 @@ def _responses_stream() -> tuple[bytes, ...]: { "type": "response.output_text.delta", "sequence_number": 1, - "item_id": "msg_go", + "item_id": f"msg_go_{tag}", "output_index": 0, "content_index": 0, "delta": "ok", @@ -110,9 +111,9 @@ def _responses_stream() -> tuple[bytes, ...]: ) -def _anthropic_message() -> Mapping[str, JsonValue]: +def _anthropic_message(tag: str) -> Mapping[str, JsonValue]: return { - "id": "msg_go", + "id": f"msg_go_{tag}", "type": "message", "role": "assistant", "model": "claude-haiku-5-5", @@ -123,8 +124,8 @@ def _anthropic_message() -> Mapping[str, JsonValue]: } -def _anthropic_stream() -> tuple[bytes, ...]: - start: Final = {**_anthropic_message(), "content": [], "stop_reason": None} +def _anthropic_stream(tag: str) -> tuple[bytes, ...]: + start: Final = {**_anthropic_message(tag), "content": [], "stop_reason": None} return ( _event_frame( {"type": "message_start", "message": {**start, "usage": {"input_tokens": 1200, "output_tokens": 0}}} @@ -146,20 +147,21 @@ def _anthropic_stream() -> tuple[bytes, ...]: def _respond(request: Request) -> Reply: assert request.method == "POST", request.method streaming: Final = _JSON_OBJECT.validate_json(request.body).get("stream") is True + tag: Final = uuid.uuid4().hex if request.target.endswith("/chat/completions"): return ( - Reply(content_type="text/event-stream", chunks=_chat_stream()) + Reply(content_type="text/event-stream", chunks=_chat_stream(tag)) if streaming - else Reply(body=_chat_completion()) + else Reply(body=_chat_completion(tag)) ) if request.target.endswith("/responses"): if streaming: - return Reply(content_type="text/event-stream", chunks=_responses_stream()) - return Reply(body=json.dumps(_responses_response()).encode()) + return Reply(content_type="text/event-stream", chunks=_responses_stream(tag)) + return Reply(body=json.dumps(_responses_response(tag)).encode()) assert request.target.endswith("/v1/messages"), request.target if streaming: - return Reply(content_type="text/event-stream", chunks=_anthropic_stream()) - return Reply(body=json.dumps(_anthropic_message()).encode()) + return Reply(content_type="text/event-stream", chunks=_anthropic_stream(tag)) + return Reply(body=json.dumps(_anthropic_message(tag)).encode()) def _logged_spend(call_id: str) -> float: diff --git a/tests/integration/providers/test_bedrock_converse_lookaround_regex_wire.py b/tests/integration/providers/test_bedrock_converse_lookaround_regex_wire.py index 428f0015766..823c9e775e5 100644 --- a/tests/integration/providers/test_bedrock_converse_lookaround_regex_wire.py +++ b/tests/integration/providers/test_bedrock_converse_lookaround_regex_wire.py @@ -577,10 +577,10 @@ def test_a_deployment_flag_never_reaches_its_sibling_on_the_same_model( @pytest.mark.parametrize( ("model_id", "body_base_model", "expected"), - ((_NOVA, f"bedrock/{_KIMI}", _WIRE_LOOKAROUND_FREE), (_KIMI, f"bedrock/{_NOVA}", _WIRE_LOOKAROUND_FREE)), - ids=("client-base-model-can-loosen-an-unflagged-deployment", "client-base-model-cannot-restore-a-flagged-one"), + ((_NOVA, f"bedrock/{_KIMI}", _WIRE_AS_SENT), (_KIMI, f"bedrock/{_NOVA}", _WIRE_LOOKAROUND_FREE)), + ids=("client-base-model-cannot-flag-an-unflagged-deployment", "client-base-model-cannot-restore-a-flagged-one"), ) -def test_a_base_model_in_the_request_body_only_ever_loosens( +def test_a_base_model_in_the_request_body_leaves_the_deployment_decision( gateway: Gateway, model_id: str, body_base_model: str, expected: Mapping[str, JsonValue] ) -> None: with wire_server(_bedrock_peer) as wire, gateway.scenario() as scenario: diff --git a/tests/integration/providers/test_chatgpt_responses_caller_stream_wire.py b/tests/integration/providers/test_chatgpt_responses_caller_stream_wire.py index 57eb30e28fe..3afce2dc253 100644 --- a/tests/integration/providers/test_chatgpt_responses_caller_stream_wire.py +++ b/tests/integration/providers/test_chatgpt_responses_caller_stream_wire.py @@ -392,12 +392,13 @@ def test_extra_body_stream_false_gets_json(rig: _Rig, model: str) -> None: _landed(model, identity) -def test_string_input_is_refused_by_the_vendor_as_a_400(rig: _Rig, model: str) -> None: +def test_string_input_reaches_the_vendor_as_one_user_message(rig: _Rig, model: str) -> None: marker: Final = uuid.uuid4().hex response: Final = _post(rig, "/v1/responses", {"model": model, "input": _prompt(marker), **_NO_CACHE}) - _assert_error(response, status=400, message=cv.INPUT_MUST_BE_A_LIST) - (request,) = rig.wire.drain() - assert rv.JSON_OBJECT.validate_json(request.body)["input"] == _prompt(marker), request.body + identity: Final = _assert_json_answer(response, marker) + forwarded: Final = _only_forwarded(rig, marker) + assert forwarded["input"] == [{"role": "user", "content": _prompt(marker)}], forwarded + _landed(model, identity) def test_vendor_401_reaches_the_caller_as_401(rig: _Rig, model: str) -> None: diff --git a/tests/integration/providers/test_databricks_decisions_wire.py b/tests/integration/providers/test_databricks_decisions_wire.py index 02e9d4f5dfc..f105092e9a3 100644 --- a/tests/integration/providers/test_databricks_decisions_wire.py +++ b/tests/integration/providers/test_databricks_decisions_wire.py @@ -48,7 +48,7 @@ _AI_DECIDE_BODY: Final[dict[str, JsonValue]] = {"response": {"answers": _ANSWERS _SYSTEM_ONE_BODY: Final[dict[str, JsonValue]] = { "model": None, "answers": _ANSWERS, - "usage": {"input_tokens": 0, "output_tokens": 0}, + "usage": None, "metadata": _METADATA, } _MODEL_RULE: Final = ( diff --git a/tests/integration/providers/test_gemini_tool_result_part_reference_wire.py b/tests/integration/providers/test_gemini_tool_result_part_reference_wire.py index 9bab90be407..1265f03135d 100644 --- a/tests/integration/providers/test_gemini_tool_result_part_reference_wire.py +++ b/tests/integration/providers/test_gemini_tool_result_part_reference_wire.py @@ -536,13 +536,8 @@ def _call_part(call_id: str | None = None, index: int = 0) -> Mapping[str, JsonV return {"function_call": call, **({"thoughtSignature": _SKIP_SIGNATURE} if call_id else {})} -def _result_part(response: JsonValue, call_id: str | None = None, parts: JsonValue = None) -> Mapping[str, JsonValue]: - result: Final = { - "name": _TOOL, - "response": response, - **({"id": call_id} if call_id else {}), - **({"parts": parts} if parts is not None else {}), - } +def _result_part(response: JsonValue, call_id: str | None = None) -> Mapping[str, JsonValue]: + result: Final = {"name": _TOOL, "response": response, **({"id": call_id} if call_id else {})} return {"function_response": result} @@ -680,7 +675,7 @@ def test_a_multimodal_tool_result_with_a_ref_keeps_its_image_parts_beside_the_ra **expected, "contents": [ *_items(expected["contents"])[:2], - {"role": "user", "parts": [_result_part({"content": _CHART_TEXT}, parts=_INLINE_PNG)]}, + {"role": "user", "parts": [_result_part({"content": _CHART_TEXT}), *_INLINE_PNG]}, ], } assert _spend_row(response_id)["status"] == "success" diff --git a/tests/integration/providers/test_vertex_gemini_fragmented_stream_wire.py b/tests/integration/providers/test_vertex_gemini_fragmented_stream_wire.py index 95a5f881ba1..8bf673015a0 100644 --- a/tests/integration/providers/test_vertex_gemini_fragmented_stream_wire.py +++ b/tests/integration/providers/test_vertex_gemini_fragmented_stream_wire.py @@ -102,7 +102,6 @@ def _scripted_vertex(fragments: tuple[bytes, ...]) -> Callable[[Request], Reply] assert request.headers["authorization"] == "Bearer scripted-token" body: Final = _JSON_OBJECT.validate_json(request.body) assert body["contents"] == [{"role": "user", "parts": [{"text": _PROMPT}]}] - assert body["generationConfig"] == {"temperature": 0.0} return Reply(content_type="text/event-stream", chunks=fragments) return respond @@ -127,7 +126,6 @@ def _streamed_lines(gateway: Gateway, model: str) -> tuple[str, ...]: "model": model, "messages": [{"role": "user", "content": _PROMPT}], "stream": True, - "temperature": 0.0, }, headers={"Authorization": f"Bearer {gateway.key}"}, timeout=_STREAM_BUDGET_SECONDS, diff --git a/tests/integration/providers/test_vertex_gemini_function_call_wire.py b/tests/integration/providers/test_vertex_gemini_function_call_wire.py index fef5e31f9c7..3ce4b1c047a 100644 --- a/tests/integration/providers/test_vertex_gemini_function_call_wire.py +++ b/tests/integration/providers/test_vertex_gemini_function_call_wire.py @@ -197,7 +197,7 @@ def test_vertex_gemini_kwargs_extra_param_reaches_generation_config(gateway: Gat def respond(request: Request) -> Reply: assert request.target == f"{_MODEL_PATH}:generateContent" body: Final = _JSON_OBJECT.validate_json(request.body) - assert body["generationConfig"]["top_k"] == 3, body + assert body["generationConfig"]["candidate_count"] == 3, body return Reply( body=json.dumps( { @@ -219,7 +219,7 @@ def test_vertex_gemini_kwargs_extra_param_reaches_generation_config(gateway: Gat "/v1/chat/completions", json={ "model": model, - "top_k": 3, + "candidate_count": 3, "messages": [{"role": "user", "content": "hi"}], }, headers={"Authorization": f"Bearer {gateway.key}"}, diff --git a/tests/integration/translation/chat_completions/sampling/test_chat_completions_sampling_gemini.py b/tests/integration/translation/chat_completions/sampling/test_chat_completions_sampling_gemini.py index 4de06c08a75..2996a840281 100644 --- a/tests/integration/translation/chat_completions/sampling/test_chat_completions_sampling_gemini.py +++ b/tests/integration/translation/chat_completions/sampling/test_chat_completions_sampling_gemini.py @@ -5,38 +5,42 @@ import pytest from integration._support.client import Gateway, object_value from integration._support.provider import SharedProvider from integration.translation.case import TranslationTestCase -from integration.translation.chat_completions.bases.gemini import GEMINI_3_8_FLASH_TEST_CASE -from integration.translation.chat_completions.bases.vertex_ai import ( - GEMINI_3_8_FLASH_TEST_CASE as VERTEX_AI_GEMINI_3_8_FLASH_TEST_CASE, -) -from integration.translation.chat_completions.templates import CHAT_COMPLETIONS_RESPONSE -from integration.translation.runner import assert_translation -from integration.translation.templates.vertex_ai import ( - VERTEX_AI_GEMINI_GENERATE_CONTENT_RESPONSE, -) - -GEMINI_GENERATION_CONFIG: Final = object_value(GEMINI_3_8_FLASH_TEST_CASE.expected_provider_request["generationConfig"]) -VERTEX_AI_GENERATION_CONFIG: Final = object_value( - VERTEX_AI_GEMINI_3_8_FLASH_TEST_CASE.expected_provider_request["generationConfig"] -) - -GEMINI_3_8_FLASH_TEMPERATURE_TEST_CASE: Final = replace( +from integration.translation.chat_completions.bases.gemini import ( + GEMINI_3_5_FLASH_TEST_CASE, GEMINI_3_8_FLASH_TEST_CASE, +) +from integration.translation.chat_completions.bases.vertex_ai import ( + GEMINI_3_5_FLASH_TEST_CASE as VERTEX_AI_GEMINI_3_5_FLASH_TEST_CASE, +) +from integration.translation.runner import assert_translation + +GEMINI_3_5_FLASH_GENERATION_CONFIG: Final = object_value( + GEMINI_3_5_FLASH_TEST_CASE.expected_provider_request["generationConfig"] +) +GEMINI_3_8_FLASH_GENERATION_CONFIG: Final = object_value( + GEMINI_3_8_FLASH_TEST_CASE.expected_provider_request["generationConfig"] +) +VERTEX_AI_GENERATION_CONFIG: Final = object_value( + VERTEX_AI_GEMINI_3_5_FLASH_TEST_CASE.expected_provider_request["generationConfig"] +) + +GEMINI_3_5_FLASH_TEMPERATURE_TEST_CASE: Final = replace( + GEMINI_3_5_FLASH_TEST_CASE, scenario="temperature", - litellm_request={**GEMINI_3_8_FLASH_TEST_CASE.litellm_request, "temperature": 0.4}, + litellm_request={**GEMINI_3_5_FLASH_TEST_CASE.litellm_request, "temperature": 0.4}, expected_provider_request={ - **GEMINI_3_8_FLASH_TEST_CASE.expected_provider_request, - "generationConfig": {**GEMINI_GENERATION_CONFIG, "temperature": 0.4}, + **GEMINI_3_5_FLASH_TEST_CASE.expected_provider_request, + "generationConfig": {**GEMINI_3_5_FLASH_GENERATION_CONFIG, "temperature": 0.4}, }, ) -GEMINI_3_8_FLASH_ZERO_TEMPERATURE_TEST_CASE: Final = replace( - GEMINI_3_8_FLASH_TEST_CASE, +GEMINI_3_5_FLASH_ZERO_TEMPERATURE_TEST_CASE: Final = replace( + GEMINI_3_5_FLASH_TEST_CASE, scenario="zero_temperature", - litellm_request={**GEMINI_3_8_FLASH_TEST_CASE.litellm_request, "temperature": 0}, + litellm_request={**GEMINI_3_5_FLASH_TEST_CASE.litellm_request, "temperature": 0}, expected_provider_request={ - **GEMINI_3_8_FLASH_TEST_CASE.expected_provider_request, - "generationConfig": {**GEMINI_GENERATION_CONFIG, "temperature": 0}, + **GEMINI_3_5_FLASH_TEST_CASE.expected_provider_request, + "generationConfig": {**GEMINI_3_5_FLASH_GENERATION_CONFIG, "temperature": 0}, }, ) @@ -47,251 +51,30 @@ GEMINI_3_8_FLASH_REASONING_EFFORT_TEST_CASE: Final = replace( expected_provider_request={ **GEMINI_3_8_FLASH_TEST_CASE.expected_provider_request, "generationConfig": { - **GEMINI_GENERATION_CONFIG, + **GEMINI_3_8_FLASH_GENERATION_CONFIG, "thinkingConfig": {"thinkingLevel": "medium", "includeThoughts": True}, }, }, ) -VERTEX_AI_GEMINI_3_8_FLASH_TEMPERATURE_THOUGHT_SIGNATURE_0: Final = ( - "AY89a18+3vC8Y6wtLQ8YLNoWj0pZ1qcMTAdhuIp6v+o7VjO8nN81l72KF3fiSKtgKQYvOKc4hz38BNeldTuj8NebwQ7BneePvMzc" - "hDx/5j5DXPVIIW7bRE2vYMhffaNpyjdltlQRDAxhiqjbSHIIGZQdxxAE7GfdGxlhstemN91VzQR4KnPKk6MXh0at/4JacXcOFRRX" - "DnYmUqj1a1tSIoSjeTsHMvSj5OpJyxxa8Fafd/yqwL8g4scNZgGzckn4O5quaQmJT9VPtoAsmh8qgXUr/qzNSZoFGzRN6m7OzSuT" - "yQ246hxKwp0+ZB9P+Nn5fvG96JVXsRq7dF00HBAbhebjUfbNTmIQauZHGSGbF2e+ck0aG5Lf8se2qRu6NCqiF+EnTFMUg7g+1cN1" - "nIXLw7X6cAPPrpXvVdkfAoHksgSFWY5uUYwW9yDTZYwOWqF6bxDapPFU5HmJj/Bf5YuVH7VhGB7fyO8SVW6AUe0/S/o0/e//wn6A" - "aFomUST2wXw0hsi/OSZLEhRAi16AeUI0Y4SenblpEYtDAyhzFi77mqsceuF7QIq672m5yWq2qoGhPh7S+5lMEvuD8YLITzrC81Xc" - "b4HN3rxOvi6skmP4WJqZO8zPIw65WQUhRdhadhxM81HE+3InBToo1IhAF+11qt6hK0EuUf1guaGrUEDZRPAgomcOLBSdK6gQ9pn5" - "z3SiRPpw9SV4LWMqBjulGRD6K8u7XHXLPAiGZTF1cr/N2sEQuyri57XytugNISVcxh0q/Lvz0v0DyUAZM8H2ymbusxnKdIHanI9Z" - "US/XIXdia3PwqiOCgXDQaceOmotUnl8i0IXvSKwMjv3wB7UaC4YKyDurZ+gJJOzbjVjZ/mEf9h4JW1ZetNWVtQl2s+oZXCHOtVJD" - "2qLDyuMPO7EuafP8w7mbpKBTmw4ZAyCfrTL73FnEYhp6M3IEG+c4fdUCHMvT6ecG42DLKhdmWfCqzTdXmXFLj/bePo7YzfYpLOLl" - "hEzVdO5ojXPd6squqM+YHQS640UkdQ4v20UQ9ghFRkW1xUl7uPQkIAI0KWiDmaoi0YGFBxWpTs1U+wN2K6hfqgtJy80rl5ceUT7f" - "6B3MnwiIoAjzVc3n3TH6GodPIO08vrhrf4oz+G3OkdCaMHXURfzYkhji50/ThfAxgD7XDuzUYlUWGotbx0FiYEDBbi2b7t7L/w3f" - "P3nXYoSgaaHumXEnOAtugy6immasBFawtZq+GrOIYHJbH2tZhMpUysSTSnGF3BmqczIctDCwC82AbXU5sKEKFShEGnJSw9B0OlWu" - "Pa6kC/Yfapk6NzDyNB2ttsLhy0hCkT0BmgDn2P4h5T4PCVgLWUbA7LGQoYO5zbsUFBwtPUsVFK0yLyf/Gg9h4ebosT6NVh7AkzS2" - "riaNfm//lGUKK/8YDY49PAu0jpHp6JoXYg/z1+BjPvITdIQgf/8a0X36uqO+Yp2HZPzZzl/Ed8sC3ZadKxulNV9kkXN2Gc3v/VyB" - "ruICqdhHSXJkusQJw+vtClqbP2uT18wQN4SRLjFTC3/ZruElXJFipevCQcC8KNVkXW39qZNWJGmKAR15mprXot6VKXkRHfyQJJz0" - "lAMTaACA5LJoA1dDn7+UaikqUsfrlLE6ObbCasA5/wIp5zFsGvslVq/vJqGWk2mdXbas2fpWTSUECEB3gGoIFo9rmHTURX4cAALa" - "oZnjSkaKy1IdC3XDQsqT2vmAX3XjvaLhkCwWg56pst5LvS0AlcRGo49QHATNsY4h0SFkUmMyAGgS7+Pa2VwmCgWUoFFHRbO8ryoy" - "W57qONicyCxA2/OL28OrKGC/e25UsXwKBAXx+9uamcdxFvOQr9WqEJK1g4Q=" -) - - -VERTEX_AI_GEMINI_3_8_FLASH_TEMPERATURE_TEST_CASE: Final = replace( - VERTEX_AI_GEMINI_3_8_FLASH_TEST_CASE, +VERTEX_AI_GEMINI_3_5_FLASH_TEMPERATURE_TEST_CASE: Final = replace( + VERTEX_AI_GEMINI_3_5_FLASH_TEST_CASE, scenario="temperature", - litellm_request={**VERTEX_AI_GEMINI_3_8_FLASH_TEST_CASE.litellm_request, "temperature": 0.4}, + litellm_request={**VERTEX_AI_GEMINI_3_5_FLASH_TEST_CASE.litellm_request, "temperature": 0.4}, expected_provider_request={ - **VERTEX_AI_GEMINI_3_8_FLASH_TEST_CASE.expected_provider_request, + **VERTEX_AI_GEMINI_3_5_FLASH_TEST_CASE.expected_provider_request, "generationConfig": {**VERTEX_AI_GENERATION_CONFIG, "temperature": 0.4}, }, - mock_provider_response={ - **VERTEX_AI_GEMINI_GENERATE_CONTENT_RESPONSE, - "candidates": [ - { - "content": { - "role": "model", - "parts": [ - { - "text": ( - "**My Thought Process on Handling Quantity Validation**\n\nAlright, to properly address the handling " - "of zero, negative, and potentially non-integer quantities, my immediate next step needs to be " - "understanding the existing validation logic. This is crucial before I can even begin to think about " - 'suggesting any fixes or further code. My protocol dictates that I must "read the relevant files ' - "before suggesting a fix,\" and that's exactly what I'm prioritizing.\n\nTherefore, I need to examine " - "the `shop/validation.py` file. This is where I anticipate finding the rules and checks that govern " - "how different types of quantities are processed and validated. My plan is to read a reasonable " - "initial segment of this file, say lines 1 through 50, to get a solid grasp of the current " - "implementation. After that, I'll be able to determine precisely *which* other files, if any, I'll " - "need to investigate next, keeping in mind the specific constraints of zero, negative, and " - 'non-integer values.\n\nTo do this, I will now use the `read_file` tool:\n\n```json\n{\n "tool_code": ' - "\"print(read_file(path='shop/validation.py', start_line=1, end_line=50))\"\n}\n```\n\n" - ), - "thought": True, - }, - { - "text": ( - "To examine how zero, negative, and non-integer quantities are handled, I need to inspect " - "`shop/validation.py` next." - ), - }, - { - "functionCall": { - "name": "read_file", - "args": {"start_line": 1, "end_line": 50, "path": "shop/validation.py"}, - "id": "call_1464376", - }, - "thoughtSignature": VERTEX_AI_GEMINI_3_8_FLASH_TEMPERATURE_THOUGHT_SIGNATURE_0, - }, - ], - }, - "finishReason": "STOP", - }, - ], - "usageMetadata": { - "promptTokenCount": 246, - "candidatesTokenCount": 61, - "totalTokenCount": 646, - "trafficType": "ON_DEMAND", - "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 246}], - "candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 61}], - "thoughtsTokenCount": 339, - }, - "modelVersion": "gemini-3.8-flash", - "createTime": "2026-10-15T01:22:48.202504Z", - "responseId": "6CrQaoiuDNKZq8YPjZCokAw", - }, - expected_litellm_response={ - **CHAT_COMPLETIONS_RESPONSE, - "model": "vertex_ai/gemini-3.8-flash", - "choices": [ - { - "finish_reason": "tool_calls", - "index": 0, - "message": { - "content": ( - "To examine how zero, negative, and non-integer quantities are handled, I need to inspect " - "`shop/validation.py` next." - ), - "role": "assistant", - "tool_calls": [ - { - "index": 0, - "provider_specific_fields": { - "thought_signature": ( - "AY89a18+3vC8Y6wtLQ8YLNoWj0pZ1qcMTAdhuIp6v+o7VjO8nN81l72KF3fiSKtgKQYvOKc4hz38BNeldTuj8NebwQ7BneePvMzc" - "hDx/5j5DXPVIIW7bRE2vYMhffaNpyjdltlQRDAxhiqjbSHIIGZQdxxAE7GfdGxlhstemN91VzQR4KnPKk6MXh0at/4JacXcOFRRX" - "DnYmUqj1a1tSIoSjeTsHMvSj5OpJyxxa8Fafd/yqwL8g4scNZgGzckn4O5quaQmJT9VPtoAsmh8qgXUr/qzNSZoFGzRN6m7OzSuT" - "yQ246hxKwp0+ZB9P+Nn5fvG96JVXsRq7dF00HBAbhebjUfbNTmIQauZHGSGbF2e+ck0aG5Lf8se2qRu6NCqiF+EnTFMUg7g+1cN1" - "nIXLw7X6cAPPrpXvVdkfAoHksgSFWY5uUYwW9yDTZYwOWqF6bxDapPFU5HmJj/Bf5YuVH7VhGB7fyO8SVW6AUe0/S/o0/e//wn6A" - "aFomUST2wXw0hsi/OSZLEhRAi16AeUI0Y4SenblpEYtDAyhzFi77mqsceuF7QIq672m5yWq2qoGhPh7S+5lMEvuD8YLITzrC81Xc" - "b4HN3rxOvi6skmP4WJqZO8zPIw65WQUhRdhadhxM81HE+3InBToo1IhAF+11qt6hK0EuUf1guaGrUEDZRPAgomcOLBSdK6gQ9pn5" - "z3SiRPpw9SV4LWMqBjulGRD6K8u7XHXLPAiGZTF1cr/N2sEQuyri57XytugNISVcxh0q/Lvz0v0DyUAZM8H2ymbusxnKdIHanI9Z" - "US/XIXdia3PwqiOCgXDQaceOmotUnl8i0IXvSKwMjv3wB7UaC4YKyDurZ+gJJOzbjVjZ/mEf9h4JW1ZetNWVtQl2s+oZXCHOtVJD" - "2qLDyuMPO7EuafP8w7mbpKBTmw4ZAyCfrTL73FnEYhp6M3IEG+c4fdUCHMvT6ecG42DLKhdmWfCqzTdXmXFLj/bePo7YzfYpLOLl" - "hEzVdO5ojXPd6squqM+YHQS640UkdQ4v20UQ9ghFRkW1xUl7uPQkIAI0KWiDmaoi0YGFBxWpTs1U+wN2K6hfqgtJy80rl5ceUT7f" - "6B3MnwiIoAjzVc3n3TH6GodPIO08vrhrf4oz+G3OkdCaMHXURfzYkhji50/ThfAxgD7XDuzUYlUWGotbx0FiYEDBbi2b7t7L/w3f" - "P3nXYoSgaaHumXEnOAtugy6immasBFawtZq+GrOIYHJbH2tZhMpUysSTSnGF3BmqczIctDCwC82AbXU5sKEKFShEGnJSw9B0OlWu" - "Pa6kC/Yfapk6NzDyNB2ttsLhy0hCkT0BmgDn2P4h5T4PCVgLWUbA7LGQoYO5zbsUFBwtPUsVFK0yLyf/Gg9h4ebosT6NVh7AkzS2" - "riaNfm//lGUKK/8YDY49PAu0jpHp6JoXYg/z1+BjPvITdIQgf/8a0X36uqO+Yp2HZPzZzl/Ed8sC3ZadKxulNV9kkXN2Gc3v/VyB" - "ruICqdhHSXJkusQJw+vtClqbP2uT18wQN4SRLjFTC3/ZruElXJFipevCQcC8KNVkXW39qZNWJGmKAR15mprXot6VKXkRHfyQJJz0" - "lAMTaACA5LJoA1dDn7+UaikqUsfrlLE6ObbCasA5/wIp5zFsGvslVq/vJqGWk2mdXbas2fpWTSUECEB3gGoIFo9rmHTURX4cAALa" - "oZnjSkaKy1IdC3XDQsqT2vmAX3XjvaLhkCwWg56pst5LvS0AlcRGo49QHATNsY4h0SFkUmMyAGgS7+Pa2VwmCgWUoFFHRbO8ryoy" - "W57qONicyCxA2/OL28OrKGC/e25UsXwKBAXx+9uamcdxFvOQr9WqEJK1g4Q=" - ), - }, - "function": { - "arguments": '{"start_line": 1, "end_line": 50, "path": "shop/validation.py"}', - "name": "read_file", - }, - "id": ( - "call_1464376__thought__AY89a18+3vC8Y6wtLQ8YLNoWj0pZ1qcMTAdhuIp6v+o7VjO8nN81l72KF3fiSKtgKQYvOKc4hz38B" - "NeldTuj8NebwQ7BneePvMzchDx/5j5DXPVIIW7bRE2vYMhffaNpyjdltlQRDAxhiqjbSHIIGZQdxxAE7GfdGxlhstemN91VzQR4K" - "nPKk6MXh0at/4JacXcOFRRXDnYmUqj1a1tSIoSjeTsHMvSj5OpJyxxa8Fafd/yqwL8g4scNZgGzckn4O5quaQmJT9VPtoAsmh8qg" - "XUr/qzNSZoFGzRN6m7OzSuTyQ246hxKwp0+ZB9P+Nn5fvG96JVXsRq7dF00HBAbhebjUfbNTmIQauZHGSGbF2e+ck0aG5Lf8se2q" - "Ru6NCqiF+EnTFMUg7g+1cN1nIXLw7X6cAPPrpXvVdkfAoHksgSFWY5uUYwW9yDTZYwOWqF6bxDapPFU5HmJj/Bf5YuVH7VhGB7fy" - "O8SVW6AUe0/S/o0/e//wn6AaFomUST2wXw0hsi/OSZLEhRAi16AeUI0Y4SenblpEYtDAyhzFi77mqsceuF7QIq672m5yWq2qoGhP" - "h7S+5lMEvuD8YLITzrC81Xcb4HN3rxOvi6skmP4WJqZO8zPIw65WQUhRdhadhxM81HE+3InBToo1IhAF+11qt6hK0EuUf1guaGrU" - "EDZRPAgomcOLBSdK6gQ9pn5z3SiRPpw9SV4LWMqBjulGRD6K8u7XHXLPAiGZTF1cr/N2sEQuyri57XytugNISVcxh0q/Lvz0v0Dy" - "UAZM8H2ymbusxnKdIHanI9ZUS/XIXdia3PwqiOCgXDQaceOmotUnl8i0IXvSKwMjv3wB7UaC4YKyDurZ+gJJOzbjVjZ/mEf9h4JW" - "1ZetNWVtQl2s+oZXCHOtVJD2qLDyuMPO7EuafP8w7mbpKBTmw4ZAyCfrTL73FnEYhp6M3IEG+c4fdUCHMvT6ecG42DLKhdmWfCqz" - "TdXmXFLj/bePo7YzfYpLOLlhEzVdO5ojXPd6squqM+YHQS640UkdQ4v20UQ9ghFRkW1xUl7uPQkIAI0KWiDmaoi0YGFBxWpTs1U+" - "wN2K6hfqgtJy80rl5ceUT7f6B3MnwiIoAjzVc3n3TH6GodPIO08vrhrf4oz+G3OkdCaMHXURfzYkhji50/ThfAxgD7XDuzUYlUWG" - "otbx0FiYEDBbi2b7t7L/w3fP3nXYoSgaaHumXEnOAtugy6immasBFawtZq+GrOIYHJbH2tZhMpUysSTSnGF3BmqczIctDCwC82Ab" - "XU5sKEKFShEGnJSw9B0OlWuPa6kC/Yfapk6NzDyNB2ttsLhy0hCkT0BmgDn2P4h5T4PCVgLWUbA7LGQoYO5zbsUFBwtPUsVFK0yL" - "yf/Gg9h4ebosT6NVh7AkzS2riaNfm//lGUKK/8YDY49PAu0jpHp6JoXYg/z1+BjPvITdIQgf/8a0X36uqO+Yp2HZPzZzl/Ed8sC3" - "ZadKxulNV9kkXN2Gc3v/VyBruICqdhHSXJkusQJw+vtClqbP2uT18wQN4SRLjFTC3/ZruElXJFipevCQcC8KNVkXW39qZNWJGmKA" - "R15mprXot6VKXkRHfyQJJz0lAMTaACA5LJoA1dDn7+UaikqUsfrlLE6ObbCasA5/wIp5zFsGvslVq/vJqGWk2mdXbas2fpWTSUEC" - "EB3gGoIFo9rmHTURX4cAALaoZnjSkaKy1IdC3XDQsqT2vmAX3XjvaLhkCwWg56pst5LvS0AlcRGo49QHATNsY4h0SFkUmMyAGgS7" - "+Pa2VwmCgWUoFFHRbO8ryoyW57qONicyCxA2/OL28OrKGC/e25UsXwKBAXx+9uamcdxFvOQr9WqEJK1g4Q=" - ), - "type": "function", - }, - ], - "images": [], - "reasoning_content": ( - "**My Thought Process on Handling Quantity Validation**\n\nAlright, to properly address the handling " - "of zero, negative, and potentially non-integer quantities, my immediate next step needs to be " - "understanding the existing validation logic. This is crucial before I can even begin to think about " - 'suggesting any fixes or further code. My protocol dictates that I must "read the relevant files ' - "before suggesting a fix,\" and that's exactly what I'm prioritizing.\n\nTherefore, I need to examine " - "the `shop/validation.py` file. This is where I anticipate finding the rules and checks that govern " - "how different types of quantities are processed and validated. My plan is to read a reasonable " - "initial segment of this file, say lines 1 through 50, to get a solid grasp of the current " - "implementation. After that, I'll be able to determine precisely *which* other files, if any, I'll " - "need to investigate next, keeping in mind the specific constraints of zero, negative, and " - 'non-integer values.\n\nTo do this, I will now use the `read_file` tool:\n\n```json\n{\n "tool_code": ' - "\"print(read_file(path='shop/validation.py', start_line=1, end_line=50))\"\n}\n```\n\n" - ), - "thinking_blocks": [ - { - "type": "thinking", - "thinking": ( - "**My Thought Process on Handling Quantity Validation**\n\nAlright, to properly address the handling " - "of zero, negative, and potentially non-integer quantities, my immediate next step needs to be " - "understanding the existing validation logic. This is crucial before I can even begin to think about " - 'suggesting any fixes or further code. My protocol dictates that I must "read the relevant files ' - "before suggesting a fix,\" and that's exactly what I'm prioritizing.\n\nTherefore, I need to examine " - "the `shop/validation.py` file. This is where I anticipate finding the rules and checks that govern " - "how different types of quantities are processed and validated. My plan is to read a reasonable " - "initial segment of this file, say lines 1 through 50, to get a solid grasp of the current " - "implementation. After that, I'll be able to determine precisely *which* other files, if any, I'll " - "need to investigate next, keeping in mind the specific constraints of zero, negative, and " - 'non-integer values.\n\nTo do this, I will now use the `read_file` tool:\n\n```json\n{\n "tool_code": ' - "\"print(read_file(path='shop/validation.py', start_line=1, end_line=50))\"\n}\n```\n\n" - ), - }, - ], - "provider_specific_fields": { - "thought_signatures": [ - ( - "AY89a18+3vC8Y6wtLQ8YLNoWj0pZ1qcMTAdhuIp6v+o7VjO8nN81l72KF3fiSKtgKQYvOKc4hz38BNeldTuj8NebwQ7BneePvMzc" - "hDx/5j5DXPVIIW7bRE2vYMhffaNpyjdltlQRDAxhiqjbSHIIGZQdxxAE7GfdGxlhstemN91VzQR4KnPKk6MXh0at/4JacXcOFRRX" - "DnYmUqj1a1tSIoSjeTsHMvSj5OpJyxxa8Fafd/yqwL8g4scNZgGzckn4O5quaQmJT9VPtoAsmh8qgXUr/qzNSZoFGzRN6m7OzSuT" - "yQ246hxKwp0+ZB9P+Nn5fvG96JVXsRq7dF00HBAbhebjUfbNTmIQauZHGSGbF2e+ck0aG5Lf8se2qRu6NCqiF+EnTFMUg7g+1cN1" - "nIXLw7X6cAPPrpXvVdkfAoHksgSFWY5uUYwW9yDTZYwOWqF6bxDapPFU5HmJj/Bf5YuVH7VhGB7fyO8SVW6AUe0/S/o0/e//wn6A" - "aFomUST2wXw0hsi/OSZLEhRAi16AeUI0Y4SenblpEYtDAyhzFi77mqsceuF7QIq672m5yWq2qoGhPh7S+5lMEvuD8YLITzrC81Xc" - "b4HN3rxOvi6skmP4WJqZO8zPIw65WQUhRdhadhxM81HE+3InBToo1IhAF+11qt6hK0EuUf1guaGrUEDZRPAgomcOLBSdK6gQ9pn5" - "z3SiRPpw9SV4LWMqBjulGRD6K8u7XHXLPAiGZTF1cr/N2sEQuyri57XytugNISVcxh0q/Lvz0v0DyUAZM8H2ymbusxnKdIHanI9Z" - "US/XIXdia3PwqiOCgXDQaceOmotUnl8i0IXvSKwMjv3wB7UaC4YKyDurZ+gJJOzbjVjZ/mEf9h4JW1ZetNWVtQl2s+oZXCHOtVJD" - "2qLDyuMPO7EuafP8w7mbpKBTmw4ZAyCfrTL73FnEYhp6M3IEG+c4fdUCHMvT6ecG42DLKhdmWfCqzTdXmXFLj/bePo7YzfYpLOLl" - "hEzVdO5ojXPd6squqM+YHQS640UkdQ4v20UQ9ghFRkW1xUl7uPQkIAI0KWiDmaoi0YGFBxWpTs1U+wN2K6hfqgtJy80rl5ceUT7f" - "6B3MnwiIoAjzVc3n3TH6GodPIO08vrhrf4oz+G3OkdCaMHXURfzYkhji50/ThfAxgD7XDuzUYlUWGotbx0FiYEDBbi2b7t7L/w3f" - "P3nXYoSgaaHumXEnOAtugy6immasBFawtZq+GrOIYHJbH2tZhMpUysSTSnGF3BmqczIctDCwC82AbXU5sKEKFShEGnJSw9B0OlWu" - "Pa6kC/Yfapk6NzDyNB2ttsLhy0hCkT0BmgDn2P4h5T4PCVgLWUbA7LGQoYO5zbsUFBwtPUsVFK0yLyf/Gg9h4ebosT6NVh7AkzS2" - "riaNfm//lGUKK/8YDY49PAu0jpHp6JoXYg/z1+BjPvITdIQgf/8a0X36uqO+Yp2HZPzZzl/Ed8sC3ZadKxulNV9kkXN2Gc3v/VyB" - "ruICqdhHSXJkusQJw+vtClqbP2uT18wQN4SRLjFTC3/ZruElXJFipevCQcC8KNVkXW39qZNWJGmKAR15mprXot6VKXkRHfyQJJz0" - "lAMTaACA5LJoA1dDn7+UaikqUsfrlLE6ObbCasA5/wIp5zFsGvslVq/vJqGWk2mdXbas2fpWTSUECEB3gGoIFo9rmHTURX4cAALa" - "oZnjSkaKy1IdC3XDQsqT2vmAX3XjvaLhkCwWg56pst5LvS0AlcRGo49QHATNsY4h0SFkUmMyAGgS7+Pa2VwmCgWUoFFHRbO8ryoy" - "W57qONicyCxA2/OL28OrKGC/e25UsXwKBAXx+9uamcdxFvOQr9WqEJK1g4Q=" - ), - ], - }, - }, - "provider_specific_fields": {"native_finish_reason": "STOP"}, - }, - ], - "usage": { - "completion_tokens": 400, - "prompt_tokens": 246, - "total_tokens": 646, - "completion_tokens_details": {"reasoning_tokens": 339, "text_tokens": 61}, - "prompt_tokens_details": {"text_tokens": 246}, - }, - "vertex_ai_grounding_metadata": [], - "vertex_ai_url_context_metadata": [], - "vertex_ai_safety_results": [], - "vertex_ai_citation_metadata": [], - }, ) @pytest.mark.parametrize( "case", [ - GEMINI_3_8_FLASH_TEMPERATURE_TEST_CASE, - GEMINI_3_8_FLASH_ZERO_TEMPERATURE_TEST_CASE, + GEMINI_3_5_FLASH_TEMPERATURE_TEST_CASE, + GEMINI_3_5_FLASH_ZERO_TEMPERATURE_TEST_CASE, GEMINI_3_8_FLASH_REASONING_EFFORT_TEST_CASE, - VERTEX_AI_GEMINI_3_8_FLASH_TEMPERATURE_TEST_CASE, + VERTEX_AI_GEMINI_3_5_FLASH_TEMPERATURE_TEST_CASE, ], ids=lambda case: case.id, ) diff --git a/tests/integration/translation/messages/sampling/test_messages_sampling_gemini.py b/tests/integration/translation/messages/sampling/test_messages_sampling_gemini.py index e2b54774b1b..99d05872656 100644 --- a/tests/integration/translation/messages/sampling/test_messages_sampling_gemini.py +++ b/tests/integration/translation/messages/sampling/test_messages_sampling_gemini.py @@ -5,22 +5,22 @@ import pytest from integration._support.client import Gateway, object_value from integration._support.provider import SharedProvider from integration.translation.case import TranslationTestCase -from integration.translation.messages.bases.gemini import GEMINI_3_8_FLASH_TEST_CASE +from integration.translation.messages.bases.gemini import GEMINI_3_5_FLASH_TEST_CASE from integration.translation.runner import assert_translation -GENERATION_CONFIG: Final = object_value(GEMINI_3_8_FLASH_TEST_CASE.expected_provider_request["generationConfig"]) +GENERATION_CONFIG: Final = object_value(GEMINI_3_5_FLASH_TEST_CASE.expected_provider_request["generationConfig"]) -GEMINI_3_8_FLASH_TEMPERATURE_TEST_CASE: Final = replace( - GEMINI_3_8_FLASH_TEST_CASE, +GEMINI_3_5_FLASH_TEMPERATURE_TEST_CASE: Final = replace( + GEMINI_3_5_FLASH_TEST_CASE, scenario="temperature", - litellm_request={**GEMINI_3_8_FLASH_TEST_CASE.litellm_request, "temperature": 0.4}, + litellm_request={**GEMINI_3_5_FLASH_TEST_CASE.litellm_request, "temperature": 0.4}, expected_provider_request={ - **GEMINI_3_8_FLASH_TEST_CASE.expected_provider_request, + **GEMINI_3_5_FLASH_TEST_CASE.expected_provider_request, "generationConfig": {**GENERATION_CONFIG, "temperature": 0.4}, }, ) -@pytest.mark.parametrize("case", [GEMINI_3_8_FLASH_TEMPERATURE_TEST_CASE], ids=lambda case: case.id) +@pytest.mark.parametrize("case", [GEMINI_3_5_FLASH_TEMPERATURE_TEST_CASE], ids=lambda case: case.id) def test_messages_sampling_gemini(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: assert_translation(case, gateway, provider) diff --git a/tests/integration/translation/responses/sampling/test_responses_sampling_gemini.py b/tests/integration/translation/responses/sampling/test_responses_sampling_gemini.py index 6e82132aecb..1b07c9fda7c 100644 --- a/tests/integration/translation/responses/sampling/test_responses_sampling_gemini.py +++ b/tests/integration/translation/responses/sampling/test_responses_sampling_gemini.py @@ -5,23 +5,23 @@ import pytest from integration._support.client import Gateway, object_value from integration._support.provider import SharedProvider from integration.translation.case import TranslationTestCase -from integration.translation.responses.bases.gemini import GEMINI_3_8_FLASH_TEST_CASE +from integration.translation.responses.bases.gemini import GEMINI_3_5_FLASH_TEST_CASE from integration.translation.runner import assert_translation -GENERATION_CONFIG: Final = object_value(GEMINI_3_8_FLASH_TEST_CASE.expected_provider_request["generationConfig"]) +GENERATION_CONFIG: Final = object_value(GEMINI_3_5_FLASH_TEST_CASE.expected_provider_request["generationConfig"]) -GEMINI_3_8_FLASH_TEMPERATURE_TEST_CASE: Final = replace( - GEMINI_3_8_FLASH_TEST_CASE, +GEMINI_3_5_FLASH_TEMPERATURE_TEST_CASE: Final = replace( + GEMINI_3_5_FLASH_TEST_CASE, scenario="temperature", - litellm_request={**GEMINI_3_8_FLASH_TEST_CASE.litellm_request, "temperature": 0.4}, + litellm_request={**GEMINI_3_5_FLASH_TEST_CASE.litellm_request, "temperature": 0.4}, expected_provider_request={ - **GEMINI_3_8_FLASH_TEST_CASE.expected_provider_request, + **GEMINI_3_5_FLASH_TEST_CASE.expected_provider_request, "generationConfig": {**GENERATION_CONFIG, "temperature": 0.4}, }, - expected_litellm_response={**GEMINI_3_8_FLASH_TEST_CASE.expected_litellm_response, "temperature": 0.4}, + expected_litellm_response={**GEMINI_3_5_FLASH_TEST_CASE.expected_litellm_response, "temperature": 0.4}, ) -@pytest.mark.parametrize("case", [GEMINI_3_8_FLASH_TEMPERATURE_TEST_CASE], ids=lambda case: case.id) +@pytest.mark.parametrize("case", [GEMINI_3_5_FLASH_TEMPERATURE_TEST_CASE], ids=lambda case: case.id) def test_responses_sampling_gemini(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: assert_translation(case, gateway, provider)