From de40c0aa39e26d318eaa3fc0337a788165862972 Mon Sep 17 00:00:00 2001 From: tessellark <349031000+tessellark@users.noreply.github.com> Date: Wed, 28 Oct 2026 03:38:21 -0700 Subject: [PATCH] test(llms): audit #47535 (#48786) Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../providers/_api_base_url_join_support.py | 9 ++ ...test_api_base_url_join_host_owned_proxy.py | 37 ++++++- .../providers/test_api_base_url_join_wire.py | 99 +++++++++++++++++-- 3 files changed, 134 insertions(+), 11 deletions(-) diff --git a/tests/integration/providers/_api_base_url_join_support.py b/tests/integration/providers/_api_base_url_join_support.py index 44bd6ec4dca..5711cc77309 100644 --- a/tests/integration/providers/_api_base_url_join_support.py +++ b/tests/integration/providers/_api_base_url_join_support.py @@ -13,6 +13,7 @@ from pydantic import JsonValue COUNTED_TOKENS: Final = 7 RELEVANCE: Final = 0.75 +AZURE_DEPLOYMENT: Final = "gpt-url-join" _USAGE: Final = {"input_tokens": 3, "output_tokens": 2} @@ -146,12 +147,20 @@ def _cohere_rerank(request: Request) -> Reply: ) +def _titan_embedding(_: Request) -> Reply: + return _json({"embedding": list(VECTOR), "inputTextTokenCount": 2}) + + _BY_PREFIX: Final[Mapping[str, Callable[[Request], Reply]]] = MappingProxyType( { "an": _anthropic, "bi": lambda request: _json(anthropic_message(marker_of(request))), "bc": _bedrock_converse, + "be": _titan_embedding, "gm": _gemini, + "vx": _gemini, + "fd": _anthropic, + "az": _openai_like(AZURE_DEPLOYMENT), "mi": _openai_like("mistral-small-latest"), "db": _openai_like("databricks-meta-llama-3-3-70b-instruct"), "co": _cohere_rerank, diff --git a/tests/integration/providers/test_api_base_url_join_host_owned_proxy.py b/tests/integration/providers/test_api_base_url_join_host_owned_proxy.py index 4a075138305..687c15283f4 100644 --- a/tests/integration/providers/test_api_base_url_join_host_owned_proxy.py +++ b/tests/integration/providers/test_api_base_url_join_host_owned_proxy.py @@ -1,6 +1,7 @@ import os from collections.abc import Callable, Iterator from pathlib import Path +from types import MappingProxyType from typing import Final import pytest @@ -8,6 +9,7 @@ import yaml from integration._support.client import JSON_OBJECT, Gateway, gateway_from_environment from integration._support.forward_proxy import ForwardProxy, refusing_forward_proxy from integration._support.process import graceful_stop_seconds, owned_proxy +from integration._support.vertex import service_account_json from integration.providers._api_base_trailing_slash_support import KEY, new_marker from pydantic import JsonValue @@ -17,6 +19,12 @@ _RERANK: Final = "host-rerank" _AZURE_CLAUDE: Final = "host-azure-claude" _AZURE_COUNT: Final = "host-azure-count" _EMPTY_BASE: Final = "host-empty-base" +_VERTEX_HOST: Final = "vertex-gateway" +_VERTEX_PROJECT: Final = "url-join-project" +_VERTEX_LOCATION: Final = "us-east5" +_VERTEX_CLAUDE_BASES: Final = MappingProxyType( + {"host-vertex-claude": f"http://{_VERTEX_HOST}", "host-vertex-claude-slash": f"http://{_VERTEX_HOST}/"} +) _SCRUBBED_PREFIXES: Final = ("ANTHROPIC_", "AZURE_AI_", "INFINITY_") _PROXY_VARIABLES: Final = frozenset({"HTTPS_PROXY", "HTTP_PROXY", "ALL_PROXY", "NO_PROXY"}) _load_yaml: Final[Callable[[str], object]] = yaml.safe_load @@ -28,7 +36,20 @@ def _deployment(name: str, model: str, api_base: str) -> dict[str, JsonValue]: return {"model_name": name, "litellm_params": {"model": model, "api_base": api_base, "api_key": KEY}} -def _config(directory: Path) -> Path: +def _vertex_claude_deployment(name: str, api_base: str, token_url: str) -> dict[str, JsonValue]: + return { + "model_name": name, + "litellm_params": { + "model": "vertex_ai/claude-haiku-5-5", + "api_base": api_base, + "vertex_project": _VERTEX_PROJECT, + "vertex_location": _VERTEX_LOCATION, + "vertex_credentials": service_account_json(_VERTEX_PROJECT, token_url), + }, + } + + +def _config(directory: Path, token_url: str) -> Path: base: Final = JSON_OBJECT.validate_python(_load_yaml(Path("tests/integration/proxy_config.yaml").read_text())) configuration: Final = { **base, @@ -38,6 +59,7 @@ def _config(directory: Path) -> Path: _deployment(_AZURE_CLAUDE, "azure_ai/claude-haiku-5-5", f"http://{_AZURE_HOST}/anthropic"), _deployment(_AZURE_COUNT, "azure_ai/claude-haiku-5-5", f"http://{_AZURE_HOST}"), _deployment(_EMPTY_BASE, "anthropic/claude-haiku-5-5", ""), + *(_vertex_claude_deployment(name, base, token_url) for name, base in _VERTEX_CLAUDE_BASES.items()), ], "router_settings": {"disable_cooldowns": True, "num_retries": 0}, } @@ -72,7 +94,7 @@ def isolated(forward: ForwardProxy, tmp_path_factory: pytest.TempPathFactory) -> "NO_PROXY": "127.0.0.1,localhost", "PROXY_BATCH_POLLING_ENABLED": "false", }, - config=_config(directory), + config=_config(directory, shared.upstream_url.rstrip("/")), remove_environment=_scrubbed(), ) as candidate: yield candidate @@ -138,3 +160,14 @@ def test_azure_claude_count_tokens_on_a_bare_host_starting_with_anthropic_adds_t def test_empty_api_base_falls_back_to_the_provider_default_host(isolated: Gateway, forward: ForwardProxy) -> None: targets: Final = _targets_after(forward, lambda: _chat(isolated, _EMPTY_BASE)) assert targets == {"api.anthropic.com:443"} + + +@pytest.mark.parametrize("model", tuple(_VERTEX_CLAUDE_BASES), ids=("no-slash", "slash")) +def test_vertex_partner_claude_on_a_bare_host_reaches_one_single_slash_url( + isolated: Gateway, forward: ForwardProxy, model: str +) -> None: + targets: Final = _targets_after(forward, lambda: _chat(isolated, model)) + assert targets == { + f"http://{_VERTEX_HOST}/v1/projects/{_VERTEX_PROJECT}/locations/{_VERTEX_LOCATION}" + "/publishers/anthropic/models/claude-haiku-5-5:rawPredict" + } diff --git a/tests/integration/providers/test_api_base_url_join_wire.py b/tests/integration/providers/test_api_base_url_join_wire.py index 42e53ff9f95..74f8dde3cf7 100644 --- a/tests/integration/providers/test_api_base_url_join_wire.py +++ b/tests/integration/providers/test_api_base_url_join_wire.py @@ -7,6 +7,7 @@ import openai import pytest from integration._support.client import Gateway, Scenario, eventually, list_value, object_value, string_value from integration._support.database import read_rows +from integration._support.vertex import service_account_json from integration._support.wire import Reply, Request, wire_server from integration.providers._api_base_trailing_slash_support import ( KEY, @@ -22,7 +23,7 @@ from integration.providers._api_base_trailing_slash_support import ( reply_text, request_json, ) -from integration.providers._api_base_url_join_support import COUNTED_TOKENS, RELEVANCE, join_peer +from integration.providers._api_base_url_join_support import AZURE_DEPLOYMENT, COUNTED_TOKENS, RELEVANCE, join_peer from pydantic import JsonValue _HAIKU: Final = "anthropic.claude-haiku-5-5" @@ -49,10 +50,20 @@ _RESPONSES_ROWS: Final = (_ANTHROPIC, _GEMINI) _EMBEDDING_ROWS: Final = ( Row("mistral/mistral-embed", "/mi/v1", "/embeddings"), Row("gemini/gemini-embedding-2", "/gm/v1beta", "/models/gemini-embedding-2:batchEmbedContents"), + Row("bedrock/amazon.titan-embed-text-v2:0", "/be", "/model/amazon.titan-embed-text-v2%3A0/invoke"), ) _TRANSCRIPTION: Final = Row("mistral/voxtral-mini-latest", "/mi/v1", "/audio/transcriptions") -_FULL_ROUTE_SUFFIXES: Final = ("", "/", "?beta=true") -_FULL_ROUTE_IDS: Final = ("exact", "slash", "query") +_FULL_ROUTE_SUFFIXES: Final = (("", ""), ("/", ""), ("?beta=true", "?beta=true"), ("/?beta=true", "?beta=true")) +_FULL_ROUTE_IDS: Final = ("exact", "slash", "query", "slash-query") +_VERTEX_PROJECT: Final = "url-join-project" +_VERTEX_LOCATION: Final = "us-east5" +_VERTEX_GEMINI_PATH: Final = ( + f"/vx/v1/projects/{_VERTEX_PROJECT}/locations/{_VERTEX_LOCATION}/publishers/google/models/gemini-3.8-flash" +) +_AZURE_V1_BASES: Final = ("/az", "/az/", "/az/openai/v1", "/az/openai/v1/") +_AZURE_V1_IDS: Final = ("host", "host-slash", "route", "route-slash") +_AZURE_ANTHROPIC_BASES: Final = ("/fd/anthropic", "/fd/anthropic/v1/messages") +_AZURE_ANTHROPIC_IDS: Final = ("anthropic-root", "messages-route") def _ids(rows: tuple[Row, ...]) -> tuple[str, ...]: @@ -65,6 +76,17 @@ def _deploy(scenario: Scenario, row: Row, api_base: str) -> str: return scenario.model(model=row.model, api_base=api_base, api_key=KEY) +def _deploy_vertex(gateway: Gateway, scenario: Scenario, model: str, api_base: str) -> str: + return scenario.model( + model=model, + api_base=api_base, + api_key=None, + vertex_project=_VERTEX_PROJECT, + vertex_location=_VERTEX_LOCATION, + vertex_credentials=service_account_json(_VERTEX_PROJECT, gateway.upstream_url.rstrip("/")), + ) + + def _openai(gateway: Gateway) -> openai.OpenAI: return openai.OpenAI(base_url=str(gateway.client.base_url) + "/v1", api_key=gateway.key, max_retries=0) @@ -80,6 +102,13 @@ def _chat(gateway: Gateway, model: str, marker: str) -> str | None: return completion.choices[0].message.content +def _streamed_chat(gateway: Gateway, model: str, marker: str) -> str: + stream: Final = _openai(gateway).chat.completions.create( + model=model, messages=[{"role": "user", "content": marker}], stream=True + ) + return "".join(chunk.choices[0].delta.content or "" for chunk in stream if chunk.choices) + + def _message_text(gateway: Gateway, model: str, marker: str) -> list[str]: message: Final = _anthropic(gateway).messages.create( model=model, max_tokens=16, messages=[{"role": "user", "content": marker}] @@ -270,22 +299,74 @@ def test_cache_hit_on_a_slash_base_skips_the_upstream_and_logs_its_own_row(gatew assert "//" not in urlsplit(string_value(hit["api_base"])).path, hit -@pytest.mark.parametrize("suffix", _FULL_ROUTE_SUFFIXES, ids=_FULL_ROUTE_IDS) -def test_chat_base_that_already_names_the_route_keeps_its_path_and_query(gateway: Gateway, suffix: str) -> None: +@pytest.mark.parametrize(("suffix", "sent"), _FULL_ROUTE_SUFFIXES, ids=_FULL_ROUTE_IDS) +def test_chat_base_that_already_names_the_route_keeps_its_path_and_query( + gateway: Gateway, suffix: str, sent: str +) -> None: marker: Final = new_marker() with wire_server(join_peer) as wire, gateway.scenario() as scenario: model: Final = _deploy(scenario, _ANTHROPIC, f"{wire.url}/an/v1/messages{suffix}") assert _chat(gateway, model, marker) == reply_text(marker) - assert_one_request(wire.drain(), f"/an/v1/messages{suffix.rstrip('/')}", marker) + assert_one_request(wire.drain(), f"/an/v1/messages{sent}", marker) -@pytest.mark.parametrize("suffix", _FULL_ROUTE_SUFFIXES, ids=_FULL_ROUTE_IDS) -def test_messages_base_that_already_names_the_route_keeps_its_path_and_query(gateway: Gateway, suffix: str) -> None: +@pytest.mark.parametrize(("suffix", "sent"), _FULL_ROUTE_SUFFIXES, ids=_FULL_ROUTE_IDS) +def test_messages_base_that_already_names_the_route_keeps_its_path_and_query( + gateway: Gateway, suffix: str, sent: str +) -> None: marker: Final = new_marker() with wire_server(join_peer) as wire, gateway.scenario() as scenario: model: Final = _deploy(scenario, _ANTHROPIC, f"{wire.url}/an/v1/messages{suffix}") assert _message_text(gateway, model, marker) == [reply_text(marker)] - assert_one_request(wire.drain(), f"/an/v1/messages{suffix.rstrip('/')}", marker) + assert_one_request(wire.drain(), f"/an/v1/messages{sent}", marker) + + +@pytest.mark.parametrize("trailing", TRAILING, ids=TRAILING_IDS) +def test_vertex_model_url_base_calls_its_method_with_no_slash_before_the_colon(gateway: Gateway, trailing: str) -> None: + marker: Final = new_marker() + with wire_server(join_peer) as wire, gateway.scenario() as scenario: + model: Final = _deploy_vertex( + gateway, scenario, "vertex_ai/gemini-3.8-flash", f"{wire.url}{_VERTEX_GEMINI_PATH}{trailing}" + ) + assert _chat(gateway, model, marker) == reply_text(marker) + assert_one_request(wire.drain(), f"{_VERTEX_GEMINI_PATH}:generateContent", marker, keyed=False) + + +@pytest.mark.parametrize("trailing", TRAILING, ids=TRAILING_IDS) +def test_vertex_model_url_base_streams_with_no_slash_before_the_colon(gateway: Gateway, trailing: str) -> None: + marker: Final = new_marker() + with wire_server(join_peer) as wire, gateway.scenario() as scenario: + model: Final = _deploy_vertex( + gateway, scenario, "vertex_ai/gemini-3.8-flash", f"{wire.url}{_VERTEX_GEMINI_PATH}{trailing}" + ) + assert _streamed_chat(gateway, model, marker) == reply_text(marker) + assert_one_request(wire.drain(), f"{_VERTEX_GEMINI_PATH}:streamGenerateContent?alt=sse", marker, keyed=False) + + +@pytest.mark.parametrize("base", _AZURE_V1_BASES, ids=_AZURE_V1_IDS) +def test_azure_v1_chat_reaches_one_openai_v1_route(gateway: Gateway, base: str) -> None: + marker: Final = new_marker() + with wire_server(join_peer) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=f"azure/{AZURE_DEPLOYMENT}", api_base=f"{wire.url}{base}", api_key=KEY, api_version="v1" + ) + assert _chat(gateway, model, marker) == reply_text(marker) + assert_one_request(wire.drain(), "/az/openai/v1/chat/completions", marker) + + +@pytest.mark.parametrize("trailing", TRAILING, ids=TRAILING_IDS) +@pytest.mark.parametrize("base", _AZURE_ANTHROPIC_BASES, ids=_AZURE_ANTHROPIC_IDS) +def test_azure_ai_count_tokens_reaches_one_count_tokens_route(gateway: Gateway, base: str, trailing: str) -> None: + marker: Final = new_marker() + with wire_server(join_peer) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model="azure_ai/claude-haiku-5-5", api_base=f"{wire.url}{base}{trailing}", api_key=KEY + ) + response: Final = gateway.request( + "POST", "/v1/messages/count_tokens", {"model": model, "messages": [{"role": "user", "content": marker}]} + ) + assert body_of(response)["input_tokens"] == COUNTED_TOKENS + assert_one_request(wire.drain(), "/fd/anthropic/v1/messages/count_tokens", marker) @pytest.mark.parametrize("suffix", ("", "/", "?tier=a"), ids=("exact", "slash", "query"))