scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-29 06:33:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 200

test(llms): audit #47535 (#48786)

Browse source 
Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
This commit is contained in:
tessellark 2026-10-28 03:38:21 -07:00 • committed by GitHub
parent b97fcf1af0
commit de40c0aa39
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 134 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
tests/integration/providers/_api_base_url_join_support.py
View file

@ -13,6 +13,7 @@ from pydantic import JsonValue
COUNTED_TOKENS: Final = 7
RELEVANCE: Final = 0.75
AZURE_DEPLOYMENT: Final = "gpt-url-join"
_USAGE: Final = {"input_tokens": 3, "output_tokens": 2}
@ -146,12 +147,20 @@ def _cohere_rerank(request: Request) -> Reply:
)
def _titan_embedding(_: Request) -> Reply:
return _json({"embedding": list(VECTOR), "inputTextTokenCount": 2})
_BY_PREFIX: Final[Mapping[str, Callable[[Request], Reply]]] = MappingProxyType(
{
"an": _anthropic,
"bi": lambda request: _json(anthropic_message(marker_of(request))),
"bc": _bedrock_converse,
"be": _titan_embedding,
"gm": _gemini,
"vx": _gemini,
"fd": _anthropic,
"az": _openai_like(AZURE_DEPLOYMENT),
"mi": _openai_like("mistral-small-latest"),
"db": _openai_like("databricks-meta-llama-3-3-70b-instruct"),
"co": _cohere_rerank,

37
tests/integration/providers/test_api_base_url_join_host_owned_proxy.py
View file

@ -1,6 +1,7 @@
import os
from collections.abc import Callable, Iterator
from pathlib import Path
from types import MappingProxyType
from typing import Final
import pytest
@ -8,6 +9,7 @@ import yaml
from integration._support.client import JSON_OBJECT, Gateway, gateway_from_environment
from integration._support.forward_proxy import ForwardProxy, refusing_forward_proxy
from integration._support.process import graceful_stop_seconds, owned_proxy
from integration._support.vertex import service_account_json
from integration.providers._api_base_trailing_slash_support import KEY, new_marker
from pydantic import JsonValue
@ -17,6 +19,12 @@ _RERANK: Final = "host-rerank"
_AZURE_CLAUDE: Final = "host-azure-claude"
_AZURE_COUNT: Final = "host-azure-count"
_EMPTY_BASE: Final = "host-empty-base"
_VERTEX_HOST: Final = "vertex-gateway"
_VERTEX_PROJECT: Final = "url-join-project"
_VERTEX_LOCATION: Final = "us-east5"
_VERTEX_CLAUDE_BASES: Final = MappingProxyType(
{"host-vertex-claude": f"http://{_VERTEX_HOST}", "host-vertex-claude-slash": f"http://{_VERTEX_HOST}/"}
)
_SCRUBBED_PREFIXES: Final = ("ANTHROPIC_", "AZURE_AI_", "INFINITY_")
_PROXY_VARIABLES: Final = frozenset({"HTTPS_PROXY", "HTTP_PROXY", "ALL_PROXY", "NO_PROXY"})
_load_yaml: Final[Callable[[str], object]] = yaml.safe_load
@ -28,7 +36,20 @@ def _deployment(name: str, model: str, api_base: str) -> dict[str, JsonValue]:
return {"model_name": name, "litellm_params": {"model": model, "api_base": api_base, "api_key": KEY}}
def _config(directory: Path) -> Path:
def _vertex_claude_deployment(name: str, api_base: str, token_url: str) -> dict[str, JsonValue]:
return {
"model_name": name,
"litellm_params": {
"model": "vertex_ai/claude-haiku-5-5",
"api_base": api_base,
"vertex_project": _VERTEX_PROJECT,
"vertex_location": _VERTEX_LOCATION,
"vertex_credentials": service_account_json(_VERTEX_PROJECT, token_url),
},
}
def _config(directory: Path, token_url: str) -> Path:
base: Final = JSON_OBJECT.validate_python(_load_yaml(Path("tests/integration/proxy_config.yaml").read_text()))
configuration: Final = {
**base,
@ -38,6 +59,7 @@ def _config(directory: Path) -> Path:
_deployment(_AZURE_CLAUDE, "azure_ai/claude-haiku-5-5", f"http://{_AZURE_HOST}/anthropic"),
_deployment(_AZURE_COUNT, "azure_ai/claude-haiku-5-5", f"http://{_AZURE_HOST}"),
_deployment(_EMPTY_BASE, "anthropic/claude-haiku-5-5", ""),
*(_vertex_claude_deployment(name, base, token_url) for name, base in _VERTEX_CLAUDE_BASES.items()),
],
"router_settings": {"disable_cooldowns": True, "num_retries": 0},
}
@ -72,7 +94,7 @@ def isolated(forward: ForwardProxy, tmp_path_factory: pytest.TempPathFactory) ->
"NO_PROXY": "127.0.0.1,localhost",
"PROXY_BATCH_POLLING_ENABLED": "false",
},
config=_config(directory),
config=_config(directory, shared.upstream_url.rstrip("/")),
remove_environment=_scrubbed(),
) as candidate:
yield candidate
@ -138,3 +160,14 @@ def test_azure_claude_count_tokens_on_a_bare_host_starting_with_anthropic_adds_t
def test_empty_api_base_falls_back_to_the_provider_default_host(isolated: Gateway, forward: ForwardProxy) -> None:
targets: Final = _targets_after(forward, lambda: _chat(isolated, _EMPTY_BASE))
assert targets == {"api.anthropic.com:443"}
@pytest.mark.parametrize("model", tuple(_VERTEX_CLAUDE_BASES), ids=("no-slash", "slash"))
def test_vertex_partner_claude_on_a_bare_host_reaches_one_single_slash_url(
isolated: Gateway, forward: ForwardProxy, model: str
) -> None:
targets: Final = _targets_after(forward, lambda: _chat(isolated, model))
assert targets == {
f"http://{_VERTEX_HOST}/v1/projects/{_VERTEX_PROJECT}/locations/{_VERTEX_LOCATION}"
"/publishers/anthropic/models/claude-haiku-5-5:rawPredict"
}

99
tests/integration/providers/test_api_base_url_join_wire.py
View file

@ -7,6 +7,7 @@ import openai
import pytest
from integration._support.client import Gateway, Scenario, eventually, list_value, object_value, string_value
from integration._support.database import read_rows
from integration._support.vertex import service_account_json
from integration._support.wire import Reply, Request, wire_server
from integration.providers._api_base_trailing_slash_support import (
KEY,
@ -22,7 +23,7 @@ from integration.providers._api_base_trailing_slash_support import (
reply_text,
request_json,
)
from integration.providers._api_base_url_join_support import COUNTED_TOKENS, RELEVANCE, join_peer
from integration.providers._api_base_url_join_support import AZURE_DEPLOYMENT, COUNTED_TOKENS, RELEVANCE, join_peer
from pydantic import JsonValue
_HAIKU: Final = "anthropic.claude-haiku-5-5"
@ -49,10 +50,20 @@ _RESPONSES_ROWS: Final = (_ANTHROPIC, _GEMINI)
_EMBEDDING_ROWS: Final = (
Row("mistral/mistral-embed", "/mi/v1", "/embeddings"),
Row("gemini/gemini-embedding-2", "/gm/v1beta", "/models/gemini-embedding-2:batchEmbedContents"),
Row("bedrock/amazon.titan-embed-text-v2:0", "/be", "/model/amazon.titan-embed-text-v2%3A0/invoke"),
)
_TRANSCRIPTION: Final = Row("mistral/voxtral-mini-latest", "/mi/v1", "/audio/transcriptions")
_FULL_ROUTE_SUFFIXES: Final = ("", "/", "?beta=true")
_FULL_ROUTE_IDS: Final = ("exact", "slash", "query")
_FULL_ROUTE_SUFFIXES: Final = (("", ""), ("/", ""), ("?beta=true", "?beta=true"), ("/?beta=true", "?beta=true"))
_FULL_ROUTE_IDS: Final = ("exact", "slash", "query", "slash-query")
_VERTEX_PROJECT: Final = "url-join-project"
_VERTEX_LOCATION: Final = "us-east5"
_VERTEX_GEMINI_PATH: Final = (
f"/vx/v1/projects/{_VERTEX_PROJECT}/locations/{_VERTEX_LOCATION}/publishers/google/models/gemini-3.8-flash"
)
_AZURE_V1_BASES: Final = ("/az", "/az/", "/az/openai/v1", "/az/openai/v1/")
_AZURE_V1_IDS: Final = ("host", "host-slash", "route", "route-slash")
_AZURE_ANTHROPIC_BASES: Final = ("/fd/anthropic", "/fd/anthropic/v1/messages")
_AZURE_ANTHROPIC_IDS: Final = ("anthropic-root", "messages-route")
def _ids(rows: tuple[Row, ...]) -> tuple[str, ...]:
@ -65,6 +76,17 @@ def _deploy(scenario: Scenario, row: Row, api_base: str) -> str:
return scenario.model(model=row.model, api_base=api_base, api_key=KEY)
def _deploy_vertex(gateway: Gateway, scenario: Scenario, model: str, api_base: str) -> str:
return scenario.model(
model=model,
api_base=api_base,
api_key=None,
vertex_project=_VERTEX_PROJECT,
vertex_location=_VERTEX_LOCATION,
vertex_credentials=service_account_json(_VERTEX_PROJECT, gateway.upstream_url.rstrip("/")),
)
def _openai(gateway: Gateway) -> openai.OpenAI:
return openai.OpenAI(base_url=str(gateway.client.base_url) + "/v1", api_key=gateway.key, max_retries=0)
@ -80,6 +102,13 @@ def _chat(gateway: Gateway, model: str, marker: str) -> str | None:
return completion.choices[0].message.content
def _streamed_chat(gateway: Gateway, model: str, marker: str) -> str:
stream: Final = _openai(gateway).chat.completions.create(
model=model, messages=[{"role": "user", "content": marker}], stream=True
)
return "".join(chunk.choices[0].delta.content or "" for chunk in stream if chunk.choices)
def _message_text(gateway: Gateway, model: str, marker: str) -> list[str]:
message: Final = _anthropic(gateway).messages.create(
model=model, max_tokens=16, messages=[{"role": "user", "content": marker}]
@ -270,22 +299,74 @@ def test_cache_hit_on_a_slash_base_skips_the_upstream_and_logs_its_own_row(gatew
assert "//" not in urlsplit(string_value(hit["api_base"])).path, hit
@pytest.mark.parametrize("suffix", _FULL_ROUTE_SUFFIXES, ids=_FULL_ROUTE_IDS)
def test_chat_base_that_already_names_the_route_keeps_its_path_and_query(gateway: Gateway, suffix: str) -> None:
@pytest.mark.parametrize(("suffix", "sent"), _FULL_ROUTE_SUFFIXES, ids=_FULL_ROUTE_IDS)
def test_chat_base_that_already_names_the_route_keeps_its_path_and_query(
gateway: Gateway, suffix: str, sent: str
) -> None:
marker: Final = new_marker()
with wire_server(join_peer) as wire, gateway.scenario() as scenario:
model: Final = _deploy(scenario, _ANTHROPIC, f"{wire.url}/an/v1/messages{suffix}")
assert _chat(gateway, model, marker) == reply_text(marker)
assert_one_request(wire.drain(), f"/an/v1/messages{suffix.rstrip('/')}", marker)
assert_one_request(wire.drain(), f"/an/v1/messages{sent}", marker)
@pytest.mark.parametrize("suffix", _FULL_ROUTE_SUFFIXES, ids=_FULL_ROUTE_IDS)
def test_messages_base_that_already_names_the_route_keeps_its_path_and_query(gateway: Gateway, suffix: str) -> None:
@pytest.mark.parametrize(("suffix", "sent"), _FULL_ROUTE_SUFFIXES, ids=_FULL_ROUTE_IDS)
def test_messages_base_that_already_names_the_route_keeps_its_path_and_query(
gateway: Gateway, suffix: str, sent: str
) -> None:
marker: Final = new_marker()
with wire_server(join_peer) as wire, gateway.scenario() as scenario:
model: Final = _deploy(scenario, _ANTHROPIC, f"{wire.url}/an/v1/messages{suffix}")
assert _message_text(gateway, model, marker) == [reply_text(marker)]
assert_one_request(wire.drain(), f"/an/v1/messages{suffix.rstrip('/')}", marker)
assert_one_request(wire.drain(), f"/an/v1/messages{sent}", marker)
@pytest.mark.parametrize("trailing", TRAILING, ids=TRAILING_IDS)
def test_vertex_model_url_base_calls_its_method_with_no_slash_before_the_colon(gateway: Gateway, trailing: str) -> None:
marker: Final = new_marker()
with wire_server(join_peer) as wire, gateway.scenario() as scenario:
model: Final = _deploy_vertex(
gateway, scenario, "vertex_ai/gemini-3.8-flash", f"{wire.url}{_VERTEX_GEMINI_PATH}{trailing}"
)
assert _chat(gateway, model, marker) == reply_text(marker)
assert_one_request(wire.drain(), f"{_VERTEX_GEMINI_PATH}:generateContent", marker, keyed=False)
@pytest.mark.parametrize("trailing", TRAILING, ids=TRAILING_IDS)
def test_vertex_model_url_base_streams_with_no_slash_before_the_colon(gateway: Gateway, trailing: str) -> None:
marker: Final = new_marker()
with wire_server(join_peer) as wire, gateway.scenario() as scenario:
model: Final = _deploy_vertex(
gateway, scenario, "vertex_ai/gemini-3.8-flash", f"{wire.url}{_VERTEX_GEMINI_PATH}{trailing}"
)
assert _streamed_chat(gateway, model, marker) == reply_text(marker)
assert_one_request(wire.drain(), f"{_VERTEX_GEMINI_PATH}:streamGenerateContent?alt=sse", marker, keyed=False)
@pytest.mark.parametrize("base", _AZURE_V1_BASES, ids=_AZURE_V1_IDS)
def test_azure_v1_chat_reaches_one_openai_v1_route(gateway: Gateway, base: str) -> None:
marker: Final = new_marker()
with wire_server(join_peer) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(
model=f"azure/{AZURE_DEPLOYMENT}", api_base=f"{wire.url}{base}", api_key=KEY, api_version="v1"
)
assert _chat(gateway, model, marker) == reply_text(marker)
assert_one_request(wire.drain(), "/az/openai/v1/chat/completions", marker)
@pytest.mark.parametrize("trailing", TRAILING, ids=TRAILING_IDS)
@pytest.mark.parametrize("base", _AZURE_ANTHROPIC_BASES, ids=_AZURE_ANTHROPIC_IDS)
def test_azure_ai_count_tokens_reaches_one_count_tokens_route(gateway: Gateway, base: str, trailing: str) -> None:
marker: Final = new_marker()
with wire_server(join_peer) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(
model="azure_ai/claude-haiku-5-5", api_base=f"{wire.url}{base}{trailing}", api_key=KEY
)
response: Final = gateway.request(
"POST", "/v1/messages/count_tokens", {"model": model, "messages": [{"role": "user", "content": marker}]}
)
assert body_of(response)["input_tokens"] == COUNTED_TOKENS
assert_one_request(wire.drain(), "/fd/anthropic/v1/messages/count_tokens", marker)
@pytest.mark.parametrize("suffix", ("", "/", "?tier=a"), ids=("exact", "slash", "query"))