scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-03 02:22:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 82

test(integration): edge-case matrices for malformed token limits and callback_settings shapes (#42895)

Browse source 
* test(integration): edge-case matrices for malformed token limits and callback_settings shapes

Extends the integration suite so two classes of issues found by gauntlet
reviews are caught end to end against a real proxy:

- non-numeric or odd model_info token limits (from /model/new and from
  config YAML) must be listed as absent on /v1/models, /models,
  /v1/models/{id} and /model/info, keep sibling models listed, and still
  serve chat
- every callback_settings shape (top level and per consumer) must let
  the proxy boot, register the configured callbacks and serve chat

Four product bugs on main surfaced by the matrices are recorded as
BUG skips per the suite convention: chat 500 and /model_group/info 500
on non-numeric token limits, a startup crash on a non-object
callback_settings, and otel silently dropped on a non-object
callback_settings.otel

* test(integration): pin the exact coerced value for numeric-edge token limits

Addresses review feedback: the numeric-edge matrix only asserted
'int or absent'. It now asserts the listed value for each case on
/v1/models, /models and /v1/models/{id}, which also lets the listing
helper drop its optional-expectation branch.
This commit is contained in:
yuneng-jiang 2026-09-24 01:05:55 -07:00 • committed by GitHub
parent d11705a24d
commit 265f874eaf
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 369 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

154
tests/integration/configuration/test_callback_settings_boot.py Normal file
View file

@ -0,0 +1,154 @@
import json
import uuid
from collections.abc import Mapping
from pathlib import Path
from typing import Final
import pytest
from pydantic import JsonValue
from tests.integration._support.client import Gateway
from tests.integration._support.process import owned_proxy
SERVING_CONSUMERS: Final = {
"compression_interception": "CompressionInterceptionLogger",
"code_interpreter_interception": "CodeInterpreterInterceptionLogger",
"websearch_interception": "WebSearchInterceptionLogger",
}
OTEL_CONSUMER: Final = {"otel": "OpenTelemetry"}
GUARDRAIL_CONSUMERS: Final = {
"presidio": "_OPTIONAL_PresidioPIIMasking",
"lakera_prompt_injection": "lakeraAI_Moderation",
}
TOP_LEVEL_SHAPES: Final = (
pytest.param({}, id="empty-object"),
pytest.param(None, id="null"),
pytest.param("otel", id="string"),
pytest.param(["otel"], id="list"),
pytest.param(True, id="bool"),
pytest.param(0, id="zero"),
)
CONSUMER_SHAPES: Final = (
pytest.param({}, id="empty-object"),
pytest.param(None, id="null"),
pytest.param("on", id="string"),
pytest.param(True, id="bool"),
pytest.param([], id="empty-list"),
pytest.param(["on"], id="list"),
pytest.param(7, id="int"),
)
TOP_LEVEL_BOOT_CRASH: Final = (
"BUG: a non-object callback_settings is stored verbatim and proxy startup crashes calling .get on it"
)
TOP_LEVEL_BOOT_CRASH_IDS: Final = frozenset({"string", "list", "bool"})
OTEL_DROPPED: Final = (
"BUG: a non-object callback_settings.otel fails dict() and the otel callback is silently not registered"
)
OTEL_DROPPED_IDS: Final = frozenset({"null", "string", "bool", "int"})
def _write_config(
directory: Path, upstream_url: str, model: str, callbacks: tuple[str, ...], callback_settings: JsonValue
) -> Path:
config: Final = directory / f"callback_settings_{uuid.uuid4().hex}.yaml"
config.write_text(
json.dumps(
{
"model_list": [
{
"model_name": model,
"litellm_params": {
"model": f"openai/{model}",
"api_base": f"{upstream_url}/v1",
"api_key": "integration-provider-key",
},
}
],
"litellm_settings": {"callbacks": list(callbacks)},
"callback_settings": callback_settings,
"general_settings": {
"master_key": "os.environ/LITELLM_MASTER_KEY",
"database_url": "os.environ/DATABASE_URL",
},
}
)
)
return config
def _assert_registered(candidate: Gateway, consumers: Mapping[str, str]) -> None:
response: Final = candidate.request("GET", "/active/callbacks")
assert response.status_code == 200, response.text
missing: Final = sorted(name for name, class_name in consumers.items() if class_name not in response.text)
assert missing == [], response.text
@pytest.mark.parametrize("callback_settings", TOP_LEVEL_SHAPES)
def test_top_level_callback_settings_shape_boots_registers_and_serves_chat(
gateway: Gateway, tmp_path: Path, callback_settings: JsonValue, request: pytest.FixtureRequest
) -> None:
if request.node.callspec.id in TOP_LEVEL_BOOT_CRASH_IDS:
pytest.skip(TOP_LEVEL_BOOT_CRASH)
consumers: Final = {**SERVING_CONSUMERS, **OTEL_CONSUMER}
model: Final = f"integration-callback-settings-{uuid.uuid4().hex}"
config: Final = _write_config(tmp_path, gateway.upstream_url, model, tuple(consumers), callback_settings)
with owned_proxy(gateway, tmp_path, {"STORE_MODEL_IN_DB": "False"}, config=config) as candidate:
_assert_registered(candidate, consumers)
reply: Final = candidate.chat(model, text=f"callback settings {uuid.uuid4().hex}")
assert reply["model"] == model, reply
@pytest.mark.parametrize("value", CONSUMER_SHAPES)
def test_serving_consumer_settings_shape_boots_registers_and_serves_chat(
gateway: Gateway, tmp_path: Path, value: JsonValue
) -> None:
model: Final = f"integration-callback-settings-{uuid.uuid4().hex}"
config: Final = _write_config(
tmp_path,
gateway.upstream_url,
model,
tuple(SERVING_CONSUMERS),
{consumer: value for consumer in SERVING_CONSUMERS},
)
with owned_proxy(gateway, tmp_path, {"STORE_MODEL_IN_DB": "False"}, config=config) as candidate:
_assert_registered(candidate, SERVING_CONSUMERS)
reply: Final = candidate.chat(model, text=f"callback settings {uuid.uuid4().hex}")
assert reply["model"] == model, reply
@pytest.mark.parametrize("value", CONSUMER_SHAPES)
def test_otel_settings_shape_boots_registers_and_serves_chat(
gateway: Gateway, tmp_path: Path, value: JsonValue, request: pytest.FixtureRequest
) -> None:
if request.node.callspec.id in OTEL_DROPPED_IDS:
pytest.skip(OTEL_DROPPED)
model: Final = f"integration-callback-settings-{uuid.uuid4().hex}"
config: Final = _write_config(tmp_path, gateway.upstream_url, model, tuple(OTEL_CONSUMER), {"otel": value})
with owned_proxy(gateway, tmp_path, {"STORE_MODEL_IN_DB": "False"}, config=config) as candidate:
_assert_registered(candidate, OTEL_CONSUMER)
reply: Final = candidate.chat(model, text=f"callback settings {uuid.uuid4().hex}")
assert reply["model"] == model, reply
@pytest.mark.parametrize("value", CONSUMER_SHAPES)
def test_guardrail_consumer_settings_shape_boots_and_registers(
gateway: Gateway, tmp_path: Path, value: JsonValue
) -> None:
model: Final = f"integration-callback-settings-{uuid.uuid4().hex}"
config: Final = _write_config(
tmp_path,
gateway.upstream_url,
model,
tuple(GUARDRAIL_CONSUMERS),
{consumer: value for consumer in GUARDRAIL_CONSUMERS},
)
environment: Final = {
"STORE_MODEL_IN_DB": "False",
"PRESIDIO_ANALYZER_API_BASE": gateway.upstream_url,
"PRESIDIO_ANONYMIZER_API_BASE": gateway.upstream_url,
}
with owned_proxy(gateway, tmp_path, environment, config=config) as candidate:
_assert_registered(candidate, GUARDRAIL_CONSUMERS)

215
tests/integration/pricing/test_model_listing_token_limits.py
View file

@ -1,9 +1,93 @@
import json
import uuid
from collections.abc import Mapping
from pathlib import Path
from typing import Final
import pytest
from integration._support.client import Gateway, object_value
from integration._support.process import owned_proxy
from pydantic import JsonValue
SIBLING_LIMITS: Final = {"max_input_tokens": 4321, "max_output_tokens": 987}
NON_NUMERIC_LIMITS: Final = (
pytest.param("", id="empty-string"),
pytest.param(" ", id="blank-string"),
pytest.param("128,000", id="thousands-separator"),
pytest.param("unlimited", id="word"),
pytest.param("NaN", id="nan-string"),
pytest.param("inf", id="inf-string"),
pytest.param([], id="empty-list"),
pytest.param([4096], id="list"),
pytest.param({}, id="empty-object"),
pytest.param({"tokens": 4096}, id="object"),
pytest.param(True, id="bool"),
pytest.param(None, id="null"),
)
NUMERIC_EDGE_LIMITS: Final = (
pytest.param(0, id="zero"),
pytest.param(-1, id="negative"),
pytest.param(1.5, id="float"),
pytest.param("1.5", id="float-string"),
pytest.param("1e9", id="exponent-string"),
pytest.param(10**12, id="huge"),
)
NUMERIC_EDGE_EXPECTED: Final = {
"zero": 0,
"negative": -1,
"float": 1,
"float-string": 1,
"exponent-string": 1_000_000_000,
"huge": 10**12,
}
MODEL_GROUP_INFO_500: Final = (
"BUG: /model_group/info returns 500 for every caller when one deployment's token limit is non-numeric"
)
CHAT_500: Final = (
"BUG: chat completions return 500 from ModelGroupInfo validation when the deployment's token limit is non-numeric"
)
MODEL_GROUP_INFO_500_IDS: Final = frozenset(
{"empty-string", "blank-string", "thousands-separator", "word", "nan-string", "inf-string"}
| {"empty-list", "list", "empty-object", "object"}
)
CHAT_500_IDS: Final = frozenset(
{"empty-string", "blank-string", "thousands-separator", "word", "empty-list", "list", "empty-object", "object"}
)
def _listed(gateway: Gateway, path: str) -> dict[str, dict[str, JsonValue]]:
entries: Final = gateway.get(path)["data"]
assert isinstance(entries, list)
return {str(object_value(entry)["id"]): object_value(entry) for entry in entries}
def _limits(entry: Mapping[str, JsonValue]) -> tuple[JsonValue, JsonValue]:
return entry.get("max_input_tokens"), entry.get("max_output_tokens")
def _assert_listing_spares_the_sibling(
gateway: Gateway, broken: str, sibling: str, broken_limits: tuple[JsonValue, JsonValue]
) -> None:
for path in ("/v1/models", "/models"):
listed: Final = _listed(gateway, path)
assert _limits(listed[sibling]) == (4321, 987), (path, listed[sibling])
assert _limits(listed[broken]) == broken_limits, (path, listed[broken])
single: Final = gateway.get(f"/v1/models/{broken}")
assert single["id"] == broken, single
assert _limits(single) == broken_limits, single
registered: Final = gateway.get("/model/info")["data"]
assert isinstance(registered, list)
assert {broken, sibling} <= {str(object_value(entry)["model_name"]) for entry in registered}
def _assert_serves_chat(gateway: Gateway, *models: str) -> None:
for model in models:
reply: Final = gateway.chat(model, text=f"token limit edge {uuid.uuid4().hex}")
assert reply["model"] == model, reply
def _listed_model(gateway: Gateway, model: str) -> dict[str, JsonValue]:
entries: Final = gateway.get("/v1/models")["data"]
@ -27,3 +111,134 @@ def test_v1_models_carries_deployment_model_info_limits_for_an_unknown_model(gat
listed: Final = _listed_model(gateway, model)
assert listed["max_input_tokens"] == 4321, listed
assert listed["max_output_tokens"] == 987, listed
def test_numeric_string_token_limit_is_coerced_to_an_int(gateway: Gateway) -> None:
with gateway.scenario() as scenario:
model: Final = scenario.model(
model=f"openai/custom-{uuid.uuid4().hex}",
model_info={"max_input_tokens": "4096", "max_output_tokens": "512"},
)
assert _limits(_listed_model(gateway, model)) == (4096, 512)
@pytest.mark.parametrize("value", NON_NUMERIC_LIMITS)
def test_non_numeric_token_limit_is_listed_as_absent_without_breaking_the_listing(
gateway: Gateway, value: JsonValue
) -> None:
with gateway.scenario() as scenario:
sibling: Final = scenario.model(model=f"openai/custom-{uuid.uuid4().hex}", model_info=SIBLING_LIMITS)
broken: Final = scenario.model(
model=f"openai/custom-{uuid.uuid4().hex}",
model_info={"max_input_tokens": value, "max_output_tokens": value},
)
_assert_listing_spares_the_sibling(gateway, broken, sibling, (None, None))
@pytest.mark.parametrize("value", NON_NUMERIC_LIMITS)
def test_non_numeric_token_limit_still_serves_chat(
gateway: Gateway, value: JsonValue, request: pytest.FixtureRequest
) -> None:
if request.node.callspec.id in CHAT_500_IDS:
pytest.skip(CHAT_500)
with gateway.scenario() as scenario:
sibling: Final = scenario.model(model=f"openai/custom-{uuid.uuid4().hex}", model_info=SIBLING_LIMITS)
broken: Final = scenario.model(
model=f"openai/custom-{uuid.uuid4().hex}",
model_info={"max_input_tokens": value, "max_output_tokens": value},
)
_assert_serves_chat(gateway, broken, sibling)
@pytest.mark.parametrize("value", NUMERIC_EDGE_LIMITS)
def test_numeric_edge_token_limit_is_listed_as_its_integer_without_breaking_the_listing(
gateway: Gateway, value: JsonValue, request: pytest.FixtureRequest
) -> None:
expected: Final = NUMERIC_EDGE_EXPECTED[request.node.callspec.id]
with gateway.scenario() as scenario:
sibling: Final = scenario.model(model=f"openai/custom-{uuid.uuid4().hex}", model_info=SIBLING_LIMITS)
broken: Final = scenario.model(
model=f"openai/custom-{uuid.uuid4().hex}",
model_info={"max_input_tokens": value, "max_output_tokens": value},
)
_assert_listing_spares_the_sibling(gateway, broken, sibling, (expected, expected))
_assert_serves_chat(gateway, broken, sibling)
@pytest.mark.parametrize("field", ("max_input_tokens", "max_output_tokens"))
def test_one_malformed_limit_does_not_disturb_the_other(gateway: Gateway, field: str) -> None:
other: Final = "max_output_tokens" if field == "max_input_tokens" else "max_input_tokens"
with gateway.scenario() as scenario:
model: Final = scenario.model(
model=f"openai/custom-{uuid.uuid4().hex}", model_info={field: "128,000", other: 2048}
)
listed: Final = _listed_model(gateway, model)
assert listed.get(field) is None, listed
assert listed[other] == 2048, listed
@pytest.mark.parametrize("value", NON_NUMERIC_LIMITS + NUMERIC_EDGE_LIMITS)
def test_malformed_token_limit_keeps_model_group_info_serving(
gateway: Gateway, value: JsonValue, request: pytest.FixtureRequest
) -> None:
if request.node.callspec.id in MODEL_GROUP_INFO_500_IDS:
pytest.skip(MODEL_GROUP_INFO_500)
with gateway.scenario() as scenario:
sibling: Final = scenario.model(model=f"openai/custom-{uuid.uuid4().hex}", model_info=SIBLING_LIMITS)
broken: Final = scenario.model(
model=f"openai/custom-{uuid.uuid4().hex}",
model_info={"max_input_tokens": value, "max_output_tokens": value},
)
groups: Final = gateway.get("/model_group/info")["data"]
assert isinstance(groups, list)
assert {broken, sibling} <= {str(object_value(group)["model_group"]) for group in groups}
single: Final = gateway.get("/model_group/info", {"model_group": broken})["data"]
assert isinstance(single, list)
assert [object_value(group)["model_group"] for group in single] == [broken]
def _yaml_deployment(name: str, upstream_url: str, model_info: Mapping[str, JsonValue]) -> dict[str, JsonValue]:
return {
"model_name": name,
"litellm_params": {
"model": f"openai/custom-{uuid.uuid4().hex}",
"api_base": f"{upstream_url}/v1",
"api_key": "integration-provider-key",
},
"model_info": dict(model_info),
}
def test_non_numeric_token_limits_in_config_yaml_are_listed_as_absent(gateway: Gateway, tmp_path: Path) -> None:
run: Final = uuid.uuid4().hex
sibling: Final = f"integration-yaml-sibling-{run}"
broken: Final = {f"integration-yaml-{parameter.id}-{run}": parameter.values[0] for parameter in NON_NUMERIC_LIMITS}
serving: Final = tuple(
f"integration-yaml-{parameter.id}-{run}" for parameter in NON_NUMERIC_LIMITS if parameter.id not in CHAT_500_IDS
)
config: Final = tmp_path / "malformed_token_limits.yaml"
config.write_text(
json.dumps(
{
"model_list": [
_yaml_deployment(sibling, gateway.upstream_url, SIBLING_LIMITS),
*(
_yaml_deployment(
name, gateway.upstream_url, {"max_input_tokens": value, "max_output_tokens": value}
)
for name, value in broken.items()
),
],
"general_settings": {
"master_key": "os.environ/LITELLM_MASTER_KEY",
"database_url": "os.environ/DATABASE_URL",
"store_model_in_db": True,
},
"router_settings": {"disable_cooldowns": True},
}
)
)
with owned_proxy(gateway, tmp_path, {}, config=config) as candidate:
for name in broken:
_assert_listing_spares_the_sibling(candidate, name, sibling, (None, None))
_assert_serves_chat(candidate, sibling, *serving)