test(integration): repair seven stale integration tests on main (#47600)

The Gemini TTS billing and text completion guardrail tests read spend rows from the owned proxy's own database, which #47213 made private. The cursor wire test derives its effort levels from the gpt-5 cost map row that #43935 now enforces, and the Azure effort check accepts the unquoted level. The pin TTL test asserts the one hour pin #46411 writes for a tool-only mark, the video callback test decodes ids with the product decoder after #44614, and the two Gemini 3.8 flash stream cost cases bill audio at the text rate their rows imply
This commit is contained in:
yuneng-jiang 2026-10-20 11:21:04 -07:00 • committed by GitHub
parent 7f0bb8424c
commit e78f7b4602
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 37 additions and 25 deletions

View file

@ -2658,7 +2658,7 @@ GEMINI_GEMINI_3_8_FLASH_STREAM_FULL_USAGE: Final = CostTrackingTestCase(
), ),
), ),
expected=ExactExpected( expected=ExactExpected(
spend=0.0076232, input_cost=0.0015572, output_cost=0.006066, prompt_tokens=8314, completion_tokens=1592 spend=0.0066182, input_cost=0.0013922, output_cost=0.005226, prompt_tokens=8314, completion_tokens=1592
), ),
) )

View file

@ -2846,7 +2846,7 @@ GEMINI_3_8_FLASH_STREAM_FULL_USAGE: Final = CostTrackingTestCase(
), ),
), ),
expected=ExactExpected( expected=ExactExpected(
spend=0.005775328, input_cost=0.001276288, output_cost=0.00449904, prompt_tokens=8314, completion_tokens=1592 spend=0.005946928, input_cost=0.001447888, output_cost=0.00449904, prompt_tokens=8314, completion_tokens=1592
), ),
) )

View file

@ -1,4 +1,3 @@
import base64
import json import json
import uuid import uuid
from collections.abc import Callable, Mapping from collections.abc import Callable, Mapping
@ -14,6 +13,8 @@ from integration._support.database import read_rows
from integration._support.process import graceful_stop_seconds, owned_proxy from integration._support.process import graceful_stop_seconds, owned_proxy
from integration._support.wire import Reply, Request, wire_server from integration._support.wire import Reply, Request, wire_server
from litellm.types.videos.utils import decode_video_id_with_provider
pytestmark: Final = pytest.mark.shares_worker_state pytestmark: Final = pytest.mark.shares_worker_state
@ -431,9 +432,8 @@ def _responses_text(body: dict[str, JsonValue]) -> str:
def _video_job(body: dict[str, JsonValue]) -> tuple[str, str]: def _video_job(body: dict[str, JsonValue]) -> tuple[str, str]:
encoded_id: Final = string_value(body["id"]).removeprefix("video_") decoded: Final = decode_video_id_with_provider(string_value(body["id"]))
decoded: Final = base64.b64decode(encoded_id).decode() return string_value(decoded.get("video_id")), string_value(body["status"])
return decoded.rsplit("video_id:", 1)[-1], string_value(body["status"])
@dataclass(frozen=True, slots=True) @dataclass(frozen=True, slots=True)

View file

@ -1,3 +1,4 @@
import re
from typing import Final from typing import Final
from integration._support.client import Gateway from integration._support.client import Gateway
@ -30,7 +31,7 @@ def test_reasoning_effort_none_is_rejected_by_litellm_before_reaching_azure(gate
) )
assert response.status_code == 400, response.text assert response.status_code == 400, response.text
assert "reasoning_effort='none'" in response.text, response.text assert re.search(r"reasoning_effort='?none\b", response.text), response.text
assert wire.drain() == () assert wire.drain() == ()

View file

@ -3,6 +3,7 @@ from __future__ import annotations
import json import json
import os import os
import uuid import uuid
from importlib.resources import files
from typing import Final from typing import Final
import pytest import pytest
@ -16,8 +17,16 @@ _MODEL: Final = "gpt-5"
_API_KEY: Final = "synthetic-cursor-key" _API_KEY: Final = "synthetic-cursor-key"
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) _JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
_PROXY_WORKERS: Final = max(1, int(os.environ.get("INTEGRATION_PROXY_WORKERS", "1"))) _PROXY_WORKERS: Final = max(1, int(os.environ.get("INTEGRATION_PROXY_WORKERS", "1")))
_THINKING_LEVELS: Final = ("none", "minimal", "low", "medium", "high", "xhigh", "max") _COST_MAP: Final = TypeAdapter(dict[str, dict[str, JsonValue]])
_CHAT_THINKING_LEVELS: Final = ("none", "minimal", "low", "medium", "high", "max") _MODEL_ROW: Final = _COST_MAP.validate_json(
(files("litellm") / "model_prices_and_context_window_backup.json").read_bytes()
)[_MODEL]
_THINKING_LEVELS: Final = tuple(
level
for level in ("none", "minimal", "low", "medium", "high", "xhigh", "max")
if _MODEL_ROW.get(f"supports_{level}_reasoning_effort") is not False
)
_CHAT_THINKING_LEVELS: Final = tuple(level for level in _THINKING_LEVELS if level != "xhigh")
_RESPONSES_SUFFIXES: Final[tuple[tuple[str, dict[str, JsonValue]], ...]] = tuple( _RESPONSES_SUFFIXES: Final[tuple[tuple[str, dict[str, JsonValue]], ...]] = tuple(
(f"-thinking-{level}", {"reasoning": {"effort": level}}) for level in _THINKING_LEVELS (f"-thinking-{level}", {"reasoning": {"effort": level}}) for level in _THINKING_LEVELS
) + ( ) + (

View file

@ -270,7 +270,7 @@ def test_chat_pin_lifetime_follows_the_longest_breakpoint_ttl_in_the_cacheable_p
spend_row_lands(response_id) spend_row_lands(response_id)
def test_chat_one_hour_mark_on_a_tool_definition_alone_writes_no_pin(rig: _Rig) -> None: def test_chat_one_hour_mark_on_a_tool_definition_alone_pins_the_served_deployment_for_an_hour(rig: _Rig) -> None:
before: Final = _before(rig) before: Final = _before(rig)
marker: Final = new_marker() marker: Final = new_marker()
tool: Final[dict[str, JsonValue]] = { tool: Final[dict[str, JsonValue]] = {
@ -281,8 +281,10 @@ def test_chat_one_hour_mark_on_a_tool_definition_alone_writes_no_pin(rig: _Rig)
response_id, served = post_chat( response_id, served = post_chat(
rig.owned.gateway, chat_body(_CLAUDE, _first_turn(new_marker(), marker, None), tools=[tool]) rig.owned.gateway, chat_body(_CLAUDE, _first_turn(new_marker(), marker, None), tools=[tool])
) )
assert served == rig.served_by(marker), served _, pin = one_new_pin(rig.redis, before)
no_pin_written(rig.redis, before, response_id) assert served == rig.served_by(marker) == pin.model_id, (served, pin)
assert lives_one_hour(pin), pin
spend_row_lands(response_id)
def _openai_client(rig: _Rig) -> openai.OpenAI: def _openai_client(rig: _Rig) -> openai.OpenAI:

View file

@ -11,7 +11,6 @@ from typing import Final, Literal
import pytest import pytest
import yaml import yaml
from integration._support.client import JSON_OBJECT, eventually, gateway_from_environment, object_value from integration._support.client import JSON_OBJECT, eventually, gateway_from_environment, object_value
from integration._support.database import read_rows
from integration._support.process import OwnedProxy, graceful_stop_seconds, owned_proxy_process from integration._support.process import OwnedProxy, graceful_stop_seconds, owned_proxy_process
from integration._support.vertex import service_account_json from integration._support.vertex import service_account_json
from integration._support.wire import Wire, wire_server from integration._support.wire import Wire, wire_server
@ -201,13 +200,15 @@ def _upstream_calls(wire: Wire, text: str) -> int:
return sum(1 for request in wire.drain() if received_text(request) == text) return sum(1 for request in wire.drain() if received_text(request) == text)
def _spend_rows(key: str, count: int) -> list[dict[str, JsonValue]]: def _spend_rows(billing: BillingProxy, key: str, count: int) -> list[dict[str, JsonValue]]:
return eventually(lambda: read_rows(SPEND_ROWS_SQL, (_hashed(key),)), lambda rows: len(rows) == count, seconds=70) return eventually(
lambda: billing.proxy.gateway.rows(SPEND_ROWS_SQL, (_hashed(key),)), lambda rows: len(rows) == count, seconds=70
)
def _key_spend(key: str, expected: float) -> JsonValue: def _key_spend(billing: BillingProxy, key: str, expected: float) -> JsonValue:
rows: Final = eventually( rows: Final = eventually(
lambda: read_rows(KEY_SPEND_SQL, (_hashed(key),)), lambda: billing.proxy.gateway.rows(KEY_SPEND_SQL, (_hashed(key),)),
lambda found: len(found) == 1 and found[0]["spend"] == pytest.approx(expected), lambda found: len(found) == 1 and found[0]["spend"] == pytest.approx(expected),
seconds=70, seconds=70,
return_last_on_timeout=True, return_last_on_timeout=True,
@ -224,7 +225,7 @@ def _chat_spend(billing: BillingProxy, model: str, key: str) -> float:
} }
response: Final = billing.proxy.gateway.request("POST", "/v1/chat/completions", body, key=key) response: Final = billing.proxy.gateway.request("POST", "/v1/chat/completions", body, key=key)
assert response.status_code == 200, response.text assert response.status_code == 200, response.text
row: Final = _spend_rows(key, 1)[0] row: Final = _spend_rows(billing, key, 1)[0]
assert (row["prompt_tokens"], row["completion_tokens"]) == (PROMPT_TOKENS, AUDIO_TOKENS), row assert (row["prompt_tokens"], row["completion_tokens"]) == (PROMPT_TOKENS, AUDIO_TOKENS), row
spend: Final = row["spend"] spend: Final = row["spend"]
assert isinstance(spend, float) and spend > 0, row assert isinstance(spend, float) and spend > 0, row
@ -261,11 +262,11 @@ def test_speech_calls_google_once_and_bills_and_logs_its_token_usage_once(
assert wav_payload(sent.body) == PCM assert wav_payload(sent.body) == PCM
assert _upstream_calls(tts_wire, text) == 1 assert _upstream_calls(tts_wire, text) == 1
chat_spend: Final = _chat_spend(uncached, MODELS[provider], scenario.key()) chat_spend: Final = _chat_spend(uncached, MODELS[provider], scenario.key())
row: Final = _spend_rows(speech_key, 1)[0] row: Final = _spend_rows(uncached, speech_key, 1)[0]
assert row["call_type"] == "aspeech", row assert row["call_type"] == "aspeech", row
assert (row["prompt_tokens"], row["completion_tokens"]) == (PROMPT_TOKENS, AUDIO_TOKENS), row assert (row["prompt_tokens"], row["completion_tokens"]) == (PROMPT_TOKENS, AUDIO_TOKENS), row
assert row["spend"] == pytest.approx(chat_spend), row assert row["spend"] == pytest.approx(chat_spend), row
assert _key_spend(speech_key, chat_spend) == pytest.approx(chat_spend) assert _key_spend(uncached, speech_key, chat_spend) == pytest.approx(chat_spend)
events: Final = _speech_events(uncached, speech_key, 1) events: Final = _speech_events(uncached, speech_key, 1)
assert [_summary(event) for event in events] == [("aspeech", BINARY_RESPONSE, PROMPT_TOKENS, AUDIO_TOKENS)] assert [_summary(event) for event in events] == [("aspeech", BINARY_RESPONSE, PROMPT_TOKENS, AUDIO_TOKENS)]
assert events[0]["response_cost"] == pytest.approx(chat_spend), events assert events[0]["response_cost"] == pytest.approx(chat_spend), events
@ -286,11 +287,11 @@ def test_cached_speech_repeat_is_a_free_cache_hit_after_a_billed_first_call(
assert _upstream_calls(tts_wire, text) == 1 assert _upstream_calls(tts_wire, text) == 1
assert repeat.cache_key, "the cached repeat carried no x-litellm-cache-key header" assert repeat.cache_key, "the cached repeat carried no x-litellm-cache-key header"
chat_spend: Final = _chat_spend(cached, model, scenario.key()) chat_spend: Final = _chat_spend(cached, model, scenario.key())
billed, hit = _spend_rows(speech_key, 2) billed, hit = _spend_rows(cached, speech_key, 2)
assert billed["cache_hit"] != "True" and billed["spend"] == pytest.approx(chat_spend), billed assert billed["cache_hit"] != "True" and billed["spend"] == pytest.approx(chat_spend), billed
assert (billed["prompt_tokens"], billed["completion_tokens"]) == (PROMPT_TOKENS, AUDIO_TOKENS), billed assert (billed["prompt_tokens"], billed["completion_tokens"]) == (PROMPT_TOKENS, AUDIO_TOKENS), billed
assert (hit["cache_hit"], hit["spend"]) == ("True", 0.0), hit assert (hit["cache_hit"], hit["spend"]) == ("True", 0.0), hit
assert _key_spend(speech_key, chat_spend) == pytest.approx(chat_spend) assert _key_spend(cached, speech_key, chat_spend) == pytest.approx(chat_spend)
events: Final = _speech_events(cached, speech_key, 2) events: Final = _speech_events(cached, speech_key, 2)
assert len(events) == 2, events assert len(events) == 2, events
hits: Final = tuple(event for event in events if event["cache_hit"] is True) hits: Final = tuple(event for event in events if event["cache_hit"] is True)
@ -325,7 +326,7 @@ def test_a_speech_reply_fills_the_deployment_tpm_limit_so_the_next_request_gets_
first_text: Final = marker() first_text: Final = marker()
first: Final = _speech(uncached, TPM_MODEL, first_text, key) first: Final = _speech(uncached, TPM_MODEL, first_text, key)
assert first.status == 200, first.body[:300] assert first.status == 200, first.body[:300]
_spend_rows(key, 1) _spend_rows(uncached, key, 1)
second_text: Final = marker() second_text: Final = marker()
second: Final = _speech(uncached, TPM_MODEL, second_text, key) second: Final = _speech(uncached, TPM_MODEL, second_text, key)
assert second.status == 429, second.body[:300] assert second.status == 429, second.body[:300]

View file

@ -16,7 +16,6 @@ from integration._support.client import (
list_value, list_value,
object_value, object_value,
) )
from integration._support.database import read_rows
from integration._support.process import OwnedProxy, WorkerHealthcheck, graceful_stop_seconds, owned_proxy_process from integration._support.process import OwnedProxy, WorkerHealthcheck, graceful_stop_seconds, owned_proxy_process
from integration._support.wire import Reply, Request, Wire, wire_server from integration._support.wire import Reply, Request, Wire, wire_server
from pydantic import JsonValue from pydantic import JsonValue
@ -202,7 +201,7 @@ def test_guarded_text_completion_stream_reaches_the_caller_whole_with_its_usage(
], stream.frames ], stream.frames
assert _posted_targets(rig.wire) == ("/v1/completions",) assert _posted_targets(rig.wire) == ("/v1/completions",)
(row,) = eventually( (row,) = eventually(
lambda: read_rows( lambda: rig.proxy.gateway.rows(
'SELECT prompt_tokens, completion_tokens, metadata FROM "LiteLLM_SpendLogs" WHERE request_id = %s', 'SELECT prompt_tokens, completion_tokens, metadata FROM "LiteLLM_SpendLogs" WHERE request_id = %s',
(identity,), (identity,),
), ),