ci: add merge smoke checks workflow with loopback-only harness and 11 curated cases (#42709)

* ci: add dashboard and core smoke checks across supported Python versions

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* ci: tighten merge smoke harness and keep mapped test diffs additive

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* ci: terminate proxy on readiness timeout and use contextlib.suppress in teardown

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: yuneng <yuneng@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
devin-ai-integration[bot] 2026-09-23 11:01:08 -07:00 • committed by GitHub
parent e73f949fbb
commit b0407ad33e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 1441 additions and 1 deletions

15
.github/merge-smoke-tests.json vendored Normal file
View file

@ -0,0 +1,15 @@
{
"cases": {
"CHAT-JSON": "tests/test_litellm/llms/openai/test_openai.py::test_acompletion_returns_json_reply_over_injected_transport",
"CHAT-TEXT-STREAM": "tests/test_litellm/llms/openai/test_openai.py::test_acompletion_streams_text_deltas_over_injected_transport",
"CHAT-TOOL-STREAM": "tests/test_litellm/llms/openai/test_openai.py::test_acompletion_streams_tool_call_arguments_over_injected_transport",
"MODEL-ALLOW": "tests/test_litellm/proxy/auth/test_auth_checks.py::test_can_object_call_model_allows_listed_model_for_key",
"MODEL-DENY": "tests/test_litellm/proxy/auth/test_auth_checks.py::test_can_object_call_model_denials_return_forbidden[key-key_model_access_denied]",
"COST-EXPLICIT": "tests/test_litellm/test_cost_calculator.py::test_completion_cost_charges_explicit_per_token_rates_over_registered_ones",
"COST-ZERO": "tests/test_litellm/test_cost_calculator.py::test_completion_cost_is_zero_when_explicit_rates_are_zero",
"LOG-CONTENT-ON": "tests/test_litellm/litellm_core_utils/test_litellm_logging.py::test_standard_logging_payload_keeps_message_content_when_message_logging_is_on",
"LOG-CONTENT-OFF": "tests/test_litellm/litellm_core_utils/test_litellm_logging.py::test_standard_logging_payload_redacts_message_content_when_message_logging_is_off",
"CALLBACK-SUCCESS": "tests/test_litellm/litellm_core_utils/test_litellm_logging.py::test_async_success_handler_delivers_standard_logging_payload_to_custom_logger",
"CALLBACK-FAILURE": "tests/test_litellm/litellm_core_utils/test_litellm_logging.py::test_async_failure_handler_delivers_failure_payload_to_custom_logger"
}
}

493
.github/scripts/run_merge_smoke.py vendored Normal file
View file

@ -0,0 +1,493 @@
#!/usr/bin/env python3
"""Merge smoke harness: bounded checks run inside a loopback-only Linux network namespace."""
# ruff: noqa: T201 # CLI harness: stdout/stderr lines are the reported result
from __future__ import annotations
import argparse
import contextlib
import http.client
import json
import os
import secrets
import signal
import socket
import subprocess
import sys
import time
from collections import Counter
from collections.abc import Sequence
from dataclasses import dataclass, field
from pathlib import Path
from types import MappingProxyType
from typing import Final, NoReturn, TextIO, cast
import pytest
EXPECTED_CASES: Final = (
"CHAT-JSON",
"CHAT-TEXT-STREAM",
"CHAT-TOOL-STREAM",
"MODEL-ALLOW",
"MODEL-DENY",
"COST-EXPLICIT",
"COST-ZERO",
"LOG-CONTENT-ON",
"LOG-CONTENT-OFF",
"CALLBACK-SUCCESS",
"CALLBACK-FAILURE",
)
@dataclass(frozen=True, slots=True)
class CheckResult:
ok: bool
detail: str = ""
@dataclass(slots=True)
class _Args:
command: str = ""
no_child: bool = False
expect: str = ""
litellm_bin: str | None = None
lite_bin: str | None = None
diagnostics_dir: str = ""
ready_deadline: float = 120.0
shutdown_deadline: float = 20.0
poll_interval: float = 0.5
manifest: str = ""
rootdir: str | None = None
def fail(reason: str) -> NoReturn:
print(f"merge-smoke: FAIL {reason}", file=sys.stderr)
sys.exit(1)
def ok(step: str) -> None:
print(f"merge-smoke: OK {step}")
def tail(path: Path, lines: int = 20) -> str:
try:
return "\n".join(path.read_text(errors="replace").splitlines()[-lines:])
except OSError as exc:
return f"<cannot read {path}: {exc}>"
def cmd_verify_isolation(args: _Args) -> int:
if os.geteuid() == 0:
fail("verify-isolation must run unprivileged (geteuid()==0)")
try:
socket.create_connection(("192.0.2.1", 9), timeout=3)
except OSError as exc:
print(f"external connect blocked as expected: errno={exc.errno} {exc}")
else:
fail("external TCP connect to 192.0.2.1:9 succeeded; namespace is not isolated")
listener: Final = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
listener.bind(("127.0.0.1", 0))
listener.listen(1)
port: Final = cast(int, listener.getsockname()[1])
client: Final = socket.create_connection(("127.0.0.1", port), timeout=5)
accepted: Final = listener.accept()
accepted[0].close()
client.close()
listener.close()
print(f"loopback connect ok on 127.0.0.1:{port}")
if not args.no_child:
proc: Final = subprocess.run(
[sys.executable, str(Path(__file__).resolve()), "verify-isolation", "--no-child"],
timeout=30,
capture_output=True,
text=True,
)
if proc.returncode != 0:
fail(f"child process did not inherit isolation: {proc.stderr.strip()}")
print("child process inherits isolation")
ok("verify-isolation")
return 0
def cmd_interpreter(args: _Args) -> int:
print(sys.version)
print(sys.executable)
actual: Final = f"{sys.version_info.major}.{sys.version_info.minor}"
if actual != args.expect:
fail(f"interpreter is {actual}, expected {args.expect}")
ok(f"interpreter {actual}")
return 0
def _run_cli(argv: Sequence[str], label: str) -> CheckResult:
try:
proc: Final = subprocess.run(list(argv), timeout=120, capture_output=True, text=True)
except subprocess.TimeoutExpired:
return CheckResult(ok=False, detail=f"{label} timed out after 120s")
sys.stdout.write(proc.stdout)
sys.stderr.write(proc.stderr)
if proc.returncode != 0:
return CheckResult(ok=False, detail=f"{label} exited {proc.returncode}")
return CheckResult(ok=True)
def cmd_cli(args: _Args) -> int:
venv_bin: Final = Path(sys.executable).parent
litellm_bin: Final = Path(args.litellm_bin) if args.litellm_bin else venv_bin / "litellm"
lite_bin: Final = Path(args.lite_bin) if args.lite_bin else venv_bin / "lite"
commands: Final = (
("import litellm", [sys.executable, "-c", "import litellm"]),
("litellm --version", [str(litellm_bin), "--version"]),
("lite version", [str(lite_bin), "version"]),
)
for label, argv in commands:
result = _run_cli(argv, label)
if not result.ok:
fail(result.detail)
ok(label)
return 0
def _free_port() -> int:
sock: Final = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.bind(("127.0.0.1", 0))
port: Final = cast(int, sock.getsockname()[1])
sock.close()
return port
_CONFIG_TEMPLATE: Final = """model_list:
- model_name: smoke-model
litellm_params:
model: openai/smoke-model
api_base: http://127.0.0.1:9/v1
api_key: synthetic-key
general_settings:
master_key: os.environ/LITELLM_MASTER_KEY
"""
def _listen_inode(port: int) -> str | None:
target: Final = f"{port:04X}"
for table in ("/proc/net/tcp", "/proc/net/tcp6"):
try:
rows = Path(table).read_text().splitlines()[1:]
except OSError:
continue
for row in rows:
cols = row.split()
if len(cols) > 9 and cols[3] == "0A" and cols[1].rsplit(":", 1)[-1] == target:
return cols[9]
return None
def _ancestors(pid: int) -> frozenset[int]:
chain: Final[set[int]] = set()
pending: Final[list[int]] = [pid]
while pending:
current = pending.pop()
if current <= 0 or current in chain:
continue
chain.add(current)
try:
stat = Path(f"/proc/{current}/stat").read_text()
except OSError:
continue
pending.append(int(stat.rpartition(")")[2].split()[1]))
return frozenset(chain)
def _socket_owner_pid(inode: str) -> int | None:
for proc_dir in Path("/proc").iterdir():
if not proc_dir.name.isdigit():
continue
fd_dir = proc_dir / "fd"
try:
for fd in fd_dir.iterdir():
try:
if os.readlink(fd) == f"socket:[{inode}]":
return int(proc_dir.name)
except OSError:
continue
except OSError:
continue
return None
def _verify_port_owner(port: int, proc: subprocess.Popen[bytes]) -> CheckResult:
inode: Final = _listen_inode(port)
if inode is None:
return CheckResult(ok=False, detail=f"no LISTEN socket found for port {port} in /proc/net/tcp")
owner: Final = _socket_owner_pid(inode)
if owner is None:
return CheckResult(ok=False, detail=f"no process owns the listen socket inode {inode} for port {port}")
if owner != proc.pid and proc.pid not in _ancestors(owner):
return CheckResult(
ok=False, detail=f"port {port} owned by pid {owner} outside the launched process group {proc.pid}"
)
if proc.poll() is not None:
return CheckResult(ok=False, detail=f"proxy exited with code {proc.returncode} after readiness")
return CheckResult(ok=True)
def cmd_proxy_startup(args: _Args) -> int:
diagnostics: Final = Path(args.diagnostics_dir)
diagnostics.mkdir(parents=True, exist_ok=True)
venv_bin: Final = Path(sys.executable).parent
litellm_bin: Final = Path(args.litellm_bin) if args.litellm_bin else venv_bin / "litellm"
port: Final = _free_port()
master_key: Final = "sk-smoke-" + secrets.token_hex(16)
config_path: Final = diagnostics / "config.yaml"
config_path.write_text(_CONFIG_TEMPLATE)
log_path: Final = diagnostics / "proxy.log"
result_path: Final = diagnostics / "result.json"
outcome: Final[dict[str, object]] = {
"port": port,
"time_to_ready_s": None,
"shutdown_s": None,
"readiness": None,
"outcome": "failed",
}
log_file: Final = log_path.open("w")
env: Final = {
**os.environ,
"LITELLM_MASTER_KEY": master_key,
"LITELLM_LOCAL_MODEL_COST_MAP": "True",
}
started: Final = time.monotonic()
proc: Final = subprocess.Popen(
[str(litellm_bin), "--config", str(config_path), "--host", "127.0.0.1", "--port", str(port)],
stdout=log_file,
stderr=subprocess.STDOUT,
start_new_session=True,
env=env,
)
body: str | None = None
last_status: int | None = None
while time.monotonic() - started < args.ready_deadline:
if proc.poll() is not None:
log_file.close()
result_path.write_text(json.dumps(outcome))
fail(f"proxy exited early with code {proc.returncode}\n{tail(log_path)}")
try:
conn = http.client.HTTPConnection("127.0.0.1", port, timeout=5)
conn.request("GET", "/health/readiness")
resp = conn.getresponse()
last_status = resp.status
candidate = resp.read().decode()
conn.close()
except (http.client.HTTPException, ConnectionError, OSError):
time.sleep(args.poll_interval)
continue
if last_status == 200:
body = candidate
break
time.sleep(args.poll_interval)
outcome["time_to_ready_s"] = round(time.monotonic() - started, 3)
if body is None:
_terminate(proc, log_file)
result_path.write_text(json.dumps(outcome))
detail = f"last status {last_status}" if last_status is not None else "no response"
fail(f"readiness not reached within {args.ready_deadline}s ({detail})\n{tail(log_path)}")
outcome["readiness"] = body
try:
readiness = cast(object, json.loads(body))
except json.JSONDecodeError:
readiness = None
if readiness != {"status": "healthy", "db": "Not connected"}:
_terminate(proc, log_file)
result_path.write_text(json.dumps(outcome))
fail(f"unexpected readiness body: {body}")
owner_check: Final = _verify_port_owner(port, proc)
if not owner_check.ok:
_terminate(proc, log_file)
result_path.write_text(json.dumps(outcome))
fail(owner_check.detail)
shutdown_started: Final = time.monotonic()
os.killpg(proc.pid, signal.SIGTERM)
try:
proc.wait(timeout=args.shutdown_deadline)
except subprocess.TimeoutExpired:
os.killpg(proc.pid, signal.SIGKILL)
proc.wait(timeout=10)
outcome["shutdown_s"] = round(time.monotonic() - shutdown_started, 3)
log_file.close()
result_path.write_text(json.dumps(outcome))
fail(f"forced kill after {args.shutdown_deadline}s\n{tail(log_path)}")
outcome["shutdown_s"] = round(time.monotonic() - shutdown_started, 3)
try:
os.killpg(proc.pid, 0)
except ProcessLookupError:
pass
else:
os.killpg(proc.pid, signal.SIGKILL)
log_file.close()
result_path.write_text(json.dumps(outcome))
fail("process group survived SIGTERM")
log_file.close()
outcome["outcome"] = "ok"
result_path.write_text(json.dumps(outcome))
ok(f"proxy-startup ready={outcome['time_to_ready_s']}s shutdown={outcome['shutdown_s']}s")
return 0
def _terminate(proc: subprocess.Popen[bytes], log_file: TextIO) -> None:
with contextlib.suppress(ProcessLookupError):
os.killpg(proc.pid, signal.SIGTERM)
try:
proc.wait(timeout=10)
except subprocess.TimeoutExpired:
with contextlib.suppress(ProcessLookupError):
os.killpg(proc.pid, signal.SIGKILL)
with contextlib.suppress(subprocess.TimeoutExpired):
proc.wait(timeout=10)
log_file.close()
def _load_manifest(path: Path) -> MappingProxyType[str, str]:
def no_duplicates(pairs: list[tuple[object, object]]) -> dict[object, object]:
seen: dict[object, object] = {}
for key, value in pairs:
if key in seen:
raise ValueError(f"duplicate key in manifest: {key}")
seen[key] = value
return seen
raw_value: object = cast(object, json.loads(path.read_text(), object_pairs_hook=no_duplicates))
if not isinstance(raw_value, dict):
raise ValueError("manifest must be an object")
loaded: Final = cast(dict[object, object], raw_value)
cases_value: object = loaded.get("cases")
if not isinstance(cases_value, dict):
raise ValueError("manifest must be an object with a 'cases' object")
cases_any: Final = cast(dict[object, object], cases_value)
cases: Final = {k: v for k, v in cases_any.items() if isinstance(k, str) and isinstance(v, str)}
if len(cases) != len(cases_any):
raise ValueError("manifest 'cases' must map string ids to string node ids")
return MappingProxyType(cases)
@dataclass(slots=True, eq=False)
class _Recorder:
collect_failed: list[str] = field(default_factory=list)
collected: tuple[str, ...] = ()
reports: dict[str, list[tuple[str, str, bool]]] = field(default_factory=dict)
def pytest_collectreport(self, report: pytest.CollectReport) -> None:
if report.failed:
self.collect_failed.append(report.nodeid)
def pytest_collection_finish(self, session: pytest.Session) -> None:
self.collected = tuple(item.nodeid for item in session.items)
def pytest_runtest_logreport(self, report: pytest.TestReport) -> None:
self.reports.setdefault(report.nodeid, []).append((report.when, report.outcome, hasattr(report, "wasxfail")))
def cmd_pytest(args: _Args) -> int:
try:
cases: Final = _load_manifest(Path(args.manifest))
except (OSError, ValueError, json.JSONDecodeError) as exc:
fail(f"manifest invalid: {exc}")
if tuple(cases) != EXPECTED_CASES:
fail(f"manifest case ids must be exactly {list(EXPECTED_CASES)} in order, got {list(cases)}")
node_ids: Final = tuple(cases.values())
if len(set(node_ids)) != len(node_ids):
fail("manifest node ids are not unique")
argv: Final = [
*node_ids,
"-p",
"no:cacheprovider",
"-p",
"no:xdist",
"-p",
"no:rerunfailures",
"-p",
"no:randomly",
"-rA",
"-q",
*(["--rootdir", args.rootdir] if args.rootdir else []),
]
recorder: Final = _Recorder()
code: Final = pytest.main(argv, plugins=[recorder])
name_of: Final = MappingProxyType({node_id: case_id for case_id, node_id in cases.items()})
problems: Final[list[str]] = []
if code != 0:
problems.append(f"pytest exit code {code}")
for failed_id in recorder.collect_failed:
problems.append(f"collection failed: {name_of.get(failed_id, failed_id)}")
expected: Final = Counter(node_ids)
collected: Final = Counter(recorder.collected)
for node_id in expected - collected:
problems.append(f"missing case {name_of[node_id]} ({node_id})")
for node_id in collected - expected:
problems.append(f"unexpected test collected: {node_id}")
for node_id, count in collected.items():
if count > 1:
problems.append(f"duplicated test id: {node_id}")
if len(recorder.collected) != len(EXPECTED_CASES):
problems.append(f"collected {len(recorder.collected)} tests, expected {len(EXPECTED_CASES)}")
rows: Final[list[tuple[str, bool]]] = []
for case_id, node_id in cases.items():
reports = recorder.reports.get(node_id, [])
case_ok = (
bool(reports)
and all(outcome == "passed" and not wasxfail for _, outcome, wasxfail in reports)
and {when for when, _, _ in reports} >= {"setup", "call", "teardown"}
)
rows.append((case_id, case_ok))
if not reports:
problems.append(f"{case_id} ({node_id}) produced no runtest reports")
continue
for when, outcome, wasxfail in reports:
if outcome != "passed":
problems.append(f"{case_id} ({node_id}) {when} outcome={outcome}")
if wasxfail:
problems.append(f"{case_id} ({node_id}) {when} was xfail/xpass")
missing_phases = {"setup", "call", "teardown"} - {when for when, _, _ in reports}
for phase in sorted(missing_phases):
problems.append(f"{case_id} ({node_id}) missing {phase} report")
for case_id, passed in rows:
print(f"{case_id} {'PASS' if passed else 'FAIL'} {cases[case_id]}")
if problems:
for problem in problems:
print(f"merge-smoke: {problem}", file=sys.stderr)
fail("pytest verdict failed")
ok("pytest 11 cases")
return 0
def main() -> int:
parser: Final = argparse.ArgumentParser(description=__doc__)
subs: Final = parser.add_subparsers(dest="command", required=True)
p_iso: Final = subs.add_parser("verify-isolation")
p_iso.add_argument("--no-child", action="store_true")
p_interp: Final = subs.add_parser("interpreter")
p_interp.add_argument("--expect", required=True)
p_cli: Final = subs.add_parser("cli")
p_cli.add_argument("--litellm-bin", default=None)
p_cli.add_argument("--lite-bin", default=None)
p_proxy: Final = subs.add_parser("proxy-startup")
p_proxy.add_argument("--diagnostics-dir", required=True)
p_proxy.add_argument("--litellm-bin", default=None)
p_proxy.add_argument("--ready-deadline", type=float, default=120)
p_proxy.add_argument("--shutdown-deadline", type=float, default=20)
p_proxy.add_argument("--poll-interval", type=float, default=0.5)
p_test: Final = subs.add_parser("pytest")
p_test.add_argument("--manifest", required=True)
p_test.add_argument("--rootdir", default=None)
args: Final = parser.parse_args(namespace=_Args())
handlers: Final = {
"verify-isolation": cmd_verify_isolation,
"interpreter": cmd_interpreter,
"cli": cmd_cli,
"proxy-startup": cmd_proxy_startup,
"pytest": cmd_pytest,
}
return handlers[args.command](args)
if __name__ == "__main__":
sys.exit(main())

View file

@ -80,6 +80,9 @@ jobs:
- name: test_e2e_changed_gate
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_changed_gate.py tests/code_coverage_tests/test_e2e_idp_stack.py
- name: Check merge smoke harness
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_merge_smoke.py
- name: router_code_coverage
run: uv run --no-sync python ./tests/code_coverage_tests/router_code_coverage.py

95
.github/workflows/test-merge-smoke.yml vendored Normal file
View file

@ -0,0 +1,95 @@
name: Merge smoke checks
on:
pull_request:
branches: [main, litellm_internal_staging, litellm_oss_staging, "litellm_**"]
workflow_dispatch:
permissions:
contents: read
concurrency:
group: merge-smoke-${{ github.event.pull_request.number || github.run_id }}
cancel-in-progress: true
jobs:
dashboard-build:
name: Dashboard build
runs-on: ubuntu-24.04
timeout-minutes: 30
steps:
- name: Checkout
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Build the dashboard stage
run: docker build --target ui-builder -f Dockerfile .
core-checks:
name: Core checks (Python ${{ matrix.python-version }})
runs-on: ubuntu-24.04
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
env:
LITELLM_LOCAL_MODEL_COST_MAP: "True"
steps:
- name: Checkout
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: ${{ matrix.python-version }}
- name: Set up uv
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Install dependencies
run: .github/scripts/uv_sync_with_retries.sh --frozen --extra proxy --extra cli --group dev --group proxy-dev --python ${{ matrix.python-version }}
- name: Create the loopback-only network namespace
run: |
sudo ip netns add smoke
sudo ip netns exec smoke ip link set lo up
cat > "${RUNNER_TEMP}/in-netns" <<'WRAP'
#!/usr/bin/env bash
set -euo pipefail
exec sudo --preserve-env=LITELLM_LOCAL_MODEL_COST_MAP ip netns exec smoke setpriv --reuid "$(id -u)" --regid "$(id -g)" --init-groups -- env HOME="${HOME}" PATH="${PATH}" "$@"
WRAP
chmod +x "${RUNNER_TEMP}/in-netns"
echo "IN_NETNS=${RUNNER_TEMP}/in-netns" >> "${GITHUB_ENV}"
- name: Verify namespace isolation
run: $IN_NETNS .venv/bin/python .github/scripts/run_merge_smoke.py verify-isolation
- name: Verify interpreter version
run: $IN_NETNS .venv/bin/python .github/scripts/run_merge_smoke.py interpreter --expect ${{ matrix.python-version }}
- name: Import and CLI checks
run: $IN_NETNS .venv/bin/python .github/scripts/run_merge_smoke.py cli
- name: Proxy startup check
run: $IN_NETNS .venv/bin/python .github/scripts/run_merge_smoke.py proxy-startup --diagnostics-dir "${RUNNER_TEMP}/smoke-diagnostics"
- name: Run curated smoke cases
run: $IN_NETNS .venv/bin/python .github/scripts/run_merge_smoke.py pytest --manifest .github/merge-smoke-tests.json
- name: Upload smoke diagnostics
if: always()
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
with:
name: merge-smoke-diagnostics-py${{ matrix.python-version }}
path: ${{ runner.temp }}/smoke-diagnostics
if-no-files-found: ignore
- name: Remove the network namespace
if: always()
run: sudo ip netns delete smoke

View file

@ -0,0 +1,402 @@
import json
import os
import stat
import subprocess
import sys
import textwrap
from pathlib import Path
from typing import Final, cast
import pytest
HARNESS: Final = Path(__file__).parents[2] / ".github" / "scripts" / "run_merge_smoke.py"
CASE_IDS: Final = (
"CHAT-JSON",
"CHAT-TEXT-STREAM",
"CHAT-TOOL-STREAM",
"MODEL-ALLOW",
"MODEL-DENY",
"COST-EXPLICIT",
"COST-ZERO",
"LOG-CONTENT-ON",
"LOG-CONTENT-OFF",
"CALLBACK-SUCCESS",
"CALLBACK-FAILURE",
)
def _write_fake_tests(root: Path, body: str) -> Path:
package: Final = root / "fake_tests"
package.mkdir()
(package / "test_cases.py").write_text(body)
return package
def _manifest(root: Path, **overrides: str) -> Path:
cases: Final[dict[str, str]] = {
case_id: f"fake_tests/test_cases.py::test_{case_id.lower().replace('-', '_')}" for case_id in CASE_IDS
}
cases.update(overrides)
path: Final = root / "manifest.json"
path.write_text(json.dumps({"cases": cases}))
return path
def _run(root: Path, *argv: str) -> subprocess.CompletedProcess[str]:
return subprocess.run(
[sys.executable, "-I", str(HARNESS), *argv],
cwd=root,
capture_output=True,
text=True,
timeout=120,
)
def _passing_tests() -> str:
return "\n".join(f"def test_{case_id.lower().replace('-', '_')}():\n assert True" for case_id in CASE_IDS)
def test_all_eleven_cases_pass(tmp_path: Path) -> None:
_write_fake_tests(tmp_path, _passing_tests())
manifest: Final = _manifest(tmp_path)
proc: Final = _run(tmp_path, "pytest", "--manifest", str(manifest), "--rootdir", str(tmp_path))
assert proc.returncode == 0, proc.stderr
assert proc.stdout.count("PASS") >= 11
for case_id in CASE_IDS:
assert f"{case_id} PASS" in proc.stdout
def test_missing_test_node_id_fails(tmp_path: Path) -> None:
_write_fake_tests(tmp_path, _passing_tests())
manifest: Final = _manifest(tmp_path, **{"COST-ZERO": "fake_tests/test_cases.py::test_does_not_exist"})
proc: Final = _run(tmp_path, "pytest", "--manifest", str(manifest), "--rootdir", str(tmp_path))
assert proc.returncode != 0
assert "COST-ZERO" in proc.stderr or "test_does_not_exist" in proc.stderr
def test_skipped_case_fails(tmp_path: Path) -> None:
_write_fake_tests(
tmp_path,
_passing_tests().replace(
"def test_cost_zero():\n assert True",
"def test_cost_zero():\n import pytest\n pytest.skip('nope')",
),
)
manifest: Final = _manifest(tmp_path)
proc: Final = _run(tmp_path, "pytest", "--manifest", str(manifest), "--rootdir", str(tmp_path))
assert proc.returncode != 0
assert "COST-ZERO" in proc.stderr
def test_xfail_case_fails(tmp_path: Path) -> None:
_write_fake_tests(
tmp_path,
"import pytest\n"
+ _passing_tests().replace(
"def test_cost_zero():\n assert True",
"@pytest.mark.xfail\ndef test_cost_zero():\n assert False",
),
)
manifest: Final = _manifest(tmp_path)
proc: Final = _run(tmp_path, "pytest", "--manifest", str(manifest), "--rootdir", str(tmp_path))
assert proc.returncode != 0
assert "COST-ZERO" in proc.stderr
def test_xpass_case_fails(tmp_path: Path) -> None:
_write_fake_tests(
tmp_path,
"import pytest\n"
+ _passing_tests().replace(
"def test_cost_zero():\n assert True",
"@pytest.mark.xfail\ndef test_cost_zero():\n assert True",
),
)
manifest: Final = _manifest(tmp_path)
proc: Final = _run(tmp_path, "pytest", "--manifest", str(manifest), "--rootdir", str(tmp_path))
assert proc.returncode != 0
assert "COST-ZERO" in proc.stderr
def test_duplicate_manifest_key_fails(tmp_path: Path) -> None:
manifest: Final = tmp_path / "manifest.json"
manifest.write_text('{"cases": {"CHAT-JSON": "a::b", "CHAT-JSON": "a::c"}}')
proc: Final = _run(tmp_path, "pytest", "--manifest", str(manifest))
assert proc.returncode != 0
assert "CHAT-JSON" in proc.stderr
def test_missing_case_id_fails(tmp_path: Path) -> None:
manifest: Final = tmp_path / "manifest.json"
cases: Final = {c: f"t::{c}" for c in CASE_IDS[:-1]}
manifest.write_text(json.dumps({"cases": cases}))
proc: Final = _run(tmp_path, "pytest", "--manifest", str(manifest))
assert proc.returncode != 0
assert "case ids" in proc.stderr
def test_extra_case_id_fails(tmp_path: Path) -> None:
manifest: Final = tmp_path / "manifest.json"
cases: Final = {c: f"t::{c}" for c in CASE_IDS}
cases["EXTRA"] = "t::x"
manifest.write_text(json.dumps({"cases": cases}))
proc: Final = _run(tmp_path, "pytest", "--manifest", str(manifest))
assert proc.returncode != 0
assert "case ids" in proc.stderr
def test_teardown_error_fails(tmp_path: Path) -> None:
body: Final = (
"import pytest\n\n@pytest.fixture\ndef boom():\n yield\n raise RuntimeError('teardown-boom')\n\n"
+ _passing_tests().replace(
"def test_cost_zero():\n assert True",
"def test_cost_zero(boom):\n assert True",
)
)
_write_fake_tests(tmp_path, body)
manifest: Final = _manifest(tmp_path)
proc: Final = _run(tmp_path, "pytest", "--manifest", str(manifest), "--rootdir", str(tmp_path))
assert proc.returncode != 0
assert "COST-ZERO" in proc.stderr
def _fake_litellm(tmp_path: Path, script: str) -> Path:
path: Final = tmp_path / "fake-litellm"
path.write_text(f"#!{sys.executable}\n" + textwrap.dedent(script))
path.chmod(path.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
return path
def test_proxy_startup_exits_early_fails(tmp_path: Path) -> None:
fake: Final = _fake_litellm(tmp_path, "import sys\nsys.exit(1)\n")
diagnostics: Final = tmp_path / "diag"
proc: Final = _run(
tmp_path,
"proxy-startup",
"--diagnostics-dir",
str(diagnostics),
"--litellm-bin",
str(fake),
)
assert proc.returncode != 0
assert "exited early" in proc.stderr
assert (diagnostics / "proxy.log").exists()
def test_proxy_startup_readiness_timeout_fails(tmp_path: Path) -> None:
fake: Final = _fake_litellm(
tmp_path,
"import os, pathlib, sys, time\npathlib.Path(sys.argv[0]).with_name('fake.pid').write_text(str(os.getpid()))\ntime.sleep(3600)\n",
)
diagnostics: Final = tmp_path / "diag"
proc: Final = _run(
tmp_path,
"proxy-startup",
"--diagnostics-dir",
str(diagnostics),
"--litellm-bin",
str(fake),
"--ready-deadline",
"3",
"--shutdown-deadline",
"2",
)
assert proc.returncode != 0
assert "readiness" in proc.stderr
assert (diagnostics / "proxy.log").exists()
with pytest.raises(ProcessLookupError):
os.kill(int((tmp_path / "fake.pid").read_text()), 0)
def test_proxy_startup_healthy_succeeds(tmp_path: Path) -> None:
fake: Final = _fake_litellm(
tmp_path,
"""
import http.server, json, sys
port = int(sys.argv[sys.argv.index("--port") + 1])
class H(http.server.BaseHTTPRequestHandler):
def do_GET(self):
body = json.dumps({"status": "healthy", "db": "Not connected"}).encode()
self.send_response(200)
self.send_header("content-type", "application/json")
self.end_headers()
self.wfile.write(body)
def log_message(self, *a):
pass
http.server.HTTPServer(("127.0.0.1", port), H).serve_forever()
""",
)
diagnostics: Final = tmp_path / "diag"
proc: Final = _run(
tmp_path,
"proxy-startup",
"--diagnostics-dir",
str(diagnostics),
"--litellm-bin",
str(fake),
"--ready-deadline",
"15",
)
assert proc.returncode == 0, proc.stderr
result: Final = cast(dict[str, object], json.loads((diagnostics / "result.json").read_text()))
assert result["outcome"] == "ok"
assert result["readiness"] == '{"status": "healthy", "db": "Not connected"}'
def test_proxy_startup_sigterm_ignored_forces_kill(tmp_path: Path) -> None:
fake: Final = _fake_litellm(
tmp_path,
"""
import http.server, json, os, pathlib, signal, sys
port = int(sys.argv[sys.argv.index("--port") + 1])
pathlib.Path(sys.argv[0]).with_name("fake.pid").write_text(str(os.getpid()))
signal.signal(signal.SIGTERM, signal.SIG_IGN)
class H(http.server.BaseHTTPRequestHandler):
def do_GET(self):
body = json.dumps({"status": "healthy", "db": "Not connected"}).encode()
self.send_response(200)
self.send_header("content-type", "application/json")
self.end_headers()
self.wfile.write(body)
def log_message(self, *a):
pass
http.server.HTTPServer(("127.0.0.1", port), H).serve_forever()
""",
)
diagnostics: Final = tmp_path / "diag"
proc: Final = _run(
tmp_path,
"proxy-startup",
"--diagnostics-dir",
str(diagnostics),
"--litellm-bin",
str(fake),
"--ready-deadline",
"15",
"--shutdown-deadline",
"2",
)
assert proc.returncode != 0
assert "forced kill" in proc.stderr
result: Final = cast(dict[str, object], json.loads((diagnostics / "result.json").read_text()))
assert result["outcome"] == "failed"
with pytest.raises(ProcessLookupError):
os.kill(int((tmp_path / "fake.pid").read_text()), 0)
def test_proxy_startup_waits_through_not_ready_status(tmp_path: Path) -> None:
fake: Final = _fake_litellm(
tmp_path,
"""
import http.server, json, sys
port = int(sys.argv[sys.argv.index("--port") + 1])
hits = [0]
class H(http.server.BaseHTTPRequestHandler):
def do_GET(self):
hits[0] += 1
if hits[0] <= 2:
self.send_response(503)
self.end_headers()
return
body = json.dumps({"status": "healthy", "db": "Not connected"}).encode()
self.send_response(200)
self.send_header("content-type", "application/json")
self.end_headers()
self.wfile.write(body)
def log_message(self, *a):
pass
http.server.HTTPServer(("127.0.0.1", port), H).serve_forever()
""",
)
diagnostics: Final = tmp_path / "diag"
proc: Final = _run(
tmp_path,
"proxy-startup",
"--diagnostics-dir",
str(diagnostics),
"--litellm-bin",
str(fake),
"--ready-deadline",
"15",
)
assert proc.returncode == 0, proc.stderr
def test_proxy_startup_wrong_body_fails(tmp_path: Path) -> None:
fake: Final = _fake_litellm(
tmp_path,
"""
import http.server, json, sys
port = int(sys.argv[sys.argv.index("--port") + 1])
class H(http.server.BaseHTTPRequestHandler):
def do_GET(self):
body = json.dumps({"status": "healthy", "db": "connected"}).encode()
self.send_response(200)
self.send_header("content-type", "application/json")
self.end_headers()
self.wfile.write(body)
def log_message(self, *a):
pass
http.server.HTTPServer(("127.0.0.1", port), H).serve_forever()
""",
)
diagnostics: Final = tmp_path / "diag"
proc: Final = _run(
tmp_path,
"proxy-startup",
"--diagnostics-dir",
str(diagnostics),
"--litellm-bin",
str(fake),
"--ready-deadline",
"15",
)
assert proc.returncode != 0
assert "connected" in proc.stderr
def test_interpreter_expect_mismatch_fails() -> None:
proc: Final = _run(Path.cwd(), "interpreter", "--expect", "9.99")
assert proc.returncode != 0
assert "9.99" in proc.stderr
def test_interpreter_expect_match_passes() -> None:
expect: Final = f"{sys.version_info.major}.{sys.version_info.minor}"
proc: Final = _run(Path.cwd(), "interpreter", "--expect", expect)
assert proc.returncode == 0
assert f"OK interpreter {expect}" in proc.stdout

View file

@ -14,6 +14,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
from mcp.types import AudioContent, CallToolResult, ImageContent, TextContent
from openai import AsyncOpenAI
from openai._legacy_response import HttpxBinaryResponseContent
import litellm
@ -8426,3 +8427,174 @@ class TestBudgetReservationBinding:
assert logging_obj.litellm_params["metadata"]["user_api_key_budget_reservation"] is reservation
assert reservation["callback_bound"] is False
@pytest.mark.asyncio
async def test_standard_logging_payload_keeps_message_content_when_message_logging_is_on(monkeypatch):
outbound: Final = asyncio.Queue()
logs: Final = asyncio.Queue()
monkeypatch.setattr(litellm, "turn_off_message_logging", False)
def respond(request: httpx.Request) -> httpx.Response:
outbound.put_nowait(json.loads(request.content))
return httpx.Response(
200,
json={
"id": "chatcmpl-smoke",
"object": "chat.completion",
"created": 0,
"model": "gpt-5.6",
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": "smoke-marker-reply"},
"finish_reason": "stop",
}
],
"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
},
)
async def capture(kwargs, response_obj, start_time, end_time):
logs.put_nowait(kwargs["standard_logging_object"])
async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as http_client:
client: Final = AsyncOpenAI(api_key="transport-only", http_client=http_client)
await litellm.acompletion(
model="openai/gpt-5.6",
api_key="transport-only",
client=client,
messages=[{"role": "user", "content": "smoke-marker-request"}],
success_callback=[capture],
num_retries=0,
max_retries=0,
)
payload: Final = await asyncio.wait_for(logs.get(), timeout=10)
request: Final = await asyncio.wait_for(outbound.get(), timeout=10)
assert outbound.empty()
assert request["messages"][0]["content"] == "smoke-marker-request"
assert payload["messages"][0]["content"] == "smoke-marker-request"
assert payload["response"]["choices"][0]["message"]["content"] == "smoke-marker-reply"
@pytest.mark.asyncio
async def test_standard_logging_payload_redacts_message_content_when_message_logging_is_off(monkeypatch):
outbound: Final = asyncio.Queue()
logs: Final = asyncio.Queue()
monkeypatch.setattr(litellm, "turn_off_message_logging", False)
def respond(request: httpx.Request) -> httpx.Response:
outbound.put_nowait(json.loads(request.content))
return httpx.Response(
200,
json={
"id": "chatcmpl-smoke",
"object": "chat.completion",
"created": 0,
"model": "gpt-5.6",
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": "smoke-marker-reply"},
"finish_reason": "stop",
}
],
"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
},
)
async def capture(kwargs, response_obj, start_time, end_time):
logs.put_nowait(kwargs["standard_logging_object"])
async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as http_client:
client: Final = AsyncOpenAI(api_key="transport-only", http_client=http_client)
await litellm.acompletion(
model="openai/gpt-5.6",
api_key="transport-only",
client=client,
messages=[{"role": "user", "content": "smoke-marker-request"}],
turn_off_message_logging=True,
success_callback=[capture],
num_retries=0,
max_retries=0,
)
payload: Final = await asyncio.wait_for(logs.get(), timeout=10)
assert outbound.qsize() == 1
assert "smoke-marker-request" not in json.dumps(payload["messages"])
assert "smoke-marker-reply" not in json.dumps(payload["response"])
assert payload["model"]
assert payload["total_tokens"] == 15
@pytest.mark.asyncio
async def test_async_success_handler_delivers_standard_logging_payload_to_custom_logger():
events: Final = asyncio.Queue()
class SuccessRecorder(CustomLogger):
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
events.put_nowait((kwargs, response_obj))
recorder: Final = SuccessRecorder()
logging_obj: Final = LitellmLogging(
model="openai/gpt-5.6",
messages=[{"role": "user", "content": "smoke-callback-request"}],
stream=False,
call_type="acompletion",
start_time=time.time(),
litellm_call_id="smoke-callback-success",
function_id="smoke-callback-success",
dynamic_async_success_callbacks=[recorder],
)
logging_obj.model_call_details["litellm_params"] = {"metadata": {}, "proxy_server_request": {}}
result: Final = ModelResponse(
model="openai/gpt-5.6",
choices=[
{"index": 0, "message": {"role": "assistant", "content": "smoke-callback-reply"}, "finish_reason": "stop"}
],
usage=litellm.Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15),
)
now: Final = datetime.datetime.now()
await logging_obj.async_success_handler(result=result, start_time=now, end_time=now, cache_hit=False)
kwargs, response_obj = await asyncio.wait_for(events.get(), timeout=10)
assert response_obj is result
payload: Final = kwargs["standard_logging_object"]
assert payload["status"] == "success"
assert payload["model"] == "openai/gpt-5.6"
assert payload["total_tokens"] == 15
assert events.empty()
@pytest.mark.asyncio
async def test_async_failure_handler_delivers_failure_payload_to_custom_logger():
events: Final = asyncio.Queue()
class FailureRecorder(CustomLogger):
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
events.put_nowait((kwargs, response_obj))
recorder: Final = FailureRecorder()
logging_obj: Final = LitellmLogging(
model="openai/gpt-5.6",
messages=[{"role": "user", "content": "smoke-callback-request"}],
stream=False,
call_type="acompletion",
start_time=time.time(),
litellm_call_id="smoke-callback-failure",
function_id="smoke-callback-failure",
dynamic_async_failure_callbacks=[recorder],
)
logging_obj.model_call_details["litellm_params"] = {"metadata": {}, "proxy_server_request": {}}
failure: Final = ValueError("smoke-failure")
now: Final = datetime.datetime.now()
await logging_obj.async_failure_handler(exception=failure, traceback_exception="", start_time=now, end_time=now)
kwargs, response_obj = await asyncio.wait_for(events.get(), timeout=10)
assert kwargs["exception"] is failure
payload: Final = kwargs["standard_logging_object"]
assert payload["status"] == "failure"
assert "smoke-failure" in payload["error_str"]
assert payload["model"] == "openai/gpt-5.6"
assert events.empty()

View file

@ -1,5 +1,12 @@
import pytest
import asyncio
import json
from typing import Final
import httpx
import pytest
from openai import AsyncOpenAI
import litellm
from litellm.llms.openai.openai import OpenAIChatCompletion
@ -50,3 +57,199 @@ def test_get_stream_options_passes_caller_stream_options_through_on_any_host(api
assert OpenAIChatCompletion().get_stream_options(stream_options=caller_options, api_base=api_base) == {
"stream_options": caller_options
}
@pytest.mark.asyncio
async def test_acompletion_returns_json_reply_over_injected_transport():
outbound: Final = asyncio.Queue()
def respond(request: httpx.Request) -> httpx.Response:
outbound.put_nowait(json.loads(request.content))
return httpx.Response(
200,
json={
"id": "chatcmpl-smoke",
"object": "chat.completion",
"created": 0,
"model": "gpt-5.6",
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": "smoke-json-reply"},
"finish_reason": "stop",
}
],
"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
},
)
async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as http_client:
client: Final = AsyncOpenAI(api_key="transport-only", http_client=http_client)
response: Final = await asyncio.wait_for(
litellm.acompletion(
model="openai/gpt-5.6",
api_key="transport-only",
client=client,
messages=[{"role": "user", "content": "smoke-json-request"}],
num_retries=0,
max_retries=0,
),
timeout=10,
)
request: Final = await asyncio.wait_for(outbound.get(), timeout=10)
assert request["model"] == "gpt-5.6"
assert request["messages"] == [{"role": "user", "content": "smoke-json-request"}]
assert not request.get("stream")
assert outbound.empty()
assert response.choices[0].message.content == "smoke-json-reply"
assert response.choices[0].finish_reason == "stop"
assert response.usage.total_tokens == 15
@pytest.mark.asyncio
async def test_acompletion_streams_text_deltas_over_injected_transport():
outbound: Final = asyncio.Queue()
def chunk(delta: dict, finish: str | None) -> bytes:
body: Final = {
"id": "chatcmpl-smoke",
"object": "chat.completion.chunk",
"created": 0,
"model": "gpt-5.6",
"choices": [{"index": 0, "delta": delta, "finish_reason": finish}],
}
return f"data: {json.dumps(body)}\n\n".encode()
def respond(request: httpx.Request) -> httpx.Response:
outbound.put_nowait(json.loads(request.content))
usage: Final = {
"id": "chatcmpl-smoke",
"object": "chat.completion.chunk",
"created": 0,
"model": "gpt-5.6",
"choices": [],
"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
}
content: Final = b"".join(
(
chunk({"role": "assistant", "content": "Hel"}, None),
chunk({"content": "lo"}, "stop"),
f"data: {json.dumps(usage)}\n\n".encode(),
b"data: [DONE]\n\n",
)
)
return httpx.Response(200, headers={"content-type": "text/event-stream"}, content=content)
async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as http_client:
client: Final = AsyncOpenAI(api_key="transport-only", http_client=http_client)
stream: Final = await litellm.acompletion(
model="openai/gpt-5.6",
api_key="transport-only",
client=client,
messages=[{"role": "user", "content": "smoke-stream-request"}],
stream=True,
num_retries=0,
max_retries=0,
)
chunks: Final = []
async def drain() -> None:
async for part in stream:
chunks.append(part)
await asyncio.wait_for(drain(), timeout=10)
request: Final = await asyncio.wait_for(outbound.get(), timeout=10)
assert request["stream"] is True
assert outbound.empty()
assert (
"".join(part.choices[0].delta.content or "" for part in chunks if part.choices and part.choices[0].delta)
== "Hello"
)
last_finish: Final = next(
part.choices[0].finish_reason for part in reversed(chunks) if part.choices and part.choices[0].finish_reason
)
assert last_finish == "stop"
@pytest.mark.asyncio
async def test_acompletion_streams_tool_call_arguments_over_injected_transport():
outbound: Final = asyncio.Queue()
tools: Final = [
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Look up weather for a city",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"],
},
},
}
]
def chunk(delta: dict, finish: str | None) -> bytes:
body: Final = {
"id": "chatcmpl-smoke",
"object": "chat.completion.chunk",
"created": 0,
"model": "gpt-5.6",
"choices": [{"index": 0, "delta": delta, "finish_reason": finish}],
}
return f"data: {json.dumps(body)}\n\n".encode()
def respond(request: httpx.Request) -> httpx.Response:
outbound.put_nowait(json.loads(request.content))
content: Final = b"".join(
(
chunk(
{
"tool_calls": [
{
"index": 0,
"id": "call-1",
"type": "function",
"function": {"name": "get_weather", "arguments": ""},
}
]
},
None,
),
chunk({"tool_calls": [{"index": 0, "function": {"arguments": '{"city":'}}]}, None),
chunk({"tool_calls": [{"index": 0, "function": {"arguments": '"Paris"}'}}]}, "tool_calls"),
b"data: [DONE]\n\n",
)
)
return httpx.Response(200, headers={"content-type": "text/event-stream"}, content=content)
async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as http_client:
client: Final = AsyncOpenAI(api_key="transport-only", http_client=http_client)
messages: Final = [{"role": "user", "content": "weather in Paris"}]
stream: Final = await litellm.acompletion(
model="openai/gpt-4o",
api_key="transport-only",
client=client,
messages=messages,
tools=tools,
stream=True,
num_retries=0,
max_retries=0,
)
chunks: Final = []
async def drain() -> None:
async for part in stream:
chunks.append(part)
await asyncio.wait_for(drain(), timeout=10)
request: Final = await asyncio.wait_for(outbound.get(), timeout=10)
assert request["stream"] is True
assert request["tools"][0]["function"]["name"] == "get_weather"
assert outbound.empty()
rebuilt: Final = litellm.stream_chunk_builder(chunks, messages=messages)
tool_call: Final = rebuilt.choices[0].message.tool_calls[0]
assert tool_call.id == "call-1"
assert tool_call.function.name == "get_weather"
assert json.loads(tool_call.function.arguments) == {"city": "Paris"}
assert rebuilt.choices[0].finish_reason == "tool_calls"

View file

@ -9314,3 +9314,14 @@ async def test_agent_key_without_an_echoed_caller_keeps_its_own_models():
await _check_caller_models(agent_key, "claude-sonnet", load_team, load_user)
assert asked == []
def test_can_object_call_model_allows_listed_model_for_key():
result: Final = _can_object_call_model(
model="allowed-model",
llm_router=None,
models=["allowed-model"],
object_type="key",
)
assert result is True

View file

@ -4633,3 +4633,49 @@ def test_gemini_live_native_audio_limits_and_capabilities_match_vendor_model_car
assert info["supports_response_schema"] is False
assert info["supports_url_context"] is False
assert info["supports_pdf_input"] is False
def test_completion_cost_charges_explicit_per_token_rates_over_registered_ones(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setitem(
litellm.model_cost,
"smoke-priced-model",
{"input_cost_per_token": 0.01, "output_cost_per_token": 0.02, "litellm_provider": "openai", "mode": "chat"},
)
response: Final = ModelResponse(
model="smoke-priced-model",
choices=[],
usage=Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150),
)
cost: Final = completion_cost(
completion_response=response,
model="smoke-priced-model",
custom_llm_provider="openai",
custom_cost_per_token={"input_cost_per_token": 0.001, "output_cost_per_token": 0.002},
)
assert cost == pytest.approx(100 * 0.001 + 50 * 0.002)
def test_completion_cost_is_zero_when_explicit_rates_are_zero(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setitem(
litellm.model_cost,
"smoke-priced-model",
{"input_cost_per_token": 0.01, "output_cost_per_token": 0.02, "litellm_provider": "openai", "mode": "chat"},
)
response: Final = ModelResponse(
model="smoke-priced-model",
choices=[],
usage=Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150),
)
cost: Final = completion_cost(
completion_response=response,
model="smoke-priced-model",
custom_llm_provider="openai",
custom_cost_per_token={"input_cost_per_token": 0.0, "output_cost_per_token": 0.0},
)
assert cost == 0.0