mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
* test(unit): reset module state only when a test changed it The proxy conftest deep-copied every snapshotted litellm attribute before each test, and the copy of litellm.model_cost alone was a median 0.12 s for each of 56,031 proxy tests on the runner. The reset now copies an attribute back only when it no longer equals the snapshot. The enterprise and MCP server conftests reloaded litellm before every test, 3.27 s and 0.21 s per test. They now use the same reset. * test(unit): remove six flaky tests They failed on main without a code change behind them between 2026-11-05 and 2026-11-14: the vector store hook SDK fallback, the concurrent user config TPM count, three Prometheus metrics server port tests and the OpenCode list cost billing test. * ci: run four heavy unit shards on 16-core runners and keep two smoke jobs off pull requests proxy-feature-endpoints, proxy-endpoints, root and llms-providers ran 1.9 to 2.2 times faster with 8 workers on a 16-core runner in a benchmark run. windows-sdk and proxy-build-from-pip still run on pushes to main and manual runs. proxy-build-from-pip installs a published release and never runs the pull request's code. * test: move 109 tests that start a process or open a socket into tests/integration tests/unit is in-process by its own rules, and these tests spawn a Python interpreter, a shell or a local TCP or HTTP server. They took 921 s of unit shard time per run, 396 s of it in the MCP client transport tests. Each one now lives in a *_real_io.py file under the integration directory for its domain and runs with that directory's group. The tests themselves are unchanged apart from one repository-root path in the CLI test. * test(unit): copy back only the dict entries a test changed A test module that registers models at import leaves litellm.model_cost four entries larger than the snapshot before every test in its directory, so the reset still deep-copied the whole price map for each of them. It now rebuilds a changed dict from the live entries that still match and copies only the rest. * ci(smoke): keep the two jobs that skip on pull requests out of the smoke gate A skipped job passing the gate is what test_unit_passed_gate guards against, so the gate goes back to requiring success from every job it needs and windows-sdk and proxy-build-from-pip stop being part of it. They still run and report on pushes to main and manual runs.
107 lines
3.7 KiB
Python
107 lines
3.7 KiB
Python
import copy
|
|
import inspect
|
|
import itertools
|
|
import warnings
|
|
from collections.abc import Mapping
|
|
from types import ModuleType
|
|
from typing import Final
|
|
|
|
_SNAPSHOT_TYPES: Final = (list, dict, set, tuple, str, int, float, bool, bytes)
|
|
_MUTABLE_CONTAINERS: Final = (list, dict, set, bytearray)
|
|
_ABSENT: Final = object()
|
|
|
|
|
|
def _holds_mutable_container(value: object) -> bool:
|
|
if isinstance(value, _MUTABLE_CONTAINERS):
|
|
return True
|
|
if isinstance(value, tuple):
|
|
return any(_holds_mutable_container(element) for element in value)
|
|
return False
|
|
|
|
|
|
def _restored_value(value: object) -> object:
|
|
return copy.deepcopy(value) if _holds_mutable_container(value) else value
|
|
|
|
|
|
def _readable_attributes(module: ModuleType) -> tuple[tuple[str, object], ...]:
|
|
return tuple((attr, value) for attr, value in tuple(vars(module).items()) if not attr.startswith("_"))
|
|
|
|
|
|
def _snapshot_of(module: ModuleType, attr: str, value: object) -> object:
|
|
try:
|
|
return _restored_value(value)
|
|
except Exception as exc:
|
|
warnings.warn(f"conftest: could not snapshot {module.__name__}.{attr}: {exc}", stacklevel=2)
|
|
return _ABSENT
|
|
|
|
|
|
def snapshot_mutable_state(module: ModuleType) -> Mapping[str, object]:
|
|
candidates: Final = (
|
|
(attr, _snapshot_of(module, attr, value))
|
|
for attr, value in _readable_attributes(module)
|
|
if value is None or isinstance(value, _SNAPSHOT_TYPES)
|
|
)
|
|
return {attr: value for attr, value in candidates if value is not _ABSENT}
|
|
|
|
|
|
def _still_matches(current: object, default: object) -> bool:
|
|
if current is default:
|
|
return True
|
|
if type(current) is not type(default):
|
|
return False
|
|
try:
|
|
return bool(current == default)
|
|
except Exception:
|
|
return False
|
|
|
|
|
|
def _restored_entries(current: dict[object, object], default: dict[object, object]) -> dict[object, object]:
|
|
return {
|
|
key: current[key]
|
|
if key in current and type(current[key]) is type(value) and current[key] == value
|
|
else _restored_value(value)
|
|
for key, value in default.items()
|
|
}
|
|
|
|
|
|
def _restored_from(current: object, default: object) -> object:
|
|
if type(current) is dict and type(default) is dict:
|
|
return _restored_entries(current, default)
|
|
return _restored_value(default)
|
|
|
|
|
|
def restore_mutable_state(module: ModuleType, snapshot: Mapping[str, object]) -> None:
|
|
live: Final = vars(module)
|
|
for attr, default in snapshot.items():
|
|
current = live.get(attr, _ABSENT)
|
|
if _still_matches(current, default):
|
|
continue
|
|
try:
|
|
setattr(module, attr, _restored_from(current, default))
|
|
except Exception as exc:
|
|
warnings.warn(f"conftest: could not restore {module.__name__}.{attr}: {exc}", stacklevel=2)
|
|
|
|
|
|
def _is_flushable(value: object) -> bool:
|
|
if inspect.isclass(value) or inspect.ismodule(value):
|
|
return False
|
|
return callable(getattr(value, "flush_cache", None))
|
|
|
|
|
|
def _flushable_attributes(module: ModuleType) -> tuple[tuple[ModuleType, str], ...]:
|
|
return tuple((module, attr) for attr, value in _readable_attributes(module) if _is_flushable(value))
|
|
|
|
|
|
def collect_flushable_caches(modules: tuple[ModuleType, ...]) -> tuple[tuple[ModuleType, str], ...]:
|
|
return tuple(itertools.chain.from_iterable(_flushable_attributes(module) for module in modules))
|
|
|
|
|
|
def flush_caches(targets: tuple[tuple[ModuleType, str], ...]) -> None:
|
|
for module, attr in targets:
|
|
flush = getattr(vars(module).get(attr), "flush_cache", None)
|
|
if not callable(flush):
|
|
continue
|
|
try:
|
|
flush()
|
|
except Exception as exc:
|
|
warnings.warn(f"conftest: flush_cache failed on {module.__name__}.{attr}: {exc}", stacklevel=2)
|