scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-13 23:11:40 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 100

perf(complexity_router): build the classifier rubric entries and response schema once per instance

Browse source 
Both are pure functions of config, which is immutable for an instance's
lifetime since editing the router rebuilds it, so the per-request create_model
call only manufactured identical class objects for the GC to chase
This commit is contained in:
Tin Chi Lo 2026-08-05 15:05:18 -07:00
parent 7f308b4d13
commit 43d3f36a14
1 changed files with 7 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
litellm/router_strategy/complexity_router/complexity_router.py
View file

@ -593,6 +593,11 @@ class ComplexityRouter(CustomLogger):
self._savings_baseline: Baseline | None = None
self._savings_baseline_derived = False
self._rubric_entries_cache: tuple[tuple[str, str], ...] = self._rubric_entries()
self._response_model_cache: type[BaseModel] = _tier_classification_model(
tuple(name for name, _ in self._rubric_entries_cache)
)
verbose_router_logger.debug("ComplexityRouter initialized for %s with tiers: %s", model_name, self.config.tiers)
def _hardest_tier_models(self) -> tuple[str, ...]:
@ -1046,8 +1051,8 @@ class ComplexityRouter(CustomLogger):
metadata: Final = _classifier_call_metadata(request_metadata)
turn_off_message_logging: Final = _effective_turn_off_message_logging(request_kwargs)
tier_entries: Final = self._rubric_entries()
response_model: Final = _tier_classification_model(tuple(name for name, _ in tier_entries))
tier_entries: Final = self._rubric_entries_cache
response_model: Final = self._response_model_cache
messages_for_call: Final = [
{
"role": "system",