From 43d3f36a1442b962518aa6883c6fe605adcb0d21 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Wed, 5 Aug 2026 15:05:18 -0700 Subject: [PATCH] perf(complexity_router): build the classifier rubric entries and response schema once per instance Both are pure functions of config, which is immutable for an instance's lifetime since editing the router rebuilds it, so the per-request create_model call only manufactured identical class objects for the GC to chase --- .../complexity_router/complexity_router.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 764a4562ba3..6144b1e337e 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -593,6 +593,11 @@ class ComplexityRouter(CustomLogger): self._savings_baseline: Baseline | None = None self._savings_baseline_derived = False + self._rubric_entries_cache: tuple[tuple[str, str], ...] = self._rubric_entries() + self._response_model_cache: type[BaseModel] = _tier_classification_model( + tuple(name for name, _ in self._rubric_entries_cache) + ) + verbose_router_logger.debug("ComplexityRouter initialized for %s with tiers: %s", model_name, self.config.tiers) def _hardest_tier_models(self) -> tuple[str, ...]: @@ -1046,8 +1051,8 @@ class ComplexityRouter(CustomLogger): metadata: Final = _classifier_call_metadata(request_metadata) turn_off_message_logging: Final = _effective_turn_off_message_logging(request_kwargs) - tier_entries: Final = self._rubric_entries() - response_model: Final = _tier_classification_model(tuple(name for name, _ in tier_entries)) + tier_entries: Final = self._rubric_entries_cache + response_model: Final = self._response_model_cache messages_for_call: Final = [ { "role": "system",