diff --git a/litellm/router_strategy/complexity_router/README.md b/litellm/router_strategy/complexity_router/README.md
index bc8df67cc28..8e0cad39561 100644
--- a/litellm/router_strategy/complexity_router/README.md
+++ b/litellm/router_strategy/complexity_router/README.md
@@ -68,6 +68,36 @@ still resolve to a deployment in `model_list`; this configuration does not creat
- abc
```
+### Heuristic v2
+
+Set `classifier_type: heuristic_v2` to classify with the bundled calibrated
+success-probability model instead of the hand-written weighted scorer
+
+```yaml
+model_list:
+ - model_name: smart-router
+ litellm_params:
+ model: auto_router/complexity_router
+ complexity_router_config:
+ classifier_type: heuristic_v2
+ tiers:
+ SIMPLE: luna
+ MEDIUM: terra
+ COMPLEX: sol
+ REASONING: sol-ultra
+```
+
+No classifier model call or per-model training data is required. The classifier
+uses global tier quality, request-type quality, and similar-request cohorts from
+the bundled UltraFeedback artifact. It estimates success at every tier, enforces
+monotonic probabilities, and returns the first tier meeting the trained 0.75
+threshold. The existing complexity-router tier pool then selects and dispatches
+a model from that tier
+
+Spend logs record `routing_decision.cause: heuristic_v2`, the detected request
+type, and all four predicted probabilities. Existing `classifier_type: heuristic`
+configurations keep the original weighted scorer unchanged
+
### Renaming the tiers
`tier_labels` puts your own vocabulary on the four tiers:
diff --git a/litellm/router_strategy/complexity_router/artifacts/ultrafeedback_tiers.json b/litellm/router_strategy/complexity_router/artifacts/ultrafeedback_tiers.json
new file mode 100644
index 00000000000..4fcb599907c
--- /dev/null
+++ b/litellm/router_strategy/complexity_router/artifacts/ultrafeedback_tiers.json
@@ -0,0 +1,4069 @@
+{
+ "schema_version": 1,
+ "global_statistics": [
+ {
+ "tier": 1,
+ "successes": 36619.0,
+ "observations": 45504.0
+ },
+ {
+ "tier": 2,
+ "successes": 59797.0,
+ "observations": 70062.0
+ },
+ {
+ "tier": 3,
+ "successes": 48604.0,
+ "observations": 52245.0
+ },
+ {
+ "tier": 4,
+ "successes": 11393.0,
+ "observations": 11561.0
+ }
+ ],
+ "domain_statistics": [
+ {
+ "tier": 1,
+ "successes": 1592.0,
+ "observations": 2211.0,
+ "request_type": "analytical_reasoning"
+ },
+ {
+ "tier": 2,
+ "successes": 2654.0,
+ "observations": 3374.0,
+ "request_type": "analytical_reasoning"
+ },
+ {
+ "tier": 3,
+ "successes": 2243.0,
+ "observations": 2481.0,
+ "request_type": "analytical_reasoning"
+ },
+ {
+ "tier": 4,
+ "successes": 538.0,
+ "observations": 546.0,
+ "request_type": "analytical_reasoning"
+ },
+ {
+ "tier": 1,
+ "successes": 750.0,
+ "observations": 1015.0,
+ "request_type": "code_generation"
+ },
+ {
+ "tier": 2,
+ "successes": 1271.0,
+ "observations": 1511.0,
+ "request_type": "code_generation"
+ },
+ {
+ "tier": 3,
+ "successes": 1030.0,
+ "observations": 1111.0,
+ "request_type": "code_generation"
+ },
+ {
+ "tier": 4,
+ "successes": 233.0,
+ "observations": 235.0,
+ "request_type": "code_generation"
+ },
+ {
+ "tier": 1,
+ "successes": 243.0,
+ "observations": 277.0,
+ "request_type": "code_understanding"
+ },
+ {
+ "tier": 2,
+ "successes": 385.0,
+ "observations": 425.0,
+ "request_type": "code_understanding"
+ },
+ {
+ "tier": 3,
+ "successes": 322.0,
+ "observations": 334.0,
+ "request_type": "code_understanding"
+ },
+ {
+ "tier": 4,
+ "successes": 74.0,
+ "observations": 76.0,
+ "request_type": "code_understanding"
+ },
+ {
+ "tier": 1,
+ "successes": 2014.0,
+ "observations": 2170.0,
+ "request_type": "factual_lookup"
+ },
+ {
+ "tier": 2,
+ "successes": 3120.0,
+ "observations": 3266.0,
+ "request_type": "factual_lookup"
+ },
+ {
+ "tier": 3,
+ "successes": 2612.0,
+ "observations": 2670.0,
+ "request_type": "factual_lookup"
+ },
+ {
+ "tier": 4,
+ "successes": 540.0,
+ "observations": 542.0,
+ "request_type": "factual_lookup"
+ },
+ {
+ "tier": 1,
+ "successes": 30571.0,
+ "observations": 38161.0,
+ "request_type": "general"
+ },
+ {
+ "tier": 2,
+ "successes": 50037.0,
+ "observations": 58821.0,
+ "request_type": "general"
+ },
+ {
+ "tier": 3,
+ "successes": 40460.0,
+ "observations": 43618.0,
+ "request_type": "general"
+ },
+ {
+ "tier": 4,
+ "successes": 9565.0,
+ "observations": 9716.0,
+ "request_type": "general"
+ },
+ {
+ "tier": 1,
+ "successes": 159.0,
+ "observations": 170.0,
+ "request_type": "technical_design"
+ },
+ {
+ "tier": 2,
+ "successes": 282.0,
+ "observations": 303.0,
+ "request_type": "technical_design"
+ },
+ {
+ "tier": 3,
+ "successes": 231.0,
+ "observations": 236.0,
+ "request_type": "technical_design"
+ },
+ {
+ "tier": 4,
+ "successes": 47.0,
+ "observations": 47.0,
+ "request_type": "technical_design"
+ },
+ {
+ "tier": 1,
+ "successes": 1290.0,
+ "observations": 1500.0,
+ "request_type": "writing"
+ },
+ {
+ "tier": 2,
+ "successes": 2048.0,
+ "observations": 2362.0,
+ "request_type": "writing"
+ },
+ {
+ "tier": 3,
+ "successes": 1706.0,
+ "observations": 1795.0,
+ "request_type": "writing"
+ },
+ {
+ "tier": 4,
+ "successes": 396.0,
+ "observations": 399.0,
+ "request_type": "writing"
+ }
+ ],
+ "cohort_statistics": [
+ {
+ "tier": 1,
+ "successes": 272.0,
+ "observations": 372.0,
+ "cohort": "analytical_reasoning|long|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 420.0,
+ "observations": 519.0,
+ "cohort": "analytical_reasoning|long|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 341.0,
+ "observations": 381.0,
+ "cohort": "analytical_reasoning|long|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 82.0,
+ "observations": 84.0,
+ "cohort": "analytical_reasoning|long|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 0.0,
+ "observations": 1.0,
+ "cohort": "analytical_reasoning|long|code=0|math=0|mc=0|intl=1"
+ },
+ {
+ "tier": 2,
+ "successes": 0.0,
+ "observations": 2.0,
+ "cohort": "analytical_reasoning|long|code=0|math=0|mc=0|intl=1"
+ },
+ {
+ "tier": 3,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "analytical_reasoning|long|code=0|math=0|mc=0|intl=1"
+ },
+ {
+ "tier": 1,
+ "successes": 11.0,
+ "observations": 18.0,
+ "cohort": "analytical_reasoning|long|code=0|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 33.0,
+ "observations": 45.0,
+ "cohort": "analytical_reasoning|long|code=0|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 28.0,
+ "observations": 34.0,
+ "cohort": "analytical_reasoning|long|code=0|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 3.0,
+ "observations": 3.0,
+ "cohort": "analytical_reasoning|long|code=0|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 131.0,
+ "observations": 176.0,
+ "cohort": "analytical_reasoning|long|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 210.0,
+ "observations": 274.0,
+ "cohort": "analytical_reasoning|long|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 187.0,
+ "observations": 209.0,
+ "cohort": "analytical_reasoning|long|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 39.0,
+ "observations": 41.0,
+ "cohort": "analytical_reasoning|long|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 4.0,
+ "observations": 9.0,
+ "cohort": "analytical_reasoning|long|code=0|math=1|mc=0|intl=1"
+ },
+ {
+ "tier": 2,
+ "successes": 11.0,
+ "observations": 13.0,
+ "cohort": "analytical_reasoning|long|code=0|math=1|mc=0|intl=1"
+ },
+ {
+ "tier": 3,
+ "successes": 6.0,
+ "observations": 7.0,
+ "cohort": "analytical_reasoning|long|code=0|math=1|mc=0|intl=1"
+ },
+ {
+ "tier": 4,
+ "successes": 3.0,
+ "observations": 3.0,
+ "cohort": "analytical_reasoning|long|code=0|math=1|mc=0|intl=1"
+ },
+ {
+ "tier": 1,
+ "successes": 10.0,
+ "observations": 15.0,
+ "cohort": "analytical_reasoning|long|code=0|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 14.0,
+ "observations": 16.0,
+ "cohort": "analytical_reasoning|long|code=0|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 15.0,
+ "observations": 16.0,
+ "cohort": "analytical_reasoning|long|code=0|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 5.0,
+ "observations": 5.0,
+ "cohort": "analytical_reasoning|long|code=0|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 15.0,
+ "observations": 20.0,
+ "cohort": "analytical_reasoning|long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 33.0,
+ "observations": 38.0,
+ "cohort": "analytical_reasoning|long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 28.0,
+ "observations": 29.0,
+ "cohort": "analytical_reasoning|long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 5.0,
+ "observations": 5.0,
+ "cohort": "analytical_reasoning|long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 1.0,
+ "observations": 2.0,
+ "cohort": "analytical_reasoning|long|code=1|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 2.0,
+ "observations": 2.0,
+ "cohort": "analytical_reasoning|long|code=1|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 26.0,
+ "observations": 34.0,
+ "cohort": "analytical_reasoning|long|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 39.0,
+ "observations": 50.0,
+ "cohort": "analytical_reasoning|long|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 42.0,
+ "observations": 45.0,
+ "cohort": "analytical_reasoning|long|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 11.0,
+ "observations": 11.0,
+ "cohort": "analytical_reasoning|long|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 2.0,
+ "observations": 3.0,
+ "cohort": "analytical_reasoning|long|code=1|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 4.0,
+ "observations": 4.0,
+ "cohort": "analytical_reasoning|long|code=1|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "analytical_reasoning|long|code=1|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 452.0,
+ "observations": 634.0,
+ "cohort": "analytical_reasoning|medium|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 745.0,
+ "observations": 962.0,
+ "cohort": "analytical_reasoning|medium|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 634.0,
+ "observations": 691.0,
+ "cohort": "analytical_reasoning|medium|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 151.0,
+ "observations": 153.0,
+ "cohort": "analytical_reasoning|medium|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 1.0,
+ "observations": 3.0,
+ "cohort": "analytical_reasoning|medium|code=0|math=0|mc=0|intl=1"
+ },
+ {
+ "tier": 2,
+ "successes": 1.0,
+ "observations": 4.0,
+ "cohort": "analytical_reasoning|medium|code=0|math=0|mc=0|intl=1"
+ },
+ {
+ "tier": 4,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "analytical_reasoning|medium|code=0|math=0|mc=0|intl=1"
+ },
+ {
+ "tier": 1,
+ "successes": 10.0,
+ "observations": 20.0,
+ "cohort": "analytical_reasoning|medium|code=0|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 30.0,
+ "observations": 44.0,
+ "cohort": "analytical_reasoning|medium|code=0|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 12.0,
+ "observations": 15.0,
+ "cohort": "analytical_reasoning|medium|code=0|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 5.0,
+ "observations": 5.0,
+ "cohort": "analytical_reasoning|medium|code=0|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 178.0,
+ "observations": 270.0,
+ "cohort": "analytical_reasoning|medium|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 304.0,
+ "observations": 402.0,
+ "cohort": "analytical_reasoning|medium|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 266.0,
+ "observations": 295.0,
+ "cohort": "analytical_reasoning|medium|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 73.0,
+ "observations": 73.0,
+ "cohort": "analytical_reasoning|medium|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 7.0,
+ "observations": 18.0,
+ "cohort": "analytical_reasoning|medium|code=0|math=1|mc=0|intl=1"
+ },
+ {
+ "tier": 2,
+ "successes": 17.0,
+ "observations": 32.0,
+ "cohort": "analytical_reasoning|medium|code=0|math=1|mc=0|intl=1"
+ },
+ {
+ "tier": 3,
+ "successes": 17.0,
+ "observations": 27.0,
+ "cohort": "analytical_reasoning|medium|code=0|math=1|mc=0|intl=1"
+ },
+ {
+ "tier": 4,
+ "successes": 7.0,
+ "observations": 7.0,
+ "cohort": "analytical_reasoning|medium|code=0|math=1|mc=0|intl=1"
+ },
+ {
+ "tier": 1,
+ "successes": 11.0,
+ "observations": 17.0,
+ "cohort": "analytical_reasoning|medium|code=0|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 17.0,
+ "observations": 25.0,
+ "cohort": "analytical_reasoning|medium|code=0|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 10.0,
+ "observations": 16.0,
+ "cohort": "analytical_reasoning|medium|code=0|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 6.0,
+ "observations": 6.0,
+ "cohort": "analytical_reasoning|medium|code=0|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 58.0,
+ "observations": 72.0,
+ "cohort": "analytical_reasoning|medium|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 90.0,
+ "observations": 103.0,
+ "cohort": "analytical_reasoning|medium|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 72.0,
+ "observations": 78.0,
+ "cohort": "analytical_reasoning|medium|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 19.0,
+ "observations": 19.0,
+ "cohort": "analytical_reasoning|medium|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "analytical_reasoning|medium|code=1|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 2.0,
+ "observations": 2.0,
+ "cohort": "analytical_reasoning|medium|code=1|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "analytical_reasoning|medium|code=1|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 37.0,
+ "observations": 56.0,
+ "cohort": "analytical_reasoning|medium|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 95.0,
+ "observations": 110.0,
+ "cohort": "analytical_reasoning|medium|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 73.0,
+ "observations": 82.0,
+ "cohort": "analytical_reasoning|medium|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 20.0,
+ "observations": 20.0,
+ "cohort": "analytical_reasoning|medium|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 69.0,
+ "observations": 83.0,
+ "cohort": "analytical_reasoning|short|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 102.0,
+ "observations": 110.0,
+ "cohort": "analytical_reasoning|short|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 93.0,
+ "observations": 98.0,
+ "cohort": "analytical_reasoning|short|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 17.0,
+ "observations": 17.0,
+ "cohort": "analytical_reasoning|short|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "analytical_reasoning|short|code=0|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 6.0,
+ "observations": 6.0,
+ "cohort": "analytical_reasoning|short|code=0|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "analytical_reasoning|short|code=0|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 21.0,
+ "observations": 32.0,
+ "cohort": "analytical_reasoning|short|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 45.0,
+ "observations": 59.0,
+ "cohort": "analytical_reasoning|short|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 45.0,
+ "observations": 48.0,
+ "cohort": "analytical_reasoning|short|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 9.0,
+ "observations": 9.0,
+ "cohort": "analytical_reasoning|short|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "analytical_reasoning|short|code=0|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 2.0,
+ "observations": 2.0,
+ "cohort": "analytical_reasoning|short|code=0|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "analytical_reasoning|short|code=0|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 7.0,
+ "observations": 9.0,
+ "cohort": "analytical_reasoning|short|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 8.0,
+ "observations": 12.0,
+ "cohort": "analytical_reasoning|short|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 11.0,
+ "observations": 11.0,
+ "cohort": "analytical_reasoning|short|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 5.0,
+ "observations": 9.0,
+ "cohort": "analytical_reasoning|short|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 9.0,
+ "observations": 11.0,
+ "cohort": "analytical_reasoning|short|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 8.0,
+ "observations": 8.0,
+ "cohort": "analytical_reasoning|short|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 136.0,
+ "observations": 176.0,
+ "cohort": "analytical_reasoning|very_long|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 221.0,
+ "observations": 267.0,
+ "cohort": "analytical_reasoning|very_long|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 172.0,
+ "observations": 194.0,
+ "cohort": "analytical_reasoning|very_long|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 37.0,
+ "observations": 39.0,
+ "cohort": "analytical_reasoning|very_long|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 6.0,
+ "observations": 11.0,
+ "cohort": "analytical_reasoning|very_long|code=0|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 10.0,
+ "observations": 15.0,
+ "cohort": "analytical_reasoning|very_long|code=0|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 7.0,
+ "observations": 8.0,
+ "cohort": "analytical_reasoning|very_long|code=0|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 2.0,
+ "observations": 2.0,
+ "cohort": "analytical_reasoning|very_long|code=0|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 68.0,
+ "observations": 84.0,
+ "cohort": "analytical_reasoning|very_long|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 103.0,
+ "observations": 136.0,
+ "cohort": "analytical_reasoning|very_long|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 113.0,
+ "observations": 119.0,
+ "cohort": "analytical_reasoning|very_long|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 21.0,
+ "observations": 21.0,
+ "cohort": "analytical_reasoning|very_long|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "analytical_reasoning|very_long|code=0|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 1.0,
+ "observations": 3.0,
+ "cohort": "analytical_reasoning|very_long|code=0|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 4.0,
+ "observations": 4.0,
+ "cohort": "analytical_reasoning|very_long|code=0|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 13.0,
+ "observations": 16.0,
+ "cohort": "analytical_reasoning|very_long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 27.0,
+ "observations": 34.0,
+ "cohort": "analytical_reasoning|very_long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 15.0,
+ "observations": 16.0,
+ "cohort": "analytical_reasoning|very_long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 6.0,
+ "observations": 6.0,
+ "cohort": "analytical_reasoning|very_long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 4.0,
+ "observations": 5.0,
+ "cohort": "analytical_reasoning|very_long|code=1|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 4.0,
+ "observations": 7.0,
+ "cohort": "analytical_reasoning|very_long|code=1|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 3.0,
+ "observations": 3.0,
+ "cohort": "analytical_reasoning|very_long|code=1|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "analytical_reasoning|very_long|code=1|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 32.0,
+ "observations": 39.0,
+ "cohort": "analytical_reasoning|very_long|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 48.0,
+ "observations": 63.0,
+ "cohort": "analytical_reasoning|very_long|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 35.0,
+ "observations": 40.0,
+ "cohort": "analytical_reasoning|very_long|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 14.0,
+ "observations": 14.0,
+ "cohort": "analytical_reasoning|very_long|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 1.0,
+ "observations": 3.0,
+ "cohort": "analytical_reasoning|very_long|code=1|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 1.0,
+ "observations": 2.0,
+ "cohort": "analytical_reasoning|very_long|code=1|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 1.0,
+ "observations": 2.0,
+ "cohort": "analytical_reasoning|very_long|code=1|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "analytical_reasoning|very_long|code=1|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 28.0,
+ "observations": 31.0,
+ "cohort": "code_generation|long|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 41.0,
+ "observations": 46.0,
+ "cohort": "code_generation|long|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 25.0,
+ "observations": 26.0,
+ "cohort": "code_generation|long|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 5.0,
+ "observations": 5.0,
+ "cohort": "code_generation|long|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 6.0,
+ "observations": 9.0,
+ "cohort": "code_generation|long|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 20.0,
+ "observations": 23.0,
+ "cohort": "code_generation|long|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 16.0,
+ "observations": 16.0,
+ "cohort": "code_generation|long|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 4.0,
+ "observations": 4.0,
+ "cohort": "code_generation|long|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 46.0,
+ "observations": 60.0,
+ "cohort": "code_generation|long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 72.0,
+ "observations": 91.0,
+ "cohort": "code_generation|long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 59.0,
+ "observations": 64.0,
+ "cohort": "code_generation|long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 17.0,
+ "observations": 17.0,
+ "cohort": "code_generation|long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 32.0,
+ "observations": 49.0,
+ "cohort": "code_generation|long|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 60.0,
+ "observations": 76.0,
+ "cohort": "code_generation|long|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 48.0,
+ "observations": 52.0,
+ "cohort": "code_generation|long|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 15.0,
+ "observations": 15.0,
+ "cohort": "code_generation|long|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 93.0,
+ "observations": 121.0,
+ "cohort": "code_generation|medium|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 149.0,
+ "observations": 170.0,
+ "cohort": "code_generation|medium|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 146.0,
+ "observations": 151.0,
+ "cohort": "code_generation|medium|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 26.0,
+ "observations": 26.0,
+ "cohort": "code_generation|medium|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 12.0,
+ "observations": 16.0,
+ "cohort": "code_generation|medium|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 22.0,
+ "observations": 25.0,
+ "cohort": "code_generation|medium|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 13.0,
+ "observations": 15.0,
+ "cohort": "code_generation|medium|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 4.0,
+ "observations": 4.0,
+ "cohort": "code_generation|medium|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 196.0,
+ "observations": 268.0,
+ "cohort": "code_generation|medium|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 312.0,
+ "observations": 370.0,
+ "cohort": "code_generation|medium|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 234.0,
+ "observations": 253.0,
+ "cohort": "code_generation|medium|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 56.0,
+ "observations": 57.0,
+ "cohort": "code_generation|medium|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 0.0,
+ "observations": 1.0,
+ "cohort": "code_generation|medium|code=1|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 2.0,
+ "observations": 2.0,
+ "cohort": "code_generation|medium|code=1|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "code_generation|medium|code=1|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 40.0,
+ "observations": 59.0,
+ "cohort": "code_generation|medium|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 73.0,
+ "observations": 92.0,
+ "cohort": "code_generation|medium|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 78.0,
+ "observations": 83.0,
+ "cohort": "code_generation|medium|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 14.0,
+ "observations": 14.0,
+ "cohort": "code_generation|medium|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 2.0,
+ "observations": 3.0,
+ "cohort": "code_generation|medium|code=1|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "code_generation|medium|code=1|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 106.0,
+ "observations": 140.0,
+ "cohort": "code_generation|short|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 220.0,
+ "observations": 247.0,
+ "cohort": "code_generation|short|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 167.0,
+ "observations": 178.0,
+ "cohort": "code_generation|short|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 43.0,
+ "observations": 43.0,
+ "cohort": "code_generation|short|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 2.0,
+ "observations": 4.0,
+ "cohort": "code_generation|short|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 7.0,
+ "observations": 9.0,
+ "cohort": "code_generation|short|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 7.0,
+ "observations": 7.0,
+ "cohort": "code_generation|short|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 147.0,
+ "observations": 199.0,
+ "cohort": "code_generation|short|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 233.0,
+ "observations": 269.0,
+ "cohort": "code_generation|short|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 189.0,
+ "observations": 209.0,
+ "cohort": "code_generation|short|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 34.0,
+ "observations": 35.0,
+ "cohort": "code_generation|short|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 7.0,
+ "observations": 10.0,
+ "cohort": "code_generation|short|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 18.0,
+ "observations": 22.0,
+ "cohort": "code_generation|short|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 10.0,
+ "observations": 10.0,
+ "cohort": "code_generation|short|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 2.0,
+ "observations": 2.0,
+ "cohort": "code_generation|short|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 2.0,
+ "observations": 4.0,
+ "cohort": "code_generation|very_long|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 5.0,
+ "observations": 8.0,
+ "cohort": "code_generation|very_long|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 7.0,
+ "observations": 8.0,
+ "cohort": "code_generation|very_long|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 2.0,
+ "observations": 2.0,
+ "cohort": "code_generation|very_long|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 3.0,
+ "observations": 5.0,
+ "cohort": "code_generation|very_long|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 3.0,
+ "observations": 3.0,
+ "cohort": "code_generation|very_long|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 2.0,
+ "observations": 2.0,
+ "cohort": "code_generation|very_long|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 7.0,
+ "observations": 9.0,
+ "cohort": "code_generation|very_long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 7.0,
+ "observations": 8.0,
+ "cohort": "code_generation|very_long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 9.0,
+ "observations": 9.0,
+ "cohort": "code_generation|very_long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 2.0,
+ "observations": 2.0,
+ "cohort": "code_generation|very_long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 24.0,
+ "observations": 33.0,
+ "cohort": "code_generation|very_long|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 25.0,
+ "observations": 45.0,
+ "cohort": "code_generation|very_long|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 19.0,
+ "observations": 27.0,
+ "cohort": "code_generation|very_long|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 7.0,
+ "observations": 7.0,
+ "cohort": "code_generation|very_long|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 2.0,
+ "observations": 2.0,
+ "cohort": "code_understanding|long|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 2.0,
+ "observations": 2.0,
+ "cohort": "code_understanding|long|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 6.0,
+ "observations": 6.0,
+ "cohort": "code_understanding|long|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 2.0,
+ "observations": 2.0,
+ "cohort": "code_understanding|long|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 3.0,
+ "observations": 5.0,
+ "cohort": "code_understanding|long|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 3.0,
+ "observations": 3.0,
+ "cohort": "code_understanding|long|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 2.0,
+ "observations": 3.0,
+ "cohort": "code_understanding|long|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "code_understanding|long|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 17.0,
+ "observations": 23.0,
+ "cohort": "code_understanding|long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 27.0,
+ "observations": 37.0,
+ "cohort": "code_understanding|long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 32.0,
+ "observations": 33.0,
+ "cohort": "code_understanding|long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 3.0,
+ "observations": 3.0,
+ "cohort": "code_understanding|long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 24.0,
+ "observations": 28.0,
+ "cohort": "code_understanding|long|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 37.0,
+ "observations": 44.0,
+ "cohort": "code_understanding|long|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 33.0,
+ "observations": 33.0,
+ "cohort": "code_understanding|long|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 7.0,
+ "observations": 7.0,
+ "cohort": "code_understanding|long|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 8.0,
+ "observations": 9.0,
+ "cohort": "code_understanding|medium|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 19.0,
+ "observations": 21.0,
+ "cohort": "code_understanding|medium|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 10.0,
+ "observations": 10.0,
+ "cohort": "code_understanding|medium|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 4.0,
+ "observations": 4.0,
+ "cohort": "code_understanding|medium|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 2.0,
+ "observations": 3.0,
+ "cohort": "code_understanding|medium|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 2.0,
+ "observations": 3.0,
+ "cohort": "code_understanding|medium|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "code_understanding|medium|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "code_understanding|medium|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 73.0,
+ "observations": 76.0,
+ "cohort": "code_understanding|medium|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 107.0,
+ "observations": 111.0,
+ "cohort": "code_understanding|medium|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 87.0,
+ "observations": 87.0,
+ "cohort": "code_understanding|medium|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 22.0,
+ "observations": 22.0,
+ "cohort": "code_understanding|medium|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 32.0,
+ "observations": 33.0,
+ "cohort": "code_understanding|medium|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 44.0,
+ "observations": 44.0,
+ "cohort": "code_understanding|medium|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 44.0,
+ "observations": 47.0,
+ "cohort": "code_understanding|medium|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 8.0,
+ "observations": 8.0,
+ "cohort": "code_understanding|medium|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 0.0,
+ "observations": 1.0,
+ "cohort": "code_understanding|medium|code=1|math=1|mc=0|intl=1"
+ },
+ {
+ "tier": 2,
+ "successes": 0.0,
+ "observations": 2.0,
+ "cohort": "code_understanding|medium|code=1|math=1|mc=0|intl=1"
+ },
+ {
+ "tier": 4,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "code_understanding|medium|code=1|math=1|mc=0|intl=1"
+ },
+ {
+ "tier": 1,
+ "successes": 15.0,
+ "observations": 16.0,
+ "cohort": "code_understanding|short|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 23.0,
+ "observations": 23.0,
+ "cohort": "code_understanding|short|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 12.0,
+ "observations": 14.0,
+ "cohort": "code_understanding|short|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 3.0,
+ "observations": 3.0,
+ "cohort": "code_understanding|short|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 2.0,
+ "observations": 3.0,
+ "cohort": "code_understanding|short|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "code_understanding|short|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 36.0,
+ "observations": 43.0,
+ "cohort": "code_understanding|short|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 71.0,
+ "observations": 74.0,
+ "cohort": "code_understanding|short|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 48.0,
+ "observations": 50.0,
+ "cohort": "code_understanding|short|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 12.0,
+ "observations": 13.0,
+ "cohort": "code_understanding|short|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 5.0,
+ "observations": 6.0,
+ "cohort": "code_understanding|short|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 10.0,
+ "observations": 10.0,
+ "cohort": "code_understanding|short|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 7.0,
+ "observations": 7.0,
+ "cohort": "code_understanding|short|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "code_understanding|short|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "code_understanding|very_long|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 0.0,
+ "observations": 1.0,
+ "cohort": "code_understanding|very_long|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 2.0,
+ "observations": 2.0,
+ "cohort": "code_understanding|very_long|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 9.0,
+ "observations": 11.0,
+ "cohort": "code_understanding|very_long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 18.0,
+ "observations": 23.0,
+ "cohort": "code_understanding|very_long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 11.0,
+ "observations": 13.0,
+ "cohort": "code_understanding|very_long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "code_understanding|very_long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 3.0,
+ "observations": 3.0,
+ "cohort": "code_understanding|very_long|code=1|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "code_understanding|very_long|code=1|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 13.0,
+ "observations": 17.0,
+ "cohort": "code_understanding|very_long|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 19.0,
+ "observations": 23.0,
+ "cohort": "code_understanding|very_long|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 27.0,
+ "observations": 28.0,
+ "cohort": "code_understanding|very_long|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 7.0,
+ "observations": 8.0,
+ "cohort": "code_understanding|very_long|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 49.0,
+ "observations": 54.0,
+ "cohort": "factual_lookup|long|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 64.0,
+ "observations": 75.0,
+ "cohort": "factual_lookup|long|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 80.0,
+ "observations": 80.0,
+ "cohort": "factual_lookup|long|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 15.0,
+ "observations": 15.0,
+ "cohort": "factual_lookup|long|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 9.0,
+ "observations": 9.0,
+ "cohort": "factual_lookup|long|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 12.0,
+ "observations": 14.0,
+ "cohort": "factual_lookup|long|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 9.0,
+ "observations": 10.0,
+ "cohort": "factual_lookup|long|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 3.0,
+ "observations": 3.0,
+ "cohort": "factual_lookup|long|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "factual_lookup|long|code=0|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 3.0,
+ "observations": 3.0,
+ "cohort": "factual_lookup|long|code=0|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 19.0,
+ "observations": 22.0,
+ "cohort": "factual_lookup|long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 35.0,
+ "observations": 38.0,
+ "cohort": "factual_lookup|long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 37.0,
+ "observations": 41.0,
+ "cohort": "factual_lookup|long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 7.0,
+ "observations": 7.0,
+ "cohort": "factual_lookup|long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 66.0,
+ "observations": 75.0,
+ "cohort": "factual_lookup|long|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 73.0,
+ "observations": 86.0,
+ "cohort": "factual_lookup|long|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 69.0,
+ "observations": 74.0,
+ "cohort": "factual_lookup|long|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 21.0,
+ "observations": 21.0,
+ "cohort": "factual_lookup|long|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "factual_lookup|long|code=1|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 2.0,
+ "observations": 2.0,
+ "cohort": "factual_lookup|long|code=1|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "factual_lookup|long|code=1|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 174.0,
+ "observations": 181.0,
+ "cohort": "factual_lookup|medium|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 204.0,
+ "observations": 215.0,
+ "cohort": "factual_lookup|medium|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 206.0,
+ "observations": 210.0,
+ "cohort": "factual_lookup|medium|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 42.0,
+ "observations": 42.0,
+ "cohort": "factual_lookup|medium|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 31.0,
+ "observations": 37.0,
+ "cohort": "factual_lookup|medium|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 41.0,
+ "observations": 48.0,
+ "cohort": "factual_lookup|medium|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 44.0,
+ "observations": 45.0,
+ "cohort": "factual_lookup|medium|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 10.0,
+ "observations": 10.0,
+ "cohort": "factual_lookup|medium|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 0.0,
+ "observations": 1.0,
+ "cohort": "factual_lookup|medium|code=0|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 2.0,
+ "observations": 2.0,
+ "cohort": "factual_lookup|medium|code=0|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "factual_lookup|medium|code=0|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 131.0,
+ "observations": 142.0,
+ "cohort": "factual_lookup|medium|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 162.0,
+ "observations": 171.0,
+ "cohort": "factual_lookup|medium|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 146.0,
+ "observations": 151.0,
+ "cohort": "factual_lookup|medium|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 24.0,
+ "observations": 24.0,
+ "cohort": "factual_lookup|medium|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 41.0,
+ "observations": 46.0,
+ "cohort": "factual_lookup|medium|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 55.0,
+ "observations": 59.0,
+ "cohort": "factual_lookup|medium|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 45.0,
+ "observations": 46.0,
+ "cohort": "factual_lookup|medium|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 9.0,
+ "observations": 9.0,
+ "cohort": "factual_lookup|medium|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 1382.0,
+ "observations": 1473.0,
+ "cohort": "factual_lookup|short|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 2290.0,
+ "observations": 2348.0,
+ "cohort": "factual_lookup|short|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 1824.0,
+ "observations": 1853.0,
+ "cohort": "factual_lookup|short|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 368.0,
+ "observations": 370.0,
+ "cohort": "factual_lookup|short|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 3.0,
+ "observations": 5.0,
+ "cohort": "factual_lookup|short|code=0|math=0|mc=0|intl=1"
+ },
+ {
+ "tier": 2,
+ "successes": 12.0,
+ "observations": 13.0,
+ "cohort": "factual_lookup|short|code=0|math=0|mc=0|intl=1"
+ },
+ {
+ "tier": 3,
+ "successes": 7.0,
+ "observations": 7.0,
+ "cohort": "factual_lookup|short|code=0|math=0|mc=0|intl=1"
+ },
+ {
+ "tier": 4,
+ "successes": 3.0,
+ "observations": 3.0,
+ "cohort": "factual_lookup|short|code=0|math=0|mc=0|intl=1"
+ },
+ {
+ "tier": 1,
+ "successes": 11.0,
+ "observations": 16.0,
+ "cohort": "factual_lookup|short|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 28.0,
+ "observations": 31.0,
+ "cohort": "factual_lookup|short|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 15.0,
+ "observations": 17.0,
+ "cohort": "factual_lookup|short|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 4.0,
+ "observations": 4.0,
+ "cohort": "factual_lookup|short|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 36.0,
+ "observations": 39.0,
+ "cohort": "factual_lookup|short|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 63.0,
+ "observations": 66.0,
+ "cohort": "factual_lookup|short|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 64.0,
+ "observations": 66.0,
+ "cohort": "factual_lookup|short|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 13.0,
+ "observations": 13.0,
+ "cohort": "factual_lookup|short|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 4.0,
+ "observations": 4.0,
+ "cohort": "factual_lookup|short|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 5.0,
+ "observations": 6.0,
+ "cohort": "factual_lookup|short|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 4.0,
+ "observations": 5.0,
+ "cohort": "factual_lookup|short|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "factual_lookup|short|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 27.0,
+ "observations": 31.0,
+ "cohort": "factual_lookup|very_long|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 29.0,
+ "observations": 37.0,
+ "cohort": "factual_lookup|very_long|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 26.0,
+ "observations": 29.0,
+ "cohort": "factual_lookup|very_long|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 11.0,
+ "observations": 11.0,
+ "cohort": "factual_lookup|very_long|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "factual_lookup|very_long|code=0|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 0.0,
+ "observations": 2.0,
+ "cohort": "factual_lookup|very_long|code=0|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "factual_lookup|very_long|code=0|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 7.0,
+ "observations": 8.0,
+ "cohort": "factual_lookup|very_long|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 17.0,
+ "observations": 20.0,
+ "cohort": "factual_lookup|very_long|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 14.0,
+ "observations": 14.0,
+ "cohort": "factual_lookup|very_long|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 2.0,
+ "observations": 2.0,
+ "cohort": "factual_lookup|very_long|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 4.0,
+ "observations": 4.0,
+ "cohort": "factual_lookup|very_long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 6.0,
+ "observations": 6.0,
+ "cohort": "factual_lookup|very_long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 5.0,
+ "observations": 5.0,
+ "cohort": "factual_lookup|very_long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "factual_lookup|very_long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 17.0,
+ "observations": 21.0,
+ "cohort": "factual_lookup|very_long|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 19.0,
+ "observations": 25.0,
+ "cohort": "factual_lookup|very_long|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 13.0,
+ "observations": 13.0,
+ "cohort": "factual_lookup|very_long|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 5.0,
+ "observations": 5.0,
+ "cohort": "factual_lookup|very_long|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 2828.0,
+ "observations": 3552.0,
+ "cohort": "general|long|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 4621.0,
+ "observations": 5551.0,
+ "cohort": "general|long|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 3628.0,
+ "observations": 3931.0,
+ "cohort": "general|long|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 911.0,
+ "observations": 922.0,
+ "cohort": "general|long|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 127.0,
+ "observations": 406.0,
+ "cohort": "general|long|code=0|math=0|mc=0|intl=1"
+ },
+ {
+ "tier": 2,
+ "successes": 250.0,
+ "observations": 583.0,
+ "cohort": "general|long|code=0|math=0|mc=0|intl=1"
+ },
+ {
+ "tier": 3,
+ "successes": 219.0,
+ "observations": 396.0,
+ "cohort": "general|long|code=0|math=0|mc=0|intl=1"
+ },
+ {
+ "tier": 4,
+ "successes": 98.0,
+ "observations": 107.0,
+ "cohort": "general|long|code=0|math=0|mc=0|intl=1"
+ },
+ {
+ "tier": 1,
+ "successes": 94.0,
+ "observations": 127.0,
+ "cohort": "general|long|code=0|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 168.0,
+ "observations": 196.0,
+ "cohort": "general|long|code=0|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 135.0,
+ "observations": 146.0,
+ "cohort": "general|long|code=0|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 34.0,
+ "observations": 35.0,
+ "cohort": "general|long|code=0|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 912.0,
+ "observations": 1219.0,
+ "cohort": "general|long|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 1417.0,
+ "observations": 1736.0,
+ "cohort": "general|long|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 1163.0,
+ "observations": 1258.0,
+ "cohort": "general|long|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 278.0,
+ "observations": 283.0,
+ "cohort": "general|long|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 33.0,
+ "observations": 122.0,
+ "cohort": "general|long|code=0|math=1|mc=0|intl=1"
+ },
+ {
+ "tier": 2,
+ "successes": 41.0,
+ "observations": 150.0,
+ "cohort": "general|long|code=0|math=1|mc=0|intl=1"
+ },
+ {
+ "tier": 3,
+ "successes": 61.0,
+ "observations": 96.0,
+ "cohort": "general|long|code=0|math=1|mc=0|intl=1"
+ },
+ {
+ "tier": 4,
+ "successes": 24.0,
+ "observations": 24.0,
+ "cohort": "general|long|code=0|math=1|mc=0|intl=1"
+ },
+ {
+ "tier": 1,
+ "successes": 16.0,
+ "observations": 24.0,
+ "cohort": "general|long|code=0|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 24.0,
+ "observations": 34.0,
+ "cohort": "general|long|code=0|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 18.0,
+ "observations": 21.0,
+ "cohort": "general|long|code=0|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 5.0,
+ "observations": 5.0,
+ "cohort": "general|long|code=0|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 271.0,
+ "observations": 348.0,
+ "cohort": "general|long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 488.0,
+ "observations": 555.0,
+ "cohort": "general|long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 369.0,
+ "observations": 386.0,
+ "cohort": "general|long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 67.0,
+ "observations": 71.0,
+ "cohort": "general|long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 1.0,
+ "observations": 3.0,
+ "cohort": "general|long|code=1|math=0|mc=0|intl=1"
+ },
+ {
+ "tier": 2,
+ "successes": 0.0,
+ "observations": 3.0,
+ "cohort": "general|long|code=1|math=0|mc=0|intl=1"
+ },
+ {
+ "tier": 3,
+ "successes": 2.0,
+ "observations": 2.0,
+ "cohort": "general|long|code=1|math=0|mc=0|intl=1"
+ },
+ {
+ "tier": 1,
+ "successes": 9.0,
+ "observations": 9.0,
+ "cohort": "general|long|code=1|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 14.0,
+ "observations": 17.0,
+ "cohort": "general|long|code=1|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 9.0,
+ "observations": 10.0,
+ "cohort": "general|long|code=1|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 4.0,
+ "observations": 4.0,
+ "cohort": "general|long|code=1|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 340.0,
+ "observations": 426.0,
+ "cohort": "general|long|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 544.0,
+ "observations": 635.0,
+ "cohort": "general|long|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 472.0,
+ "observations": 491.0,
+ "cohort": "general|long|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 120.0,
+ "observations": 120.0,
+ "cohort": "general|long|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "general|long|code=1|math=1|mc=0|intl=1"
+ },
+ {
+ "tier": 3,
+ "successes": 2.0,
+ "observations": 2.0,
+ "cohort": "general|long|code=1|math=1|mc=0|intl=1"
+ },
+ {
+ "tier": 4,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "general|long|code=1|math=1|mc=0|intl=1"
+ },
+ {
+ "tier": 1,
+ "successes": 0.0,
+ "observations": 2.0,
+ "cohort": "general|long|code=1|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "general|long|code=1|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 0.0,
+ "observations": 1.0,
+ "cohort": "general|long|code=1|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 8332.0,
+ "observations": 10133.0,
+ "cohort": "general|medium|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 13225.0,
+ "observations": 15509.0,
+ "cohort": "general|medium|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 10614.0,
+ "observations": 11347.0,
+ "cohort": "general|medium|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 2499.0,
+ "observations": 2531.0,
+ "cohort": "general|medium|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 437.0,
+ "observations": 1294.0,
+ "cohort": "general|medium|code=0|math=0|mc=0|intl=1"
+ },
+ {
+ "tier": 2,
+ "successes": 863.0,
+ "observations": 1932.0,
+ "cohort": "general|medium|code=0|math=0|mc=0|intl=1"
+ },
+ {
+ "tier": 3,
+ "successes": 738.0,
+ "observations": 1269.0,
+ "cohort": "general|medium|code=0|math=0|mc=0|intl=1"
+ },
+ {
+ "tier": 4,
+ "successes": 296.0,
+ "observations": 325.0,
+ "cohort": "general|medium|code=0|math=0|mc=0|intl=1"
+ },
+ {
+ "tier": 1,
+ "successes": 117.0,
+ "observations": 173.0,
+ "cohort": "general|medium|code=0|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 183.0,
+ "observations": 252.0,
+ "cohort": "general|medium|code=0|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 171.0,
+ "observations": 191.0,
+ "cohort": "general|medium|code=0|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 47.0,
+ "observations": 52.0,
+ "cohort": "general|medium|code=0|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 927.0,
+ "observations": 1353.0,
+ "cohort": "general|medium|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 1550.0,
+ "observations": 2023.0,
+ "cohort": "general|medium|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 1273.0,
+ "observations": 1430.0,
+ "cohort": "general|medium|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 348.0,
+ "observations": 354.0,
+ "cohort": "general|medium|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 60.0,
+ "observations": 255.0,
+ "cohort": "general|medium|code=0|math=1|mc=0|intl=1"
+ },
+ {
+ "tier": 2,
+ "successes": 143.0,
+ "observations": 377.0,
+ "cohort": "general|medium|code=0|math=1|mc=0|intl=1"
+ },
+ {
+ "tier": 3,
+ "successes": 145.0,
+ "observations": 243.0,
+ "cohort": "general|medium|code=0|math=1|mc=0|intl=1"
+ },
+ {
+ "tier": 4,
+ "successes": 58.0,
+ "observations": 61.0,
+ "cohort": "general|medium|code=0|math=1|mc=0|intl=1"
+ },
+ {
+ "tier": 1,
+ "successes": 27.0,
+ "observations": 47.0,
+ "cohort": "general|medium|code=0|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 30.0,
+ "observations": 48.0,
+ "cohort": "general|medium|code=0|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 30.0,
+ "observations": 36.0,
+ "cohort": "general|medium|code=0|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 5.0,
+ "observations": 5.0,
+ "cohort": "general|medium|code=0|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 1119.0,
+ "observations": 1348.0,
+ "cohort": "general|medium|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 1816.0,
+ "observations": 2024.0,
+ "cohort": "general|medium|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 1472.0,
+ "observations": 1552.0,
+ "cohort": "general|medium|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 301.0,
+ "observations": 304.0,
+ "cohort": "general|medium|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "general|medium|code=1|math=0|mc=0|intl=1"
+ },
+ {
+ "tier": 2,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "general|medium|code=1|math=0|mc=0|intl=1"
+ },
+ {
+ "tier": 3,
+ "successes": 2.0,
+ "observations": 2.0,
+ "cohort": "general|medium|code=1|math=0|mc=0|intl=1"
+ },
+ {
+ "tier": 1,
+ "successes": 8.0,
+ "observations": 10.0,
+ "cohort": "general|medium|code=1|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 9.0,
+ "observations": 12.0,
+ "cohort": "general|medium|code=1|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 8.0,
+ "observations": 9.0,
+ "cohort": "general|medium|code=1|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "general|medium|code=1|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 349.0,
+ "observations": 439.0,
+ "cohort": "general|medium|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 549.0,
+ "observations": 622.0,
+ "cohort": "general|medium|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 495.0,
+ "observations": 526.0,
+ "cohort": "general|medium|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 111.0,
+ "observations": 113.0,
+ "cohort": "general|medium|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "general|medium|code=1|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 4.0,
+ "observations": 6.0,
+ "cohort": "general|medium|code=1|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 4.0,
+ "observations": 4.0,
+ "cohort": "general|medium|code=1|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "general|medium|code=1|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 11635.0,
+ "observations": 12910.0,
+ "cohort": "general|short|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 19249.0,
+ "observations": 20591.0,
+ "cohort": "general|short|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 15415.0,
+ "observations": 15867.0,
+ "cohort": "general|short|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 3362.0,
+ "observations": 3392.0,
+ "cohort": "general|short|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 68.0,
+ "observations": 155.0,
+ "cohort": "general|short|code=0|math=0|mc=0|intl=1"
+ },
+ {
+ "tier": 2,
+ "successes": 113.0,
+ "observations": 202.0,
+ "cohort": "general|short|code=0|math=0|mc=0|intl=1"
+ },
+ {
+ "tier": 3,
+ "successes": 97.0,
+ "observations": 124.0,
+ "cohort": "general|short|code=0|math=0|mc=0|intl=1"
+ },
+ {
+ "tier": 4,
+ "successes": 31.0,
+ "observations": 31.0,
+ "cohort": "general|short|code=0|math=0|mc=0|intl=1"
+ },
+ {
+ "tier": 1,
+ "successes": 6.0,
+ "observations": 11.0,
+ "cohort": "general|short|code=0|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 20.0,
+ "observations": 21.0,
+ "cohort": "general|short|code=0|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 23.0,
+ "observations": 24.0,
+ "cohort": "general|short|code=0|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 4.0,
+ "observations": 4.0,
+ "cohort": "general|short|code=0|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 199.0,
+ "observations": 270.0,
+ "cohort": "general|short|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 334.0,
+ "observations": 414.0,
+ "cohort": "general|short|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 309.0,
+ "observations": 337.0,
+ "cohort": "general|short|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 75.0,
+ "observations": 75.0,
+ "cohort": "general|short|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 3.0,
+ "observations": 5.0,
+ "cohort": "general|short|code=0|math=1|mc=0|intl=1"
+ },
+ {
+ "tier": 2,
+ "successes": 1.0,
+ "observations": 5.0,
+ "cohort": "general|short|code=0|math=1|mc=0|intl=1"
+ },
+ {
+ "tier": 3,
+ "successes": 2.0,
+ "observations": 2.0,
+ "cohort": "general|short|code=0|math=1|mc=0|intl=1"
+ },
+ {
+ "tier": 1,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "general|short|code=0|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 2.0,
+ "observations": 2.0,
+ "cohort": "general|short|code=0|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "general|short|code=0|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 629.0,
+ "observations": 778.0,
+ "cohort": "general|short|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 1131.0,
+ "observations": 1286.0,
+ "cohort": "general|short|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 919.0,
+ "observations": 989.0,
+ "cohort": "general|short|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 227.0,
+ "observations": 227.0,
+ "cohort": "general|short|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 3.0,
+ "observations": 4.0,
+ "cohort": "general|short|code=1|math=0|mc=0|intl=1"
+ },
+ {
+ "tier": 2,
+ "successes": 3.0,
+ "observations": 3.0,
+ "cohort": "general|short|code=1|math=0|mc=0|intl=1"
+ },
+ {
+ "tier": 3,
+ "successes": 4.0,
+ "observations": 5.0,
+ "cohort": "general|short|code=1|math=0|mc=0|intl=1"
+ },
+ {
+ "tier": 1,
+ "successes": 2.0,
+ "observations": 2.0,
+ "cohort": "general|short|code=1|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 2.0,
+ "observations": 2.0,
+ "cohort": "general|short|code=1|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 28.0,
+ "observations": 37.0,
+ "cohort": "general|short|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 54.0,
+ "observations": 64.0,
+ "cohort": "general|short|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 41.0,
+ "observations": 46.0,
+ "cohort": "general|short|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 9.0,
+ "observations": 9.0,
+ "cohort": "general|short|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 1066.0,
+ "observations": 1437.0,
+ "cohort": "general|very_long|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 1639.0,
+ "observations": 2007.0,
+ "cohort": "general|very_long|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 1397.0,
+ "observations": 1507.0,
+ "cohort": "general|very_long|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 333.0,
+ "observations": 341.0,
+ "cohort": "general|very_long|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 3.0,
+ "observations": 6.0,
+ "cohort": "general|very_long|code=0|math=0|mc=0|intl=1"
+ },
+ {
+ "tier": 2,
+ "successes": 2.0,
+ "observations": 4.0,
+ "cohort": "general|very_long|code=0|math=0|mc=0|intl=1"
+ },
+ {
+ "tier": 3,
+ "successes": 3.0,
+ "observations": 3.0,
+ "cohort": "general|very_long|code=0|math=0|mc=0|intl=1"
+ },
+ {
+ "tier": 4,
+ "successes": 3.0,
+ "observations": 3.0,
+ "cohort": "general|very_long|code=0|math=0|mc=0|intl=1"
+ },
+ {
+ "tier": 1,
+ "successes": 65.0,
+ "observations": 89.0,
+ "cohort": "general|very_long|code=0|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 119.0,
+ "observations": 139.0,
+ "cohort": "general|very_long|code=0|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 98.0,
+ "observations": 103.0,
+ "cohort": "general|very_long|code=0|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 21.0,
+ "observations": 21.0,
+ "cohort": "general|very_long|code=0|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 403.0,
+ "observations": 546.0,
+ "cohort": "general|very_long|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 700.0,
+ "observations": 875.0,
+ "cohort": "general|very_long|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 556.0,
+ "observations": 612.0,
+ "cohort": "general|very_long|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 133.0,
+ "observations": 135.0,
+ "cohort": "general|very_long|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 17.0,
+ "observations": 27.0,
+ "cohort": "general|very_long|code=0|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 19.0,
+ "observations": 33.0,
+ "cohort": "general|very_long|code=0|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 25.0,
+ "observations": 27.0,
+ "cohort": "general|very_long|code=0|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 9.0,
+ "observations": 9.0,
+ "cohort": "general|very_long|code=0|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 210.0,
+ "observations": 280.0,
+ "cohort": "general|very_long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 307.0,
+ "observations": 378.0,
+ "cohort": "general|very_long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 258.0,
+ "observations": 289.0,
+ "cohort": "general|very_long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 64.0,
+ "observations": 65.0,
+ "cohort": "general|very_long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 17.0,
+ "observations": 23.0,
+ "cohort": "general|very_long|code=1|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 31.0,
+ "observations": 39.0,
+ "cohort": "general|very_long|code=1|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 16.0,
+ "observations": 19.0,
+ "cohort": "general|very_long|code=1|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 3.0,
+ "observations": 3.0,
+ "cohort": "general|very_long|code=1|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 202.0,
+ "observations": 282.0,
+ "cohort": "general|very_long|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 361.0,
+ "observations": 478.0,
+ "cohort": "general|very_long|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 262.0,
+ "observations": 310.0,
+ "cohort": "general|very_long|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 82.0,
+ "observations": 82.0,
+ "cohort": "general|very_long|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 4.0,
+ "observations": 5.0,
+ "cohort": "general|very_long|code=1|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 9.0,
+ "observations": 11.0,
+ "cohort": "general|very_long|code=1|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 4.0,
+ "observations": 4.0,
+ "cohort": "general|very_long|code=1|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 19.0,
+ "observations": 21.0,
+ "cohort": "technical_design|long|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 23.0,
+ "observations": 25.0,
+ "cohort": "technical_design|long|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 28.0,
+ "observations": 29.0,
+ "cohort": "technical_design|long|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 5.0,
+ "observations": 5.0,
+ "cohort": "technical_design|long|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 2.0,
+ "observations": 2.0,
+ "cohort": "technical_design|long|code=0|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "technical_design|long|code=0|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "technical_design|long|code=0|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 3.0,
+ "observations": 3.0,
+ "cohort": "technical_design|long|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 8.0,
+ "observations": 9.0,
+ "cohort": "technical_design|long|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 9.0,
+ "observations": 9.0,
+ "cohort": "technical_design|long|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 3.0,
+ "observations": 3.0,
+ "cohort": "technical_design|long|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 14.0,
+ "observations": 15.0,
+ "cohort": "technical_design|long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 24.0,
+ "observations": 25.0,
+ "cohort": "technical_design|long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 18.0,
+ "observations": 18.0,
+ "cohort": "technical_design|long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 2.0,
+ "observations": 2.0,
+ "cohort": "technical_design|long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 2.0,
+ "observations": 2.0,
+ "cohort": "technical_design|long|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 3.0,
+ "observations": 3.0,
+ "cohort": "technical_design|long|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 2.0,
+ "observations": 2.0,
+ "cohort": "technical_design|long|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "technical_design|long|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 44.0,
+ "observations": 45.0,
+ "cohort": "technical_design|medium|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 84.0,
+ "observations": 86.0,
+ "cohort": "technical_design|medium|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 74.0,
+ "observations": 74.0,
+ "cohort": "technical_design|medium|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 15.0,
+ "observations": 15.0,
+ "cohort": "technical_design|medium|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 3.0,
+ "observations": 3.0,
+ "cohort": "technical_design|medium|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 6.0,
+ "observations": 6.0,
+ "cohort": "technical_design|medium|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 3.0,
+ "observations": 3.0,
+ "cohort": "technical_design|medium|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 23.0,
+ "observations": 24.0,
+ "cohort": "technical_design|medium|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 47.0,
+ "observations": 53.0,
+ "cohort": "technical_design|medium|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 28.0,
+ "observations": 29.0,
+ "cohort": "technical_design|medium|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 10.0,
+ "observations": 10.0,
+ "cohort": "technical_design|medium|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 4.0,
+ "observations": 4.0,
+ "cohort": "technical_design|medium|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 7.0,
+ "observations": 7.0,
+ "cohort": "technical_design|medium|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 7.0,
+ "observations": 8.0,
+ "cohort": "technical_design|medium|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "technical_design|medium|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 13.0,
+ "observations": 13.0,
+ "cohort": "technical_design|short|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 31.0,
+ "observations": 31.0,
+ "cohort": "technical_design|short|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 23.0,
+ "observations": 23.0,
+ "cohort": "technical_design|short|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 5.0,
+ "observations": 5.0,
+ "cohort": "technical_design|short|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 2.0,
+ "observations": 2.0,
+ "cohort": "technical_design|short|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 3.0,
+ "observations": 4.0,
+ "cohort": "technical_design|short|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 2.0,
+ "observations": 2.0,
+ "cohort": "technical_design|short|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 10.0,
+ "observations": 15.0,
+ "cohort": "technical_design|very_long|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 26.0,
+ "observations": 30.0,
+ "cohort": "technical_design|very_long|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 19.0,
+ "observations": 21.0,
+ "cohort": "technical_design|very_long|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 2.0,
+ "observations": 2.0,
+ "cohort": "technical_design|very_long|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 3.0,
+ "observations": 3.0,
+ "cohort": "technical_design|very_long|code=0|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 2.0,
+ "observations": 2.0,
+ "cohort": "technical_design|very_long|code=0|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 2.0,
+ "observations": 2.0,
+ "cohort": "technical_design|very_long|code=0|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "technical_design|very_long|code=0|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 5.0,
+ "observations": 6.0,
+ "cohort": "technical_design|very_long|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 6.0,
+ "observations": 7.0,
+ "cohort": "technical_design|very_long|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 6.0,
+ "observations": 6.0,
+ "cohort": "technical_design|very_long|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "technical_design|very_long|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "technical_design|very_long|code=0|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 4.0,
+ "observations": 6.0,
+ "cohort": "technical_design|very_long|code=0|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "technical_design|very_long|code=0|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 8.0,
+ "observations": 8.0,
+ "cohort": "technical_design|very_long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 5.0,
+ "observations": 6.0,
+ "cohort": "technical_design|very_long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 5.0,
+ "observations": 5.0,
+ "cohort": "technical_design|very_long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "technical_design|very_long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 3.0,
+ "observations": 3.0,
+ "cohort": "technical_design|very_long|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 2.0,
+ "observations": 2.0,
+ "cohort": "technical_design|very_long|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 3.0,
+ "observations": 3.0,
+ "cohort": "technical_design|very_long|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 176.0,
+ "observations": 226.0,
+ "cohort": "writing|long|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 267.0,
+ "observations": 329.0,
+ "cohort": "writing|long|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 229.0,
+ "observations": 245.0,
+ "cohort": "writing|long|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 51.0,
+ "observations": 52.0,
+ "cohort": "writing|long|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 0.0,
+ "observations": 2.0,
+ "cohort": "writing|long|code=0|math=0|mc=0|intl=1"
+ },
+ {
+ "tier": 2,
+ "successes": 1.0,
+ "observations": 2.0,
+ "cohort": "writing|long|code=0|math=0|mc=0|intl=1"
+ },
+ {
+ "tier": 3,
+ "successes": 3.0,
+ "observations": 3.0,
+ "cohort": "writing|long|code=0|math=0|mc=0|intl=1"
+ },
+ {
+ "tier": 4,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "writing|long|code=0|math=0|mc=0|intl=1"
+ },
+ {
+ "tier": 1,
+ "successes": 3.0,
+ "observations": 5.0,
+ "cohort": "writing|long|code=0|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 4.0,
+ "observations": 5.0,
+ "cohort": "writing|long|code=0|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 2.0,
+ "observations": 2.0,
+ "cohort": "writing|long|code=0|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 36.0,
+ "observations": 47.0,
+ "cohort": "writing|long|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 53.0,
+ "observations": 59.0,
+ "cohort": "writing|long|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 47.0,
+ "observations": 51.0,
+ "cohort": "writing|long|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 14.0,
+ "observations": 15.0,
+ "cohort": "writing|long|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 17.0,
+ "observations": 21.0,
+ "cohort": "writing|long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 27.0,
+ "observations": 32.0,
+ "cohort": "writing|long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 24.0,
+ "observations": 24.0,
+ "cohort": "writing|long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 3.0,
+ "observations": 3.0,
+ "cohort": "writing|long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 1.0,
+ "observations": 2.0,
+ "cohort": "writing|long|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 1.0,
+ "observations": 2.0,
+ "cohort": "writing|long|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 6.0,
+ "observations": 6.0,
+ "cohort": "writing|long|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 2.0,
+ "observations": 2.0,
+ "cohort": "writing|long|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 541.0,
+ "observations": 598.0,
+ "cohort": "writing|medium|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 892.0,
+ "observations": 997.0,
+ "cohort": "writing|medium|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 728.0,
+ "observations": 756.0,
+ "cohort": "writing|medium|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 164.0,
+ "observations": 165.0,
+ "cohort": "writing|medium|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 2.0,
+ "observations": 7.0,
+ "cohort": "writing|medium|code=0|math=0|mc=0|intl=1"
+ },
+ {
+ "tier": 2,
+ "successes": 7.0,
+ "observations": 15.0,
+ "cohort": "writing|medium|code=0|math=0|mc=0|intl=1"
+ },
+ {
+ "tier": 3,
+ "successes": 7.0,
+ "observations": 10.0,
+ "cohort": "writing|medium|code=0|math=0|mc=0|intl=1"
+ },
+ {
+ "tier": 4,
+ "successes": 4.0,
+ "observations": 4.0,
+ "cohort": "writing|medium|code=0|math=0|mc=0|intl=1"
+ },
+ {
+ "tier": 1,
+ "successes": 2.0,
+ "observations": 2.0,
+ "cohort": "writing|medium|code=0|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "writing|medium|code=0|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "writing|medium|code=0|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 25.0,
+ "observations": 35.0,
+ "cohort": "writing|medium|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 41.0,
+ "observations": 63.0,
+ "cohort": "writing|medium|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 39.0,
+ "observations": 46.0,
+ "cohort": "writing|medium|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 12.0,
+ "observations": 12.0,
+ "cohort": "writing|medium|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 1.0,
+ "observations": 2.0,
+ "cohort": "writing|medium|code=0|math=1|mc=0|intl=1"
+ },
+ {
+ "tier": 2,
+ "successes": 2.0,
+ "observations": 4.0,
+ "cohort": "writing|medium|code=0|math=1|mc=0|intl=1"
+ },
+ {
+ "tier": 3,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "writing|medium|code=0|math=1|mc=0|intl=1"
+ },
+ {
+ "tier": 4,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "writing|medium|code=0|math=1|mc=0|intl=1"
+ },
+ {
+ "tier": 1,
+ "successes": 2.0,
+ "observations": 2.0,
+ "cohort": "writing|medium|code=0|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 2.0,
+ "observations": 2.0,
+ "cohort": "writing|medium|code=0|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 24.0,
+ "observations": 27.0,
+ "cohort": "writing|medium|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 49.0,
+ "observations": 56.0,
+ "cohort": "writing|medium|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 46.0,
+ "observations": 49.0,
+ "cohort": "writing|medium|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 8.0,
+ "observations": 8.0,
+ "cohort": "writing|medium|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 11.0,
+ "observations": 13.0,
+ "cohort": "writing|medium|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 13.0,
+ "observations": 13.0,
+ "cohort": "writing|medium|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 10.0,
+ "observations": 10.0,
+ "cohort": "writing|medium|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 327.0,
+ "observations": 353.0,
+ "cohort": "writing|short|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 508.0,
+ "observations": 555.0,
+ "cohort": "writing|short|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 415.0,
+ "observations": 432.0,
+ "cohort": "writing|short|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 92.0,
+ "observations": 92.0,
+ "cohort": "writing|short|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 2.0,
+ "observations": 2.0,
+ "cohort": "writing|short|code=0|math=0|mc=0|intl=1"
+ },
+ {
+ "tier": 3,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "writing|short|code=0|math=0|mc=0|intl=1"
+ },
+ {
+ "tier": 4,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "writing|short|code=0|math=0|mc=0|intl=1"
+ },
+ {
+ "tier": 1,
+ "successes": 4.0,
+ "observations": 4.0,
+ "cohort": "writing|short|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 7.0,
+ "observations": 7.0,
+ "cohort": "writing|short|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 3.0,
+ "observations": 3.0,
+ "cohort": "writing|short|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 2.0,
+ "observations": 2.0,
+ "cohort": "writing|short|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 2.0,
+ "observations": 3.0,
+ "cohort": "writing|short|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 8.0,
+ "observations": 9.0,
+ "cohort": "writing|short|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 4.0,
+ "observations": 4.0,
+ "cohort": "writing|short|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "writing|short|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "writing|short|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 2.0,
+ "observations": 2.0,
+ "cohort": "writing|short|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 63.0,
+ "observations": 85.0,
+ "cohort": "writing|very_long|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 111.0,
+ "observations": 139.0,
+ "cohort": "writing|very_long|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 95.0,
+ "observations": 99.0,
+ "cohort": "writing|very_long|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 29.0,
+ "observations": 29.0,
+ "cohort": "writing|very_long|code=0|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 3.0,
+ "observations": 5.0,
+ "cohort": "writing|very_long|code=0|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 1.0,
+ "observations": 2.0,
+ "cohort": "writing|very_long|code=0|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 4.0,
+ "observations": 4.0,
+ "cohort": "writing|very_long|code=0|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "writing|very_long|code=0|math=0|mc=1|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 26.0,
+ "observations": 36.0,
+ "cohort": "writing|very_long|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 30.0,
+ "observations": 44.0,
+ "cohort": "writing|very_long|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 23.0,
+ "observations": 28.0,
+ "cohort": "writing|very_long|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 8.0,
+ "observations": 8.0,
+ "cohort": "writing|very_long|code=0|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 2.0,
+ "observations": 2.0,
+ "cohort": "writing|very_long|code=0|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "writing|very_long|code=0|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 1.0,
+ "observations": 1.0,
+ "cohort": "writing|very_long|code=0|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 14.0,
+ "observations": 14.0,
+ "cohort": "writing|very_long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 14.0,
+ "observations": 16.0,
+ "cohort": "writing|very_long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 6.0,
+ "observations": 7.0,
+ "cohort": "writing|very_long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 4,
+ "successes": 3.0,
+ "observations": 3.0,
+ "cohort": "writing|very_long|code=1|math=0|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 5.0,
+ "observations": 5.0,
+ "cohort": "writing|very_long|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 7.0,
+ "observations": 8.0,
+ "cohort": "writing|very_long|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 7.0,
+ "observations": 7.0,
+ "cohort": "writing|very_long|code=1|math=1|mc=0|intl=0"
+ },
+ {
+ "tier": 1,
+ "successes": 0.0,
+ "observations": 1.0,
+ "cohort": "writing|very_long|code=1|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 2,
+ "successes": 2.0,
+ "observations": 2.0,
+ "cohort": "writing|very_long|code=1|math=1|mc=1|intl=0"
+ },
+ {
+ "tier": 3,
+ "successes": 0.0,
+ "observations": 1.0,
+ "cohort": "writing|very_long|code=1|math=1|mc=1|intl=0"
+ }
+ ],
+ "domain_prior_mass": 200.0,
+ "cohort_prior_mass": 20.0,
+ "routing_threshold": 0.75,
+ "datasets": [
+ {
+ "name": "openbmb/UltraFeedback",
+ "url": "https://huggingface.co/datasets/openbmb/UltraFeedback",
+ "license": "MIT",
+ "rows": 255864,
+ "success_definition": "UltraFeedback overall_score >= 4"
+ }
+ ],
+ "success_definition": "UltraFeedback overall_score >= 4",
+ "split_method": "sha256(prompt): 70% train, 15% validation, 15% test"
+}
diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py
index 577cee0920d..a123a75dd81 100644
--- a/litellm/router_strategy/complexity_router/complexity_router.py
+++ b/litellm/router_strategy/complexity_router/complexity_router.py
@@ -33,6 +33,11 @@ from litellm.litellm_core_utils.internal_call_metadata import forwarded_internal
from litellm.litellm_core_utils.prompt_templates.common_utils import request_contains_image_content
from litellm.litellm_core_utils.sensitive_data_masker import mask_credentials_in_payload
from litellm.llms.base_llm.base_utils import type_to_response_format_param
+from litellm.router_strategy.adaptive_router.classifier import classify_prompt
+from litellm.router_strategy.complexity_router.tier_predictor import (
+ TierSuccessPredictor,
+ resolve_tier_artifact,
+)
from litellm.types.utils import (
AUTOROUTER_CLASSIFIER_CALL_ORIGIN,
ModelResponse,
@@ -790,6 +795,7 @@ class ClassificationOutcome(NamedTuple):
signals: tuple[str, ...]
cause: Literal[
"heuristic_scorer",
+ "heuristic_v2",
"reasoning_override",
"llm_classifier",
"heuristic_first_short_circuit",
@@ -978,6 +984,11 @@ class ComplexityRouter(CustomLogger):
if llm_classifier_configured
else None
)
+ self._tier_success_predictor: TierSuccessPredictor | None = (
+ TierSuccessPredictor(resolve_tier_artifact(self.config.heuristic_v2_artifact))
+ if self.config.classifier_type == "heuristic_v2"
+ else None
+ )
verbose_router_logger.debug("ComplexityRouter initialized for %s with tiers: %s", model_name, self.config.tiers)
@@ -1350,6 +1361,8 @@ class ComplexityRouter(CustomLogger):
custom tier set, and classifier_fallback otherwise decides between the heuristic scorer and
default_model. The outcome's `cause` reports which path actually ran.
"""
+ if self.config.classifier_type == "heuristic_v2":
+ return self._classify_with_heuristic_v2(prompt)
if self.config.classifier_type == "custom":
return await self._classify_with_plugin(prompt, system_prompt, request_kwargs, raw_messages)
if self.config.classifier_type == "heuristic_first" and self.config.classifier_llm_config is not None:
@@ -1359,6 +1372,24 @@ class ComplexityRouter(CustomLogger):
return ClassificationOutcome(tier=tier, score=score, signals=signals, cause=cause)
return await self._llm_classifier_outcome(prompt, system_prompt, request_kwargs, messages)
+ def _classify_with_heuristic_v2(self, prompt: str) -> ClassificationOutcome:
+ predictor: Final = self._tier_success_predictor
+ if predictor is None:
+ raise ValueError("heuristic v2 predictor is not configured")
+ request_type: Final = classify_prompt(prompt)
+ prediction: Final = predictor.predict(prompt, request_type)
+ tier: Final = TIER_SEVERITY_ORDER[prediction.required_tier - 1]
+ probability_signals: Final = tuple(
+ f"tier-probability:{candidate.value.lower()}={prediction.probabilities[index]:.6f}"
+ for index, candidate in enumerate(TIER_SEVERITY_ORDER, start=1)
+ )
+ return ClassificationOutcome(
+ tier=tier,
+ score=None,
+ signals=(f"request-type:{request_type.value}", *probability_signals),
+ cause="heuristic_v2",
+ )
+
async def _classify_heuristic_first(
self,
prompt: str,
diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py
index 70aeecb31c6..fdf2a3a0b39 100644
--- a/litellm/router_strategy/complexity_router/config.py
+++ b/litellm/router_strategy/complexity_router/config.py
@@ -14,6 +14,8 @@ from pydantic import BaseModel, ConfigDict, Field, SkipValidation, field_seriali
from litellm.types.router import AdaptiveRouterWeights, ClassifierPlugin, RoutingPlugin
+from .tier_predictor import TrainedTierArtifact
+
class ComplexityTier(str, Enum):
"""Complexity tiers for routing decisions."""
@@ -625,12 +627,19 @@ class ComplexityRouterConfig(BaseModel):
)
# Classifier strategy
- classifier_type: Literal["heuristic", "llm", "custom", "heuristic_first"] = Field(
+ classifier_type: Literal["heuristic", "heuristic_v2", "llm", "custom", "heuristic_first"] = Field(
default="heuristic",
description=(
- "Classification strategy: local regex/keyword scoring, an LLM call, a custom classifier "
- "plugin, or 'heuristic_first', which scores locally and only pays for the LLM classifier "
- "when the local scorer does not confidently land a cheap tier"
+ "Classification strategy: local regex/keyword scoring, the bundled trained four-tier heuristic, "
+ "an LLM call, a custom classifier plugin, or 'heuristic_first', which scores locally and only pays "
+ "for the LLM classifier when the local scorer does not confidently land a cheap tier"
+ ),
+ )
+ heuristic_v2_artifact: TrainedTierArtifact | Literal["ultrafeedback"] = Field(
+ default="ultrafeedback",
+ description=(
+ "Success-probability artifact used by classifier_type 'heuristic_v2'. The bundled "
+ "UltraFeedback artifact is selected by default; an inline trained artifact may replace it"
),
)
classifier_llm_config: ClassifierLLMConfig | None = Field(
@@ -1248,10 +1257,10 @@ class ComplexityRouterConfig(BaseModel):
)
if duplicated:
raise ValueError(f"tier_definitions names must be unique (case-insensitive): {', '.join(duplicated)}")
- if self.classifier_type in ("heuristic", "heuristic_first"):
+ if self.classifier_type in ("heuristic", "heuristic_v2", "heuristic_first"):
raise ValueError(
"tier_definitions requires classifier_type 'llm' or 'custom': the heuristic scorer only "
- "produces the built-in tiers"
+ "produces the four built-in tiers, as does heuristic_v2"
)
conflicts: Final = self._tier_definition_conflicts()
if conflicts:
diff --git a/litellm/router_strategy/complexity_router/tier_predictor.py b/litellm/router_strategy/complexity_router/tier_predictor.py
new file mode 100644
index 00000000000..764f6e6ad56
--- /dev/null
+++ b/litellm/router_strategy/complexity_router/tier_predictor.py
@@ -0,0 +1,156 @@
+from __future__ import annotations
+
+import re
+from collections.abc import Mapping
+from dataclasses import dataclass
+from pathlib import Path
+from types import MappingProxyType
+from typing import Final, Literal
+
+from pydantic import BaseModel, Field, model_validator
+
+from litellm.types.router import RequestType
+
+
+class TierGlobalStatistic(BaseModel):
+ tier: int = Field(ge=1, le=4)
+ successes: float = Field(ge=0.0)
+ observations: float = Field(gt=0.0)
+
+ @model_validator(mode="after")
+ def _successes_do_not_exceed_observations(self) -> TierGlobalStatistic:
+ if self.successes > self.observations:
+ raise ValueError("successes cannot exceed observations")
+ return self
+
+
+class TierDomainStatistic(TierGlobalStatistic):
+ request_type: RequestType
+
+
+class TierCohortStatistic(TierGlobalStatistic):
+ cohort: str = Field(min_length=1)
+
+
+class TierDataset(BaseModel):
+ name: str = Field(min_length=1)
+ url: str = Field(min_length=1)
+ license: str = Field(min_length=1)
+ rows: int = Field(gt=0)
+ success_definition: str = Field(default="quality score meets the dataset success threshold", min_length=1)
+
+
+class TrainedTierArtifact(BaseModel):
+ schema_version: Literal[1] = 1
+ global_statistics: tuple[TierGlobalStatistic, ...]
+ domain_statistics: tuple[TierDomainStatistic, ...] = ()
+ cohort_statistics: tuple[TierCohortStatistic, ...] = ()
+ domain_prior_mass: float = Field(default=200.0, gt=0.0)
+ cohort_prior_mass: float = Field(default=20.0, gt=0.0)
+ routing_threshold: float = Field(default=0.75, ge=0.0, le=1.0)
+ datasets: tuple[TierDataset, ...] = ()
+ success_definition: str = Field(default="quality score meets the dataset success threshold", min_length=1)
+ split_method: str = Field(default="sha256(prompt): 70% train, 15% validation, 15% test", min_length=1)
+
+ @model_validator(mode="after")
+ def _statistics_are_unique(self) -> TrainedTierArtifact:
+ global_tiers: Final = tuple(stat.tier for stat in self.global_statistics)
+ if frozenset(global_tiers) != frozenset((1, 2, 3, 4)) or len(global_tiers) != 4:
+ raise ValueError("global statistics must contain each tier exactly once")
+ domain_keys: Final = tuple((stat.request_type, stat.tier) for stat in self.domain_statistics)
+ if len(domain_keys) != len(frozenset(domain_keys)):
+ raise ValueError("domain statistics must contain unique request_type and tier pairs")
+ cohort_keys: Final = tuple((stat.cohort, stat.tier) for stat in self.cohort_statistics)
+ if len(cohort_keys) != len(frozenset(cohort_keys)):
+ raise ValueError("cohort statistics must contain unique cohort and tier pairs")
+ return self
+
+
+_CODE_PATTERN: Final = re.compile(
+ r"```|\b(def|class|function|python|javascript|typescript|sql|code)\b",
+ re.IGNORECASE,
+)
+_MATH_PATTERN: Final = re.compile(
+ r"\b(solve|calculate|equation|probability|theorem|proof|integral)\b|[$=]",
+ re.IGNORECASE,
+)
+_MULTIPLE_CHOICE_PATTERN: Final = re.compile(r"(?:^|\s)[A-D][.)]\s")
+_TIERS: Final = (1, 2, 3, 4)
+_BUILTIN_ARTIFACTS: Final = MappingProxyType({"ultrafeedback": "ultrafeedback_tiers.json"})
+
+
+def resolve_tier_artifact(artifact: TrainedTierArtifact | str) -> TrainedTierArtifact:
+ if isinstance(artifact, TrainedTierArtifact):
+ return artifact
+ filename: Final = _BUILTIN_ARTIFACTS.get(artifact)
+ if filename is None:
+ raise ValueError(f"unknown complexity router tier artifact: {artifact}")
+ path: Final = Path(__file__).with_name("artifacts") / filename
+ return TrainedTierArtifact.model_validate_json(path.read_text())
+
+
+def similarity_cohort(prompt: str, request_type: RequestType) -> str:
+ length: Final = len(prompt)
+ length_bucket: Final = (
+ "short" if length < 200 else "medium" if length < 800 else "long" if length < 2000 else "very_long"
+ )
+ code: Final = int(bool(_CODE_PATTERN.search(prompt)))
+ math: Final = int(bool(_MATH_PATTERN.search(prompt)))
+ multiple_choice: Final = int(bool(_MULTIPLE_CHOICE_PATTERN.search(prompt)))
+ non_ascii: Final = int(sum(ord(character) > 127 for character in prompt) / max(1, length) > 0.1)
+ return f"{request_type.value}|{length_bucket}|code={code}|math={math}|mc={multiple_choice}|intl={non_ascii}"
+
+
+@dataclass(frozen=True, slots=True)
+class TierPrediction:
+ probabilities: Mapping[int, float]
+ required_tier: int
+
+
+class TierSuccessPredictor:
+ def __init__(self, artifact: TrainedTierArtifact) -> None:
+ self._artifact = artifact
+ self._global: Mapping[int, TierGlobalStatistic] = MappingProxyType(
+ {stat.tier: stat for stat in artifact.global_statistics}
+ )
+ self._domain: Mapping[tuple[RequestType, int], TierDomainStatistic] = MappingProxyType(
+ {(stat.request_type, stat.tier): stat for stat in artifact.domain_statistics}
+ )
+ self._cohort: Mapping[tuple[str, int], TierCohortStatistic] = MappingProxyType(
+ {(stat.cohort, stat.tier): stat for stat in artifact.cohort_statistics}
+ )
+
+ @property
+ def routing_threshold(self) -> float:
+ return self._artifact.routing_threshold
+
+ def predict(self, prompt: str, request_type: RequestType) -> TierPrediction:
+ cohort: Final = similarity_cohort(prompt, request_type)
+ raw: Final = tuple(self._probability(tier, request_type, cohort) for tier in _TIERS)
+ monotonic: Final = tuple(max(raw[:index]) for index in range(1, len(raw) + 1))
+ probabilities: Final[Mapping[int, float]] = MappingProxyType(
+ {int(tier): probability for tier, probability in zip(_TIERS, monotonic)}
+ )
+ required_tier: Final = next(
+ (tier for tier in _TIERS if probabilities[tier] >= self._artifact.routing_threshold),
+ 4,
+ )
+ return TierPrediction(probabilities=probabilities, required_tier=required_tier)
+
+ def _probability(self, tier: int, request_type: RequestType, cohort: str) -> float:
+ global_stat: Final = self._global[tier]
+ global_mean: Final = (global_stat.successes + 1.0) / (global_stat.observations + 2.0)
+ domain_stat: Final = self._domain.get((request_type, tier))
+ domain_mean: Final = self._posterior_mean(domain_stat, self._artifact.domain_prior_mass, global_mean)
+ cohort_stat: Final = self._cohort.get((cohort, tier))
+ return self._posterior_mean(cohort_stat, self._artifact.cohort_prior_mass, domain_mean)
+
+ @staticmethod
+ def _posterior_mean(
+ statistic: TierGlobalStatistic | None,
+ prior_mass: float,
+ prior_mean: float,
+ ) -> float:
+ if statistic is None:
+ return prior_mean
+ return (statistic.successes + prior_mass * prior_mean) / (statistic.observations + prior_mass)
diff --git a/litellm/types/utils.py b/litellm/types/utils.py
index 5783a39b30c..09c01873a9d 100644
--- a/litellm/types/utils.py
+++ b/litellm/types/utils.py
@@ -2839,6 +2839,7 @@ class StandardLoggingRoutingDecisionTierBoundaries(TypedDict):
RoutingDecisionCause = Literal[
"heuristic_scorer",
+ "heuristic_v2",
# The scorer found 2+ reasoning markers and forced REASONING regardless of score.
# A distinct cause rather than a marker inside `signals`, because it is the fact
# that tells a reader the score did NOT choose the tier; encoding it as free text
diff --git a/pyproject.toml b/pyproject.toml
index 60162544612..d0e5723d1cb 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -278,7 +278,10 @@ bindings = "pyo3"
features = ["extension-module"]
profile = "release"
editable-profile = "dev"
-include = ["litellm/proxy/_experimental/out/**"]
+include = [
+ "litellm/proxy/_experimental/out/**",
+ "litellm/router_strategy/complexity_router/artifacts/*.json",
+]
exclude = [
"litellm/proxy/enterprise",
"litellm/proxy/enterprise/**",
diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py
index 1ec8be88c9b..82659571de4 100644
--- a/tests/test_litellm/router_strategy/test_complexity_router.py
+++ b/tests/test_litellm/router_strategy/test_complexity_router.py
@@ -12,7 +12,6 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from pydantic import ValidationError
-
import litellm
from litellm import Router
from litellm._logging import verbose_router_logger
@@ -34,10 +33,14 @@ from litellm.router_strategy.complexity_router.config import (
DEFAULT_CLASSIFIER_CONTEXT_WINDOW_SIZE,
DEFAULT_COMPLEXITY_CONFIG,
DEFAULT_TECHNICAL_KEYWORDS,
+ ClassificationRubric,
ClassifierLLMConfig,
ComplexityRouterConfig,
ComplexityTier,
- ClassificationRubric,
+)
+from litellm.router_strategy.complexity_router.tier_predictor import (
+ TierGlobalStatistic,
+ TrainedTierArtifact,
)
from litellm.types.router import (
Deployment,
@@ -46,6 +49,16 @@ from litellm.types.router import (
)
+def _heuristic_v2_artifact() -> TrainedTierArtifact:
+ return TrainedTierArtifact(
+ global_statistics=tuple(
+ TierGlobalStatistic(tier=tier, successes=successes, observations=100)
+ for tier, successes in enumerate((10, 20, 90, 99), start=1)
+ ),
+ routing_threshold=0.8,
+ )
+
+
@pytest.fixture
def mock_router_instance():
"""Create a mock LiteLLM Router instance."""
@@ -1696,6 +1709,59 @@ class TestLLMClassifier:
assert outcome.cause == "heuristic_scorer"
assert outcome.score is not None
+ @pytest.mark.asyncio
+ async def test_heuristic_v2_routes_directly_to_predicted_builtin_tier(self, mock_router_instance):
+ router = ComplexityRouter(
+ model_name="tier-router",
+ litellm_router_instance=mock_router_instance,
+ complexity_router_config={
+ "classifier_type": "heuristic_v2",
+ "heuristic_v2_artifact": _heuristic_v2_artifact(),
+ "tiers": {
+ "SIMPLE": "simple-model",
+ "MEDIUM": "medium-model",
+ "COMPLEX": "complex-model",
+ "REASONING": "reasoning-model",
+ },
+ },
+ )
+
+ response = await router.async_pre_routing_hook(
+ model="tier-router",
+ request_kwargs={},
+ messages=[{"role": "user", "content": "Handle this new request"}],
+ )
+
+ assert response is not None
+ assert response.model == "complex-model"
+ assert response.routing_decision["tier"] == "COMPLEX"
+ assert response.routing_decision["cause"] == "heuristic_v2"
+ assert response.routing_decision["signals"] == [
+ "request-type:general",
+ "tier-probability:simple=0.107843",
+ "tier-probability:medium=0.205882",
+ "tier-probability:complex=0.892157",
+ "tier-probability:reasoning=0.980392",
+ ]
+
+ def test_heuristic_v2_needs_no_classifier_model(self):
+ config = ComplexityRouterConfig(classifier_type="heuristic_v2")
+
+ assert config.classifier_llm_config is None
+ assert config.heuristic_v2_artifact == "ultrafeedback"
+
+ def test_heuristic_v2_rejects_custom_tier_definitions(self):
+ with pytest.raises(ValidationError, match="as does heuristic_v2"):
+ ComplexityRouterConfig(
+ classifier_type="heuristic_v2",
+ tier_definitions=(
+ {"name": "low", "description": "easy work"},
+ {"name": "high", "description": "hard work"},
+ ),
+ tiers={"low": "cheap", "high": "expensive"},
+ fallback_tier="high",
+ )
+
@pytest.mark.asyncio
async def test_aclassify_llm_success_routes_by_llm_verdict(self, llm_complexity_router, mock_router_instance):
"""A well-formed structured LLM response should decide the tier directly.
diff --git a/tests/test_litellm/router_strategy/test_complexity_tier_predictor.py b/tests/test_litellm/router_strategy/test_complexity_tier_predictor.py
new file mode 100644
index 00000000000..5bbe0fb5669
--- /dev/null
+++ b/tests/test_litellm/router_strategy/test_complexity_tier_predictor.py
@@ -0,0 +1,91 @@
+from typing import Final
+
+import pytest
+
+from litellm.router_strategy.complexity_router.tier_predictor import (
+ TierCohortStatistic,
+ TierDomainStatistic,
+ TierGlobalStatistic,
+ TierSuccessPredictor,
+ TrainedTierArtifact,
+ resolve_tier_artifact,
+ similarity_cohort,
+)
+from litellm.types.router import RequestType
+
+
+def _artifact(
+ global_successes: tuple[float, float, float, float] = (4.0, 5.0, 6.0, 7.0),
+ threshold: float = 0.75,
+ domain_statistics: tuple[TierDomainStatistic, ...] = (),
+ cohort_statistics: tuple[TierCohortStatistic, ...] = (),
+) -> TrainedTierArtifact:
+ return TrainedTierArtifact(
+ global_statistics=tuple(
+ TierGlobalStatistic(tier=tier, successes=successes, observations=10.0)
+ for tier, successes in enumerate(global_successes, start=1)
+ ),
+ domain_statistics=domain_statistics,
+ cohort_statistics=cohort_statistics,
+ domain_prior_mass=10.0,
+ cohort_prior_mass=10.0,
+ routing_threshold=threshold,
+ )
+
+
+def test_predictions_are_monotonic_across_tiers() -> None:
+ predictor: Final = TierSuccessPredictor(_artifact(global_successes=(9.0, 2.0, 7.0, 6.0)))
+
+ prediction: Final = predictor.predict("hello", RequestType.GENERAL)
+
+ probabilities: Final = tuple(prediction.probabilities.values())
+ assert probabilities == tuple(sorted(probabilities))
+
+
+def test_domain_and_cohort_statistics_back_off_hierarchically() -> None:
+ matching_cohort: Final = similarity_cohort("hello", RequestType.GENERAL)
+ artifact: Final = _artifact(
+ global_successes=(1.0, 5.0, 6.0, 7.0),
+ domain_statistics=(
+ TierDomainStatistic(
+ tier=1,
+ request_type=RequestType.GENERAL,
+ successes=10.0,
+ observations=10.0,
+ ),
+ ),
+ cohort_statistics=(
+ TierCohortStatistic(
+ tier=1,
+ cohort=matching_cohort,
+ successes=0.0,
+ observations=10.0,
+ ),
+ ),
+ )
+ predictor: Final = TierSuccessPredictor(artifact)
+
+ cohort_probability: Final = predictor.predict("hello", RequestType.GENERAL).probabilities[1]
+ domain_probability: Final = predictor.predict("hello " * 100, RequestType.GENERAL).probabilities[1]
+ global_probability: Final = predictor.predict("hello", RequestType.WRITING).probabilities[1]
+
+ assert cohort_probability == pytest.approx(7.0 / 24.0)
+ assert domain_probability == pytest.approx(7.0 / 12.0)
+ assert global_probability == pytest.approx(1.0 / 6.0)
+
+
+def test_selects_first_tier_above_probability_threshold() -> None:
+ predictor: Final = TierSuccessPredictor(_artifact(global_successes=(4.0, 6.0, 8.0, 9.0), threshold=0.7))
+
+ prediction: Final = predictor.predict("hello", RequestType.GENERAL)
+
+ assert prediction.required_tier == 3
+
+
+def test_builtin_ultrafeedback_artifact_is_loadable() -> None:
+ artifact: Final = resolve_tier_artifact("ultrafeedback")
+
+ assert artifact.routing_threshold == 0.75
+ assert artifact.domain_prior_mass == 200.0
+ assert artifact.cohort_prior_mass == 20.0
+ assert artifact.datasets[0].license == "MIT"
diff --git a/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx
index 96c93306611..c2ed4b1f55a 100644
--- a/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx
@@ -43,6 +43,10 @@ const DEFAULT_SCORING_EXPLANATION =
"The router scores each request across 7 dimensions: token count, code presence, reasoning markers, technical " +
"terms, simple indicators, multi-step patterns, and question complexity. The weighted score determines the tier:";
+const HEURISTIC_V2_EXPLANATION =
+ "The router estimates success probability for all four tiers with the bundled calibrated model, then selects " +
+ "the first tier that meets its trained threshold. It runs locally with no classifier API call.";
+
const CLASSIFIER_TIMEOUT_ID = "classifier-timeout-ms";
const CLASSIFIER_CONTEXT_WINDOW_SIZE_ID = "classifier-context-window-size";
const CLASSIFIER_CONTEXT_BUDGET_CHARS_ID = "classifier-context-budget-chars";
@@ -62,6 +66,7 @@ const CUSTOM_PROMPT_WITH_DEFAULT_MODEL_FALLBACK =
* at all, so the panel must not keep implying a score is involved on either router.
*/
const scoringExplanation = (value: ComplexityRouterConfigValue): string => {
+ if (value.classifier_type === "heuristic_v2") return HEURISTIC_V2_EXPLANATION;
const usesCustomPrompt =
usesLlmClassifier(value.classifier_type) && Boolean(value.classifier_llm_config?.system_prompt?.trim());
if (!usesCustomPrompt) return DEFAULT_SCORING_EXPLANATION;
@@ -179,6 +184,17 @@ const ClassifierTypeRadios: React.FC<{
+
+
+