mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(models): price the cheaperinference long-context tier at the catalog threshold
The catalog reports a 271999-token threshold for gpt-5.6-sol, gpt-5.6-terra and gpt-5.6-luna. The sync rounded it to the above_272k_tokens band, so a request of exactly 272000 tokens was charged the base rate while the gateway charges the long-context rate. litellm reads the threshold out of the field name, in the _above_128k_tokens as well as the _above_271999_tokens spelling, so the three entries now carry ..._above_271999_tokens. gpt-6-astra keeps above_272k_tokens: its threshold is 272000. The tolerance and the unmappable-threshold warning are gone, because every threshold is now expressible. Also in this commit: - model_prices_and_context_window.schema.json regenerated with ci_cd/generate_model_prices_schema.py - catalog fixture re-recorded (pricing_version sha256:6d2520b9..., checked 2026-09-17T10:00:38Z): four models now report a context length, and gpt-5.4-nano's output rate moved in the sixth digit 36 sync tests pass, including a new one that asserts litellm's own _parse_above_token_threshold reads back every generated field name.
This commit is contained in:
parent
7580857501
commit
c556ff5036
6 changed files with 796 additions and 2106 deletions
|
|
@ -2965,6 +2965,7 @@
|
|||
"cache_read_input_token_cost": 8.5e-09,
|
||||
"input_cost_per_token": 4e-08,
|
||||
"litellm_provider": "cheaperinference",
|
||||
"max_input_tokens": 131072,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2e-07,
|
||||
"source": "https://cheaperinference.com/docs",
|
||||
|
|
@ -3051,18 +3052,18 @@
|
|||
},
|
||||
"cheaperinference/gpt-5.6-luna": {
|
||||
"cache_creation_input_token_cost": 1e-07,
|
||||
"cache_creation_input_token_cost_above_272k_tokens": 2e-07,
|
||||
"cache_creation_input_token_cost_above_271999_tokens": 2e-07,
|
||||
"cache_read_input_token_cost": 8e-09,
|
||||
"cache_read_input_token_cost_above_272k_tokens": 1.6e-08,
|
||||
"cache_read_input_token_cost_above_271999_tokens": 1.6e-08,
|
||||
"input_cost_per_token": 8e-08,
|
||||
"input_cost_per_token_above_272k_tokens": 1.6e-07,
|
||||
"input_cost_per_token_above_271999_tokens": 1.6e-07,
|
||||
"litellm_provider": "cheaperinference",
|
||||
"max_input_tokens": 1050000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 4.8e-07,
|
||||
"output_cost_per_token_above_272k_tokens": 7.2e-07,
|
||||
"output_cost_per_token_above_271999_tokens": 7.2e-07,
|
||||
"source": "https://cheaperinference.com/docs",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
|
|
@ -3077,18 +3078,18 @@
|
|||
},
|
||||
"cheaperinference/gpt-5.6-sol": {
|
||||
"cache_creation_input_token_cost": 1.25e-06,
|
||||
"cache_creation_input_token_cost_above_272k_tokens": 2.5e-06,
|
||||
"cache_creation_input_token_cost_above_271999_tokens": 2.5e-06,
|
||||
"cache_read_input_token_cost": 1e-07,
|
||||
"cache_read_input_token_cost_above_272k_tokens": 2e-07,
|
||||
"cache_read_input_token_cost_above_271999_tokens": 2e-07,
|
||||
"input_cost_per_token": 1e-06,
|
||||
"input_cost_per_token_above_272k_tokens": 2e-06,
|
||||
"input_cost_per_token_above_271999_tokens": 2e-06,
|
||||
"litellm_provider": "cheaperinference",
|
||||
"max_input_tokens": 1050000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 5e-06,
|
||||
"output_cost_per_token_above_272k_tokens": 7.5e-06,
|
||||
"output_cost_per_token_above_271999_tokens": 7.5e-06,
|
||||
"source": "https://cheaperinference.com/docs",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
|
|
@ -3103,18 +3104,18 @@
|
|||
},
|
||||
"cheaperinference/gpt-5.6-terra": {
|
||||
"cache_creation_input_token_cost": 1e-06,
|
||||
"cache_creation_input_token_cost_above_272k_tokens": 2e-06,
|
||||
"cache_creation_input_token_cost_above_271999_tokens": 2e-06,
|
||||
"cache_read_input_token_cost": 8e-08,
|
||||
"cache_read_input_token_cost_above_272k_tokens": 1.6e-07,
|
||||
"cache_read_input_token_cost_above_271999_tokens": 1.6e-07,
|
||||
"input_cost_per_token": 8e-07,
|
||||
"input_cost_per_token_above_272k_tokens": 1.6e-06,
|
||||
"input_cost_per_token_above_271999_tokens": 1.6e-06,
|
||||
"litellm_provider": "cheaperinference",
|
||||
"max_input_tokens": 1050000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 4.8e-06,
|
||||
"output_cost_per_token_above_272k_tokens": 7.2e-06,
|
||||
"output_cost_per_token_above_271999_tokens": 7.2e-06,
|
||||
"source": "https://cheaperinference.com/docs",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
|
|
@ -3573,6 +3574,7 @@
|
|||
"cache_read_input_token_cost": 6.375e-08,
|
||||
"input_cost_per_token": 6.375e-07,
|
||||
"litellm_provider": "cheaperinference",
|
||||
"max_input_tokens": 1048576,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3.1875e-06,
|
||||
"source": "https://cheaperinference.com/docs",
|
||||
|
|
@ -3738,6 +3740,7 @@
|
|||
"cache_read_input_token_cost": 2.1e-08,
|
||||
"input_cost_per_token": 2.1e-07,
|
||||
"litellm_provider": "cheaperinference",
|
||||
"max_input_tokens": 1048576,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.75e-06,
|
||||
"source": "https://cheaperinference.com/docs",
|
||||
|
|
@ -3756,6 +3759,7 @@
|
|||
"cache_read_input_token_cost": 8.499e-09,
|
||||
"input_cost_per_token": 8.5e-08,
|
||||
"litellm_provider": "cheaperinference",
|
||||
"max_input_tokens": 1047576,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3.4e-07,
|
||||
"source": "https://cheaperinference.com/docs",
|
||||
|
|
@ -3841,7 +3845,7 @@
|
|||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.007109e-06,
|
||||
"output_cost_per_token": 1.00711e-06,
|
||||
"source": "https://cheaperinference.com/docs",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
|
|
|
|||
|
|
@ -2965,6 +2965,7 @@
|
|||
"cache_read_input_token_cost": 8.5e-09,
|
||||
"input_cost_per_token": 4e-08,
|
||||
"litellm_provider": "cheaperinference",
|
||||
"max_input_tokens": 131072,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2e-07,
|
||||
"source": "https://cheaperinference.com/docs",
|
||||
|
|
@ -3051,18 +3052,18 @@
|
|||
},
|
||||
"cheaperinference/gpt-5.6-luna": {
|
||||
"cache_creation_input_token_cost": 1e-07,
|
||||
"cache_creation_input_token_cost_above_272k_tokens": 2e-07,
|
||||
"cache_creation_input_token_cost_above_271999_tokens": 2e-07,
|
||||
"cache_read_input_token_cost": 8e-09,
|
||||
"cache_read_input_token_cost_above_272k_tokens": 1.6e-08,
|
||||
"cache_read_input_token_cost_above_271999_tokens": 1.6e-08,
|
||||
"input_cost_per_token": 8e-08,
|
||||
"input_cost_per_token_above_272k_tokens": 1.6e-07,
|
||||
"input_cost_per_token_above_271999_tokens": 1.6e-07,
|
||||
"litellm_provider": "cheaperinference",
|
||||
"max_input_tokens": 1050000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 4.8e-07,
|
||||
"output_cost_per_token_above_272k_tokens": 7.2e-07,
|
||||
"output_cost_per_token_above_271999_tokens": 7.2e-07,
|
||||
"source": "https://cheaperinference.com/docs",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
|
|
@ -3077,18 +3078,18 @@
|
|||
},
|
||||
"cheaperinference/gpt-5.6-sol": {
|
||||
"cache_creation_input_token_cost": 1.25e-06,
|
||||
"cache_creation_input_token_cost_above_272k_tokens": 2.5e-06,
|
||||
"cache_creation_input_token_cost_above_271999_tokens": 2.5e-06,
|
||||
"cache_read_input_token_cost": 1e-07,
|
||||
"cache_read_input_token_cost_above_272k_tokens": 2e-07,
|
||||
"cache_read_input_token_cost_above_271999_tokens": 2e-07,
|
||||
"input_cost_per_token": 1e-06,
|
||||
"input_cost_per_token_above_272k_tokens": 2e-06,
|
||||
"input_cost_per_token_above_271999_tokens": 2e-06,
|
||||
"litellm_provider": "cheaperinference",
|
||||
"max_input_tokens": 1050000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 5e-06,
|
||||
"output_cost_per_token_above_272k_tokens": 7.5e-06,
|
||||
"output_cost_per_token_above_271999_tokens": 7.5e-06,
|
||||
"source": "https://cheaperinference.com/docs",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
|
|
@ -3103,18 +3104,18 @@
|
|||
},
|
||||
"cheaperinference/gpt-5.6-terra": {
|
||||
"cache_creation_input_token_cost": 1e-06,
|
||||
"cache_creation_input_token_cost_above_272k_tokens": 2e-06,
|
||||
"cache_creation_input_token_cost_above_271999_tokens": 2e-06,
|
||||
"cache_read_input_token_cost": 8e-08,
|
||||
"cache_read_input_token_cost_above_272k_tokens": 1.6e-07,
|
||||
"cache_read_input_token_cost_above_271999_tokens": 1.6e-07,
|
||||
"input_cost_per_token": 8e-07,
|
||||
"input_cost_per_token_above_272k_tokens": 1.6e-06,
|
||||
"input_cost_per_token_above_271999_tokens": 1.6e-06,
|
||||
"litellm_provider": "cheaperinference",
|
||||
"max_input_tokens": 1050000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 4.8e-06,
|
||||
"output_cost_per_token_above_272k_tokens": 7.2e-06,
|
||||
"output_cost_per_token_above_271999_tokens": 7.2e-06,
|
||||
"source": "https://cheaperinference.com/docs",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
|
|
@ -3573,6 +3574,7 @@
|
|||
"cache_read_input_token_cost": 6.375e-08,
|
||||
"input_cost_per_token": 6.375e-07,
|
||||
"litellm_provider": "cheaperinference",
|
||||
"max_input_tokens": 1048576,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3.1875e-06,
|
||||
"source": "https://cheaperinference.com/docs",
|
||||
|
|
@ -3738,6 +3740,7 @@
|
|||
"cache_read_input_token_cost": 2.1e-08,
|
||||
"input_cost_per_token": 2.1e-07,
|
||||
"litellm_provider": "cheaperinference",
|
||||
"max_input_tokens": 1048576,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.75e-06,
|
||||
"source": "https://cheaperinference.com/docs",
|
||||
|
|
@ -3756,6 +3759,7 @@
|
|||
"cache_read_input_token_cost": 8.499e-09,
|
||||
"input_cost_per_token": 8.5e-08,
|
||||
"litellm_provider": "cheaperinference",
|
||||
"max_input_tokens": 1047576,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3.4e-07,
|
||||
"source": "https://cheaperinference.com/docs",
|
||||
|
|
@ -3841,7 +3845,7 @@
|
|||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.007109e-06,
|
||||
"output_cost_per_token": 1.00711e-06,
|
||||
"source": "https://cheaperinference.com/docs",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
|
|
|
|||
|
|
@ -104,6 +104,11 @@
|
|||
"minimum": 0,
|
||||
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
|
||||
},
|
||||
"cache_creation_input_token_cost_above_271999_tokens": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
|
||||
},
|
||||
"cache_creation_input_token_cost_above_272k_tokens": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
|
|
@ -158,6 +163,11 @@
|
|||
"minimum": 0,
|
||||
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
|
||||
},
|
||||
"cache_read_input_token_cost_above_271999_tokens": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
|
||||
},
|
||||
"cache_read_input_token_cost_above_272k_tokens": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
|
|
@ -317,6 +327,11 @@
|
|||
"minimum": 0,
|
||||
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
|
||||
},
|
||||
"input_cost_per_token_above_271999_tokens": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
|
||||
},
|
||||
"input_cost_per_token_above_272k_tokens": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
|
|
@ -507,6 +522,11 @@
|
|||
"minimum": 0,
|
||||
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
|
||||
},
|
||||
"output_cost_per_token_above_271999_tokens": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
|
||||
},
|
||||
"output_cost_per_token_above_272k_tokens": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
|
|
|
|||
|
|
@ -7,9 +7,8 @@ generated PR body; ``--write`` applies the changes to the root cost map and its
|
|||
Policy highlights:
|
||||
- Only price fields (input/output/cache read/cache write, and their long-context variants) and the two
|
||||
limit fields (max_input_tokens, max_output_tokens/max_tokens) are synced from the catalog.
|
||||
- The catalog's own long-context threshold picks the cost-map band. A threshold that matches no band
|
||||
litellm understands leaves that entry untouched and is surfaced as a warning, so a moved threshold can
|
||||
never be written out under the wrong band.
|
||||
- The catalog's own long-context threshold names the cost-map band, at its exact value, so a moved
|
||||
threshold is never rounded to a neighbouring band.
|
||||
- Capability flags (supports_vision, supports_reasoning, supports_tool_choice, and so on), litellm_provider,
|
||||
mode, source, and supported_endpoints are curated by hand and are never touched by this script.
|
||||
- A catalog model with no matching registry entry is surfaced as a warning for a human to add, never
|
||||
|
|
@ -82,32 +81,33 @@ BASE_PRICE_FIELDS: Final = (
|
|||
"cache_read_input_token_cost",
|
||||
"cache_creation_input_token_cost",
|
||||
)
|
||||
TIERED_BANDS: Final = {
|
||||
# litellm takes the threshold from the field name, in either the `_above_128k_tokens`
|
||||
# or the `_above_271999_tokens` spelling (litellm/litellm_core_utils/llm_cost_calc/utils.py).
|
||||
# The k-form is reused for the thresholds that already have one in the cost map, so a
|
||||
# catalog threshold of any size is written out at its exact value.
|
||||
CANONICAL_BANDS: Final = {
|
||||
128_000: "above_128k_tokens",
|
||||
200_000: "above_200k_tokens",
|
||||
256_000: "above_256k_tokens",
|
||||
272_000: "above_272k_tokens",
|
||||
512_000: "above_512k_tokens",
|
||||
}
|
||||
BAND_TOLERANCE_TOKENS: Final = 1
|
||||
TIERED_PRICE_FIELDS: Final = tuple(
|
||||
f"{field}_{band}" for band in TIERED_BANDS.values() for field in BASE_PRICE_FIELDS
|
||||
)
|
||||
SYNCED_PRICE_FIELDS: Final = BASE_PRICE_FIELDS + TIERED_PRICE_FIELDS
|
||||
TIERED_FIELD_PREFIXES: Final = tuple(f"{field}_above_" for field in BASE_PRICE_FIELDS)
|
||||
|
||||
|
||||
def per_token(price_per_million: str) -> float:
|
||||
return float(f"{float(price_per_million) / 1e6:.6g}")
|
||||
|
||||
|
||||
def band_for_threshold(threshold: int) -> str | None:
|
||||
for supported, band in TIERED_BANDS.items():
|
||||
if abs(threshold - supported) <= BAND_TOLERANCE_TOKENS:
|
||||
return band
|
||||
return None
|
||||
def band_for_threshold(threshold: int) -> str:
|
||||
return CANONICAL_BANDS.get(threshold, f"above_{threshold}_tokens")
|
||||
|
||||
|
||||
def _price_fields(pricing: CatalogPricing) -> tuple[RegistryEntry, int | None]:
|
||||
def is_tiered_price_field(name: str) -> bool:
|
||||
return name.endswith("_tokens") and name.startswith(TIERED_FIELD_PREFIXES)
|
||||
|
||||
|
||||
def _price_fields(pricing: CatalogPricing) -> RegistryEntry:
|
||||
base: Final[RegistryEntry] = {
|
||||
"input_cost_per_token": per_token(pricing.input_per_million),
|
||||
"output_cost_per_token": per_token(pricing.output_per_million),
|
||||
|
|
@ -115,18 +115,16 @@ def _price_fields(pricing: CatalogPricing) -> tuple[RegistryEntry, int | None]:
|
|||
"cache_creation_input_token_cost": per_token(pricing.cache_write_input_per_million),
|
||||
}
|
||||
if pricing.above_threshold is None:
|
||||
return base, None
|
||||
return base
|
||||
above: Final = pricing.above_threshold
|
||||
band: Final = band_for_threshold(above.input_token_price_threshold)
|
||||
if band is None:
|
||||
return base, above.input_token_price_threshold
|
||||
return {
|
||||
**base,
|
||||
f"input_cost_per_token_{band}": per_token(above.input_per_million),
|
||||
f"output_cost_per_token_{band}": per_token(above.output_per_million),
|
||||
f"cache_read_input_token_cost_{band}": per_token(above.cache_read_input_per_million),
|
||||
f"cache_creation_input_token_cost_{band}": per_token(above.cache_write_input_per_million),
|
||||
}, None
|
||||
}
|
||||
|
||||
|
||||
def _limit_fields(model: CatalogModel) -> RegistryEntry:
|
||||
|
|
@ -139,9 +137,8 @@ def _limit_fields(model: CatalogModel) -> RegistryEntry:
|
|||
return fields
|
||||
|
||||
|
||||
def _synced_fields(model: CatalogModel) -> tuple[RegistryEntry, int | None]:
|
||||
prices, unmappable_threshold = _price_fields(model.pricing)
|
||||
return {**prices, **_limit_fields(model)}, unmappable_threshold
|
||||
def _synced_fields(model: CatalogModel) -> RegistryEntry:
|
||||
return {**_price_fields(model.pricing), **_limit_fields(model)}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -150,7 +147,6 @@ class SyncOutcome:
|
|||
updated: tuple[str, ...] = ()
|
||||
unpriced_new_models: tuple[str, ...] = ()
|
||||
missing_from_catalog: tuple[str, ...] = ()
|
||||
unmappable_thresholds: tuple[str, ...] = ()
|
||||
|
||||
@property
|
||||
def has_changes(self) -> bool:
|
||||
|
|
@ -161,7 +157,6 @@ class SyncOutcome:
|
|||
return (
|
||||
tuple(f"new in the catalog, not registered: {key}" for key in self.unpriced_new_models)
|
||||
+ tuple(f"registered but gone from the catalog: {key}" for key in self.missing_from_catalog)
|
||||
+ tuple(f"long-context threshold matches no cost-map band, entry left untouched: {key}" for key in self.unmappable_thresholds)
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -171,7 +166,6 @@ def compute_sync(cost_map: CostMap, catalog: Sequence[CatalogModel]) -> SyncOutc
|
|||
|
||||
updated: Final[list[str]] = []
|
||||
missing: Final[list[str]] = []
|
||||
unmappable: Final[list[str]] = []
|
||||
result: Final[CostMap] = dict(cost_map)
|
||||
|
||||
for model_id, key in sorted(registry_ids.items()):
|
||||
|
|
@ -182,11 +176,8 @@ def compute_sync(cost_map: CostMap, catalog: Sequence[CatalogModel]) -> SyncOutc
|
|||
entry = result.get(key)
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
desired, unmappable_threshold = _synced_fields(model)
|
||||
if unmappable_threshold is not None:
|
||||
unmappable.append(f"{key} (threshold {unmappable_threshold})")
|
||||
continue
|
||||
stale_tiered: Final = tuple(f for f in TIERED_PRICE_FIELDS if f not in desired)
|
||||
desired: Final = _synced_fields(model)
|
||||
stale_tiered: Final = tuple(f for f in entry if is_tiered_price_field(f) and f not in desired)
|
||||
changes: Final = tuple(
|
||||
f"{name}: {entry.get(name)!r} -> {value!r}" for name, value in desired.items() if entry.get(name) != value
|
||||
) + tuple(f"{name}: {entry[name]!r} removed (not priced in that band any more)" for name in stale_tiered if name in entry)
|
||||
|
|
@ -205,7 +196,6 @@ def compute_sync(cost_map: CostMap, catalog: Sequence[CatalogModel]) -> SyncOutc
|
|||
updated=tuple(updated),
|
||||
unpriced_new_models=unpriced_new,
|
||||
missing_from_catalog=tuple(sorted(missing)),
|
||||
unmappable_thresholds=tuple(sorted(unmappable)),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -224,15 +214,13 @@ def render_pr_body(outcome: SyncOutcome) -> str:
|
|||
f"{_section_block('New in the catalog, not yet registered (needs a human review for capabilities)', outcome.unpriced_new_models)}"
|
||||
"\n"
|
||||
f"{_section_block('Registered but missing from the catalog', outcome.missing_from_catalog)}"
|
||||
"\n"
|
||||
f"{_section_block('Long-context threshold with no matching cost-map band (left untouched)', outcome.unmappable_thresholds)}"
|
||||
)
|
||||
|
||||
|
||||
def render_summary(outcome: SyncOutcome) -> str:
|
||||
return (
|
||||
f"updated={len(outcome.updated)} new_unregistered={len(outcome.unpriced_new_models)} "
|
||||
f"missing={len(outcome.missing_from_catalog)} unmappable_thresholds={len(outcome.unmappable_thresholds)}"
|
||||
f"missing={len(outcome.missing_from_catalog)}"
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -1,9 +1,18 @@
|
|||
"""Tests for scripts/sync_cheaperinference_models.py.
|
||||
|
||||
``fixtures/cheaperinference_sync/models.json`` is a recorded ``GET /v1/models``
|
||||
response, trimmed to the fields the sync reads, with the gateway's own
|
||||
``pricing_version`` and ``pricing_checked_at`` kept for provenance.
|
||||
"""
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import _parse_above_token_threshold
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
SCRIPT = ROOT / "scripts" / "sync_cheaperinference_models.py"
|
||||
FIXTURES = Path(__file__).resolve().parent / "fixtures" / "cheaperinference_sync"
|
||||
|
|
@ -88,39 +97,44 @@ def test_a_new_catalog_model_is_flagged_but_never_auto_added() -> None:
|
|||
assert "cheaperinference/seedance-2.0" not in outcome.cost_map
|
||||
|
||||
|
||||
def test_a_model_losing_its_pricing_tier_drops_the_above_272k_fields() -> None:
|
||||
def test_a_model_losing_its_pricing_tier_drops_its_above_threshold_fields() -> None:
|
||||
with_stale_tier = json.loads(json.dumps(REGISTERED_CHEAPERINFERENCE))
|
||||
with_stale_tier["cheaperinference/kimi-k3"]["input_cost_per_token_above_272k_tokens"] = 5.0
|
||||
outcome = sync.compute_sync(with_stale_tier, RECORDED_CATALOG)
|
||||
assert "input_cost_per_token_above_272k_tokens" not in outcome.cost_map["cheaperinference/kimi-k3"]
|
||||
|
||||
|
||||
def test_null_context_length_and_max_output_tokens_do_not_overwrite_curated_limits() -> None:
|
||||
"""gpt-oss-120b reports null limits in the live catalog, so the curated ones must survive."""
|
||||
def test_a_null_limit_in_the_catalog_never_overwrites_the_registry() -> None:
|
||||
"""gpt-oss-120b reports a null max_output_tokens, so no output limit may appear."""
|
||||
registered = json.loads(json.dumps(REGISTERED_CHEAPERINFERENCE))
|
||||
assert "cheaperinference/gpt-oss-120b" in registered
|
||||
curated_max_input = registered["cheaperinference/gpt-oss-120b"].get("max_input_tokens")
|
||||
assert "max_output_tokens" not in registered["cheaperinference/gpt-oss-120b"]
|
||||
|
||||
outcome = sync.compute_sync(registered, RECORDED_CATALOG)
|
||||
assert outcome.cost_map["cheaperinference/gpt-oss-120b"].get("max_input_tokens") == curated_max_input
|
||||
|
||||
assert "max_output_tokens" not in outcome.cost_map["cheaperinference/gpt-oss-120b"]
|
||||
assert "max_tokens" not in outcome.cost_map["cheaperinference/gpt-oss-120b"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"threshold,band",
|
||||
[
|
||||
(272_000, "above_272k_tokens"),
|
||||
(271_999, "above_272k_tokens"),
|
||||
(200_000, "above_200k_tokens"),
|
||||
(128_000, "above_128k_tokens"),
|
||||
(512_000, "above_512k_tokens"),
|
||||
(271_999, "above_271999_tokens"),
|
||||
(300_000, "above_300000_tokens"),
|
||||
],
|
||||
)
|
||||
def test_catalog_thresholds_map_to_the_matching_cost_map_band(threshold: int, band: str) -> None:
|
||||
def test_a_catalog_threshold_names_its_own_band(threshold: int, band: str) -> None:
|
||||
assert sync.band_for_threshold(threshold) == band
|
||||
|
||||
|
||||
@pytest.mark.parametrize("threshold", [0, 100_000, 300_000, 999_999])
|
||||
def test_a_threshold_outside_every_band_maps_to_nothing(threshold: int) -> None:
|
||||
assert sync.band_for_threshold(threshold) is None
|
||||
@pytest.mark.parametrize("threshold", [100_000, 128_000, 200_000, 271_999, 272_000, 300_000, 512_000, 999_999])
|
||||
def test_litellm_reads_back_the_exact_threshold_from_the_generated_field(threshold: int) -> None:
|
||||
field = f"input_cost_per_token_{sync.band_for_threshold(threshold)}"
|
||||
|
||||
assert _parse_above_token_threshold(field) == threshold
|
||||
|
||||
|
||||
def _catalog_with_threshold(model_id: str, threshold: int) -> list:
|
||||
|
|
@ -134,16 +148,14 @@ def _catalog_with_threshold(model_id: str, threshold: int) -> list:
|
|||
return sync.load_catalog(json.dumps(entries).encode())
|
||||
|
||||
|
||||
def test_a_threshold_with_no_band_leaves_the_entry_untouched_and_is_flagged() -> None:
|
||||
def test_a_threshold_outside_the_k_form_bands_is_written_at_its_exact_value() -> None:
|
||||
catalog = _catalog_with_threshold("gpt-5.6-luna", 300_000)
|
||||
perturbed = json.loads(json.dumps(REGISTERED_CHEAPERINFERENCE))
|
||||
perturbed["cheaperinference/gpt-5.6-luna"]["input_cost_per_token"] = 999.0
|
||||
|
||||
outcome = sync.compute_sync(perturbed, catalog)
|
||||
outcome = sync.compute_sync(REGISTERED_CHEAPERINFERENCE, catalog)
|
||||
|
||||
assert outcome.unmappable_thresholds == ("cheaperinference/gpt-5.6-luna (threshold 300000)",)
|
||||
assert outcome.updated == ()
|
||||
assert outcome.cost_map["cheaperinference/gpt-5.6-luna"]["input_cost_per_token"] == 999.0
|
||||
entry = outcome.cost_map["cheaperinference/gpt-5.6-luna"]
|
||||
assert entry["input_cost_per_token_above_300000_tokens"] == pytest.approx(0.16e-06)
|
||||
assert "input_cost_per_token_above_271999_tokens" not in entry
|
||||
|
||||
|
||||
def test_a_moved_threshold_rewrites_the_band_and_drops_the_old_one() -> None:
|
||||
|
|
@ -153,8 +165,7 @@ def test_a_moved_threshold_rewrites_the_band_and_drops_the_old_one() -> None:
|
|||
|
||||
entry = outcome.cost_map["cheaperinference/gpt-5.6-luna"]
|
||||
assert entry["input_cost_per_token_above_200k_tokens"] == pytest.approx(0.16e-06)
|
||||
assert "input_cost_per_token_above_272k_tokens" not in entry
|
||||
assert outcome.unmappable_thresholds == ()
|
||||
assert "input_cost_per_token_above_271999_tokens" not in entry
|
||||
|
||||
|
||||
def test_warnings_name_every_kind_of_catalog_drift() -> None:
|
||||
|
|
@ -162,13 +173,11 @@ def test_warnings_name_every_kind_of_catalog_drift() -> None:
|
|||
cost_map={},
|
||||
unpriced_new_models=("cheaperinference/brand-new",),
|
||||
missing_from_catalog=("cheaperinference/retired",),
|
||||
unmappable_thresholds=("cheaperinference/odd (threshold 300000)",),
|
||||
)
|
||||
|
||||
assert outcome.warnings == (
|
||||
"new in the catalog, not registered: cheaperinference/brand-new",
|
||||
"registered but gone from the catalog: cheaperinference/retired",
|
||||
"long-context threshold matches no cost-map band, entry left untouched: cheaperinference/odd (threshold 300000)",
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue