diff --git a/.github/scripts/auto_update_price_and_context_window_file.py b/.github/scripts/auto_update_price_and_context_window_file.py index 5ff93ef4294..f9a36f0270d 100644 --- a/.github/scripts/auto_update_price_and_context_window_file.py +++ b/.github/scripts/auto_update_price_and_context_window_file.py @@ -97,7 +97,7 @@ def _modality_flags(input_mods: list) -> dict: } -def transform_friendli_data(data: list, local_data: dict) -> dict: +def transform_friendli_data(data: list, local_data: dict) -> dict[str, dict[str, object]]: transformed: dict[str, dict] = {} if not data: return transformed @@ -165,25 +165,26 @@ def transform_friendli_data(data: list, local_data: dict) -> dict: return transformed # Synchronize local data with remote data -def _stale_friendli_keys(local_data: dict[str, Any], remote_data: dict[str, Any]) -> set[str]: - """friendliai/* keys absent from a non-empty catalog must not survive the sync: - Friendli delists deprecated serverless models, so registry entries the catalog - no longer returns are dead — proxies keep billing stale prices against a model - that fails at the provider. Restricted to the friendliai prefix and gated on a - non-empty remote so a failed or empty fetch can never wipe entries.""" - if not remote_data: - return set() - return { - key - for key in local_data - if key.startswith(f"{FRIENDLI_PROVIDER}/") and key not in remote_data - } +def _stale_friendli_keys( + local_keys: frozenset[str], + friendli_keys_in_remote: frozenset[str], +) -> frozenset[str]: + """friendliai/* keys absent from the transformed Friendli catalog are dead: + Friendli delists deprecated serverless models, so a registry entry the + catalog no longer returns bills stale prices against a model that fails + at the provider. Legacy pre-sync rows (no `source`) are included since + scoping is by key prefix, not by provenance""" + prefix = f"{FRIENDLI_PROVIDER}/" + return frozenset( + key for key in local_keys if key.startswith(prefix) and key not in friendli_keys_in_remote + ) def sync_local_data_with_remote( - local_data: dict[str, Any], - remote_data: dict[str, Any], + local_data: dict[str, dict[str, object]], + remote_data: dict[str, dict[str, object]], replace_keys: frozenset[str] = frozenset(), + catalog_dropped_keys: frozenset[str] = frozenset(), ) -> None: # Update existing keys in local_data with values from remote_data # (replace_keys entries are swapped wholesale so a field the remote catalog @@ -198,11 +199,8 @@ def sync_local_data_with_remote( for key in (set(remote_data) - set(local_data)): local_data[key] = remote_data[key] - # Drop friendliai keys the live catalog no longer returns (see docstring); - # other providers keep their entries — OpenRouter/Vercel list rows can lag - # or delist while users still route to them - for key in _stale_friendli_keys(local_data, remote_data): - del local_data[key] + for key in catalog_dropped_keys: + local_data.pop(key, None) # Write data to the json file def write_to_file(file_path, data): @@ -327,14 +325,33 @@ def main(): vercel_data = transform_vercel_ai_gateway_data(vercel_data) friendli_data = asyncio.run(fetch_data(FRIENDLI_API_URL)) + friendli_fetch_failed = friendli_data is None friendli_data = transform_friendli_data(friendli_data, local_data) - + # Combine both datasets all_remote_data = {**openrouter_data, **vercel_data, **friendli_data} + # Deletion only runs when the Friendli catalog itself was fetched: if the + # catalog is unreachable, absence proves nothing and deleting entries would + # wipe live models. OpenRouter/Vercel rows never qualify for deletion; their + # gateways can delist rows users still route to + catalog_dropped_keys: frozenset[str] = ( + frozenset() + if friendli_fetch_failed + else _stale_friendli_keys( + local_keys=frozenset(local_data or {}), + friendli_keys_in_remote=frozenset(friendli_data), + ) + ) + # If both local and openrouter data are available, synchronize and save if local_data and all_remote_data: - sync_local_data_with_remote(local_data, all_remote_data, replace_keys=frozenset(friendli_data)) + sync_local_data_with_remote( + local_data, + all_remote_data, + replace_keys=frozenset(friendli_data), + catalog_dropped_keys=catalog_dropped_keys, + ) write_to_file(local_file_path, local_data) else: print("Failed to fetch model data from either local file or URL.") diff --git a/tests/test_litellm/test_auto_update_price_and_context_window_file.py b/tests/test_litellm/test_auto_update_price_and_context_window_file.py index 17e917cb283..896002139e0 100644 --- a/tests/test_litellm/test_auto_update_price_and_context_window_file.py +++ b/tests/test_litellm/test_auto_update_price_and_context_window_file.py @@ -229,13 +229,24 @@ def test_sync_drops_friendli_entries_absent_from_catalog(sync_module): "openrouter/still-listed": {"input_cost_per_token": 1e-06}, } remote = sync_module.transform_friendli_data([_reasoning_model()], local) - sync_module.sync_local_data_with_remote(local, remote, replace_keys=frozenset(remote)) + dropped = sync_module._stale_friendli_keys( + local_keys=frozenset(local), friendli_keys_in_remote=frozenset(remote) + ) + sync_module.sync_local_data_with_remote( + local, remote, replace_keys=frozenset(remote), catalog_dropped_keys=dropped + ) assert "friendliai/dead-model" not in local assert "friendliai/zai-org/GLM-Test" in local - # non-friendli entries are never deleted by absence assert "openrouter/still-listed" in local +def test_stale_keys_scoped_to_friendli_prefix(sync_module): + local_keys = frozenset({"friendliai/gone", "openrouter/also-absent", "zhipuai/glm-test"}) + remote_keys = frozenset({"friendliai/zai-org/GLM-Test"}) + dropped = sync_module._stale_friendli_keys(local_keys, remote_keys) + assert dropped == frozenset({"friendliai/gone"}) + + def test_sync_keeps_all_entries_when_friendli_fetch_fails_or_is_empty(sync_module): local = { "friendliai/zai-org/GLM-Test": {"litellm_provider": "friendliai"}, @@ -249,11 +260,30 @@ def test_sync_keeps_all_entries_when_friendli_fetch_fails_or_is_empty(sync_modul assert "openrouter/kept" in local +def test_sync_deletes_nothing_when_friendli_fetch_fails_but_other_catalogs_succeed(sync_module): + # merged remote is non-empty (openrouter rows), friendli catalog unreachable: + # absence from the friendli catalog proves nothing, so its entries must survive + local = { + "friendliai/zai-org/GLM-Test": {"litellm_provider": "friendliai"}, + "openrouter/live-model": {"input_cost_per_token": 1e-06}, + } + remote = {"openrouter/live-model": {"input_cost_per_token": 2e-06}} + sync_module.sync_local_data_with_remote( + local, remote, replace_keys=frozenset(), catalog_dropped_keys=frozenset() + ) + assert "friendliai/zai-org/GLM-Test" in local + assert "openrouter/live-model" in local + + def test_sync_drops_legacy_friendli_entries_without_source_field(sync_module): - # pre-sync rows (no `source`) must be cleaned up too, not just synced ones local = { "friendliai/legacy-model": {"litellm_provider": "friendliai", "max_tokens": 8192}, } remote = sync_module.transform_friendli_data([_reasoning_model()], local) - sync_module.sync_local_data_with_remote(local, remote, replace_keys=frozenset(remote)) + dropped = sync_module._stale_friendli_keys( + local_keys=frozenset(local), friendli_keys_in_remote=frozenset(remote) + ) + sync_module.sync_local_data_with_remote( + local, remote, replace_keys=frozenset(remote), catalog_dropped_keys=dropped + ) assert "friendliai/legacy-model" not in local