From dfd991d381e8fab0cff9997916c5469f30114a7d Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Tue, 10 Feb 2026 14:24:22 -0800 Subject: [PATCH] test model cost map resilience --- .../test_model_cost_map_resilience.py | 166 ++++++++++++------ 1 file changed, 113 insertions(+), 53 deletions(-) diff --git a/tests/llm_translation/test_model_cost_map_resilience.py b/tests/llm_translation/test_model_cost_map_resilience.py index 93fe731a796..e918c89b6bd 100644 --- a/tests/llm_translation/test_model_cost_map_resilience.py +++ b/tests/llm_translation/test_model_cost_map_resilience.py @@ -27,22 +27,38 @@ from litellm.litellm_core_utils.get_model_cost_map import ( ) -class TestValidateModelCostMap: - """Unit tests for the validate_model_cost_map helper.""" +class TestCheckIsValidDict: + """Unit tests for _check_is_valid_dict.""" def test_should_reject_non_dict(self): - """Non-dict fetched map should fail validation.""" - assert GetModelCostMap.validate_model_cost_map(fetched_map="not a dict", backup_map={}) is False + """Non-dict should fail.""" + assert GetModelCostMap._check_is_valid_dict("not a dict") is False - def test_should_reject_empty_map(self): - """Empty fetched map should fail validation.""" - assert GetModelCostMap.validate_model_cost_map(fetched_map={}, backup_map={}) is False + def test_should_reject_empty_dict(self): + """Empty dict should fail.""" + assert GetModelCostMap._check_is_valid_dict({}) is False + + def test_should_reject_list(self): + """List should fail.""" + assert GetModelCostMap._check_is_valid_dict([1, 2, 3]) is False + + def test_should_reject_none(self): + """None should fail.""" + assert GetModelCostMap._check_is_valid_dict(None) is False + + def test_should_accept_non_empty_dict(self): + """Non-empty dict should pass.""" + assert GetModelCostMap._check_is_valid_dict({"model": {}}) is True + + +class TestCheckModelCountNotReduced: + """Unit tests for _check_model_count_not_reduced.""" def test_should_reject_too_few_models(self): """Fetched map with fewer models than min_model_count should fail.""" small_map = {f"model-{i}": {} for i in range(5)} assert ( - GetModelCostMap.validate_model_cost_map( + GetModelCostMap._check_model_count_not_reduced( fetched_map=small_map, backup_map={}, min_model_count=10 ) is False @@ -52,6 +68,61 @@ class TestValidateModelCostMap: """Fetched map that shrunk >50% vs backup should fail.""" backup = {f"model-{i}": {} for i in range(100)} fetched = {f"model-{i}": {} for i in range(40)} # 40% of backup + assert ( + GetModelCostMap._check_model_count_not_reduced( + fetched_map=fetched, backup_map=backup, min_model_count=10 + ) + is False + ) + + def test_should_accept_when_above_threshold(self): + """Fetched map at 60% of backup (above 50% threshold) should pass.""" + backup = {f"model-{i}": {} for i in range(100)} + fetched = {f"model-{i}": {} for i in range(60)} + assert ( + GetModelCostMap._check_model_count_not_reduced( + fetched_map=fetched, backup_map=backup, min_model_count=10 + ) + is True + ) + + def test_should_accept_growth(self): + """Fetched map larger than backup should pass.""" + backup = {f"model-{i}": {} for i in range(100)} + fetched = {f"model-{i}": {} for i in range(120)} + assert ( + GetModelCostMap._check_model_count_not_reduced( + fetched_map=fetched, backup_map=backup, min_model_count=10 + ) + is True + ) + + def test_should_accept_with_empty_backup(self): + """When backup is empty, only min_model_count matters.""" + fetched = {f"model-{i}": {} for i in range(15)} + assert ( + GetModelCostMap._check_model_count_not_reduced( + fetched_map=fetched, backup_map={}, min_model_count=10 + ) + is True + ) + + +class TestValidateModelCostMap: + """Unit tests for validate_model_cost_map (combines both checks).""" + + def test_should_reject_non_dict(self): + """Non-dict should fail at check 1.""" + assert GetModelCostMap.validate_model_cost_map(fetched_map="not a dict", backup_map={}) is False + + def test_should_reject_empty_map(self): + """Empty dict should fail at check 1.""" + assert GetModelCostMap.validate_model_cost_map(fetched_map={}, backup_map={}) is False + + def test_should_reject_significant_shrinkage(self): + """Should fail at check 2 (shrinkage).""" + backup = {f"model-{i}": {} for i in range(100)} + fetched = {f"model-{i}": {} for i in range(40)} assert ( GetModelCostMap.validate_model_cost_map( fetched_map=fetched, backup_map=backup, min_model_count=10 @@ -60,7 +131,7 @@ class TestValidateModelCostMap: ) def test_should_accept_valid_map(self): - """A fetched map with enough models that hasn't shrunk should pass.""" + """Should pass both checks.""" backup = {f"model-{i}": {} for i in range(100)} fetched = {f"model-{i}": {} for i in range(120)} assert ( @@ -71,7 +142,7 @@ class TestValidateModelCostMap: ) def test_should_accept_equal_size_map(self): - """A fetched map equal in size to backup should pass.""" + """Equal size should pass both checks.""" backup = {f"model-{i}": {} for i in range(100)} fetched = {f"model-{i}": {} for i in range(100)} assert ( @@ -145,66 +216,55 @@ class TestGetModelCostMapFallback: assert len(result) > 0 -class TestCompletionWithBadModelCostMap: +class TestBadHostedModelCostMap: """ - Simulates a bad model cost map and verifies litellm.completion() - still works (it catches cost map errors silently) and - litellm.get_model_info() raises the expected error. + Simulates the hosted model cost map being bad (invalid JSON / corrupted). + + When the hosted map is bad, get_model_cost_map() falls back to the local + backup. These tests verify that after fallback: + - get_model_info() still works for models in the backup + - litellm.completion() still works """ - def test_should_raise_model_not_mapped_with_empty_cost_map(self): + def test_should_model_info_pass_after_bad_hosted_map(self): """ - With an empty model cost map, get_model_info should raise - 'This model isn't mapped yet' for any model. + If the hosted map is bad, get_model_cost_map falls back to the local + backup. get_model_info should still work for models in the backup. """ - original = litellm.model_cost - litellm.model_cost = {} - try: - with pytest.raises(Exception, match="This model isn't mapped yet"): - litellm.get_model_info("azure/gpt-5.2") - finally: - litellm.model_cost = original + mock_response = MagicMock() + mock_response.raise_for_status = MagicMock() + mock_response.json.side_effect = json.JSONDecodeError("bad json", "", 0) + + with patch("httpx.get", return_value=mock_response): + fallback_map = get_model_cost_map("https://fake-url.com/bad.json") - def test_should_raise_model_not_mapped_for_missing_model(self): - """ - With a cost map that has some models but not the requested one, - get_model_info should raise 'This model isn't mapped yet'. - """ original = litellm.model_cost - litellm.model_cost = { - "gpt-4": { - "litellm_provider": "openai", - "max_tokens": 8192, - "input_cost_per_token": 0.00003, - "output_cost_per_token": 0.00006, - "mode": "chat", - } - } + litellm.model_cost = fallback_map try: - # gpt-4 should work - info = litellm.get_model_info("gpt-4") + # gpt-4o is in every backup — should work fine + info = litellm.get_model_info("gpt-4o") assert info is not None - - # azure/gpt-5.2 is not in the map — should fail - with pytest.raises(Exception, match="This model isn't mapped yet"): - litellm.get_model_info("azure/gpt-5.2") + assert info["input_cost_per_token"] > 0 finally: litellm.model_cost = original @pytest.mark.asyncio - async def test_should_complete_even_with_bad_cost_map(self): + async def test_should_completion_pass_after_bad_hosted_map(self): """ - litellm.completion() should NOT fail even when the model cost map - is empty. It catches get_model_info errors internally and proceeds - to the API call. The cost tracking will be wrong, but the LLM - request itself should succeed. + If the hosted map is bad, litellm.completion() should still work. + The fallback backup is used for cost tracking, and the LLM call + itself is never blocked by cost map issues. + """ + mock_response = MagicMock() + mock_response.raise_for_status = MagicMock() + mock_response.json.side_effect = json.JSONDecodeError("bad json", "", 0) + + with patch("httpx.get", return_value=mock_response): + fallback_map = get_model_cost_map("https://fake-url.com/bad.json") - This is the key behavior: completion() is resilient to cost map failures. - """ original = litellm.model_cost - litellm.model_cost = {} + litellm.model_cost = fallback_map try: - # This should still work — completion catches model_info errors response = litellm.completion( model="azure/gpt-4o-mini", messages=[{"role": "user", "content": "say hi"}],