From 73568e0173027f73e80fa33a09d0716a641a9aaf Mon Sep 17 00:00:00 2001 From: pfolex Date: Fri, 23 Oct 2026 21:16:37 -0400 Subject: [PATCH] feat(models): add Groq Llama 3.3 70B Versatile, Llama 3.1 8B Instant, and Llama 3.3 70B Specdec to cost map --- ...odel_prices_and_context_window_backup.json | 42 +++++++ model_prices_and_context_window.json | 42 +++++++ tests/unit/test_groq_llama_model_metadata.py | 103 ++++++++++++++++++ 3 files changed, 187 insertions(+) create mode 100644 tests/unit/test_groq_llama_model_metadata.py diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 840576e8225..691712a02b1 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -39227,6 +39227,48 @@ "supports_response_schema": true, "supports_vision": true }, + "groq/llama-3.1-8b-instant": { + "input_cost_per_token": 5e-08, + "litellm_provider": "groq", + "max_input_tokens": 131072, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 8e-08, + "source": "https://groq.com/pricing", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, + "groq/llama-3.3-70b-specdec": { + "input_cost_per_token": 5.9e-07, + "litellm_provider": "groq", + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 9.9e-07, + "source": "https://groq.com/pricing", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, + "groq/llama-3.3-70b-versatile": { + "input_cost_per_token": 5.9e-07, + "litellm_provider": "groq", + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 7.9e-07, + "source": "https://groq.com/pricing", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, "groq/llama-guard-3-8b": { "deprecation_date": "2025-06-06", "input_cost_per_token": 2e-07, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 840576e8225..691712a02b1 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -39227,6 +39227,48 @@ "supports_response_schema": true, "supports_vision": true }, + "groq/llama-3.1-8b-instant": { + "input_cost_per_token": 5e-08, + "litellm_provider": "groq", + "max_input_tokens": 131072, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 8e-08, + "source": "https://groq.com/pricing", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, + "groq/llama-3.3-70b-specdec": { + "input_cost_per_token": 5.9e-07, + "litellm_provider": "groq", + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 9.9e-07, + "source": "https://groq.com/pricing", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, + "groq/llama-3.3-70b-versatile": { + "input_cost_per_token": 5.9e-07, + "litellm_provider": "groq", + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 7.9e-07, + "source": "https://groq.com/pricing", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, "groq/llama-guard-3-8b": { "deprecation_date": "2025-06-06", "input_cost_per_token": 2e-07, diff --git a/tests/unit/test_groq_llama_model_metadata.py b/tests/unit/test_groq_llama_model_metadata.py new file mode 100644 index 00000000000..887612e4393 --- /dev/null +++ b/tests/unit/test_groq_llama_model_metadata.py @@ -0,0 +1,103 @@ +"""Tests for Groq Llama 3 model metadata in model_prices_and_context_window.json.""" + +import json +from pathlib import Path + +import pytest + +import litellm + +REPO_ROOT = Path(__file__).parents[2] +MAIN_COST_MAP = REPO_ROOT / "model_prices_and_context_window.json" +BACKUP_COST_MAP = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json" + +GROQ_LLAMA_MODELS = [ + "groq/llama-3.1-8b-instant", + "groq/llama-3.3-70b-specdec", + "groq/llama-3.3-70b-versatile", +] + + +@pytest.fixture(autouse=True) +def local_model_cost_map(monkeypatch): + """Ensure tests run against local repository cost map.""" + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + litellm.get_model_info.cache_clear() + yield + litellm.get_model_info.cache_clear() + + +@pytest.mark.parametrize("model", GROQ_LLAMA_MODELS) +def test_groq_llama_metadata_in_main_and_backup(model: str): + """Verify each model exists in both main and backup JSON files and matches.""" + main_data = json.loads(MAIN_COST_MAP.read_text()) + backup_data = json.loads(BACKUP_COST_MAP.read_text()) + + assert model in main_data, f"{model} missing from {MAIN_COST_MAP.name}" + assert model in backup_data, f"{model} missing from {BACKUP_COST_MAP.name}" + assert main_data[model] == backup_data[model] + + info = litellm.get_model_info(model) + assert info["litellm_provider"] == "groq" + assert info["mode"] == "chat" + assert info["supports_function_calling"] is True + assert info["supports_parallel_function_calling"] is True + assert info["supports_response_schema"] is True + assert info["supports_tool_choice"] is True + + +def test_groq_llama_3_1_8b_instant(): + """Verify specific rates and context window for Llama 3.1 8B Instant on Groq.""" + info = litellm.get_model_info("groq/llama-3.1-8b-instant") + assert info["max_input_tokens"] == 131072 + assert info["max_output_tokens"] == 8192 + assert info["max_tokens"] == 8192 + assert info["input_cost_per_token"] == 5e-08 + assert info["output_cost_per_token"] == 8e-08 + + # Calculate cost for 1,000,000 prompt + 1,000,000 completion tokens + prompt_cost, completion_cost = litellm.cost_per_token( + model="groq/llama-3.1-8b-instant", + prompt_tokens=1_000_000, + completion_tokens=1_000_000, + ) + assert abs(prompt_cost - 0.05) < 1e-6 + assert abs(completion_cost - 0.08) < 1e-6 + + +def test_groq_llama_3_3_70b_versatile(): + """Verify specific rates and context window for Llama 3.3 70B Versatile on Groq.""" + info = litellm.get_model_info("groq/llama-3.3-70b-versatile") + assert info["max_input_tokens"] == 131072 + assert info["max_output_tokens"] == 32768 + assert info["max_tokens"] == 32768 + assert info["input_cost_per_token"] == 5.9e-07 + assert info["output_cost_per_token"] == 7.9e-07 + + # Calculate cost for 1,000,000 prompt + 1,000,000 completion tokens + prompt_cost, completion_cost = litellm.cost_per_token( + model="groq/llama-3.3-70b-versatile", + prompt_tokens=1_000_000, + completion_tokens=1_000_000, + ) + assert abs(prompt_cost - 0.59) < 1e-6 + assert abs(completion_cost - 0.79) < 1e-6 + + +def test_groq_llama_3_3_70b_specdec(): + """Verify specific rates and context window for Llama 3.3 70B Speculative Decoding on Groq.""" + info = litellm.get_model_info("groq/llama-3.3-70b-specdec") + assert info["max_input_tokens"] == 8192 + assert info["max_output_tokens"] == 8192 + assert info["max_tokens"] == 8192 + assert info["input_cost_per_token"] == 5.9e-07 + assert info["output_cost_per_token"] == 9.9e-07 + + prompt_cost, completion_cost = litellm.cost_per_token( + model="groq/llama-3.3-70b-specdec", + prompt_tokens=1_000_000, + completion_tokens=1_000_000, + ) + assert abs(prompt_cost - 0.59) < 1e-6 + assert abs(completion_cost - 0.99) < 1e-6