diff --git a/litellm/__init__.py b/litellm/__init__.py index cc5fbc1e72e..b3e80bfece5 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -7,6 +7,7 @@ _apply_process_defaults() ### INIT VARIABLES ######################### import threading import os +import datetime from collections.abc import Mapping, Sequence from typing import ( @@ -62,7 +63,6 @@ from litellm.constants import ( modelscope_models, empower_models, together_ai_models, - baseten_models, WANDB_MODELS, REPEATED_STREAMING_CHUNK_LIMIT, request_timeout, @@ -624,6 +624,7 @@ bedrock_models: Set = set() bedrock_converse_models: Set = set(BEDROCK_CONVERSE_MODELS) fal_ai_models: Set = set() fireworks_ai_models: Set = set() +baseten_models: Set[str] = set() fireworks_ai_embedding_models: Set = set() deepinfra_models: Set = set() perplexity_models: Set = set() @@ -742,6 +743,16 @@ def is_openai_finetune_model(key: str) -> bool: return key.startswith("ft:") and not key.count(":") > 1 +def _is_past_deprecation_date(value: Mapping[str, object]) -> bool: + deprecation_date: Final = value.get("deprecation_date") + if not isinstance(deprecation_date, str): + return False + try: + return datetime.date.fromisoformat(deprecation_date) <= datetime.date.today() + except ValueError: + return False + + def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, object]]) -> None: for key, value in tuple(model_cost_map.items()): if value.get("litellm_provider") == "openai" and not is_openai_finetune_model(key): @@ -842,6 +853,11 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje # ignore the 'up-to', '-to-' model names -> not real models. just for cost tracking based on model params. if "-to-" not in key and "fireworks-ai-default" not in key: fireworks_ai_models.add(key) + elif value.get("litellm_provider") == "baseten": + if _is_past_deprecation_date(value): + baseten_models.discard(key) + else: + baseten_models.add(key) elif value.get("litellm_provider") == "fireworks_ai-embedding-models": # ignore the 'up-to', '-to-' model names -> not real models. just for cost tracking based on model params. if "-to-" not in key: diff --git a/litellm/constants.py b/litellm/constants.py index da95b97036a..5473a3c6390 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1256,14 +1256,6 @@ together_ai_models: Final[set] = set( # supports all together ai models, just pass in the model id e.g. completion(model="together_computer/replit_code_3b",...) -baseten_models: Final[set] = set( - [ - "qvv0xeq", - "q841o8w", - "31dxrj3", - ] -) # FALCON 7B # WizardLM # Mosaic ML - featherless_ai_models: Final[set] = set( [ "featherless-ai/Qwerky-72B", diff --git a/litellm/llms/baseten/chat.py b/litellm/llms/baseten/chat.py index e92f47c41c0..381d487c36d 100644 --- a/litellm/llms/baseten/chat.py +++ b/litellm/llms/baseten/chat.py @@ -1,6 +1,10 @@ from typing import Final +import litellm from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig +from litellm.secret_managers.main import get_secret_str + +BASETEN_MODEL_API_BASE: Final = "https://inference.baseten.co/v1" class BasetenConfig(OpenAIGPTConfig): @@ -89,11 +93,39 @@ class BasetenConfig(OpenAIGPTConfig): Get the OpenAI compatible provider info for Baseten """ # Default to Model API - default_api_base: Final = "https://inference.baseten.co/v1" + default_api_base: Final = BASETEN_MODEL_API_BASE default_api_key: Final = api_key or "BASETEN_API_KEY" return default_api_base, default_api_key + @staticmethod + def get_api_key(api_key: str | None = None) -> str | None: + return api_key or get_secret_str("BASETEN_API_KEY") + + @staticmethod + def get_api_base(api_base: str | None = None) -> str | None: + return api_base or get_secret_str("BASETEN_API_BASE") or BASETEN_MODEL_API_BASE + + def get_models(self, api_key: str | None = None, api_base: str | None = None) -> list[str]: + """ + Calls Baseten's Model API `/v1/models` endpoint and returns the live catalog. + """ + resolved_api_key: Final = self.get_api_key(api_key) + if resolved_api_key is None: + raise ValueError( + "BASETEN_API_KEY is not set. Please set the environment variable, to query Baseten's `/models` endpoint." + ) + resolved_api_base: Final = (self.get_api_base(api_base) or BASETEN_MODEL_API_BASE).rstrip("/") + response: Final = litellm.module_level_client.get( # pyright: ignore[reportUnknownMemberType] # untyped params + url=f"{resolved_api_base.removesuffix('/v1')}/v1/models", + headers={"Authorization": f"Bearer {resolved_api_key}"}, + ) + if response.status_code != 200: + raise ValueError( + f"Failed to fetch models from Baseten. Status code: {response.status_code}, Response: {response.text}" + ) + return ["baseten/" + model["id"] for model in response.json()["data"]] + @staticmethod def is_dedicated_deployment(model: str) -> bool: """ @@ -118,4 +150,4 @@ class BasetenConfig(OpenAIGPTConfig): return f"https://model-{model_id}.api.baseten.co/environments/production/sync/v1" else: # Use Model API - return "https://inference.baseten.co/v1" + return BASETEN_MODEL_API_BASE diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index e32eb4ba747..e0ace9c4f39 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -33684,24 +33684,28 @@ "output_cost_per_token": 1.2e-06 }, "baseten/MiniMaxAI/MiniMax-M2.5": { + "deprecation_date": "2026-10-16", "input_cost_per_token": 3e-07, "litellm_provider": "baseten", "mode": "chat", "output_cost_per_token": 1.2e-06 }, "baseten/nvidia/Nemotron-120B-A12B": { + "deprecation_date": "2026-10-16", "input_cost_per_token": 3e-07, "litellm_provider": "baseten", "mode": "chat", "output_cost_per_token": 7.5e-07 }, "baseten/zai-org/GLM-5": { + "deprecation_date": "2026-10-16", "input_cost_per_token": 9.5e-07, "litellm_provider": "baseten", "mode": "chat", "output_cost_per_token": 3.15e-06 }, "baseten/zai-org/GLM-4.7": { + "deprecation_date": "2026-10-16", "cache_read_input_token_cost": 1.2e-07, "input_cost_per_token": 6e-07, "litellm_provider": "baseten", @@ -33719,24 +33723,28 @@ "supports_vision": false }, "baseten/zai-org/GLM-4.6": { + "deprecation_date": "2026-10-16", "input_cost_per_token": 6e-07, "litellm_provider": "baseten", "mode": "chat", "output_cost_per_token": 2.2e-06 }, "baseten/moonshotai/Kimi-K2.5": { + "deprecation_date": "2026-10-16", "input_cost_per_token": 6e-07, "litellm_provider": "baseten", "mode": "chat", "output_cost_per_token": 3e-06 }, "baseten/moonshotai/Kimi-K2-Thinking": { + "deprecation_date": "2026-10-16", "input_cost_per_token": 6e-07, "litellm_provider": "baseten", "mode": "chat", "output_cost_per_token": 2.5e-06 }, "baseten/moonshotai/Kimi-K2-Instruct-0905": { + "deprecation_date": "2026-10-16", "input_cost_per_token": 6e-07, "litellm_provider": "baseten", "mode": "chat", @@ -33760,12 +33768,14 @@ "supports_vision": false }, "baseten/deepseek-ai/DeepSeek-V3.1": { + "deprecation_date": "2026-10-16", "input_cost_per_token": 5e-07, "litellm_provider": "baseten", "mode": "chat", "output_cost_per_token": 1.5e-06 }, "baseten/deepseek-ai/DeepSeek-V3-0324": { + "deprecation_date": "2026-10-16", "input_cost_per_token": 7.7e-07, "litellm_provider": "baseten", "mode": "chat", @@ -72125,6 +72135,7 @@ "supports_vision": true }, "baseten/moonshotai/Kimi-K2.6": { + "deprecation_date": "2026-10-16", "cache_read_input_token_cost": 1.6e-07, "input_cost_per_token": 9.5e-07, "litellm_provider": "baseten", @@ -72142,6 +72153,7 @@ "supports_vision": true }, "baseten/moonshotai/Kimi-K2.7-Code": { + "deprecation_date": "2026-10-16", "cache_read_input_token_cost": 1.6e-07, "input_cost_per_token": 9.5e-07, "litellm_provider": "baseten", @@ -72193,6 +72205,7 @@ "supports_vision": false }, "baseten/thinkingmachines/inkling": { + "deprecation_date": "2026-10-16", "cache_read_input_token_cost": 1.7e-07, "input_cost_per_token": 1e-06, "litellm_provider": "baseten", @@ -72210,6 +72223,7 @@ "supports_vision": true }, "baseten/thinkingmachines/inkling-small": { + "deprecation_date": "2026-10-16", "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 5e-07, "litellm_provider": "baseten", @@ -72278,6 +72292,7 @@ "supports_vision": false }, "baseten/deepseek-ai/DeepSeek-V4-Pro": { + "deprecation_date": "2026-10-16", "cache_read_input_token_cost": 1.45e-07, "input_cost_per_token": 1.74e-06, "litellm_provider": "baseten", diff --git a/litellm/utils.py b/litellm/utils.py index 66d78a03075..ff2c9012ec8 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -9704,6 +9704,8 @@ class ProviderConfigManager: ) -> BaseLLMModelInfo | None: if LlmProviders.FIREWORKS_AI == provider: return litellm.FireworksAIConfig() + elif LlmProviders.BASETEN == provider: + return litellm.BasetenConfig() elif LlmProviders.OPENAI == provider: return litellm.OpenAIGPTConfig() elif LlmProviders.GEMINI == provider: diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index e32eb4ba747..e0ace9c4f39 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -33684,24 +33684,28 @@ "output_cost_per_token": 1.2e-06 }, "baseten/MiniMaxAI/MiniMax-M2.5": { + "deprecation_date": "2026-10-16", "input_cost_per_token": 3e-07, "litellm_provider": "baseten", "mode": "chat", "output_cost_per_token": 1.2e-06 }, "baseten/nvidia/Nemotron-120B-A12B": { + "deprecation_date": "2026-10-16", "input_cost_per_token": 3e-07, "litellm_provider": "baseten", "mode": "chat", "output_cost_per_token": 7.5e-07 }, "baseten/zai-org/GLM-5": { + "deprecation_date": "2026-10-16", "input_cost_per_token": 9.5e-07, "litellm_provider": "baseten", "mode": "chat", "output_cost_per_token": 3.15e-06 }, "baseten/zai-org/GLM-4.7": { + "deprecation_date": "2026-10-16", "cache_read_input_token_cost": 1.2e-07, "input_cost_per_token": 6e-07, "litellm_provider": "baseten", @@ -33719,24 +33723,28 @@ "supports_vision": false }, "baseten/zai-org/GLM-4.6": { + "deprecation_date": "2026-10-16", "input_cost_per_token": 6e-07, "litellm_provider": "baseten", "mode": "chat", "output_cost_per_token": 2.2e-06 }, "baseten/moonshotai/Kimi-K2.5": { + "deprecation_date": "2026-10-16", "input_cost_per_token": 6e-07, "litellm_provider": "baseten", "mode": "chat", "output_cost_per_token": 3e-06 }, "baseten/moonshotai/Kimi-K2-Thinking": { + "deprecation_date": "2026-10-16", "input_cost_per_token": 6e-07, "litellm_provider": "baseten", "mode": "chat", "output_cost_per_token": 2.5e-06 }, "baseten/moonshotai/Kimi-K2-Instruct-0905": { + "deprecation_date": "2026-10-16", "input_cost_per_token": 6e-07, "litellm_provider": "baseten", "mode": "chat", @@ -33760,12 +33768,14 @@ "supports_vision": false }, "baseten/deepseek-ai/DeepSeek-V3.1": { + "deprecation_date": "2026-10-16", "input_cost_per_token": 5e-07, "litellm_provider": "baseten", "mode": "chat", "output_cost_per_token": 1.5e-06 }, "baseten/deepseek-ai/DeepSeek-V3-0324": { + "deprecation_date": "2026-10-16", "input_cost_per_token": 7.7e-07, "litellm_provider": "baseten", "mode": "chat", @@ -72125,6 +72135,7 @@ "supports_vision": true }, "baseten/moonshotai/Kimi-K2.6": { + "deprecation_date": "2026-10-16", "cache_read_input_token_cost": 1.6e-07, "input_cost_per_token": 9.5e-07, "litellm_provider": "baseten", @@ -72142,6 +72153,7 @@ "supports_vision": true }, "baseten/moonshotai/Kimi-K2.7-Code": { + "deprecation_date": "2026-10-16", "cache_read_input_token_cost": 1.6e-07, "input_cost_per_token": 9.5e-07, "litellm_provider": "baseten", @@ -72193,6 +72205,7 @@ "supports_vision": false }, "baseten/thinkingmachines/inkling": { + "deprecation_date": "2026-10-16", "cache_read_input_token_cost": 1.7e-07, "input_cost_per_token": 1e-06, "litellm_provider": "baseten", @@ -72210,6 +72223,7 @@ "supports_vision": true }, "baseten/thinkingmachines/inkling-small": { + "deprecation_date": "2026-10-16", "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 5e-07, "litellm_provider": "baseten", @@ -72278,6 +72292,7 @@ "supports_vision": false }, "baseten/deepseek-ai/DeepSeek-V4-Pro": { + "deprecation_date": "2026-10-16", "cache_read_input_token_cost": 1.45e-07, "input_cost_per_token": 1.74e-06, "litellm_provider": "baseten", diff --git a/tests/unit/llms/baseten/chat/test_baseten_completions.py b/tests/unit/llms/baseten/chat/test_baseten_completions.py index 7cac16cca1c..4f135a1a106 100644 --- a/tests/unit/llms/baseten/chat/test_baseten_completions.py +++ b/tests/unit/llms/baseten/chat/test_baseten_completions.py @@ -1,8 +1,14 @@ import os from typing import Final -import pytest from unittest.mock import patch + +import httpx +import pytest +import respx + +import litellm from litellm.llms.baseten.chat import BasetenConfig +from litellm.utils import get_valid_models class TestBasetenRouting: @@ -50,6 +56,107 @@ class TestBasetenModelAPI: assert api_key == "test-key" +class TestBasetenModelDiscovery: + """Test Baseten model listing for `baseten/*` wildcard expansion and /v1/models""" + + def test_models_by_provider_comes_from_registry(self): + """Static listing uses the model cost map, not the retired 2023 deployment IDs.""" + baseten_models: Final = litellm.models_by_provider["baseten"] + servable_registry_models: Final = { + key + for key, value in litellm.model_cost.items() + if value.get("litellm_provider") == "baseten" and not litellm._is_past_deprecation_date(value) + } + + assert servable_registry_models + assert servable_registry_models <= set(baseten_models) + for retired_id in ("qvv0xeq", "q841o8w", "31dxrj3"): + assert retired_id not in baseten_models + + def test_models_by_provider_skips_deprecated_models(self): + """Retired Model APIs stay in the cost map for spend tracking but are not listed.""" + assert litellm.model_cost["baseten/zai-org/GLM-5"]["deprecation_date"] + assert "baseten/zai-org/GLM-5" not in litellm.models_by_provider["baseten"] + + def test_cost_map_reload_drops_a_newly_retired_model(self): + """A refreshed cost map that marks a listed model retired removes it without a restart.""" + model: Final = "baseten/acme/reload-probe" + try: + litellm.add_known_models({model: {"litellm_provider": "baseten"}}) + assert model in litellm.models_by_provider["baseten"] + + litellm.add_known_models({model: {"litellm_provider": "baseten", "deprecation_date": "2000-01-01"}}) + + assert model not in litellm.models_by_provider["baseten"] + finally: + litellm.baseten_models.discard(model) + litellm.add_known_models({}) + + @pytest.mark.parametrize( + "value, expected", + [ + ({}, False), + ({"deprecation_date": "2000-01-01"}, True), + ({"deprecation_date": "2999-01-01"}, False), + ({"deprecation_date": "not-a-date"}, False), + ], + ) + def test_is_past_deprecation_date(self, value, expected): + assert litellm._is_past_deprecation_date(value) is expected + + def test_get_models_calls_model_api(self, respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.get("https://inference.baseten.co/v1/models").mock( + return_value=httpx.Response( + 200, + json={ + "object": "list", + "data": [ + {"id": "zai-org/GLM-5", "object": "model"}, + {"id": "moonshotai/Kimi-K2.5", "object": "model"}, + ], + }, + ) + ) + + models: Final = BasetenConfig().get_models(api_key="test-key") + + assert models == ["baseten/zai-org/GLM-5", "baseten/moonshotai/Kimi-K2.5"] + assert route.calls.last.request.headers["Authorization"] == "Bearer test-key" + + def test_get_models_respects_custom_api_base(self, respx_mock: respx.MockRouter) -> None: + respx_mock.get("https://custom.baseten.example/v1/models").mock( + return_value=httpx.Response(200, json={"data": [{"id": "zai-org/GLM-5"}]}) + ) + + models: Final = BasetenConfig().get_models(api_key="test-key", api_base="https://custom.baseten.example/v1/") + + assert models == ["baseten/zai-org/GLM-5"] + + @patch.dict(os.environ, {}, clear=True) + def test_get_models_requires_api_key(self): + with pytest.raises(ValueError, match="BASETEN_API_KEY"): + BasetenConfig().get_models() + + def test_get_models_raises_on_error_status(self, respx_mock: respx.MockRouter) -> None: + respx_mock.get("https://inference.baseten.co/v1/models").mock( + return_value=httpx.Response(403, text="forbidden") + ) + + with pytest.raises(ValueError, match="403"): + BasetenConfig().get_models(api_key="test-key") + + @patch.dict(os.environ, {"BASETEN_API_KEY": "test-key"}) + def test_get_valid_models_check_provider_endpoint(self, respx_mock: respx.MockRouter) -> None: + """check_provider_endpoint=True returns the live catalog instead of the static list.""" + respx_mock.get("https://inference.baseten.co/v1/models").mock( + return_value=httpx.Response(200, json={"data": [{"id": "brand-new/model"}]}) + ) + + models: Final = get_valid_models(check_provider_endpoint=True, custom_llm_provider="baseten") + + assert models == ["baseten/brand-new/model"] + + class TestBasetenReasoningEffort: def test_reasoning_effort_passthrough_with_drop_params(self): config: Final = BasetenConfig()