diff --git a/litellm/__init__.py b/litellm/__init__.py index 77fa48625d9..6f4e1da1652 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1795,6 +1795,12 @@ if TYPE_CHECKING: from .llms.fireworks_ai.embed.fireworks_ai_transformation import ( FireworksAIEmbeddingConfig as FireworksAIEmbeddingConfig, ) + from .llms.fireworks_ai.messages.transformation import ( + FireworksAIMessagesConfig as FireworksAIMessagesConfig, + ) + from .llms.fireworks_ai.responses.transformation import ( + FireworksAIResponsesConfig as FireworksAIResponsesConfig, + ) from .llms.friendliai.chat.transformation import ( FriendliaiChatConfig as FriendliaiChatConfig, ) diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index 119e62a5b38..863cad694a2 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -1011,6 +1011,14 @@ _LLM_CONFIGS_IMPORT_MAP = { ".llms.fireworks_ai.embed.fireworks_ai_transformation", "FireworksAIEmbeddingConfig", ), + "FireworksAIMessagesConfig": ( + ".llms.fireworks_ai.messages.transformation", + "FireworksAIMessagesConfig", + ), + "FireworksAIResponsesConfig": ( + ".llms.fireworks_ai.responses.transformation", + "FireworksAIResponsesConfig", + ), "FriendliaiChatConfig": ( ".llms.friendliai.chat.transformation", "FriendliaiChatConfig", diff --git a/litellm/llms/fireworks_ai/chat/transformation.py b/litellm/llms/fireworks_ai/chat/transformation.py index ed6d167a118..f79e9467041 100644 --- a/litellm/llms/fireworks_ai/chat/transformation.py +++ b/litellm/llms/fireworks_ai/chat/transformation.py @@ -4,6 +4,7 @@ from typing import Any, List, Literal, Optional, Tuple, Union, cast import httpx import litellm +from litellm._logging import verbose_logger from litellm._uuid import uuid from litellm.constants import RESPONSE_FORMAT_TOOL_NAME from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -15,7 +16,6 @@ from litellm.types.llms.openai import ( AllMessageValues, ChatCompletionImageObject, ChatCompletionToolParam, - OpenAIChatCompletionToolParam, ) from litellm.types.utils import ( ChatCompletionMessageToolCall, @@ -27,7 +27,6 @@ from litellm.types.utils import ( ) from litellm.utils import ( supports_function_calling, - supports_reasoning, supports_tool_choice, ) @@ -39,9 +38,54 @@ class FireworksAIConfig(OpenAIGPTConfig): """ Reference: https://docs.fireworks.ai/api-reference/post-chatcompletions - The class `FireworksAIConfig` provides configuration for the Fireworks's Chat Completions API interface. Below are the parameters: + Fireworks AI Chat Completions API configuration. + + Fireworks is largely OpenAI-compatible. The tweaks below document where + this config diverges from the base ``OpenAIGPTConfig``: + + Request transforms + ------------------ + - **Model name prefixing**: bare model names are expanded to + ``accounts/fireworks/models/`` before sending. + - **Document inlining**: ``#transform=inline`` is appended to non-data + image URLs so non-vision models can process documents/images. Skipped + for ``data:`` URLs and vision models. Controllable via + ``disable_add_transform_inline_image_block``. + - **File → image migration**: ``file`` content parts (PDFs) are + converted to ``image_url`` parts for inlining. + - **Field stripping**: ``cache_control`` and ``provider_specific_fields`` + are removed from messages (Fireworks rejects them). + + Response transforms + ------------------- + - **Model prefixing**: the returned model name is prefixed with + ``fireworks_ai/``. + - **Tool-calls-in-content workaround**: some older Fireworks models + (e.g. Llama-v3p3-70b) return tool calls as a JSON string in the + ``content`` field with ``tool_calls: null``. The response handler + detects this and moves it to the proper ``tool_calls`` field. + (Newer models like Kimi, MiniMax, GLM, DeepSeek return tool calls + correctly.) + + Parameter handling + ------------------ + - All standard OpenAI params are passed through (``tool_choice``, + ``response_format``, ``max_completion_tokens``, ``strict`` in tools). + - Additional Fireworks-supported params: ``top_k``, ``top_logprobs``, + ``seed``, ``logit_bias``, ``parallel_tool_calls``, ``thinking``, + ``prompt_truncate_length``, ``context_length_exceeded_behavior``. + - ``tools`` and ``tool_choice`` are conditionally added based on + capability flags from ``model_prices_and_context_window.json``. + For unlisted models (custom/fine-tuned), ``get_provider_info`` + defaults both to allowed. + - ``reasoning_effort`` is always passed through — the Fireworks API + accepts it on all models and handles unsupported cases itself. """ + @property + def custom_llm_provider(self) -> Optional[str]: + return "fireworks_ai" + tools: Optional[list] = None tool_choice: Optional[Union[str, dict]] = None max_tokens: Optional[int] = None @@ -90,7 +134,6 @@ class FireworksAIConfig(OpenAIGPTConfig): return super().get_config() def get_supported_openai_params(self, model: str): - # Base parameters supported by all models supported_params = [ "stream", "max_completion_tokens", @@ -105,22 +148,22 @@ class FireworksAIConfig(OpenAIGPTConfig): "response_format", "user", "logprobs", + "top_logprobs", + "seed", + "logit_bias", + "parallel_tool_calls", + "thinking", + "reasoning_effort", "prompt_truncate_length", "context_length_exceeded_behavior", ] - # Only add tools for models that support function calling if supports_function_calling(model=model, custom_llm_provider="fireworks_ai"): supported_params.append("tools") - # Only add tool_choice for models that explicitly support it if supports_tool_choice(model=model, custom_llm_provider="fireworks_ai"): supported_params.append("tool_choice") - # Only add reasoning_effort for models that support it - if supports_reasoning(model=model, custom_llm_provider="fireworks_ai"): - supported_params.append("reasoning_effort") - return supported_params def map_openai_params( @@ -131,38 +174,12 @@ class FireworksAIConfig(OpenAIGPTConfig): drop_params: bool, ) -> dict: supported_openai_params = self.get_supported_openai_params(model=model) - is_tools_set = any( - param == "tools" and value is not None - for param, value in non_default_params.items() - ) for param, value in non_default_params.items(): if param == "tool_choice": - if value == "required": - # relevant issue: https://github.com/BerriAI/litellm/issues/4416 - optional_params["tool_choice"] = "any" - else: - # pass through the value of tool choice - optional_params["tool_choice"] = value + optional_params["tool_choice"] = value elif param == "response_format": - if ( - is_tools_set - ): # fireworks ai doesn't support tools and response_format together - optional_params = self._add_response_format_to_tools( - optional_params=optional_params, - value=value, - is_response_format_supported=False, - enforce_tool_choice=False, # tools and response_format are both set, don't enforce tool_choice - ) - elif "json_schema" in value: - optional_params["response_format"] = { - "type": "json_object", - "schema": value["json_schema"]["schema"], - } - else: - optional_params["response_format"] = value - elif param == "max_completion_tokens": - optional_params["max_tokens"] = value + optional_params["response_format"] = value elif param in supported_openai_params: if value is not None: optional_params[param] = value @@ -197,14 +214,6 @@ class FireworksAIConfig(OpenAIGPTConfig): content["image_url"]["url"] = f"{url}#transform=inline" return content - def _transform_tools( - self, tools: List[OpenAIChatCompletionToolParam] - ) -> List[OpenAIChatCompletionToolParam]: - for tool in tools: - if tool.get("type") == "function": - tool["function"].pop("strict", None) - return tools - def _transform_messages_helper( self, messages: List[AllMessageValues], model: str, litellm_params: dict ) -> List[AllMessageValues]: @@ -249,48 +258,14 @@ class FireworksAIConfig(OpenAIGPTConfig): return messages def get_provider_info(self, model: str) -> ProviderSpecificModelInfo: - # Models that support reasoning_effort - reasoning_supported_models = [ - "qwen3-8b", - "qwen3-32b", - "qwen3-coder-480b-a35b-instruct", - "deepseek-v3p1", - "deepseek-v3p2", - "glm-4p5", - "glm-4p5-air", - "glm-4p6", - "gpt-oss-120b", - "gpt-oss-20b", - ] - - # Normalize model name - remove prefix if present - normalized_model = model - if model.startswith("fireworks_ai/"): - normalized_model = model.replace("fireworks_ai/", "") - if normalized_model.startswith("accounts/fireworks/models/"): - normalized_model = normalized_model.replace( - "accounts/fireworks/models/", "" - ) - - # Check if model supports reasoning - supports_reasoning_value = any( - reasoning_model in normalized_model - for reasoning_model in reasoning_supported_models - ) - - provider_specific_model_info: ProviderSpecificModelInfo = { + return { "supports_function_calling": True, + "supports_tool_choice": True, "supports_prompt_caching": True, # https://docs.fireworks.ai/guides/prompt-caching "supports_pdf_input": True, # via document inlining "supports_vision": True, # via document inlining } - # Only include supports_reasoning if True - if supports_reasoning_value: - provider_specific_model_info["supports_reasoning"] = True - - return provider_specific_model_info - def transform_request( self, model: str, @@ -304,9 +279,6 @@ class FireworksAIConfig(OpenAIGPTConfig): messages = self._transform_messages_helper( messages=messages, model=model, litellm_params=litellm_params ) - if "tools" in optional_params and optional_params["tools"] is not None: - tools = self._transform_tools(tools=optional_params["tools"]) - optional_params["tools"] = tools return super().transform_request( model=model, messages=messages, @@ -419,37 +391,94 @@ class FireworksAIConfig(OpenAIGPTConfig): ) return api_base, dynamic_api_key - def get_models(self, api_key: Optional[str] = None, api_base: Optional[str] = None): - api_base, api_key = self._get_openai_compatible_provider_info( - api_base=api_base, api_key=api_key - ) - if api_base is None or api_key is None: + def get_models( + self, api_key: Optional[str] = None, api_base: Optional[str] = None + ) -> List[str]: + """ + Fetches available models from Fireworks AI. + + Uses the Management API ``/v1/accounts/{account_id}/models`` endpoint + (documented at https://docs.fireworks.ai/api-reference/list-models). + + - Always queries ``accounts/fireworks/models`` with + ``filter=supports_serverless=true`` to list publicly available + serverless models. + - If ``FIREWORKS_ACCOUNT_ID`` is set, also queries the user's account + for dedicated deployments and merges both lists. + """ + api_key = self.get_api_key(api_key) + if api_key is None: raise ValueError( - "FIREWORKS_API_BASE or FIREWORKS_API_KEY is not set. Please set the environment variable, to query Fireworks AI's `/models` endpoint." + "Fireworks AI API key is not set. Please set FIREWORKS_API_KEY " + "(or FIREWORKS_AI_API_KEY / FIREWORKSAI_API_KEY / FIREWORKS_AI_TOKEN)." ) - account_id = get_secret_str("FIREWORKS_ACCOUNT_ID") - if account_id is None: - raise ValueError( - "FIREWORKS_ACCOUNT_ID is not set. Please set the environment variable, to query Fireworks AI's `/models` endpoint." + base_url = "https://api.fireworks.ai" + headers = {"Authorization": f"Bearer {api_key}"} + seen: set = set() + result: List[str] = [] + + for account, query_filter in self._get_model_list_targets(): + self._fetch_models_from_account( + base_url, account, query_filter, headers, seen, result ) - base = api_base.rstrip("/") - if base.endswith("/v1"): - base = base[: -len("/v1")] - response = litellm.module_level_client.get( - url=f"{base}/v1/accounts/{account_id}/models", - headers={"Authorization": f"Bearer {api_key}"}, - ) + return result - if response.status_code != 200: - raise ValueError( - f"Failed to fetch models from Fireworks AI. Status code: {response.status_code}, Response: {response.json()}" + @staticmethod + def _get_model_list_targets() -> List[tuple]: + """Return (account_id, filter) pairs to query.""" + targets: List[tuple] = [ + ("fireworks", "supports_serverless=true"), + ] + user_account = get_secret_str("FIREWORKS_ACCOUNT_ID") + if user_account and user_account != "fireworks": + targets.append((user_account, None)) + return targets + + @staticmethod + def _fetch_models_from_account( + base_url: str, + account_id: str, + query_filter: Optional[str], + headers: dict, + seen: set, + result: List[str], + ) -> None: + """Paginate through /v1/accounts/{account_id}/models and append unique models.""" + page_token: Optional[str] = None + while True: + params: dict = {"pageSize": "200"} + if query_filter: + params["filter"] = query_filter + if page_token: + params["pageToken"] = page_token + + response = litellm.module_level_client.get( + url=f"{base_url}/v1/accounts/{account_id}/models", + headers=headers, + params=params, ) + if response.status_code != 200: + verbose_logger.warning( + "Failed to fetch models from Fireworks AI account '%s'. " + "Status %d: %s", + account_id, + response.status_code, + response.text, + ) + break - models = response.json()["models"] + data = response.json() + for model in data.get("models", []): + name = model.get("name", "") + if name and name not in seen: + seen.add(name) + result.append("fireworks_ai/" + name) - return ["fireworks_ai/" + model["name"] for model in models] + page_token = data.get("nextPageToken") + if not page_token: + break @staticmethod def get_api_key(api_key: Optional[str] = None) -> Optional[str]: diff --git a/litellm/llms/fireworks_ai/cost_calculator.py b/litellm/llms/fireworks_ai/cost_calculator.py index 46026f266d6..27152f8936b 100644 --- a/litellm/llms/fireworks_ai/cost_calculator.py +++ b/litellm/llms/fireworks_ai/cost_calculator.py @@ -10,8 +10,8 @@ from litellm.constants import ( FIREWORKS_AI_56_B_MOE, FIREWORKS_AI_176_B_MOE, ) +from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token from litellm.types.utils import Usage -from litellm.utils import get_model_info # Extract the number of billion parameters from the model name @@ -58,6 +58,11 @@ def cost_per_token(model: str, usage: Usage) -> Tuple[float, float]: """ Calculates the cost per token for a given model, prompt tokens, and completion tokens. + Routes through ``generic_cost_per_token`` so cache-token and reasoning-token + pricing are picked up automatically. Falls back to the parameter-size + heuristic (``fireworks-ai-up-to-4b`` etc.) when the model is not present in + ``model_prices_and_context_window.json``. + Input: - model: str, the model name without provider prefix - usage: LiteLLM Usage block, containing anthropic caching information @@ -65,22 +70,12 @@ def cost_per_token(model: str, usage: Usage) -> Tuple[float, float]: Returns: Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd """ - ## check if model mapped, else use default pricing try: - model_info = get_model_info(model=model, custom_llm_provider="fireworks_ai") + return generic_cost_per_token( + model=model, usage=usage, custom_llm_provider="fireworks_ai" + ) except Exception: base_model = get_base_model_for_pricing(model_name=model) - - ## GET MODEL INFO - model_info = get_model_info( - model=base_model, custom_llm_provider="fireworks_ai" + return generic_cost_per_token( + model=base_model, usage=usage, custom_llm_provider="fireworks_ai" ) - - ## CALCULATE INPUT COST - - prompt_cost: float = usage["prompt_tokens"] * model_info["input_cost_per_token"] - - ## CALCULATE OUTPUT COST - completion_cost = usage["completion_tokens"] * model_info["output_cost_per_token"] - - return prompt_cost, completion_cost diff --git a/litellm/llms/fireworks_ai/messages/transformation.py b/litellm/llms/fireworks_ai/messages/transformation.py new file mode 100644 index 00000000000..6f4256f1758 --- /dev/null +++ b/litellm/llms/fireworks_ai/messages/transformation.py @@ -0,0 +1,62 @@ +""" +FireworksAIMessagesConfig for Anthropic-compatible Messages API support. +https://docs.fireworks.ai/api-reference/anthropic-messages + +""" + +from typing import Any, List, Optional, Tuple + +from litellm.llms.anthropic.experimental_pass_through.messages.transformation import ( + AnthropicMessagesConfig, +) +from litellm.secret_managers.main import get_secret_str + + +class FireworksAIMessagesConfig(AnthropicMessagesConfig): + @property + def custom_llm_provider(self) -> Optional[str]: + return "fireworks_ai" + + def validate_anthropic_messages_environment( + self, + headers: dict, + model: str, + messages: List[Any], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> Tuple[dict, Optional[str]]: + api_key = api_key or ( + get_secret_str("FIREWORKS_API_KEY") + or get_secret_str("FIREWORKS_AI_API_KEY") + or get_secret_str("FIREWORKSAI_API_KEY") + or get_secret_str("FIREWORKS_AI_TOKEN") + ) + if api_key and "Authorization" not in headers: + headers["Authorization"] = f"Bearer {api_key}" + if "content-type" not in headers: + headers["content-type"] = "application/json" + return headers, api_base + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + api_base = ( + api_base + or get_secret_str("FIREWORKS_API_BASE") + or "https://api.fireworks.ai/inference/v1" + ) + api_base = api_base.rstrip("/") + if not api_base.endswith("/v1/messages"): + if api_base.endswith("/v1"): + api_base = f"{api_base}/messages" + else: + api_base = f"{api_base}/v1/messages" + return api_base diff --git a/litellm/llms/fireworks_ai/responses/transformation.py b/litellm/llms/fireworks_ai/responses/transformation.py new file mode 100644 index 00000000000..bdcb0d4ecab --- /dev/null +++ b/litellm/llms/fireworks_ai/responses/transformation.py @@ -0,0 +1,54 @@ +""" +FireworksAIResponsesConfig for OpenAI-compatible Responses API support. +https://docs.fireworks.ai/api-reference/post-responses +""" + +from typing import Optional, Union + +from litellm.llms.openai_like.responses.transformation import ( + OpenAILikeResponsesConfig, +) +from litellm.secret_managers.main import get_secret_str +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import LlmProviders + + +class FireworksAIResponsesConfig(OpenAILikeResponsesConfig): + @property + def custom_llm_provider(self) -> Union[str, LlmProviders]: # type: ignore[override] + return "fireworks_ai" + + def validate_environment( + self, + headers: dict, + model: str, + litellm_params: Optional[GenericLiteLLMParams] = None, + ) -> dict: + litellm_params = litellm_params or GenericLiteLLMParams() + api_key = litellm_params.api_key or ( + get_secret_str("FIREWORKS_API_KEY") + or get_secret_str("FIREWORKS_AI_API_KEY") + or get_secret_str("FIREWORKSAI_API_KEY") + or get_secret_str("FIREWORKS_AI_TOKEN") + ) + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + return headers + + def get_complete_url( + self, + api_base: Optional[str], + litellm_params: dict, + ) -> str: + api_base = ( + api_base + or get_secret_str("FIREWORKS_API_BASE") + or "https://api.fireworks.ai/inference/v1" + ) + api_base = api_base.rstrip("/") + if not api_base.endswith("/responses"): + if api_base.endswith("/v1"): + api_base = f"{api_base}/responses" + else: + api_base = f"{api_base}/v1/responses" + return api_base diff --git a/litellm/utils.py b/litellm/utils.py index e1ad1db63ef..7a4ab7fb5a2 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8438,6 +8438,8 @@ class ProviderConfigManager: ) return MinimaxMessagesConfig() + elif litellm.LlmProviders.FIREWORKS_AI == provider: + return litellm.FireworksAIMessagesConfig() return None @staticmethod @@ -8583,6 +8585,8 @@ class ProviderConfigManager: return litellm.OpenRouterResponsesAPIConfig() elif litellm.LlmProviders.HOSTED_VLLM == provider: return litellm.HostedVLLMResponsesAPIConfig() + elif litellm.LlmProviders.FIREWORKS_AI == provider: + return litellm.FireworksAIResponsesConfig() return None @staticmethod diff --git a/tests/llm_translation/test_fireworks_ai_translation.py b/tests/llm_translation/test_fireworks_ai_translation.py index 1cc6aabdca8..1297e389979 100644 --- a/tests/llm_translation/test_fireworks_ai_translation.py +++ b/tests/llm_translation/test_fireworks_ai_translation.py @@ -16,11 +16,11 @@ fireworks = FireworksAIConfig() def test_map_openai_params_tool_choice(): - # Test case 1: tool_choice is "required" + # Test case 1: tool_choice "required" is passed through natively result = fireworks.map_openai_params( {"tool_choice": "required"}, {}, "some_model", drop_params=False ) - assert result == {"tool_choice": "any"} + assert result == {"tool_choice": "required"} # Test case 2: tool_choice is "auto" result = fireworks.map_openai_params( @@ -43,12 +43,10 @@ def test_map_openai_params_tool_choice(): def test_map_response_format(): """ - Test that the response format is translated correctly. + Test that the response format is passed through as-is. - h/t to https://github.com/DaveDeCaprio (@DaveDeCaprio) for the test case - - Relevant Issue: https://github.com/BerriAI/litellm/issues/6797 - Fireworks AI Ref: https://docs.fireworks.ai/structured-responses/structured-response-formatting#step-1-import-libraries + Fireworks now natively supports the OpenAI json_schema response_format. + Ref: https://docs.fireworks.ai/api-reference/post-chatcompletions """ response_format = { "type": "json_schema", @@ -65,16 +63,7 @@ def test_map_response_format(): result = fireworks.map_openai_params( {"response_format": response_format}, {}, "some_model", drop_params=False ) - assert result == { - "response_format": { - "type": "json_object", - "schema": { - "properties": {"result": {"type": "boolean"}}, - "required": ["result"], - "type": "object", - }, - } - } + assert result == {"response_format": response_format} class TestFireworksAIAudioTranscription(BaseLLMAudioTranscriptionTest): diff --git a/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py b/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py index 323443b2e15..a35d8465f76 100644 --- a/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py +++ b/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py @@ -94,20 +94,21 @@ def test_supports_reasoning_effort(): def test_get_supported_openai_params_reasoning_effort(): - """Test that reasoning_effort is only included in supported params for models that support it.""" + """Test that reasoning_effort is always included — Fireworks accepts it on all models.""" config = FireworksAIConfig() - # Model that supports reasoning_effort + # reasoning_effort should be present for reasoning models supported_params = config.get_supported_openai_params( "fireworks_ai/accounts/fireworks/models/qwen3-8b" ) assert "reasoning_effort" in supported_params - # Model that doesn't support reasoning_effort - unsupported_params = config.get_supported_openai_params( + # reasoning_effort should also be present for non-reasoning models + # (Fireworks API accepts it; unsupported models simply ignore it) + other_params = config.get_supported_openai_params( "fireworks_ai/accounts/fireworks/models/llama-v3-70b-instruct" ) - assert "reasoning_effort" not in unsupported_params + assert "reasoning_effort" in other_params def test_add_transform_inline_image_block_skips_data_urls(): @@ -145,37 +146,17 @@ def test_add_transform_inline_image_block_skips_data_urls(): ), "https URL should get #transform=inline" -@pytest.mark.parametrize( - "api_base, expected_url_prefix", - [ - ( - "https://api.fireworks.ai/inference/v1", - "https://api.fireworks.ai/inference/v1/accounts/", - ), - ( - "https://api.fireworks.ai/inference/v1/", - "https://api.fireworks.ai/inference/v1/accounts/", - ), - ( - "https://custom-host.example.com/v1", - "https://custom-host.example.com/v1/accounts/", - ), - ( - "https://custom-host.example.com/api", - "https://custom-host.example.com/api/v1/accounts/", - ), - ], - ids=["default", "trailing-slash", "custom-with-v1", "custom-without-v1"], -) -def test_get_models_url_no_double_v1(api_base, expected_url_prefix): - """Ensure get_models never produces a /v1/v1/ URL segment (fixes #23106).""" +def test_get_models_serverless_only(): + """Without FIREWORKS_ACCOUNT_ID, get_models queries only the fireworks public account.""" config = FireworksAIConfig() - account_id = "fireworks" mock_response = MagicMock() mock_response.status_code = 200 mock_response.json.return_value = { - "models": [{"name": "accounts/fireworks/models/llama-v3-70b"}] + "models": [ + {"name": "accounts/fireworks/models/deepseek-v3p2"}, + {"name": "accounts/fireworks/models/qwen3-8b"}, + ] } with ( @@ -184,23 +165,97 @@ def test_get_models_url_no_double_v1(api_base, expected_url_prefix): ) as mock_get, patch( "litellm.llms.fireworks_ai.chat.transformation.get_secret_str", - side_effect=lambda key: { - "FIREWORKS_API_KEY": "test-key", - "FIREWORKS_API_BASE": api_base, - "FIREWORKS_ACCOUNT_ID": account_id, - }.get(key), + return_value=None, ), ): - result = config.get_models(api_key="test-key", api_base=api_base) + result = config.get_models(api_key="test-key") + # Should call the management API for the fireworks public account + mock_get.assert_called_once() called_url = mock_get.call_args.kwargs.get("url") or mock_get.call_args[1].get( "url", "" ) - assert "/v1/v1/" not in called_url, f"Double /v1/ detected in URL: {called_url}" - assert called_url.startswith( - expected_url_prefix - ), f"URL {called_url} does not start with {expected_url_prefix}" - assert result == ["fireworks_ai/accounts/fireworks/models/llama-v3-70b"] + assert called_url == "https://api.fireworks.ai/v1/accounts/fireworks/models" + called_params = mock_get.call_args.kwargs.get("params", {}) + assert called_params.get("filter") == "supports_serverless=true" + assert result == [ + "fireworks_ai/accounts/fireworks/models/deepseek-v3p2", + "fireworks_ai/accounts/fireworks/models/qwen3-8b", + ] + + +def test_get_models_with_account_id(): + """With FIREWORKS_ACCOUNT_ID set, get_models queries both public and user accounts.""" + config = FireworksAIConfig() + + serverless_response = MagicMock() + serverless_response.status_code = 200 + serverless_response.json.return_value = { + "models": [{"name": "accounts/fireworks/models/deepseek-v3p2"}] + } + + user_response = MagicMock() + user_response.status_code = 200 + user_response.json.return_value = { + "models": [{"name": "accounts/myteam/models/my-finetuned-llama"}] + } + + with ( + patch( + "litellm.module_level_client.get", + side_effect=[serverless_response, user_response], + ) as mock_get, + patch( + "litellm.llms.fireworks_ai.chat.transformation.get_secret_str", + side_effect=lambda key: "myteam" if key == "FIREWORKS_ACCOUNT_ID" else None, + ), + ): + result = config.get_models(api_key="test-key") + + assert mock_get.call_count == 2 + # First call: fireworks public serverless + url1 = mock_get.call_args_list[0].kwargs.get("url", "") + assert url1 == "https://api.fireworks.ai/v1/accounts/fireworks/models" + # Second call: user account + url2 = mock_get.call_args_list[1].kwargs.get("url", "") + assert url2 == "https://api.fireworks.ai/v1/accounts/myteam/models" + assert result == [ + "fireworks_ai/accounts/fireworks/models/deepseek-v3p2", + "fireworks_ai/accounts/myteam/models/my-finetuned-llama", + ] + + +def test_get_models_deduplicates(): + """Models appearing in both public and user accounts are not duplicated.""" + config = FireworksAIConfig() + + response = MagicMock() + response.status_code = 200 + response.json.return_value = { + "models": [{"name": "accounts/fireworks/models/deepseek-v3p2"}] + } + + with ( + patch("litellm.module_level_client.get", return_value=response), + patch( + "litellm.llms.fireworks_ai.chat.transformation.get_secret_str", + return_value=None, + ), + ): + result = config.get_models(api_key="test-key") + assert result == ["fireworks_ai/accounts/fireworks/models/deepseek-v3p2"] + assert len(result) == 1 + + +def test_get_models_no_api_key_raises(): + """get_models raises ValueError when no API key is available.""" + config = FireworksAIConfig() + with patch( + "litellm.llms.fireworks_ai.chat.transformation.get_secret_str", + return_value=None, + ): + with pytest.raises(ValueError, match="API key is not set"): + config.get_models() def test_transform_messages_helper_removes_provider_specific_fields(): @@ -232,3 +287,9 @@ def test_transform_messages_helper_removes_provider_specific_fields(): ) for msg in out: assert "provider_specific_fields" not in msg + + +def test_custom_llm_provider_property(): + """Test that the custom_llm_provider property returns 'fireworks_ai'.""" + config = FireworksAIConfig() + assert config.custom_llm_provider == "fireworks_ai" diff --git a/tests/test_litellm/llms/fireworks_ai/cost_calculator/test_fireworks_ai_cost_calculator.py b/tests/test_litellm/llms/fireworks_ai/cost_calculator/test_fireworks_ai_cost_calculator.py new file mode 100644 index 00000000000..7333ef7e00d --- /dev/null +++ b/tests/test_litellm/llms/fireworks_ai/cost_calculator/test_fireworks_ai_cost_calculator.py @@ -0,0 +1,84 @@ +import os +import sys +from unittest.mock import patch + +import pytest + +sys.path.insert( + 0, os.path.abspath("../../../../..") +) # Adds the parent directory to the system path + +from litellm.llms.fireworks_ai.cost_calculator import ( + cost_per_token, + get_base_model_for_pricing, +) +from litellm.types.utils import Usage + + +class TestGetBaseModelForPricing: + """Tests for the parameter-size heuristic.""" + + def test_should_return_up_to_4b_for_small_model(self): + assert ( + get_base_model_for_pricing("llama-3b-instruct") == "fireworks-ai-up-to-4b" + ) + + def test_should_return_4_1b_to_16b_for_medium_model(self): + assert ( + get_base_model_for_pricing("llama-8b-instruct") + == "fireworks-ai-4.1b-to-16b" + ) + + def test_should_return_above_16b_for_large_model(self): + assert ( + get_base_model_for_pricing("llama-70b-instruct") == "fireworks-ai-above-16b" + ) + + def test_should_return_moe_up_to_56b(self): + assert ( + get_base_model_for_pricing("mixtral-8x7b-instruct") + == "fireworks-ai-moe-up-to-56b" + ) + + def test_should_return_moe_56b_to_176b(self): + assert ( + get_base_model_for_pricing("mixtral-8x22b-instruct") + == "fireworks-ai-56b-to-176b" + ) + + def test_should_return_default_for_unknown(self): + assert get_base_model_for_pricing("some-random-model") == "fireworks-ai-default" + + +class TestCostPerToken: + """Tests for cost_per_token with generic_cost_per_token and fallback.""" + + def test_should_calculate_cost_for_mapped_model(self): + """Mapped models (in model_prices_and_context_window.json) should succeed.""" + usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150) + prompt_cost, completion_cost = cost_per_token( + model="accounts/fireworks/models/llama-v3p1-405b-instruct", + usage=usage, + ) + assert prompt_cost >= 0 + assert completion_cost >= 0 + + def test_should_fallback_for_unmapped_model(self): + """Unmapped models should fall back to the size-heuristic path.""" + usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150) + prompt_cost, completion_cost = cost_per_token( + model="accounts/fireworks/models/custom-3b-finetune", + usage=usage, + ) + assert prompt_cost >= 0 + assert completion_cost >= 0 + + def test_should_handle_zero_tokens(self): + """Zero-token usage should yield zero cost.""" + usage = Usage(prompt_tokens=0, completion_tokens=0, total_tokens=0) + prompt_cost, completion_cost = cost_per_token( + model="accounts/fireworks/models/llama-v3p1-405b-instruct", + usage=usage, + ) + assert prompt_cost == 0.0 + assert completion_cost == 0.0 diff --git a/tests/test_litellm/llms/fireworks_ai/messages/test_fireworks_ai_messages_transformation.py b/tests/test_litellm/llms/fireworks_ai/messages/test_fireworks_ai_messages_transformation.py new file mode 100644 index 00000000000..56638088321 --- /dev/null +++ b/tests/test_litellm/llms/fireworks_ai/messages/test_fireworks_ai_messages_transformation.py @@ -0,0 +1,185 @@ +import os +import sys +from unittest.mock import patch + +import pytest + +sys.path.insert( + 0, os.path.abspath("../../../../..") +) # Adds the parent directory to the system path + +from litellm.llms.fireworks_ai.messages.transformation import ( + FireworksAIMessagesConfig, +) + + +@pytest.fixture +def config(): + return FireworksAIMessagesConfig() + + +class TestValidateAnthropicMessagesEnvironment: + """Tests for validate_anthropic_messages_environment.""" + + def test_should_set_api_key_from_explicit_param(self, config): + headers, api_base = config.validate_anthropic_messages_environment( + headers={}, + model="claude-3-5-sonnet", + messages=[], + optional_params={}, + litellm_params={}, + api_key="explicit-key", + ) + assert headers["Authorization"] == "Bearer explicit-key" + + def test_should_set_api_key_from_fireworks_api_key_env(self, config): + with patch( + "litellm.llms.fireworks_ai.messages.transformation.get_secret_str", + side_effect=lambda key: "env-key" if key == "FIREWORKS_API_KEY" else None, + ): + headers, _ = config.validate_anthropic_messages_environment( + headers={}, + model="claude-3-5-sonnet", + messages=[], + optional_params={}, + litellm_params={}, + ) + assert headers["Authorization"] == "Bearer env-key" + + def test_should_set_api_key_from_fireworks_ai_api_key_env(self, config): + with patch( + "litellm.llms.fireworks_ai.messages.transformation.get_secret_str", + side_effect=lambda key: ( + "ai-env-key" if key == "FIREWORKS_AI_API_KEY" else None + ), + ): + headers, _ = config.validate_anthropic_messages_environment( + headers={}, + model="claude-3-5-sonnet", + messages=[], + optional_params={}, + litellm_params={}, + ) + assert headers["Authorization"] == "Bearer ai-env-key" + + def test_should_not_overwrite_existing_authorization(self, config): + headers, _ = config.validate_anthropic_messages_environment( + headers={"Authorization": "Bearer pre-existing"}, + model="claude-3-5-sonnet", + messages=[], + optional_params={}, + litellm_params={}, + api_key="new-key", + ) + assert headers["Authorization"] == "Bearer pre-existing" + + def test_should_set_default_content_type(self, config): + headers, _ = config.validate_anthropic_messages_environment( + headers={}, + model="claude-3-5-sonnet", + messages=[], + optional_params={}, + litellm_params={}, + api_key="key", + ) + assert headers["content-type"] == "application/json" + assert "anthropic-version" not in headers + + def test_should_not_overwrite_existing_content_type(self, config): + headers, _ = config.validate_anthropic_messages_environment( + headers={"content-type": "application/xml"}, + model="claude-3-5-sonnet", + messages=[], + optional_params={}, + litellm_params={}, + api_key="key", + ) + assert headers["content-type"] == "application/xml" + + def test_should_pass_through_api_base(self, config): + _, api_base = config.validate_anthropic_messages_environment( + headers={}, + model="claude-3-5-sonnet", + messages=[], + optional_params={}, + litellm_params={}, + api_key="key", + api_base="https://custom.example.com", + ) + assert api_base == "https://custom.example.com" + + +class TestGetCompleteUrl: + """Tests for get_complete_url.""" + + def test_should_use_default_base_url(self, config): + with patch( + "litellm.llms.fireworks_ai.messages.transformation.get_secret_str", + return_value=None, + ): + url = config.get_complete_url( + api_base=None, + api_key=None, + model="claude-3-5-sonnet", + optional_params={}, + litellm_params={}, + ) + assert url == "https://api.fireworks.ai/inference/v1/messages" + + def test_should_append_messages_to_v1_base(self, config): + url = config.get_complete_url( + api_base="https://custom.example.com/v1", + api_key=None, + model="claude-3-5-sonnet", + optional_params={}, + litellm_params={}, + ) + assert url == "https://custom.example.com/v1/messages" + + def test_should_append_v1_messages_to_bare_base(self, config): + url = config.get_complete_url( + api_base="https://custom.example.com/api", + api_key=None, + model="claude-3-5-sonnet", + optional_params={}, + litellm_params={}, + ) + assert url == "https://custom.example.com/api/v1/messages" + + def test_should_not_duplicate_v1_messages_suffix(self, config): + url = config.get_complete_url( + api_base="https://custom.example.com/v1/messages", + api_key=None, + model="claude-3-5-sonnet", + optional_params={}, + litellm_params={}, + ) + assert url == "https://custom.example.com/v1/messages" + + def test_should_strip_trailing_slash(self, config): + url = config.get_complete_url( + api_base="https://api.fireworks.ai/inference/v1/", + api_key=None, + model="claude-3-5-sonnet", + optional_params={}, + litellm_params={}, + ) + assert url == "https://api.fireworks.ai/inference/v1/messages" + + def test_should_use_fireworks_api_base_env(self, config): + with patch( + "litellm.llms.fireworks_ai.messages.transformation.get_secret_str", + side_effect=lambda key: ( + "https://env-base.example.com/v1" + if key == "FIREWORKS_API_BASE" + else None + ), + ): + url = config.get_complete_url( + api_base=None, + api_key=None, + model="claude-3-5-sonnet", + optional_params={}, + litellm_params={}, + ) + assert url == "https://env-base.example.com/v1/messages" diff --git a/tests/test_litellm/llms/fireworks_ai/responses/test_fireworks_ai_responses_transformation.py b/tests/test_litellm/llms/fireworks_ai/responses/test_fireworks_ai_responses_transformation.py new file mode 100644 index 00000000000..7b8c4e11c2e --- /dev/null +++ b/tests/test_litellm/llms/fireworks_ai/responses/test_fireworks_ai_responses_transformation.py @@ -0,0 +1,138 @@ +import os +import sys +from unittest.mock import patch + +import pytest + +sys.path.insert( + 0, os.path.abspath("../../../../..") +) # Adds the parent directory to the system path + +from litellm.llms.fireworks_ai.responses.transformation import ( + FireworksAIResponsesConfig, +) +from litellm.types.router import GenericLiteLLMParams + + +@pytest.fixture +def config(): + return FireworksAIResponsesConfig() + + +class TestCustomLlmProvider: + """Tests for the custom_llm_provider property.""" + + def test_should_return_fireworks_ai(self, config): + assert config.custom_llm_provider == "fireworks_ai" + + +class TestValidateEnvironment: + """Tests for validate_environment.""" + + def test_should_set_auth_header_from_litellm_params(self, config): + params = GenericLiteLLMParams(api_key="param-key") + headers = config.validate_environment( + headers={}, + model="accounts/fireworks/models/llama-v3-70b", + litellm_params=params, + ) + assert headers["Authorization"] == "Bearer param-key" + + def test_should_set_auth_header_from_fireworks_api_key_env(self, config): + with patch( + "litellm.llms.fireworks_ai.responses.transformation.get_secret_str", + side_effect=lambda key: "env-key" if key == "FIREWORKS_API_KEY" else None, + ): + headers = config.validate_environment( + headers={}, + model="accounts/fireworks/models/llama-v3-70b", + ) + assert headers["Authorization"] == "Bearer env-key" + + def test_should_set_auth_header_from_fireworks_ai_api_key_env(self, config): + with patch( + "litellm.llms.fireworks_ai.responses.transformation.get_secret_str", + side_effect=lambda key: ( + "ai-env-key" if key == "FIREWORKS_AI_API_KEY" else None + ), + ): + headers = config.validate_environment( + headers={}, + model="accounts/fireworks/models/llama-v3-70b", + ) + assert headers["Authorization"] == "Bearer ai-env-key" + + def test_should_not_set_auth_header_when_no_key(self, config): + with patch( + "litellm.llms.fireworks_ai.responses.transformation.get_secret_str", + return_value=None, + ): + headers = config.validate_environment( + headers={}, + model="accounts/fireworks/models/llama-v3-70b", + ) + assert "Authorization" not in headers + + def test_should_use_default_litellm_params_when_none(self, config): + with patch( + "litellm.llms.fireworks_ai.responses.transformation.get_secret_str", + side_effect=lambda key: "env-key" if key == "FIREWORKS_API_KEY" else None, + ): + headers = config.validate_environment( + headers={}, + model="accounts/fireworks/models/llama-v3-70b", + litellm_params=None, + ) + assert headers["Authorization"] == "Bearer env-key" + + +class TestGetCompleteUrl: + """Tests for get_complete_url.""" + + def test_should_use_default_base_url(self, config): + with patch( + "litellm.llms.fireworks_ai.responses.transformation.get_secret_str", + return_value=None, + ): + url = config.get_complete_url(api_base=None, litellm_params={}) + assert url == "https://api.fireworks.ai/inference/v1/responses" + + def test_should_append_responses_to_custom_base(self, config): + url = config.get_complete_url( + api_base="https://custom.example.com/v1", + litellm_params={}, + ) + assert url == "https://custom.example.com/v1/responses" + + def test_should_append_v1_responses_to_bare_base(self, config): + url = config.get_complete_url( + api_base="https://custom.example.com/api", + litellm_params={}, + ) + assert url == "https://custom.example.com/api/v1/responses" + + def test_should_not_duplicate_responses_suffix(self, config): + url = config.get_complete_url( + api_base="https://custom.example.com/v1/responses", + litellm_params={}, + ) + assert url == "https://custom.example.com/v1/responses" + + def test_should_strip_trailing_slash(self, config): + url = config.get_complete_url( + api_base="https://api.fireworks.ai/inference/v1/", + litellm_params={}, + ) + assert url == "https://api.fireworks.ai/inference/v1/responses" + + def test_should_use_fireworks_api_base_env(self, config): + with patch( + "litellm.llms.fireworks_ai.responses.transformation.get_secret_str", + side_effect=lambda key: ( + "https://env-base.example.com/v1" + if key == "FIREWORKS_API_BASE" + else None + ), + ): + url = config.get_complete_url(api_base=None, litellm_params={}) + assert url == "https://env-base.example.com/v1/responses"