diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 7fd767abf59..54e5e26764c 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -71,13 +71,21 @@ jobs: # a lookup key into the haveibeenpwned range API (the protocol mandates # SHA-1) and the digest itself never leaves the proxy beyond its first 5 # characters. - - name: Filter SARIF (OCI sha256, HIBP sha1) + # Finally it fires on the Fireworks shared-session affinity token in + # litellm/llms/fireworks_ai/common_utils.py, where a credential is hashed so + # concurrent requests on one Fireworks key map to a stable x-session-affinity + # value and their shared prompt prefix hits the replica cache. Nothing is + # authenticated by that digest; it is a cache-partition key, and SHA-256 is + # kept because two credentials must not collide onto one token. CodeQL flags + # it because the flow starts at a lookup key literally named `api_key`. + - name: Filter SARIF (OCI sha256, HIBP sha1, Fireworks affinity token) if: matrix.language == 'python' uses: advanced-security/filter-sarif@2da736ff05ef065cb2894ac6892e47b5eac2c3c0 # v1.1 with: patterns: | -litellm/llms/oci/common_utils.py:py/weak-sensitive-data-hashing -litellm/proxy/auth/password_policy.py:py/weak-sensitive-data-hashing + -litellm/llms/fireworks_ai/common_utils.py:py/weak-sensitive-data-hashing input: sarif-results/python.sarif output: sarif-results/python.sarif diff --git a/litellm/litellm_core_utils/get_litellm_params.py b/litellm/litellm_core_utils/get_litellm_params.py index 85557090ae9..48f418f01cc 100644 --- a/litellm/litellm_core_utils/get_litellm_params.py +++ b/litellm/litellm_core_utils/get_litellm_params.py @@ -82,6 +82,7 @@ OPTIONAL_KWARGS_KEYS: Final = ( "fireworks_forward_user_id", "responses_api_version", PROVIDER_AFFINITY_HEADER_KWARG_KEY, + "fireworks_shared_session_affinity", } ) | AWS_CREDENTIAL_KWARGS_KEYS diff --git a/litellm/llms/fireworks_ai/common_utils.py b/litellm/llms/fireworks_ai/common_utils.py index 10314d3fac4..399d22d7b10 100644 --- a/litellm/llms/fireworks_ai/common_utils.py +++ b/litellm/llms/fireworks_ai/common_utils.py @@ -1,6 +1,7 @@ +import hashlib from collections.abc import Mapping from types import MappingProxyType -from typing import Final +from typing import Final, cast from httpx import Headers @@ -15,27 +16,76 @@ class FireworksAIException(BaseLLMException): pass +SHARED_SESSION_AFFINITY_PARAM: Final = "fireworks_shared_session_affinity" + + +def _lookup(mapping: object, key: str) -> object | None: + if isinstance(mapping, Mapping): + return cast("Mapping[str, object]", mapping).get(key) + return None + + def get_fireworks_session_id(litellm_params: Mapping[str, object]) -> str | None: """ Session id to send as `x-session-affinity`, or None when the caller gave none. + Priority: + 1. Explicit `litellm_session_id` or `session_id` in litellm_params + 2. Explicit `session_id` in metadata + 3. Shared affinity, when `fireworks_shared_session_affinity` is set: a stable digest of + the caller's LiteLLM user key (proxy) or Fireworks credential (direct SDK) + Deliberately does not fall back to `litellm_trace_id`, and ignores session ids the proxy generated for a request that had none: both are per request, so using them pins every request to a different Fireworks node and prompt caching never hits. """ params: Final = litellm_params metadata: Final = params.get("metadata") - if isinstance(metadata, Mapping) and metadata.get(SESSION_ID_GENERATED_METADATA_KEY): + + has_generated_session_id: Final = _lookup(metadata, SESSION_ID_GENERATED_METADATA_KEY) + if not has_generated_session_id: + for key in ("litellm_session_id", "session_id"): + value = params.get(key) + if value: + return str(value) + session_from_metadata: Final = _lookup(metadata, "session_id") + if session_from_metadata: + return str(session_from_metadata) + + shared_affinity: Final = params.get(SHARED_SESSION_AFFINITY_PARAM) or _lookup( + metadata, SHARED_SESSION_AFFINITY_PARAM + ) + if not shared_affinity: return None - for key in ("litellm_session_id", "session_id"): - value = params.get(key) - if value: - return str(value) - if isinstance(metadata, Mapping): - value = metadata.get("session_id") - if value: - return str(value) - return None + + user_key_hash: Final = _lookup(metadata, "user_api_key_hash") + if isinstance(user_key_hash, str) and user_key_hash: + # `user_api_key_hash` is not guaranteed to be a digest: the proxy leaves opaque + # custom-auth keys and OAuth bearer tokens unhashed, so hashing it here is what + # keeps a raw proxy credential from riding to Fireworks in a request header. + return f"litellm-user-{shared_affinity_token(user_key_hash)}" + + configured: Final = params.get("api_key") + credential: Final = configured if isinstance(configured, str) and configured else resolve_fireworks_api_key(None) + if credential is None: + return None + return shared_affinity_token(credential) + + +def shared_affinity_token(credential: str) -> str: + """ + Opaque, stable routing token for one credential. + + This is a cache-isolation identifier, not a password hash: it is sent to Fireworks as + `x-session-affinity` so concurrent requests on one credential land on one replica and + their shared prompt prefix hits the cache. It is never used to authenticate, and it + leaks nothing beyond a digest that Fireworks could compute itself from its own key. + Collision resistance is however load bearing: two distinct credentials must not share + a token, or one account's cache becomes readable by another, so the digest must stay a + strong hash. `usedforsecurity=False` declares that no authentication property rests on + it (and keeps it available under FIPS builds). + """ + return hashlib.sha256(credential.encode(), usedforsecurity=False).hexdigest() def with_fireworks_session_affinity( diff --git a/litellm/main.py b/litellm/main.py index 327a7a1c4b5..625031035f2 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -5978,6 +5978,7 @@ def completion( key: kwargs[key] for key in ( *AWS_CREDENTIAL_KWARGS_KEYS, + "fireworks_shared_session_affinity", *ANTHROPIC_WIF_KWARGS_KEYS, *OPENAI_WIF_KWARGS_KEYS, *OAUTH_TOKEN_EXCHANGE_KWARGS_KEYS, diff --git a/litellm/types/litellm_params.py b/litellm/types/litellm_params.py index 089f21a33c1..2cd41d3d2e4 100644 --- a/litellm/types/litellm_params.py +++ b/litellm/types/litellm_params.py @@ -91,6 +91,7 @@ class ProviderConnection: litellm_credential_name: str | None = None configurable_clientside_auth_params: "Sequence[str | ConfigurableClientsideParamsCustomAuth] | None" = None use_xai_oauth: bool | None = None + fireworks_shared_session_affinity: bool | None = None github_copilot_auth_type: str | None = None github_copilot_host: str | None = None github_copilot_user_session: object | None = None diff --git a/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py b/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py index 04b791cf289..73daa4a8b33 100644 --- a/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py +++ b/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py @@ -12,7 +12,7 @@ from litellm.constants import SESSION_ID_GENERATED_METADATA_KEY from litellm.litellm_core_utils.get_supported_openai_params import get_supported_openai_params from litellm.llms.custom_httpx.http_handler import HTTPHandler from litellm.llms.fireworks_ai.chat.transformation import FireworksAIConfig -from litellm.llms.fireworks_ai.common_utils import get_fireworks_session_id +from litellm.llms.fireworks_ai.common_utils import get_fireworks_session_id, shared_affinity_token from litellm.types.utils import ( ChatCompletionMessageToolCall, Function, @@ -221,16 +221,13 @@ def test_validate_environment_raises_without_api_key(monkeypatch): def test_get_fireworks_session_id_prefers_litellm_session_id_over_trace_id(): assert ( - get_fireworks_session_id( - {"litellm_session_id": "session-123", "litellm_trace_id": "trace-123"} - ) + get_fireworks_session_id({"litellm_session_id": "session-123", "litellm_trace_id": "trace-123"}) == "session-123" ) -def test_get_fireworks_session_id_ignores_proxy_generated_session_id(): - """general_settings.missing_session_id: generate stamps a fresh id per request; sending it - as x-session-affinity would pin every request to a different node.""" +def test_get_fireworks_session_id_ignores_proxy_generated_session_id_without_shared_affinity(): + """Without shared affinity, a generated session id must be ignored.""" assert ( get_fireworks_session_id( { @@ -243,6 +240,142 @@ def test_get_fireworks_session_id_ignores_proxy_generated_session_id(): ) +def test_get_fireworks_session_id_uses_shared_affinity_even_with_generated_id(): + """With shared affinity, the stable token wins even when the proxy generated a fresh id.""" + litellm_params = { + "litellm_session_id": "generated-1", + "fireworks_shared_session_affinity": True, + "metadata": { + "session_id": "generated-1", + SESSION_ID_GENERATED_METADATA_KEY: True, + "user_api_key_hash": "stable-user-hash", + }, + } + assert get_fireworks_session_id(litellm_params) == f"litellm-user-{shared_affinity_token('stable-user-hash')}" + + +def test_get_fireworks_session_id_falls_back_to_credential_token(): + litellm_params = { + "fireworks_shared_session_affinity": True, + "api_key": "fw-api-key", + "metadata": {}, + } + assert get_fireworks_session_id(litellm_params) == shared_affinity_token("fw-api-key") + + +def test_get_fireworks_session_id_uses_metadata_nested_shared_affinity(): + litellm_params = {"metadata": {"fireworks_shared_session_affinity": True, "user_api_key_hash": "stable-user-hash"}} + assert get_fireworks_session_id(litellm_params) == f"litellm-user-{shared_affinity_token('stable-user-hash')}" + + +def test_get_fireworks_session_id_never_emits_a_raw_proxy_credential(): + """A proxy user key is not always stored hashed: opaque custom-auth keys and OAuth + bearer tokens pass through `_safe_hash_litellm_api_key` unchanged, so the affinity + token must hash the value again rather than forwarding it verbatim.""" + raw_token = "opaque-custom-auth-token-abc123" + litellm_params = {"metadata": {"fireworks_shared_session_affinity": True, "user_api_key_hash": raw_token}} + token = get_fireworks_session_id(litellm_params) + assert token is not None + assert raw_token not in token + assert token == f"litellm-user-{shared_affinity_token(raw_token)}" + + +def test_get_fireworks_session_id_explicit_overrides_shared(): + litellm_params = { + "litellm_session_id": "explicit-session", + "metadata": {"fireworks_shared_session_affinity": True, "user_api_key_hash": "stable-user-hash"}, + } + assert get_fireworks_session_id(litellm_params) == "explicit-session" + + +def test_get_fireworks_session_id_is_none_without_the_flag_or_a_credential(): + assert get_fireworks_session_id({"metadata": {"user_api_key_hash": "stable-user-hash"}}) is None + + +def test_shared_affinity_token_is_stable_and_credential_specific(): + assert shared_affinity_token("fw-a") == shared_affinity_token("fw-a") + assert shared_affinity_token("fw-a") != shared_affinity_token("fw-b") + + +def test_top_level_shared_affinity_kwarg_sets_header_without_body_leak(): + """Full pipeline: litellm.completion(fireworks_shared_session_affinity=True) must set + x-session-affinity (hashed off the Fireworks credential) and must NOT put the flag + into the request body, which Fireworks rejects.""" + model = "accounts/fireworks/models/llama-v3p1-8b-instruct" + reply = { + "id": "c1", + "object": "chat.completion", + "created": 1, + "model": model, + "choices": [{"index": 0, "message": {"role": "assistant", "content": "Hi"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } + raw_response = MagicMock() + raw_response.status_code = 200 + raw_response.headers = {} + raw_response.text = json.dumps(reply) + raw_response.json = lambda: reply + + captured: dict = {} + client = MagicMock(spec=HTTPHandler) + + def post(*args, **kwargs): + captured["headers"] = dict(kwargs.get("headers") or {}) + captured["body"] = json.loads(kwargs.get("data") or "{}") + return raw_response + + client.post.side_effect = post + + litellm.completion( + model=f"fireworks_ai/{model}", + messages=[{"role": "user", "content": "hi"}], + api_key="fw-test-key", + client=client, + fireworks_shared_session_affinity=True, + ) + + headers = {k.lower(): v for k, v in captured["headers"].items()} + assert headers.get("x-session-affinity") == shared_affinity_token("fw-test-key") + assert "fireworks_shared_session_affinity" not in captured["body"] + + +def test_shared_affinity_kwarg_is_absent_from_the_request_body(): + """A run with the flag off must not carry the field anywhere in the request body, and a + run with it on must reach the same body, so the flag is never serialized to Fireworks.""" + model = "accounts/fireworks/models/llama-v3p1-8b-instruct" + reply = { + "id": "c1", + "object": "chat.completion", + "created": 1, + "model": model, + "choices": [{"index": 0, "message": {"role": "assistant", "content": "Hi"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } + raw_response = MagicMock() + raw_response.status_code = 200 + raw_response.headers = {} + raw_response.text = json.dumps(reply) + raw_response.json = lambda: reply + + bodies: list = [] + + def run(**extra): + client = MagicMock(spec=HTTPHandler) + client.post.side_effect = lambda *a, **kw: (bodies.append(json.loads(kw.get("data") or "{}")), raw_response)[1] + litellm.completion( + model=f"fireworks_ai/{model}", + messages=[{"role": "user", "content": "hi"}], + api_key="fw-test-key", + client=client, + **extra, + ) + + run() + run(fireworks_shared_session_affinity=True) + + assert bodies[0] == bodies[1] + + def test_handle_message_content_with_tool_calls(): config = FireworksAIConfig() message = Message( @@ -275,25 +408,18 @@ def test_handle_message_content_with_tool_calls(): }, } ] - updated_message = config._handle_message_content_with_tool_calls( - message, tool_calls - ) + updated_message = config._handle_message_content_with_tool_calls(message, tool_calls) assert updated_message.tool_calls is not None assert len(updated_message.tool_calls) == 1 assert updated_message.tool_calls[0].function.name == "get_current_weather" - assert ( - updated_message.tool_calls[0].function.arguments - == expected_tool_call.function.arguments - ) + assert updated_message.tool_calls[0].function.arguments == expected_tool_call.function.arguments def test_get_supported_openai_params_reasoning_effort(): """Test that reasoning_effort is only included in supported params for models that support it.""" config = FireworksAIConfig() - supported_params = config.get_supported_openai_params( - "fireworks_ai/accounts/fireworks/models/glm-5p1" - ) + supported_params = config.get_supported_openai_params("fireworks_ai/accounts/fireworks/models/glm-5p1") assert "reasoning_effort" in supported_params assert "thinking" in supported_params @@ -308,9 +434,7 @@ def test_get_supported_openai_params_parallel_tool_calls(): """Test that parallel_tool_calls is included for models that support function calling.""" config = FireworksAIConfig() - supported_params = config.get_supported_openai_params( - "fireworks_ai/accounts/fireworks/models/glm-5p1" - ) + supported_params = config.get_supported_openai_params("fireworks_ai/accounts/fireworks/models/glm-5p1") assert "parallel_tool_calls" in supported_params assert "tools" in supported_params assert "tool_choice" in supported_params @@ -324,9 +448,7 @@ def test_get_supported_openai_params_parallel_tool_calls(): def test_get_supported_openai_params_short_model_name_resolves_account_prefixed_entry(): config = FireworksAIConfig() - supported_params = config.get_supported_openai_params( - "fireworks_ai/deepseek-v4-pro-0813" - ) + supported_params = config.get_supported_openai_params("fireworks_ai/deepseek-v4-pro-0813") assert "tool_choice" in supported_params assert "reasoning_effort" in supported_params @@ -335,9 +457,7 @@ def test_get_supported_openai_params_short_model_name_resolves_account_prefixed_ def test_get_supported_openai_params_preserves_generic_reasoning_fallback(): config = FireworksAIConfig() - supported_params = config.get_supported_openai_params( - "fireworks_ai/accounts/fireworks/models/glm-5p3-flash" - ) + supported_params = config.get_supported_openai_params("fireworks_ai/accounts/fireworks/models/glm-5p3-flash") assert "reasoning_effort" in supported_params @@ -404,14 +524,10 @@ def test_get_models_url_no_double_v1(api_base, expected_url_prefix): mock_response = MagicMock() mock_response.status_code = 200 - mock_response.json.return_value = { - "models": [{"name": "accounts/fireworks/models/llama-v3-70b"}] - } + mock_response.json.return_value = {"models": [{"name": "accounts/fireworks/models/llama-v3-70b"}]} with ( - patch( - "litellm.module_level_client.get", return_value=mock_response - ) as mock_get, + patch("litellm.module_level_client.get", return_value=mock_response) as mock_get, patch( "litellm.llms.fireworks_ai.chat.transformation.get_secret_str", side_effect=lambda key: { @@ -423,13 +539,9 @@ def test_get_models_url_no_double_v1(api_base, expected_url_prefix): ): result = config.get_models(api_key="test-key", api_base=api_base) - called_url = mock_get.call_args.kwargs.get("url") or mock_get.call_args[1].get( - "url", "" - ) + called_url = mock_get.call_args.kwargs.get("url") or mock_get.call_args[1].get("url", "") assert "/v1/v1/" not in called_url, f"Double /v1/ detected in URL: {called_url}" - assert called_url.startswith( - expected_url_prefix - ), f"URL {called_url} does not start with {expected_url_prefix}" + assert called_url.startswith(expected_url_prefix), f"URL {called_url} does not start with {expected_url_prefix}" assert result == ["fireworks_ai/accounts/fireworks/models/llama-v3-70b"] @@ -457,9 +569,7 @@ def test_transform_messages_helper_removes_provider_specific_fields(): }, ] # Call helper - out = config._transform_messages_helper( - messages, model="fireworks/test", litellm_params={} - ) + out = config._transform_messages_helper(messages, model="fireworks/test", litellm_params={}) for msg in out: assert "provider_specific_fields" not in msg @@ -480,15 +590,11 @@ def test_transform_messages_helper_strips_thinking_blocks_but_keeps_reasoning_co { "role": "assistant", "content": "I can help.", - "thinking_blocks": [ - {"type": "thinking", "thinking": "internal", "signature": ""} - ], + "thinking_blocks": [{"type": "thinking", "thinking": "internal", "signature": ""}], "reasoning_content": "internal", }, ] - out = config._transform_messages_helper( - messages, model="accounts/fireworks/models/glm-5p1", litellm_params={} - ) + out = config._transform_messages_helper(messages, model="accounts/fireworks/models/glm-5p1", litellm_params={}) assert "thinking_blocks" not in out[1] assert out[1]["reasoning_content"] == "internal" assert out[1]["content"] == "I can help." @@ -1006,9 +1112,7 @@ def test_transform_messages_helper_rejects_file_blocks(): litellm.BadRequestError, match="Fireworks AI chat completions does not support file content blocks", ): - config._transform_messages_helper( - messages, model="accounts/fireworks/models/kimi-k2p6", litellm_params={} - ) + config._transform_messages_helper(messages, model="accounts/fireworks/models/kimi-k2p6", litellm_params={}) def test_transform_messages_helper_rejects_non_vision_image_inputs(): @@ -1020,18 +1124,14 @@ def test_transform_messages_helper_rejects_non_vision_image_inputs(): {"type": "text", "text": "Describe this"}, { "type": "image_url", - "image_url": { - "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE=" - }, + "image_url": {"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE="}, }, ], } ] with pytest.raises(litellm.BadRequestError, match="does not support image inputs"): - config._transform_messages_helper( - messages, model="accounts/fireworks/models/glm-5p2", litellm_params={} - ) + config._transform_messages_helper(messages, model="accounts/fireworks/models/glm-5p2", litellm_params={}) def test_transform_messages_helper_allows_vision_image_inputs(): @@ -1043,9 +1143,7 @@ def test_transform_messages_helper_allows_vision_image_inputs(): {"type": "text", "text": "Describe this"}, { "type": "image_url", - "image_url": { - "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE=" - }, + "image_url": {"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE="}, }, ], } @@ -1069,9 +1167,7 @@ def test_image_inputs_not_rejected_for_fuzzy_non_vision_match(): custom_model = "accounts/myorg/models/custom-glm-5p2" assert config._get_model_cost_capability(custom_model, "supports_vision") is False - assert ( - config._get_model_cost_capability_exact(custom_model, "supports_vision") is None - ) + assert config._get_model_cost_capability_exact(custom_model, "supports_vision") is None messages = [ { @@ -1079,16 +1175,12 @@ def test_image_inputs_not_rejected_for_fuzzy_non_vision_match(): "content": [ { "type": "image_url", - "image_url": { - "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE=" - }, + "image_url": {"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE="}, }, ], } ] - out = config._transform_messages_helper( - messages, model=custom_model, litellm_params={} - ) + out = config._transform_messages_helper(messages, model=custom_model, litellm_params={}) assert out == messages @@ -1101,9 +1193,7 @@ def test_transform_messages_helper_skips_non_dict_content(): } ] - out = config._transform_messages_helper( - messages, model="accounts/fireworks/models/glm-5p2", litellm_params={} - ) + out = config._transform_messages_helper(messages, model="accounts/fireworks/models/glm-5p2", litellm_params={}) assert out == messages @@ -1343,9 +1433,7 @@ def test_streaming_surfaces_fireworks_response_fields(): surfaced: dict = {} for chunk in stream: fields = getattr(chunk, "provider_specific_fields", None) or {} - surfaced.update( - {k: v for k, v in fields.items() if k.startswith("fireworks_")} - ) + surfaced.update({k: v for k, v in fields.items() if k.startswith("fireworks_")}) assert surfaced["fireworks_token_ids"] == [[123]] assert surfaced["fireworks_raw_outputs"] == [raw_output] @@ -1398,9 +1486,7 @@ def test_transform_request_direct_route_passthrough(): def test_map_extra_body_params_translates_truncate_prompt_tokens(): config = FireworksAIConfig() - result = config.map_extra_body_params( - {"extra_body": {"truncate_prompt_tokens": 4096}}, _REASONING_MODEL - ) + result = config.map_extra_body_params({"extra_body": {"truncate_prompt_tokens": 4096}}, _REASONING_MODEL) assert result == {"prompt_truncate_len": 4096} @@ -1559,9 +1645,7 @@ def test_map_extra_body_params_non_dict_chat_template_kwargs_dropped(): def test_map_extra_body_params_guided_json(): config = FireworksAIConfig() schema = {"type": "object", "properties": {"x": {"type": "string"}}} - result = config.map_extra_body_params( - {"extra_body": {"guided_json": schema}}, _REASONING_MODEL - ) + result = config.map_extra_body_params({"extra_body": {"guided_json": schema}}, _REASONING_MODEL) assert result == { "response_format": { "type": "json_schema", @@ -1572,16 +1656,10 @@ def test_map_extra_body_params_guided_json(): def test_map_extra_body_params_guided_grammar_and_choice(): config = FireworksAIConfig() - grammar = config.map_extra_body_params( - {"extra_body": {"guided_grammar": "root ::= 'hello'"}}, _REASONING_MODEL - ) - assert grammar == { - "response_format": {"type": "grammar", "grammar": "root ::= 'hello'"} - } + grammar = config.map_extra_body_params({"extra_body": {"guided_grammar": "root ::= 'hello'"}}, _REASONING_MODEL) + assert grammar == {"response_format": {"type": "grammar", "grammar": "root ::= 'hello'"}} - choice = config.map_extra_body_params( - {"extra_body": {"guided_choice": ["yes", "no"]}}, _REASONING_MODEL - ) + choice = config.map_extra_body_params({"extra_body": {"guided_choice": ["yes", "no"]}}, _REASONING_MODEL) assert choice == { "response_format": { "type": "json_schema", @@ -1667,9 +1745,7 @@ def test_map_extra_body_params_strips_unsupported_nim_vllm_params(param, value, config = FireworksAIConfig() with caplog.at_level(logging.DEBUG): - result = config.map_extra_body_params( - {"extra_body": {param: value}}, _REASONING_MODEL - ) + result = config.map_extra_body_params({"extra_body": {param: value}}, _REASONING_MODEL) assert result == {} assert param in caplog.text @@ -1741,15 +1817,10 @@ def test_nim_vllm_extras_translated_end_to_end_in_request_body(): assert request_body["top_k"] == 40 - - def test_streaming_preserves_selected_model_for_private_accounting(): from litellm.llms.custom_httpx.http_handler import HTTPHandler - requested_route = ( - "accounts/fireworks/routers/firerouter/" - "kimi-k3/deepseek-v4-pro-0813/deepseek-v4-flash-0731" - ) + requested_route = "accounts/fireworks/routers/firerouter/kimi-k3/deepseek-v4-pro-0813/deepseek-v4-flash-0731" selected_model = "deepseek-v4-flash-0731" sse_lines = [ "data: " @@ -1803,19 +1874,14 @@ def test_streaming_preserves_selected_model_for_private_accounting(): assert chunks assert {chunk.model for chunk in chunks} == {requested_route} - assert { - chunk._hidden_params.get("provider_response_model") for chunk in chunks - } == {selected_model} + assert {chunk._hidden_params.get("provider_response_model") for chunk in chunks} == {selected_model} assembled = litellm.stream_chunk_builder(chunks=chunks) assert assembled is not None assert assembled.model == requested_route assert assembled._hidden_params["provider_response_model"] == selected_model selected_model_info = litellm.model_cost[f"fireworks_ai/{selected_model}"] - expected_cost = ( - 5 * selected_model_info["input_cost_per_token"] - + selected_model_info["output_cost_per_token"] - ) + expected_cost = 5 * selected_model_info["input_cost_per_token"] + selected_model_info["output_cost_per_token"] assert litellm.completion_cost( completion_response=assembled, custom_llm_provider="fireworks_ai", diff --git a/tests/unit/types/test_litellm_params.py b/tests/unit/types/test_litellm_params.py index b0086662f1c..440cf710226 100644 --- a/tests/unit/types/test_litellm_params.py +++ b/tests/unit/types/test_litellm_params.py @@ -94,6 +94,7 @@ CONNECTION_NAMES: Final = ( "github_copilot_host", "configurable_clientside_auth_params", "use_xai_oauth", + "fireworks_shared_session_affinity", "fireworks_forward_user_id", "aws_batch_role_arn", "aws_bedrock_control_plane_endpoint",