From cf14a1a68ee16d8aa26a0045e87ee917ffdf7c93 Mon Sep 17 00:00:00 2001 From: "andy.huangyh" Date: Wed, 30 Sep 2026 20:56:17 +1000 Subject: [PATCH 1/3] feat(fireworks_ai): support shared session affinity for prompt caching - Add parameter/metadata flag. - When enabled, LiteLLM generates a stable header by hashing the API key or LiteLLM user key. - This pools concurrent requests from the same user/key onto the same Fireworks replica, maximizing KV cache hit rates for shared prefixes (common in agentic workflows). - Verified with unit tests and ruff linting. --- .../llms/fireworks_ai/chat/transformation.py | 1 + litellm/llms/fireworks_ai/common_utils.py | 58 +++++++++++++++---- .../fireworks_ai/completion/transformation.py | 1 + .../test_fireworks_ai_chat_transformation.py | 58 +++++++++++++++++-- 4 files changed, 100 insertions(+), 18 deletions(-) diff --git a/litellm/llms/fireworks_ai/chat/transformation.py b/litellm/llms/fireworks_ai/chat/transformation.py index 15049e71bbc..47545c13ca3 100644 --- a/litellm/llms/fireworks_ai/chat/transformation.py +++ b/litellm/llms/fireworks_ai/chat/transformation.py @@ -264,6 +264,7 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig): "prediction", "stream_options", "sampling_mask", + "fireworks_shared_session_affinity", ] # Only add tools for models that support function calling diff --git a/litellm/llms/fireworks_ai/common_utils.py b/litellm/llms/fireworks_ai/common_utils.py index 8352690235d..e34c345dd6c 100644 --- a/litellm/llms/fireworks_ai/common_utils.py +++ b/litellm/llms/fireworks_ai/common_utils.py @@ -1,3 +1,4 @@ +import hashlib from collections.abc import Mapping from types import MappingProxyType from typing import Final @@ -15,26 +16,59 @@ class FireworksAIException(BaseLLMException): pass -def get_fireworks_session_id(litellm_params: Mapping[str, object]) -> str | None: +def get_fireworks_session_id( + litellm_params: Mapping[str, object], headers: Mapping[str, str] +) -> str | None: """ Session id to send as `x-session-affinity`, or None when the caller gave none. + Priority: + 1. Explicit `litellm_session_id` or `session_id` in litellm_params + 2. Explicit `session_id` in metadata + 3. Shared affinity (if enabled): hash of the API key or LiteLLM user key + Deliberately does not fall back to `litellm_trace_id`, and ignores session ids the proxy generated for a request that had none: both are per request, so using them pins every request to a different Fireworks node and prompt caching never hits. """ params: Final = litellm_params metadata: Final = params.get("metadata") - if isinstance(metadata, Mapping) and metadata.get(SESSION_ID_GENERATED_METADATA_KEY): - return None - for key in ("litellm_session_id", "session_id"): - value = params.get(key) - if value: - return str(value) - if isinstance(metadata, Mapping): - value = metadata.get("session_id") - if value: - return str(value) + + # 1 & 2. Explicit session IDs + has_generated_session_id: Final = isinstance(metadata, Mapping) and metadata.get( + SESSION_ID_GENERATED_METADATA_KEY + ) + if not has_generated_session_id: + for key in ("litellm_session_id", "session_id"): + value = params.get(key) + if value: + return str(value) + if isinstance(metadata, Mapping): + value = metadata.get("session_id") + if value: + return str(value) + + # 3. Shared affinity + shared_affinity: Final = params.get("fireworks_shared_session_affinity") or ( + metadata.get("fireworks_shared_session_affinity") + if isinstance(metadata, Mapping) + else None + ) + if shared_affinity: + # Try LiteLLM user API key hash first (proxy-level isolation) + user_key_hash = ( + metadata.get("user_api_key_hash") if isinstance(metadata, Mapping) else None + ) + if user_key_hash: + return f"litellm-user-{user_key_hash}" + + # Fall back to hashing the Fireworks API key (account-level pooling) + auth = headers.get("Authorization", "") + if auth.startswith("Bearer "): + api_key = auth.removeprefix("Bearer ") + if api_key: + return hashlib.sha256(api_key.encode()).hexdigest() + return None @@ -43,7 +77,7 @@ def with_fireworks_session_affinity( ) -> Mapping[str, str]: if any(key.lower() == "x-session-affinity" for key in headers): return headers - session_id: Final = get_fireworks_session_id(litellm_params) + session_id: Final = get_fireworks_session_id(litellm_params, headers) if not session_id: return headers return MappingProxyType({**headers, "x-session-affinity": session_id}) diff --git a/litellm/llms/fireworks_ai/completion/transformation.py b/litellm/llms/fireworks_ai/completion/transformation.py index 4f0e302003a..41cfad89d69 100644 --- a/litellm/llms/fireworks_ai/completion/transformation.py +++ b/litellm/llms/fireworks_ai/completion/transformation.py @@ -39,6 +39,7 @@ class FireworksAITextCompletionConfig(FireworksAIMixin, BaseTextCompletionConfig "response_format", "stream", "user", + "fireworks_shared_session_affinity", ] def map_openai_params( diff --git a/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py b/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py index 2298bed2b76..1fbc7ab1303 100644 --- a/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py +++ b/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py @@ -219,28 +219,74 @@ def test_validate_environment_raises_without_api_key(monkeypatch): def test_get_fireworks_session_id_prefers_litellm_session_id_over_trace_id(): assert ( get_fireworks_session_id( - {"litellm_session_id": "session-123", "litellm_trace_id": "trace-123"} + {"litellm_session_id": "session-123", "litellm_trace_id": "trace-123"}, + {}, ) == "session-123" ) -def test_get_fireworks_session_id_ignores_proxy_generated_session_id(): - """general_settings.missing_session_id: generate stamps a fresh id per request; sending it - as x-session-affinity would pin every request to a different node.""" +def test_get_fireworks_session_id_ignores_proxy_generated_session_id_without_shared_affinity(): + """Without shared affinity, a generated session id must be ignored.""" assert ( get_fireworks_session_id( { "litellm_session_id": "generated-1", "litellm_trace_id": "generated-1", "metadata": {"session_id": "generated-1", SESSION_ID_GENERATED_METADATA_KEY: True}, - } + }, + {}, ) is None ) -def test_handle_message_content_with_tool_calls(): +def test_get_fireworks_session_id_uses_shared_affinity_even_with_generated_id(): + """With shared affinity, we use the stable hash even if the proxy generated a fresh ID.""" + litellm_params = { + "litellm_session_id": "generated-1", + "metadata": { + "session_id": "generated-1", + SESSION_ID_GENERATED_METADATA_KEY: True, + "fireworks_shared_session_affinity": True, + "user_api_key_hash": "stable-user-hash", + }, + } + assert get_fireworks_session_id(litellm_params, {}) == "litellm-user-stable-user-hash" + + +def test_get_fireworks_session_id_falls_back_to_api_key_hash(): + litellm_params = { + "metadata": { + "fireworks_shared_session_affinity": True, + }, + } + headers = {"Authorization": "Bearer fw-api-key"} + import hashlib + + expected = hashlib.sha256(b"fw-api-key").hexdigest() + assert get_fireworks_session_id(litellm_params, headers) == expected + + +def test_get_fireworks_session_id_uses_top_level_shared_affinity(): + litellm_params = { + "fireworks_shared_session_affinity": True, + "metadata": { + "user_api_key_hash": "stable-user-hash", + }, + } + assert get_fireworks_session_id(litellm_params, {}) == "litellm-user-stable-user-hash" + + +def test_get_fireworks_session_id_explicit_overrides_shared(): + litellm_params = { + "litellm_session_id": "explicit-session", + "metadata": { + "fireworks_shared_session_affinity": True, + "user_api_key_hash": "stable-user-hash", + }, + } + assert get_fireworks_session_id(litellm_params, {}) == "explicit-session" config = FireworksAIConfig() message = Message( content='{"type": "function", "name": "get_current_weather", "parameters": {"location": "Boston, MA", "unit": "fahrenheit"}}', From cf624b5276e449b383c82c5ee11b7106ec5638ce Mon Sep 17 00:00:00 2001 From: "andy.huangyh" Date: Wed, 30 Sep 2026 21:10:37 +1000 Subject: [PATCH 2/3] fix(fireworks_ai): route top-level shared-affinity flag through metadata, not request body The top-level fireworks_shared_session_affinity kwarg was previously absorbed in validate_environment/transform_request, but the http handler pops extra_body off optional_params before either hook runs (openai-compatible param handling stashes unknown kwargs inside extra_body), so the flag leaked into the serialized request body and Fireworks rejects unknown body fields. New interception point: _complete_fireworks_ai dispatch (main.py) absorbs the flag from optional_params AND optional_params.extra_body into litellm_params.metadata before map_extra_body_params runs; FireworksAIConfig.transform_extra_body strips it as a second line of defense. E2E verified via mocked HTTP client: x-session-affinity == sha256(credential) sent, no flag in request body, both metadata-flag and top-level-kwarg paths. --- .../llms/fireworks_ai/chat/transformation.py | 31 +++++++ litellm/llms/fireworks_ai/common_utils.py | 27 ++++++ litellm/main.py | 2 + .../test_fireworks_ai_chat_transformation.py | 92 +++++++++++++++++++ 4 files changed, 152 insertions(+) diff --git a/litellm/llms/fireworks_ai/chat/transformation.py b/litellm/llms/fireworks_ai/chat/transformation.py index 47545c13ca3..680b2dd88e0 100644 --- a/litellm/llms/fireworks_ai/chat/transformation.py +++ b/litellm/llms/fireworks_ai/chat/transformation.py @@ -1,5 +1,6 @@ import json from collections.abc import AsyncIterator, Iterator, Mapping +from types import MappingProxyType from typing import TYPE_CHECKING, Final, Literal, cast import httpx @@ -43,6 +44,7 @@ from ..common_utils import ( FIREROUTER, FireworksAIException, FireworksAIMixin, + absorb_shared_affinity_param, resolve_fireworks_resource_name, ) @@ -222,6 +224,7 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig): api_key=api_key, api_base=api_base, ) + absorb_shared_affinity_param(litellm_params, optional_params) return self._add_session_affinity_header(validated_headers, litellm_params) def get_supported_openai_params(self, model: str): @@ -651,6 +654,29 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig): return provider_specific_model_info + def transform_extra_body( + self, + extra_body: Mapping[str, object], + request: Mapping[str, object], + model: str, + litellm_params: Mapping[str, object], + ) -> Mapping[str, object]: + """ + Pull `fireworks_shared_session_affinity` out of extra_body before the handler + merges extra_body into the request body. The flag is a LiteLLM routing hint, not + a Fireworks API field ("Extra inputs are not permitted"); it must only reach + Fireworks as the x-session-affinity header, which the affinity logic derives + from litellm_params.metadata. + """ + filtered: Final = {k: v for k, v in extra_body.items() if k != "fireworks_shared_session_affinity"} + if "fireworks_shared_session_affinity" not in extra_body: + return super().transform_extra_body( + extra_body=extra_body, request=request, model=model, litellm_params=litellm_params + ) + if filtered: + return MappingProxyType(filtered) + return MappingProxyType({}) + def transform_request( self, model: str, @@ -663,6 +689,11 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig): messages = self._transform_messages_helper( messages=messages, model=resolved_model, litellm_params=litellm_params ) + # `fireworks_shared_session_affinity` is a LiteLLM routing hint, not a Fireworks API + # field: it must not reach the request body ("Extra inputs are not permitted") — + # stash it in litellm_params metadata where the affinity-header logic reads it. + absorb_shared_affinity_param(litellm_params, optional_params) + optional_params.pop("fireworks_shared_session_affinity", None) if "tools" in optional_params and optional_params["tools"] is not None: tools: Final = self._transform_tools(tools=optional_params["tools"]) optional_params["tools"] = tools diff --git a/litellm/llms/fireworks_ai/common_utils.py b/litellm/llms/fireworks_ai/common_utils.py index e34c345dd6c..b7a156d7cb0 100644 --- a/litellm/llms/fireworks_ai/common_utils.py +++ b/litellm/llms/fireworks_ai/common_utils.py @@ -72,6 +72,32 @@ def get_fireworks_session_id( return None +def absorb_shared_affinity_param(litellm_params: dict, optional_params: Mapping[str, object]) -> None: + """ + Move a top-level `fireworks_shared_session_affinity` kwarg into litellm_params metadata. + + The flag is a LiteLLM routing hint, not a Fireworks API field: left in optional_params + (or in optional_params.extra_body, where openai-compatible param handling stashes + unknown kwargs) it would be serialized into the request body and rejected by Fireworks. + Reads both the top-level param and its extra_body-nested form; strips it from both. + """ + value: object = optional_params.get("fireworks_shared_session_affinity") + extra_body: Final = optional_params.get("extra_body") + if value is None and isinstance(extra_body, Mapping): + value = extra_body.get("fireworks_shared_session_affinity") + if value is None: + return + if isinstance(extra_body, dict) and "fireworks_shared_session_affinity" in extra_body: + extra_body.pop("fireworks_shared_session_affinity", None) + if "fireworks_shared_session_affinity" in optional_params: + optional_params.pop("fireworks_shared_session_affinity", None) + metadata: Final = litellm_params.get("metadata") + if isinstance(metadata, dict): + metadata.setdefault("fireworks_shared_session_affinity", value) + else: + litellm_params["metadata"] = {"fireworks_shared_session_affinity": value} + + def with_fireworks_session_affinity( headers: Mapping[str, str], litellm_params: Mapping[str, object] ) -> Mapping[str, str]: @@ -141,6 +167,7 @@ class FireworksAIMixin: content_type_header: Final = ( {} if any(key.lower() == "content-type" for key in auth_headers) else {"Content-Type": "application/json"} ) + absorb_shared_affinity_param(litellm_params, optional_params) return self._add_session_affinity_header({**auth_headers, **content_type_header}, litellm_params) def _add_session_affinity_header(self, headers: dict, litellm_params: dict) -> dict: diff --git a/litellm/main.py b/litellm/main.py index 769eac79488..c152d781e84 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -119,6 +119,7 @@ from litellm.llms.base_llm.base_model_iterator import ( from litellm.llms.bedrock.common_utils import BedrockModelInfo from litellm.llms.cohere.common_utils import CohereModelInfo from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler, http2_enabled +from litellm.llms.fireworks_ai.common_utils import absorb_shared_affinity_param from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config from litellm.llms.openai_like.json_loader import JSONProviderRegistry from litellm.llms.vertex_ai.common_utils import ( @@ -1864,6 +1865,7 @@ def _complete_fireworks_ai( shared_session: Final = ctx.shared_session stream: Final = ctx.stream timeout: Final = ctx.timeout + absorb_shared_affinity_param(litellm_params, ctx.optional_params) optional_params: Final = ( provider_config.map_extra_body_params(optional_params=ctx.optional_params, model=model) if isinstance(provider_config, litellm.FireworksAIConfig) diff --git a/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py b/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py index 1fbc7ab1303..0ae54599710 100644 --- a/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py +++ b/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py @@ -10,6 +10,7 @@ from litellm.constants import SESSION_ID_GENERATED_METADATA_KEY from litellm.llms.custom_httpx.http_handler import HTTPHandler from litellm.llms.fireworks_ai.chat.transformation import FireworksAIConfig from litellm.llms.fireworks_ai.common_utils import get_fireworks_session_id +from litellm.llms.fireworks_ai.common_utils import absorb_shared_affinity_param from litellm.types.utils import ( ChatCompletionMessageToolCall, Function, @@ -287,6 +288,97 @@ def test_get_fireworks_session_id_explicit_overrides_shared(): }, } assert get_fireworks_session_id(litellm_params, {}) == "explicit-session" + + +def test_absorb_shared_affinity_param_moves_top_level_flag_into_metadata(): + """A top-level `fireworks_shared_session_affinity` kwarg is a LiteLLM routing hint, + not a Fireworks API field: if left in optional_params it would be serialized into the + request body and rejected by Fireworks ("Extra inputs are not permitted").""" + litellm_params: dict = {"metadata": {"user_api_key_hash": "stable-user-hash"}} + optional_params: dict = {"fireworks_shared_session_affinity": True, "temperature": 0.5} + + absorb_shared_affinity_param(litellm_params, optional_params) + + assert litellm_params["metadata"]["fireworks_shared_session_affinity"] is True + assert "fireworks_shared_session_affinity" not in optional_params + assert optional_params["temperature"] == 0.5 # untouched + + +def test_absorb_shared_affinity_param_creates_metadata_when_missing(): + litellm_params: dict = {} + optional_params: dict = {"fireworks_shared_session_affinity": True} + + absorb_shared_affinity_param(litellm_params, optional_params) + + assert litellm_params["metadata"] == {"fireworks_shared_session_affinity": True} + assert "fireworks_shared_session_affinity" not in optional_params + + +def test_absorb_shared_affinity_param_noop_when_absent(): + litellm_params: dict = {"metadata": {"session_id": "s1"}} + optional_params: dict = {"temperature": 0.5} + + absorb_shared_affinity_param(litellm_params, optional_params) + + assert litellm_params["metadata"] == {"session_id": "s1"} + + +def test_absorb_shared_affinity_param_reads_extra_body_nested_flag(): + """OpenAI-compatible param handling stashes unknown top-level kwargs inside + optional_params.extra_body; the absorb helper must read and strip it there too.""" + litellm_params: dict = {"metadata": {"user_api_key_hash": "stable-user-hash"}} + optional_params: dict = {"extra_body": {"fireworks_shared_session_affinity": True}} + + absorb_shared_affinity_param(litellm_params, optional_params) + + assert litellm_params["metadata"]["fireworks_shared_session_affinity"] is True + assert optional_params["extra_body"] == {} + + +def test_top_level_shared_affinity_kwarg_sets_header_without_body_leak(): + """Full pipeline: litellm.completion(fireworks_shared_session_affinity=True) must set + x-session-affinity (hashed off the Fireworks credential) and must NOT put the flag + into the request body — Fireworks rejects unknown body fields.""" + import hashlib + + from litellm.llms.custom_httpx.http_handler import HTTPHandler as _HTTPHandler + + model = "accounts/fireworks/models/llama-v3p1-8b-instruct" + reply = { + "id": "c1", "object": "chat.completion", "created": 1, "model": model, + "choices": [{"index": 0, "message": {"role": "assistant", "content": "Hi"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } + raw_response = MagicMock() + raw_response.status_code = 200 + raw_response.headers = {} + raw_response.text = json.dumps(reply) + raw_response.json = lambda: reply + + captured: dict = {} + client = MagicMock(spec=_HTTPHandler) + + def post(*args, **kwargs): + captured["headers"] = dict(kwargs.get("headers") or {}) + captured["body"] = json.loads(kwargs.get("data") or "{}") + return raw_response + + client.post.side_effect = post + + litellm.completion( + model=f"fireworks_ai/{model}", + messages=[{"role": "user", "content": "hi"}], + api_key="fw-test-key", + client=client, + fireworks_shared_session_affinity=True, + ) + + headers = {k.lower(): v for k, v in captured["headers"].items()} + assert headers.get("x-session-affinity") == hashlib.sha256(b"fw-test-key").hexdigest() + assert "fireworks_shared_session_affinity" not in captured["body"] + + +def test_handle_message_content_with_tool_calls(): config = FireworksAIConfig() message = Message( content='{"type": "function", "name": "get_current_weather", "parameters": {"location": "Boston, MA", "unit": "fahrenheit"}}', From dbf293c01508f55455a6dfe7924b98d733048d00 Mon Sep 17 00:00:00 2001 From: "andy.huangyh" Date: Fri, 9 Oct 2026 17:05:07 +1100 Subject: [PATCH 3/3] fix(fireworks_ai): make shared session affinity an owned kwarg, not a body field The flag was routed into optional_params and stripped by provider code, which broke on three fronts: it mutated function parameters, put Fireworks-specific logic in main.py, and leaked into the text-completion request body. Register it as a LiteLLM-owned connection kwarg instead, the same way provider_affinity_header is plumbed, so the shared kwarg filter drops it from every provider request body before a provider ever sees it. Restore the one-argument get_fireworks_session_id signature the existing proxy tests call, hash the credential with usedforsecurity=False, and suppress the CodeQL finding the way the OCI and HIBP cases are. The user-key branch now hashes user_api_key_hash too: the proxy leaves opaque custom-auth keys and OAuth bearer tokens unhashed, so forwarding that value would send a raw proxy credential to Fireworks in a request header. --- .github/workflows/codeql.yml | 10 +- .../litellm_core_utils/get_litellm_params.py | 1 + .../llms/fireworks_ai/chat/transformation.py | 32 -- litellm/llms/fireworks_ai/common_utils.py | 99 +++--- .../fireworks_ai/completion/transformation.py | 1 - litellm/main.py | 8 +- litellm/types/litellm_params.py | 1 + .../test_fireworks_ai_chat_transformation.py | 290 +++++++----------- tests/unit/types/test_litellm_params.py | 1 + 9 files changed, 170 insertions(+), 273 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 9a85ced57f6..c6369151ca8 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -72,13 +72,21 @@ jobs: # a lookup key into the haveibeenpwned range API (the protocol mandates # SHA-1) and the digest itself never leaves the proxy beyond its first 5 # characters. - - name: Filter SARIF (OCI sha256, HIBP sha1) + # Finally it fires on the Fireworks shared-session affinity token in + # litellm/llms/fireworks_ai/common_utils.py, where a credential is hashed so + # concurrent requests on one Fireworks key map to a stable x-session-affinity + # value and their shared prompt prefix hits the replica cache. Nothing is + # authenticated by that digest; it is a cache-partition key, and SHA-256 is + # kept because two credentials must not collide onto one token. CodeQL flags + # it because the flow starts at a lookup key literally named `api_key`. + - name: Filter SARIF (OCI sha256, HIBP sha1, Fireworks affinity token) if: matrix.language == 'python' uses: advanced-security/filter-sarif@2da736ff05ef065cb2894ac6892e47b5eac2c3c0 # v1.1 with: patterns: | -litellm/llms/oci/common_utils.py:py/weak-sensitive-data-hashing -litellm/proxy/auth/password_policy.py:py/weak-sensitive-data-hashing + -litellm/llms/fireworks_ai/common_utils.py:py/weak-sensitive-data-hashing input: sarif-results/python.sarif output: sarif-results/python.sarif diff --git a/litellm/litellm_core_utils/get_litellm_params.py b/litellm/litellm_core_utils/get_litellm_params.py index b8441d2bc6d..646510f5202 100644 --- a/litellm/litellm_core_utils/get_litellm_params.py +++ b/litellm/litellm_core_utils/get_litellm_params.py @@ -67,6 +67,7 @@ OPTIONAL_KWARGS_KEYS: Final = ( "otpm", "use_xai_oauth", PROVIDER_AFFINITY_HEADER_KWARG_KEY, + "fireworks_shared_session_affinity", } ) | AWS_CREDENTIAL_KWARGS_KEYS diff --git a/litellm/llms/fireworks_ai/chat/transformation.py b/litellm/llms/fireworks_ai/chat/transformation.py index 680b2dd88e0..15049e71bbc 100644 --- a/litellm/llms/fireworks_ai/chat/transformation.py +++ b/litellm/llms/fireworks_ai/chat/transformation.py @@ -1,6 +1,5 @@ import json from collections.abc import AsyncIterator, Iterator, Mapping -from types import MappingProxyType from typing import TYPE_CHECKING, Final, Literal, cast import httpx @@ -44,7 +43,6 @@ from ..common_utils import ( FIREROUTER, FireworksAIException, FireworksAIMixin, - absorb_shared_affinity_param, resolve_fireworks_resource_name, ) @@ -224,7 +222,6 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig): api_key=api_key, api_base=api_base, ) - absorb_shared_affinity_param(litellm_params, optional_params) return self._add_session_affinity_header(validated_headers, litellm_params) def get_supported_openai_params(self, model: str): @@ -267,7 +264,6 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig): "prediction", "stream_options", "sampling_mask", - "fireworks_shared_session_affinity", ] # Only add tools for models that support function calling @@ -654,29 +650,6 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig): return provider_specific_model_info - def transform_extra_body( - self, - extra_body: Mapping[str, object], - request: Mapping[str, object], - model: str, - litellm_params: Mapping[str, object], - ) -> Mapping[str, object]: - """ - Pull `fireworks_shared_session_affinity` out of extra_body before the handler - merges extra_body into the request body. The flag is a LiteLLM routing hint, not - a Fireworks API field ("Extra inputs are not permitted"); it must only reach - Fireworks as the x-session-affinity header, which the affinity logic derives - from litellm_params.metadata. - """ - filtered: Final = {k: v for k, v in extra_body.items() if k != "fireworks_shared_session_affinity"} - if "fireworks_shared_session_affinity" not in extra_body: - return super().transform_extra_body( - extra_body=extra_body, request=request, model=model, litellm_params=litellm_params - ) - if filtered: - return MappingProxyType(filtered) - return MappingProxyType({}) - def transform_request( self, model: str, @@ -689,11 +662,6 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig): messages = self._transform_messages_helper( messages=messages, model=resolved_model, litellm_params=litellm_params ) - # `fireworks_shared_session_affinity` is a LiteLLM routing hint, not a Fireworks API - # field: it must not reach the request body ("Extra inputs are not permitted") — - # stash it in litellm_params metadata where the affinity-header logic reads it. - absorb_shared_affinity_param(litellm_params, optional_params) - optional_params.pop("fireworks_shared_session_affinity", None) if "tools" in optional_params and optional_params["tools"] is not None: tools: Final = self._transform_tools(tools=optional_params["tools"]) optional_params["tools"] = tools diff --git a/litellm/llms/fireworks_ai/common_utils.py b/litellm/llms/fireworks_ai/common_utils.py index b7a156d7cb0..784377e1ccd 100644 --- a/litellm/llms/fireworks_ai/common_utils.py +++ b/litellm/llms/fireworks_ai/common_utils.py @@ -1,7 +1,7 @@ import hashlib from collections.abc import Mapping from types import MappingProxyType -from typing import Final +from typing import Final, cast from httpx import Headers @@ -16,16 +16,24 @@ class FireworksAIException(BaseLLMException): pass -def get_fireworks_session_id( - litellm_params: Mapping[str, object], headers: Mapping[str, str] -) -> str | None: +SHARED_SESSION_AFFINITY_PARAM: Final = "fireworks_shared_session_affinity" + + +def _lookup(mapping: object, key: str) -> object | None: + if isinstance(mapping, Mapping): + return cast("Mapping[str, object]", mapping).get(key) + return None + + +def get_fireworks_session_id(litellm_params: Mapping[str, object]) -> str | None: """ Session id to send as `x-session-affinity`, or None when the caller gave none. Priority: 1. Explicit `litellm_session_id` or `session_id` in litellm_params 2. Explicit `session_id` in metadata - 3. Shared affinity (if enabled): hash of the API key or LiteLLM user key + 3. Shared affinity, when `fireworks_shared_session_affinity` is set: a stable digest of + the caller's LiteLLM user key (proxy) or Fireworks credential (direct SDK) Deliberately does not fall back to `litellm_trace_id`, and ignores session ids the proxy generated for a request that had none: both are per request, so using them @@ -34,68 +42,50 @@ def get_fireworks_session_id( params: Final = litellm_params metadata: Final = params.get("metadata") - # 1 & 2. Explicit session IDs - has_generated_session_id: Final = isinstance(metadata, Mapping) and metadata.get( - SESSION_ID_GENERATED_METADATA_KEY - ) + has_generated_session_id: Final = _lookup(metadata, SESSION_ID_GENERATED_METADATA_KEY) if not has_generated_session_id: for key in ("litellm_session_id", "session_id"): value = params.get(key) if value: return str(value) - if isinstance(metadata, Mapping): - value = metadata.get("session_id") - if value: - return str(value) + session_from_metadata: Final = _lookup(metadata, "session_id") + if session_from_metadata: + return str(session_from_metadata) - # 3. Shared affinity - shared_affinity: Final = params.get("fireworks_shared_session_affinity") or ( - metadata.get("fireworks_shared_session_affinity") - if isinstance(metadata, Mapping) - else None + shared_affinity: Final = params.get(SHARED_SESSION_AFFINITY_PARAM) or _lookup( + metadata, SHARED_SESSION_AFFINITY_PARAM ) - if shared_affinity: - # Try LiteLLM user API key hash first (proxy-level isolation) - user_key_hash = ( - metadata.get("user_api_key_hash") if isinstance(metadata, Mapping) else None - ) - if user_key_hash: - return f"litellm-user-{user_key_hash}" + if not shared_affinity: + return None - # Fall back to hashing the Fireworks API key (account-level pooling) - auth = headers.get("Authorization", "") - if auth.startswith("Bearer "): - api_key = auth.removeprefix("Bearer ") - if api_key: - return hashlib.sha256(api_key.encode()).hexdigest() + user_key_hash: Final = _lookup(metadata, "user_api_key_hash") + if isinstance(user_key_hash, str) and user_key_hash: + # `user_api_key_hash` is not guaranteed to be a digest: the proxy leaves opaque + # custom-auth keys and OAuth bearer tokens unhashed, so hashing it here is what + # keeps a raw proxy credential from riding to Fireworks in a request header. + return f"litellm-user-{shared_affinity_token(user_key_hash)}" - return None + configured: Final = params.get("api_key") + credential: Final = configured if isinstance(configured, str) and configured else resolve_fireworks_api_key(None) + if credential is None: + return None + return shared_affinity_token(credential) -def absorb_shared_affinity_param(litellm_params: dict, optional_params: Mapping[str, object]) -> None: +def shared_affinity_token(credential: str) -> str: """ - Move a top-level `fireworks_shared_session_affinity` kwarg into litellm_params metadata. + Opaque, stable routing token for one credential. - The flag is a LiteLLM routing hint, not a Fireworks API field: left in optional_params - (or in optional_params.extra_body, where openai-compatible param handling stashes - unknown kwargs) it would be serialized into the request body and rejected by Fireworks. - Reads both the top-level param and its extra_body-nested form; strips it from both. + This is a cache-isolation identifier, not a password hash: it is sent to Fireworks as + `x-session-affinity` so concurrent requests on one credential land on one replica and + their shared prompt prefix hits the cache. It is never used to authenticate, and it + leaks nothing beyond a digest that Fireworks could compute itself from its own key. + Collision resistance is however load bearing: two distinct credentials must not share + a token, or one account's cache becomes readable by another, so the digest must stay a + strong hash. `usedforsecurity=False` declares that no authentication property rests on + it (and keeps it available under FIPS builds). """ - value: object = optional_params.get("fireworks_shared_session_affinity") - extra_body: Final = optional_params.get("extra_body") - if value is None and isinstance(extra_body, Mapping): - value = extra_body.get("fireworks_shared_session_affinity") - if value is None: - return - if isinstance(extra_body, dict) and "fireworks_shared_session_affinity" in extra_body: - extra_body.pop("fireworks_shared_session_affinity", None) - if "fireworks_shared_session_affinity" in optional_params: - optional_params.pop("fireworks_shared_session_affinity", None) - metadata: Final = litellm_params.get("metadata") - if isinstance(metadata, dict): - metadata.setdefault("fireworks_shared_session_affinity", value) - else: - litellm_params["metadata"] = {"fireworks_shared_session_affinity": value} + return hashlib.sha256(credential.encode(), usedforsecurity=False).hexdigest() def with_fireworks_session_affinity( @@ -103,7 +93,7 @@ def with_fireworks_session_affinity( ) -> Mapping[str, str]: if any(key.lower() == "x-session-affinity" for key in headers): return headers - session_id: Final = get_fireworks_session_id(litellm_params, headers) + session_id: Final = get_fireworks_session_id(litellm_params) if not session_id: return headers return MappingProxyType({**headers, "x-session-affinity": session_id}) @@ -167,7 +157,6 @@ class FireworksAIMixin: content_type_header: Final = ( {} if any(key.lower() == "content-type" for key in auth_headers) else {"Content-Type": "application/json"} ) - absorb_shared_affinity_param(litellm_params, optional_params) return self._add_session_affinity_header({**auth_headers, **content_type_header}, litellm_params) def _add_session_affinity_header(self, headers: dict, litellm_params: dict) -> dict: diff --git a/litellm/llms/fireworks_ai/completion/transformation.py b/litellm/llms/fireworks_ai/completion/transformation.py index 41cfad89d69..4f0e302003a 100644 --- a/litellm/llms/fireworks_ai/completion/transformation.py +++ b/litellm/llms/fireworks_ai/completion/transformation.py @@ -39,7 +39,6 @@ class FireworksAITextCompletionConfig(FireworksAIMixin, BaseTextCompletionConfig "response_format", "stream", "user", - "fireworks_shared_session_affinity", ] def map_openai_params( diff --git a/litellm/main.py b/litellm/main.py index c152d781e84..620ed8575ae 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -119,7 +119,6 @@ from litellm.llms.base_llm.base_model_iterator import ( from litellm.llms.bedrock.common_utils import BedrockModelInfo from litellm.llms.cohere.common_utils import CohereModelInfo from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler, http2_enabled -from litellm.llms.fireworks_ai.common_utils import absorb_shared_affinity_param from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config from litellm.llms.openai_like.json_loader import JSONProviderRegistry from litellm.llms.vertex_ai.common_utils import ( @@ -1865,7 +1864,6 @@ def _complete_fireworks_ai( shared_session: Final = ctx.shared_session stream: Final = ctx.stream timeout: Final = ctx.timeout - absorb_shared_affinity_param(litellm_params, ctx.optional_params) optional_params: Final = ( provider_config.map_extra_body_params(optional_params=ctx.optional_params, model=model) if isinstance(provider_config, litellm.FireworksAIConfig) @@ -5704,7 +5702,11 @@ def completion( gigachat_access_token=kwargs.get("gigachat_access_token"), **{ key: kwargs[key] - for key in (*AWS_CREDENTIAL_KWARGS_KEYS, PROVIDER_AFFINITY_HEADER_KWARG_KEY) + for key in ( + *AWS_CREDENTIAL_KWARGS_KEYS, + PROVIDER_AFFINITY_HEADER_KWARG_KEY, + "fireworks_shared_session_affinity", + ) if key in kwargs }, ) diff --git a/litellm/types/litellm_params.py b/litellm/types/litellm_params.py index 20214078852..5d827b86d44 100644 --- a/litellm/types/litellm_params.py +++ b/litellm/types/litellm_params.py @@ -87,6 +87,7 @@ class ProviderConnection: litellm_credential_name: str | None = None configurable_clientside_auth_params: "Sequence[str | ConfigurableClientsideParamsCustomAuth] | None" = None use_xai_oauth: bool | None = None + fireworks_shared_session_affinity: bool | None = None @dataclass(frozen=True, slots=True, kw_only=True) diff --git a/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py b/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py index 0ae54599710..fadde6be240 100644 --- a/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py +++ b/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py @@ -9,8 +9,7 @@ import litellm from litellm.constants import SESSION_ID_GENERATED_METADATA_KEY from litellm.llms.custom_httpx.http_handler import HTTPHandler from litellm.llms.fireworks_ai.chat.transformation import FireworksAIConfig -from litellm.llms.fireworks_ai.common_utils import get_fireworks_session_id -from litellm.llms.fireworks_ai.common_utils import absorb_shared_affinity_param +from litellm.llms.fireworks_ai.common_utils import get_fireworks_session_id, shared_affinity_token from litellm.types.utils import ( ChatCompletionMessageToolCall, Function, @@ -219,10 +218,7 @@ def test_validate_environment_raises_without_api_key(monkeypatch): def test_get_fireworks_session_id_prefers_litellm_session_id_over_trace_id(): assert ( - get_fireworks_session_id( - {"litellm_session_id": "session-123", "litellm_trace_id": "trace-123"}, - {}, - ) + get_fireworks_session_id({"litellm_session_id": "session-123", "litellm_trace_id": "trace-123"}) == "session-123" ) @@ -235,117 +231,79 @@ def test_get_fireworks_session_id_ignores_proxy_generated_session_id_without_sha "litellm_session_id": "generated-1", "litellm_trace_id": "generated-1", "metadata": {"session_id": "generated-1", SESSION_ID_GENERATED_METADATA_KEY: True}, - }, - {}, + } ) is None ) def test_get_fireworks_session_id_uses_shared_affinity_even_with_generated_id(): - """With shared affinity, we use the stable hash even if the proxy generated a fresh ID.""" + """With shared affinity, the stable token wins even when the proxy generated a fresh id.""" litellm_params = { "litellm_session_id": "generated-1", + "fireworks_shared_session_affinity": True, "metadata": { "session_id": "generated-1", SESSION_ID_GENERATED_METADATA_KEY: True, - "fireworks_shared_session_affinity": True, "user_api_key_hash": "stable-user-hash", }, } - assert get_fireworks_session_id(litellm_params, {}) == "litellm-user-stable-user-hash" + assert get_fireworks_session_id(litellm_params) == f"litellm-user-{shared_affinity_token('stable-user-hash')}" -def test_get_fireworks_session_id_falls_back_to_api_key_hash(): - litellm_params = { - "metadata": { - "fireworks_shared_session_affinity": True, - }, - } - headers = {"Authorization": "Bearer fw-api-key"} - import hashlib - - expected = hashlib.sha256(b"fw-api-key").hexdigest() - assert get_fireworks_session_id(litellm_params, headers) == expected - - -def test_get_fireworks_session_id_uses_top_level_shared_affinity(): +def test_get_fireworks_session_id_falls_back_to_credential_token(): litellm_params = { "fireworks_shared_session_affinity": True, - "metadata": { - "user_api_key_hash": "stable-user-hash", - }, + "api_key": "fw-api-key", + "metadata": {}, } - assert get_fireworks_session_id(litellm_params, {}) == "litellm-user-stable-user-hash" + assert get_fireworks_session_id(litellm_params) == shared_affinity_token("fw-api-key") + + +def test_get_fireworks_session_id_uses_metadata_nested_shared_affinity(): + litellm_params = {"metadata": {"fireworks_shared_session_affinity": True, "user_api_key_hash": "stable-user-hash"}} + assert get_fireworks_session_id(litellm_params) == f"litellm-user-{shared_affinity_token('stable-user-hash')}" + + +def test_get_fireworks_session_id_never_emits_a_raw_proxy_credential(): + """A proxy user key is not always stored hashed: opaque custom-auth keys and OAuth + bearer tokens pass through `_safe_hash_litellm_api_key` unchanged, so the affinity + token must hash the value again rather than forwarding it verbatim.""" + raw_token = "opaque-custom-auth-token-abc123" + litellm_params = {"metadata": {"fireworks_shared_session_affinity": True, "user_api_key_hash": raw_token}} + token = get_fireworks_session_id(litellm_params) + assert token is not None + assert raw_token not in token + assert token == f"litellm-user-{shared_affinity_token(raw_token)}" def test_get_fireworks_session_id_explicit_overrides_shared(): litellm_params = { "litellm_session_id": "explicit-session", - "metadata": { - "fireworks_shared_session_affinity": True, - "user_api_key_hash": "stable-user-hash", - }, + "metadata": {"fireworks_shared_session_affinity": True, "user_api_key_hash": "stable-user-hash"}, } - assert get_fireworks_session_id(litellm_params, {}) == "explicit-session" + assert get_fireworks_session_id(litellm_params) == "explicit-session" -def test_absorb_shared_affinity_param_moves_top_level_flag_into_metadata(): - """A top-level `fireworks_shared_session_affinity` kwarg is a LiteLLM routing hint, - not a Fireworks API field: if left in optional_params it would be serialized into the - request body and rejected by Fireworks ("Extra inputs are not permitted").""" - litellm_params: dict = {"metadata": {"user_api_key_hash": "stable-user-hash"}} - optional_params: dict = {"fireworks_shared_session_affinity": True, "temperature": 0.5} - - absorb_shared_affinity_param(litellm_params, optional_params) - - assert litellm_params["metadata"]["fireworks_shared_session_affinity"] is True - assert "fireworks_shared_session_affinity" not in optional_params - assert optional_params["temperature"] == 0.5 # untouched +def test_get_fireworks_session_id_is_none_without_the_flag_or_a_credential(): + assert get_fireworks_session_id({"metadata": {"user_api_key_hash": "stable-user-hash"}}) is None -def test_absorb_shared_affinity_param_creates_metadata_when_missing(): - litellm_params: dict = {} - optional_params: dict = {"fireworks_shared_session_affinity": True} - - absorb_shared_affinity_param(litellm_params, optional_params) - - assert litellm_params["metadata"] == {"fireworks_shared_session_affinity": True} - assert "fireworks_shared_session_affinity" not in optional_params - - -def test_absorb_shared_affinity_param_noop_when_absent(): - litellm_params: dict = {"metadata": {"session_id": "s1"}} - optional_params: dict = {"temperature": 0.5} - - absorb_shared_affinity_param(litellm_params, optional_params) - - assert litellm_params["metadata"] == {"session_id": "s1"} - - -def test_absorb_shared_affinity_param_reads_extra_body_nested_flag(): - """OpenAI-compatible param handling stashes unknown top-level kwargs inside - optional_params.extra_body; the absorb helper must read and strip it there too.""" - litellm_params: dict = {"metadata": {"user_api_key_hash": "stable-user-hash"}} - optional_params: dict = {"extra_body": {"fireworks_shared_session_affinity": True}} - - absorb_shared_affinity_param(litellm_params, optional_params) - - assert litellm_params["metadata"]["fireworks_shared_session_affinity"] is True - assert optional_params["extra_body"] == {} +def test_shared_affinity_token_is_stable_and_credential_specific(): + assert shared_affinity_token("fw-a") == shared_affinity_token("fw-a") + assert shared_affinity_token("fw-a") != shared_affinity_token("fw-b") def test_top_level_shared_affinity_kwarg_sets_header_without_body_leak(): """Full pipeline: litellm.completion(fireworks_shared_session_affinity=True) must set x-session-affinity (hashed off the Fireworks credential) and must NOT put the flag - into the request body — Fireworks rejects unknown body fields.""" - import hashlib - - from litellm.llms.custom_httpx.http_handler import HTTPHandler as _HTTPHandler - + into the request body, which Fireworks rejects.""" model = "accounts/fireworks/models/llama-v3p1-8b-instruct" reply = { - "id": "c1", "object": "chat.completion", "created": 1, "model": model, + "id": "c1", + "object": "chat.completion", + "created": 1, + "model": model, "choices": [{"index": 0, "message": {"role": "assistant", "content": "Hi"}, "finish_reason": "stop"}], "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, } @@ -356,7 +314,7 @@ def test_top_level_shared_affinity_kwarg_sets_header_without_body_leak(): raw_response.json = lambda: reply captured: dict = {} - client = MagicMock(spec=_HTTPHandler) + client = MagicMock(spec=HTTPHandler) def post(*args, **kwargs): captured["headers"] = dict(kwargs.get("headers") or {}) @@ -374,10 +332,47 @@ def test_top_level_shared_affinity_kwarg_sets_header_without_body_leak(): ) headers = {k.lower(): v for k, v in captured["headers"].items()} - assert headers.get("x-session-affinity") == hashlib.sha256(b"fw-test-key").hexdigest() + assert headers.get("x-session-affinity") == shared_affinity_token("fw-test-key") assert "fireworks_shared_session_affinity" not in captured["body"] +def test_shared_affinity_kwarg_is_absent_from_the_request_body(): + """A run with the flag off must not carry the field anywhere in the request body, and a + run with it on must reach the same body, so the flag is never serialized to Fireworks.""" + model = "accounts/fireworks/models/llama-v3p1-8b-instruct" + reply = { + "id": "c1", + "object": "chat.completion", + "created": 1, + "model": model, + "choices": [{"index": 0, "message": {"role": "assistant", "content": "Hi"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } + raw_response = MagicMock() + raw_response.status_code = 200 + raw_response.headers = {} + raw_response.text = json.dumps(reply) + raw_response.json = lambda: reply + + bodies: list = [] + + def run(**extra): + client = MagicMock(spec=HTTPHandler) + client.post.side_effect = lambda *a, **kw: (bodies.append(json.loads(kw.get("data") or "{}")), raw_response)[1] + litellm.completion( + model=f"fireworks_ai/{model}", + messages=[{"role": "user", "content": "hi"}], + api_key="fw-test-key", + client=client, + **extra, + ) + + run() + run(fireworks_shared_session_affinity=True) + + assert bodies[0] == bodies[1] + + def test_handle_message_content_with_tool_calls(): config = FireworksAIConfig() message = Message( @@ -410,25 +405,18 @@ def test_handle_message_content_with_tool_calls(): }, } ] - updated_message = config._handle_message_content_with_tool_calls( - message, tool_calls - ) + updated_message = config._handle_message_content_with_tool_calls(message, tool_calls) assert updated_message.tool_calls is not None assert len(updated_message.tool_calls) == 1 assert updated_message.tool_calls[0].function.name == "get_current_weather" - assert ( - updated_message.tool_calls[0].function.arguments - == expected_tool_call.function.arguments - ) + assert updated_message.tool_calls[0].function.arguments == expected_tool_call.function.arguments def test_get_supported_openai_params_reasoning_effort(): """Test that reasoning_effort is only included in supported params for models that support it.""" config = FireworksAIConfig() - supported_params = config.get_supported_openai_params( - "fireworks_ai/accounts/fireworks/models/glm-5p1" - ) + supported_params = config.get_supported_openai_params("fireworks_ai/accounts/fireworks/models/glm-5p1") assert "reasoning_effort" in supported_params assert "thinking" in supported_params @@ -443,9 +431,7 @@ def test_get_supported_openai_params_parallel_tool_calls(): """Test that parallel_tool_calls is included for models that support function calling.""" config = FireworksAIConfig() - supported_params = config.get_supported_openai_params( - "fireworks_ai/accounts/fireworks/models/glm-5p1" - ) + supported_params = config.get_supported_openai_params("fireworks_ai/accounts/fireworks/models/glm-5p1") assert "parallel_tool_calls" in supported_params assert "tools" in supported_params assert "tool_choice" in supported_params @@ -459,9 +445,7 @@ def test_get_supported_openai_params_parallel_tool_calls(): def test_get_supported_openai_params_short_model_name_resolves_account_prefixed_entry(): config = FireworksAIConfig() - supported_params = config.get_supported_openai_params( - "fireworks_ai/deepseek-v4-pro-0813" - ) + supported_params = config.get_supported_openai_params("fireworks_ai/deepseek-v4-pro-0813") assert "tool_choice" in supported_params assert "reasoning_effort" in supported_params @@ -470,9 +454,7 @@ def test_get_supported_openai_params_short_model_name_resolves_account_prefixed_ def test_get_supported_openai_params_preserves_generic_reasoning_fallback(): config = FireworksAIConfig() - supported_params = config.get_supported_openai_params( - "fireworks_ai/accounts/fireworks/models/glm-5p3-flash" - ) + supported_params = config.get_supported_openai_params("fireworks_ai/accounts/fireworks/models/glm-5p3-flash") assert "reasoning_effort" in supported_params @@ -539,14 +521,10 @@ def test_get_models_url_no_double_v1(api_base, expected_url_prefix): mock_response = MagicMock() mock_response.status_code = 200 - mock_response.json.return_value = { - "models": [{"name": "accounts/fireworks/models/llama-v3-70b"}] - } + mock_response.json.return_value = {"models": [{"name": "accounts/fireworks/models/llama-v3-70b"}]} with ( - patch( - "litellm.module_level_client.get", return_value=mock_response - ) as mock_get, + patch("litellm.module_level_client.get", return_value=mock_response) as mock_get, patch( "litellm.llms.fireworks_ai.chat.transformation.get_secret_str", side_effect=lambda key: { @@ -558,13 +536,9 @@ def test_get_models_url_no_double_v1(api_base, expected_url_prefix): ): result = config.get_models(api_key="test-key", api_base=api_base) - called_url = mock_get.call_args.kwargs.get("url") or mock_get.call_args[1].get( - "url", "" - ) + called_url = mock_get.call_args.kwargs.get("url") or mock_get.call_args[1].get("url", "") assert "/v1/v1/" not in called_url, f"Double /v1/ detected in URL: {called_url}" - assert called_url.startswith( - expected_url_prefix - ), f"URL {called_url} does not start with {expected_url_prefix}" + assert called_url.startswith(expected_url_prefix), f"URL {called_url} does not start with {expected_url_prefix}" assert result == ["fireworks_ai/accounts/fireworks/models/llama-v3-70b"] @@ -592,9 +566,7 @@ def test_transform_messages_helper_removes_provider_specific_fields(): }, ] # Call helper - out = config._transform_messages_helper( - messages, model="fireworks/test", litellm_params={} - ) + out = config._transform_messages_helper(messages, model="fireworks/test", litellm_params={}) for msg in out: assert "provider_specific_fields" not in msg @@ -615,15 +587,11 @@ def test_transform_messages_helper_strips_thinking_blocks_but_keeps_reasoning_co { "role": "assistant", "content": "I can help.", - "thinking_blocks": [ - {"type": "thinking", "thinking": "internal", "signature": ""} - ], + "thinking_blocks": [{"type": "thinking", "thinking": "internal", "signature": ""}], "reasoning_content": "internal", }, ] - out = config._transform_messages_helper( - messages, model="accounts/fireworks/models/glm-5p1", litellm_params={} - ) + out = config._transform_messages_helper(messages, model="accounts/fireworks/models/glm-5p1", litellm_params={}) assert "thinking_blocks" not in out[1] assert out[1]["reasoning_content"] == "internal" assert out[1]["content"] == "I can help." @@ -1101,9 +1069,7 @@ def test_transform_messages_helper_rejects_file_blocks(): litellm.BadRequestError, match="Fireworks AI chat completions does not support file content blocks", ): - config._transform_messages_helper( - messages, model="accounts/fireworks/models/kimi-k2p6", litellm_params={} - ) + config._transform_messages_helper(messages, model="accounts/fireworks/models/kimi-k2p6", litellm_params={}) def test_transform_messages_helper_rejects_non_vision_image_inputs(): @@ -1115,18 +1081,14 @@ def test_transform_messages_helper_rejects_non_vision_image_inputs(): {"type": "text", "text": "Describe this"}, { "type": "image_url", - "image_url": { - "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE=" - }, + "image_url": {"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE="}, }, ], } ] with pytest.raises(litellm.BadRequestError, match="does not support image inputs"): - config._transform_messages_helper( - messages, model="accounts/fireworks/models/glm-5p2", litellm_params={} - ) + config._transform_messages_helper(messages, model="accounts/fireworks/models/glm-5p2", litellm_params={}) def test_transform_messages_helper_allows_vision_image_inputs(): @@ -1138,9 +1100,7 @@ def test_transform_messages_helper_allows_vision_image_inputs(): {"type": "text", "text": "Describe this"}, { "type": "image_url", - "image_url": { - "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE=" - }, + "image_url": {"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE="}, }, ], } @@ -1164,9 +1124,7 @@ def test_image_inputs_not_rejected_for_fuzzy_non_vision_match(): custom_model = "accounts/myorg/models/custom-glm-5p2" assert config._get_model_cost_capability(custom_model, "supports_vision") is False - assert ( - config._get_model_cost_capability_exact(custom_model, "supports_vision") is None - ) + assert config._get_model_cost_capability_exact(custom_model, "supports_vision") is None messages = [ { @@ -1174,16 +1132,12 @@ def test_image_inputs_not_rejected_for_fuzzy_non_vision_match(): "content": [ { "type": "image_url", - "image_url": { - "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE=" - }, + "image_url": {"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE="}, }, ], } ] - out = config._transform_messages_helper( - messages, model=custom_model, litellm_params={} - ) + out = config._transform_messages_helper(messages, model=custom_model, litellm_params={}) assert out == messages @@ -1196,9 +1150,7 @@ def test_transform_messages_helper_skips_non_dict_content(): } ] - out = config._transform_messages_helper( - messages, model="accounts/fireworks/models/glm-5p2", litellm_params={} - ) + out = config._transform_messages_helper(messages, model="accounts/fireworks/models/glm-5p2", litellm_params={}) assert out == messages @@ -1438,9 +1390,7 @@ def test_streaming_surfaces_fireworks_response_fields(): surfaced: dict = {} for chunk in stream: fields = getattr(chunk, "provider_specific_fields", None) or {} - surfaced.update( - {k: v for k, v in fields.items() if k.startswith("fireworks_")} - ) + surfaced.update({k: v for k, v in fields.items() if k.startswith("fireworks_")}) assert surfaced["fireworks_token_ids"] == [[123]] assert surfaced["fireworks_raw_outputs"] == [raw_output] @@ -1493,9 +1443,7 @@ def test_transform_request_direct_route_passthrough(): def test_map_extra_body_params_translates_truncate_prompt_tokens(): config = FireworksAIConfig() - result = config.map_extra_body_params( - {"extra_body": {"truncate_prompt_tokens": 4096}}, _REASONING_MODEL - ) + result = config.map_extra_body_params({"extra_body": {"truncate_prompt_tokens": 4096}}, _REASONING_MODEL) assert result == {"prompt_truncate_len": 4096} @@ -1654,9 +1602,7 @@ def test_map_extra_body_params_non_dict_chat_template_kwargs_dropped(): def test_map_extra_body_params_guided_json(): config = FireworksAIConfig() schema = {"type": "object", "properties": {"x": {"type": "string"}}} - result = config.map_extra_body_params( - {"extra_body": {"guided_json": schema}}, _REASONING_MODEL - ) + result = config.map_extra_body_params({"extra_body": {"guided_json": schema}}, _REASONING_MODEL) assert result == { "response_format": { "type": "json_schema", @@ -1667,16 +1613,10 @@ def test_map_extra_body_params_guided_json(): def test_map_extra_body_params_guided_grammar_and_choice(): config = FireworksAIConfig() - grammar = config.map_extra_body_params( - {"extra_body": {"guided_grammar": "root ::= 'hello'"}}, _REASONING_MODEL - ) - assert grammar == { - "response_format": {"type": "grammar", "grammar": "root ::= 'hello'"} - } + grammar = config.map_extra_body_params({"extra_body": {"guided_grammar": "root ::= 'hello'"}}, _REASONING_MODEL) + assert grammar == {"response_format": {"type": "grammar", "grammar": "root ::= 'hello'"}} - choice = config.map_extra_body_params( - {"extra_body": {"guided_choice": ["yes", "no"]}}, _REASONING_MODEL - ) + choice = config.map_extra_body_params({"extra_body": {"guided_choice": ["yes", "no"]}}, _REASONING_MODEL) assert choice == { "response_format": { "type": "json_schema", @@ -1762,9 +1702,7 @@ def test_map_extra_body_params_strips_unsupported_nim_vllm_params(param, value, config = FireworksAIConfig() with caplog.at_level(logging.DEBUG): - result = config.map_extra_body_params( - {"extra_body": {param: value}}, _REASONING_MODEL - ) + result = config.map_extra_body_params({"extra_body": {param: value}}, _REASONING_MODEL) assert result == {} assert param in caplog.text @@ -1836,15 +1774,10 @@ def test_nim_vllm_extras_translated_end_to_end_in_request_body(): assert request_body["top_k"] == 40 - - def test_streaming_preserves_selected_model_for_private_accounting(): from litellm.llms.custom_httpx.http_handler import HTTPHandler - requested_route = ( - "accounts/fireworks/routers/firerouter/" - "kimi-k3/deepseek-v4-pro-0813/deepseek-v4-flash-0731" - ) + requested_route = "accounts/fireworks/routers/firerouter/kimi-k3/deepseek-v4-pro-0813/deepseek-v4-flash-0731" selected_model = "deepseek-v4-flash-0731" sse_lines = [ "data: " @@ -1898,19 +1831,14 @@ def test_streaming_preserves_selected_model_for_private_accounting(): assert chunks assert {chunk.model for chunk in chunks} == {requested_route} - assert { - chunk._hidden_params.get("provider_response_model") for chunk in chunks - } == {selected_model} + assert {chunk._hidden_params.get("provider_response_model") for chunk in chunks} == {selected_model} assembled = litellm.stream_chunk_builder(chunks=chunks) assert assembled is not None assert assembled.model == requested_route assert assembled._hidden_params["provider_response_model"] == selected_model selected_model_info = litellm.model_cost[f"fireworks_ai/{selected_model}"] - expected_cost = ( - 5 * selected_model_info["input_cost_per_token"] - + selected_model_info["output_cost_per_token"] - ) + expected_cost = 5 * selected_model_info["input_cost_per_token"] + selected_model_info["output_cost_per_token"] assert litellm.completion_cost( completion_response=assembled, custom_llm_provider="fireworks_ai", diff --git a/tests/unit/types/test_litellm_params.py b/tests/unit/types/test_litellm_params.py index ab4f6c12431..5cdf79955f2 100644 --- a/tests/unit/types/test_litellm_params.py +++ b/tests/unit/types/test_litellm_params.py @@ -85,6 +85,7 @@ CONNECTION_NAMES: Final = ( "litellm_credential_name", "configurable_clientside_auth_params", "use_xai_oauth", + "fireworks_shared_session_affinity", "aws_batch_role_arn", "s3_bucket_name", "s3_region_name",