fix(fireworks_ai): make shared session affinity an owned kwarg, not a body field

The flag was routed into optional_params and stripped by provider code, which
broke on three fronts: it mutated function parameters, put Fireworks-specific
logic in main.py, and leaked into the text-completion request body. Register it
as a LiteLLM-owned connection kwarg instead, the same way provider_affinity_header
is plumbed, so the shared kwarg filter drops it from every provider request body
before a provider ever sees it.

Restore the one-argument get_fireworks_session_id signature the existing proxy
tests call, hash the credential with usedforsecurity=False, and suppress the
CodeQL finding the way the OCI and HIBP cases are.

The user-key branch now hashes user_api_key_hash too: the proxy leaves opaque
custom-auth keys and OAuth bearer tokens unhashed, so forwarding that value would
send a raw proxy credential to Fireworks in a request header.
This commit is contained in:
andy.huangyh 2026-10-09 17:05:07 +11:00
parent cf624b5276
commit dbf293c015
9 changed files with 170 additions and 273 deletions

View file

@ -72,13 +72,21 @@ jobs:
# a lookup key into the haveibeenpwned range API (the protocol mandates
# SHA-1) and the digest itself never leaves the proxy beyond its first 5
# characters.
- name: Filter SARIF (OCI sha256, HIBP sha1)
# Finally it fires on the Fireworks shared-session affinity token in
# litellm/llms/fireworks_ai/common_utils.py, where a credential is hashed so
# concurrent requests on one Fireworks key map to a stable x-session-affinity
# value and their shared prompt prefix hits the replica cache. Nothing is
# authenticated by that digest; it is a cache-partition key, and SHA-256 is
# kept because two credentials must not collide onto one token. CodeQL flags
# it because the flow starts at a lookup key literally named `api_key`.
- name: Filter SARIF (OCI sha256, HIBP sha1, Fireworks affinity token)
if: matrix.language == 'python'
uses: advanced-security/filter-sarif@2da736ff05ef065cb2894ac6892e47b5eac2c3c0 # v1.1
with:
patterns: |
-litellm/llms/oci/common_utils.py:py/weak-sensitive-data-hashing
-litellm/proxy/auth/password_policy.py:py/weak-sensitive-data-hashing
-litellm/llms/fireworks_ai/common_utils.py:py/weak-sensitive-data-hashing
input: sarif-results/python.sarif
output: sarif-results/python.sarif

View file

@ -67,6 +67,7 @@ OPTIONAL_KWARGS_KEYS: Final = (
"otpm",
"use_xai_oauth",
PROVIDER_AFFINITY_HEADER_KWARG_KEY,
"fireworks_shared_session_affinity",
}
)
| AWS_CREDENTIAL_KWARGS_KEYS

View file

@ -1,6 +1,5 @@
import json
from collections.abc import AsyncIterator, Iterator, Mapping
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Literal, cast
import httpx
@ -44,7 +43,6 @@ from ..common_utils import (
FIREROUTER,
FireworksAIException,
FireworksAIMixin,
absorb_shared_affinity_param,
resolve_fireworks_resource_name,
)
@ -224,7 +222,6 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
api_key=api_key,
api_base=api_base,
)
absorb_shared_affinity_param(litellm_params, optional_params)
return self._add_session_affinity_header(validated_headers, litellm_params)
def get_supported_openai_params(self, model: str):
@ -267,7 +264,6 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
"prediction",
"stream_options",
"sampling_mask",
"fireworks_shared_session_affinity",
]
# Only add tools for models that support function calling
@ -654,29 +650,6 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
return provider_specific_model_info
def transform_extra_body(
self,
extra_body: Mapping[str, object],
request: Mapping[str, object],
model: str,
litellm_params: Mapping[str, object],
) -> Mapping[str, object]:
"""
Pull `fireworks_shared_session_affinity` out of extra_body before the handler
merges extra_body into the request body. The flag is a LiteLLM routing hint, not
a Fireworks API field ("Extra inputs are not permitted"); it must only reach
Fireworks as the x-session-affinity header, which the affinity logic derives
from litellm_params.metadata.
"""
filtered: Final = {k: v for k, v in extra_body.items() if k != "fireworks_shared_session_affinity"}
if "fireworks_shared_session_affinity" not in extra_body:
return super().transform_extra_body(
extra_body=extra_body, request=request, model=model, litellm_params=litellm_params
)
if filtered:
return MappingProxyType(filtered)
return MappingProxyType({})
def transform_request(
self,
model: str,
@ -689,11 +662,6 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
messages = self._transform_messages_helper(
messages=messages, model=resolved_model, litellm_params=litellm_params
)
# `fireworks_shared_session_affinity` is a LiteLLM routing hint, not a Fireworks API
# field: it must not reach the request body ("Extra inputs are not permitted") —
# stash it in litellm_params metadata where the affinity-header logic reads it.
absorb_shared_affinity_param(litellm_params, optional_params)
optional_params.pop("fireworks_shared_session_affinity", None)
if "tools" in optional_params and optional_params["tools"] is not None:
tools: Final = self._transform_tools(tools=optional_params["tools"])
optional_params["tools"] = tools

View file

@ -1,7 +1,7 @@
import hashlib
from collections.abc import Mapping
from types import MappingProxyType
from typing import Final
from typing import Final, cast
from httpx import Headers
@ -16,16 +16,24 @@ class FireworksAIException(BaseLLMException):
pass
def get_fireworks_session_id(
litellm_params: Mapping[str, object], headers: Mapping[str, str]
) -> str | None:
SHARED_SESSION_AFFINITY_PARAM: Final = "fireworks_shared_session_affinity"
def _lookup(mapping: object, key: str) -> object | None:
if isinstance(mapping, Mapping):
return cast("Mapping[str, object]", mapping).get(key)
return None
def get_fireworks_session_id(litellm_params: Mapping[str, object]) -> str | None:
"""
Session id to send as `x-session-affinity`, or None when the caller gave none.
Priority:
1. Explicit `litellm_session_id` or `session_id` in litellm_params
2. Explicit `session_id` in metadata
3. Shared affinity (if enabled): hash of the API key or LiteLLM user key
3. Shared affinity, when `fireworks_shared_session_affinity` is set: a stable digest of
the caller's LiteLLM user key (proxy) or Fireworks credential (direct SDK)
Deliberately does not fall back to `litellm_trace_id`, and ignores session ids the
proxy generated for a request that had none: both are per request, so using them
@ -34,68 +42,50 @@ def get_fireworks_session_id(
params: Final = litellm_params
metadata: Final = params.get("metadata")
# 1 & 2. Explicit session IDs
has_generated_session_id: Final = isinstance(metadata, Mapping) and metadata.get(
SESSION_ID_GENERATED_METADATA_KEY
)
has_generated_session_id: Final = _lookup(metadata, SESSION_ID_GENERATED_METADATA_KEY)
if not has_generated_session_id:
for key in ("litellm_session_id", "session_id"):
value = params.get(key)
if value:
return str(value)
if isinstance(metadata, Mapping):
value = metadata.get("session_id")
if value:
return str(value)
session_from_metadata: Final = _lookup(metadata, "session_id")
if session_from_metadata:
return str(session_from_metadata)
# 3. Shared affinity
shared_affinity: Final = params.get("fireworks_shared_session_affinity") or (
metadata.get("fireworks_shared_session_affinity")
if isinstance(metadata, Mapping)
else None
shared_affinity: Final = params.get(SHARED_SESSION_AFFINITY_PARAM) or _lookup(
metadata, SHARED_SESSION_AFFINITY_PARAM
)
if shared_affinity:
# Try LiteLLM user API key hash first (proxy-level isolation)
user_key_hash = (
metadata.get("user_api_key_hash") if isinstance(metadata, Mapping) else None
)
if user_key_hash:
return f"litellm-user-{user_key_hash}"
if not shared_affinity:
return None
# Fall back to hashing the Fireworks API key (account-level pooling)
auth = headers.get("Authorization", "")
if auth.startswith("Bearer "):
api_key = auth.removeprefix("Bearer ")
if api_key:
return hashlib.sha256(api_key.encode()).hexdigest()
user_key_hash: Final = _lookup(metadata, "user_api_key_hash")
if isinstance(user_key_hash, str) and user_key_hash:
# `user_api_key_hash` is not guaranteed to be a digest: the proxy leaves opaque
# custom-auth keys and OAuth bearer tokens unhashed, so hashing it here is what
# keeps a raw proxy credential from riding to Fireworks in a request header.
return f"litellm-user-{shared_affinity_token(user_key_hash)}"
return None
configured: Final = params.get("api_key")
credential: Final = configured if isinstance(configured, str) and configured else resolve_fireworks_api_key(None)
if credential is None:
return None
return shared_affinity_token(credential)
def absorb_shared_affinity_param(litellm_params: dict, optional_params: Mapping[str, object]) -> None:
def shared_affinity_token(credential: str) -> str:
"""
Move a top-level `fireworks_shared_session_affinity` kwarg into litellm_params metadata.
Opaque, stable routing token for one credential.
The flag is a LiteLLM routing hint, not a Fireworks API field: left in optional_params
(or in optional_params.extra_body, where openai-compatible param handling stashes
unknown kwargs) it would be serialized into the request body and rejected by Fireworks.
Reads both the top-level param and its extra_body-nested form; strips it from both.
This is a cache-isolation identifier, not a password hash: it is sent to Fireworks as
`x-session-affinity` so concurrent requests on one credential land on one replica and
their shared prompt prefix hits the cache. It is never used to authenticate, and it
leaks nothing beyond a digest that Fireworks could compute itself from its own key.
Collision resistance is however load bearing: two distinct credentials must not share
a token, or one account's cache becomes readable by another, so the digest must stay a
strong hash. `usedforsecurity=False` declares that no authentication property rests on
it (and keeps it available under FIPS builds).
"""
value: object = optional_params.get("fireworks_shared_session_affinity")
extra_body: Final = optional_params.get("extra_body")
if value is None and isinstance(extra_body, Mapping):
value = extra_body.get("fireworks_shared_session_affinity")
if value is None:
return
if isinstance(extra_body, dict) and "fireworks_shared_session_affinity" in extra_body:
extra_body.pop("fireworks_shared_session_affinity", None)
if "fireworks_shared_session_affinity" in optional_params:
optional_params.pop("fireworks_shared_session_affinity", None)
metadata: Final = litellm_params.get("metadata")
if isinstance(metadata, dict):
metadata.setdefault("fireworks_shared_session_affinity", value)
else:
litellm_params["metadata"] = {"fireworks_shared_session_affinity": value}
return hashlib.sha256(credential.encode(), usedforsecurity=False).hexdigest()
def with_fireworks_session_affinity(
@ -103,7 +93,7 @@ def with_fireworks_session_affinity(
) -> Mapping[str, str]:
if any(key.lower() == "x-session-affinity" for key in headers):
return headers
session_id: Final = get_fireworks_session_id(litellm_params, headers)
session_id: Final = get_fireworks_session_id(litellm_params)
if not session_id:
return headers
return MappingProxyType({**headers, "x-session-affinity": session_id})
@ -167,7 +157,6 @@ class FireworksAIMixin:
content_type_header: Final = (
{} if any(key.lower() == "content-type" for key in auth_headers) else {"Content-Type": "application/json"}
)
absorb_shared_affinity_param(litellm_params, optional_params)
return self._add_session_affinity_header({**auth_headers, **content_type_header}, litellm_params)
def _add_session_affinity_header(self, headers: dict, litellm_params: dict) -> dict:

View file

@ -39,7 +39,6 @@ class FireworksAITextCompletionConfig(FireworksAIMixin, BaseTextCompletionConfig
"response_format",
"stream",
"user",
"fireworks_shared_session_affinity",
]
def map_openai_params(

View file

@ -119,7 +119,6 @@ from litellm.llms.base_llm.base_model_iterator import (
from litellm.llms.bedrock.common_utils import BedrockModelInfo
from litellm.llms.cohere.common_utils import CohereModelInfo
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler, http2_enabled
from litellm.llms.fireworks_ai.common_utils import absorb_shared_affinity_param
from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
from litellm.llms.vertex_ai.common_utils import (
@ -1865,7 +1864,6 @@ def _complete_fireworks_ai(
shared_session: Final = ctx.shared_session
stream: Final = ctx.stream
timeout: Final = ctx.timeout
absorb_shared_affinity_param(litellm_params, ctx.optional_params)
optional_params: Final = (
provider_config.map_extra_body_params(optional_params=ctx.optional_params, model=model)
if isinstance(provider_config, litellm.FireworksAIConfig)
@ -5704,7 +5702,11 @@ def completion(
gigachat_access_token=kwargs.get("gigachat_access_token"),
**{
key: kwargs[key]
for key in (*AWS_CREDENTIAL_KWARGS_KEYS, PROVIDER_AFFINITY_HEADER_KWARG_KEY)
for key in (
*AWS_CREDENTIAL_KWARGS_KEYS,
PROVIDER_AFFINITY_HEADER_KWARG_KEY,
"fireworks_shared_session_affinity",
)
if key in kwargs
},
)

View file

@ -87,6 +87,7 @@ class ProviderConnection:
litellm_credential_name: str | None = None
configurable_clientside_auth_params: "Sequence[str | ConfigurableClientsideParamsCustomAuth] | None" = None
use_xai_oauth: bool | None = None
fireworks_shared_session_affinity: bool | None = None
@dataclass(frozen=True, slots=True, kw_only=True)

View file

@ -9,8 +9,7 @@ import litellm
from litellm.constants import SESSION_ID_GENERATED_METADATA_KEY
from litellm.llms.custom_httpx.http_handler import HTTPHandler
from litellm.llms.fireworks_ai.chat.transformation import FireworksAIConfig
from litellm.llms.fireworks_ai.common_utils import get_fireworks_session_id
from litellm.llms.fireworks_ai.common_utils import absorb_shared_affinity_param
from litellm.llms.fireworks_ai.common_utils import get_fireworks_session_id, shared_affinity_token
from litellm.types.utils import (
ChatCompletionMessageToolCall,
Function,
@ -219,10 +218,7 @@ def test_validate_environment_raises_without_api_key(monkeypatch):
def test_get_fireworks_session_id_prefers_litellm_session_id_over_trace_id():
assert (
get_fireworks_session_id(
{"litellm_session_id": "session-123", "litellm_trace_id": "trace-123"},
{},
)
get_fireworks_session_id({"litellm_session_id": "session-123", "litellm_trace_id": "trace-123"})
== "session-123"
)
@ -235,117 +231,79 @@ def test_get_fireworks_session_id_ignores_proxy_generated_session_id_without_sha
"litellm_session_id": "generated-1",
"litellm_trace_id": "generated-1",
"metadata": {"session_id": "generated-1", SESSION_ID_GENERATED_METADATA_KEY: True},
},
{},
}
)
is None
)
def test_get_fireworks_session_id_uses_shared_affinity_even_with_generated_id():
"""With shared affinity, we use the stable hash even if the proxy generated a fresh ID."""
"""With shared affinity, the stable token wins even when the proxy generated a fresh id."""
litellm_params = {
"litellm_session_id": "generated-1",
"fireworks_shared_session_affinity": True,
"metadata": {
"session_id": "generated-1",
SESSION_ID_GENERATED_METADATA_KEY: True,
"fireworks_shared_session_affinity": True,
"user_api_key_hash": "stable-user-hash",
},
}
assert get_fireworks_session_id(litellm_params, {}) == "litellm-user-stable-user-hash"
assert get_fireworks_session_id(litellm_params) == f"litellm-user-{shared_affinity_token('stable-user-hash')}"
def test_get_fireworks_session_id_falls_back_to_api_key_hash():
litellm_params = {
"metadata": {
"fireworks_shared_session_affinity": True,
},
}
headers = {"Authorization": "Bearer fw-api-key"}
import hashlib
expected = hashlib.sha256(b"fw-api-key").hexdigest()
assert get_fireworks_session_id(litellm_params, headers) == expected
def test_get_fireworks_session_id_uses_top_level_shared_affinity():
def test_get_fireworks_session_id_falls_back_to_credential_token():
litellm_params = {
"fireworks_shared_session_affinity": True,
"metadata": {
"user_api_key_hash": "stable-user-hash",
},
"api_key": "fw-api-key",
"metadata": {},
}
assert get_fireworks_session_id(litellm_params, {}) == "litellm-user-stable-user-hash"
assert get_fireworks_session_id(litellm_params) == shared_affinity_token("fw-api-key")
def test_get_fireworks_session_id_uses_metadata_nested_shared_affinity():
litellm_params = {"metadata": {"fireworks_shared_session_affinity": True, "user_api_key_hash": "stable-user-hash"}}
assert get_fireworks_session_id(litellm_params) == f"litellm-user-{shared_affinity_token('stable-user-hash')}"
def test_get_fireworks_session_id_never_emits_a_raw_proxy_credential():
"""A proxy user key is not always stored hashed: opaque custom-auth keys and OAuth
bearer tokens pass through `_safe_hash_litellm_api_key` unchanged, so the affinity
token must hash the value again rather than forwarding it verbatim."""
raw_token = "opaque-custom-auth-token-abc123"
litellm_params = {"metadata": {"fireworks_shared_session_affinity": True, "user_api_key_hash": raw_token}}
token = get_fireworks_session_id(litellm_params)
assert token is not None
assert raw_token not in token
assert token == f"litellm-user-{shared_affinity_token(raw_token)}"
def test_get_fireworks_session_id_explicit_overrides_shared():
litellm_params = {
"litellm_session_id": "explicit-session",
"metadata": {
"fireworks_shared_session_affinity": True,
"user_api_key_hash": "stable-user-hash",
},
"metadata": {"fireworks_shared_session_affinity": True, "user_api_key_hash": "stable-user-hash"},
}
assert get_fireworks_session_id(litellm_params, {}) == "explicit-session"
assert get_fireworks_session_id(litellm_params) == "explicit-session"
def test_absorb_shared_affinity_param_moves_top_level_flag_into_metadata():
"""A top-level `fireworks_shared_session_affinity` kwarg is a LiteLLM routing hint,
not a Fireworks API field: if left in optional_params it would be serialized into the
request body and rejected by Fireworks ("Extra inputs are not permitted")."""
litellm_params: dict = {"metadata": {"user_api_key_hash": "stable-user-hash"}}
optional_params: dict = {"fireworks_shared_session_affinity": True, "temperature": 0.5}
absorb_shared_affinity_param(litellm_params, optional_params)
assert litellm_params["metadata"]["fireworks_shared_session_affinity"] is True
assert "fireworks_shared_session_affinity" not in optional_params
assert optional_params["temperature"] == 0.5 # untouched
def test_get_fireworks_session_id_is_none_without_the_flag_or_a_credential():
assert get_fireworks_session_id({"metadata": {"user_api_key_hash": "stable-user-hash"}}) is None
def test_absorb_shared_affinity_param_creates_metadata_when_missing():
litellm_params: dict = {}
optional_params: dict = {"fireworks_shared_session_affinity": True}
absorb_shared_affinity_param(litellm_params, optional_params)
assert litellm_params["metadata"] == {"fireworks_shared_session_affinity": True}
assert "fireworks_shared_session_affinity" not in optional_params
def test_absorb_shared_affinity_param_noop_when_absent():
litellm_params: dict = {"metadata": {"session_id": "s1"}}
optional_params: dict = {"temperature": 0.5}
absorb_shared_affinity_param(litellm_params, optional_params)
assert litellm_params["metadata"] == {"session_id": "s1"}
def test_absorb_shared_affinity_param_reads_extra_body_nested_flag():
"""OpenAI-compatible param handling stashes unknown top-level kwargs inside
optional_params.extra_body; the absorb helper must read and strip it there too."""
litellm_params: dict = {"metadata": {"user_api_key_hash": "stable-user-hash"}}
optional_params: dict = {"extra_body": {"fireworks_shared_session_affinity": True}}
absorb_shared_affinity_param(litellm_params, optional_params)
assert litellm_params["metadata"]["fireworks_shared_session_affinity"] is True
assert optional_params["extra_body"] == {}
def test_shared_affinity_token_is_stable_and_credential_specific():
assert shared_affinity_token("fw-a") == shared_affinity_token("fw-a")
assert shared_affinity_token("fw-a") != shared_affinity_token("fw-b")
def test_top_level_shared_affinity_kwarg_sets_header_without_body_leak():
"""Full pipeline: litellm.completion(fireworks_shared_session_affinity=True) must set
x-session-affinity (hashed off the Fireworks credential) and must NOT put the flag
into the request body — Fireworks rejects unknown body fields."""
import hashlib
from litellm.llms.custom_httpx.http_handler import HTTPHandler as _HTTPHandler
into the request body, which Fireworks rejects."""
model = "accounts/fireworks/models/llama-v3p1-8b-instruct"
reply = {
"id": "c1", "object": "chat.completion", "created": 1, "model": model,
"id": "c1",
"object": "chat.completion",
"created": 1,
"model": model,
"choices": [{"index": 0, "message": {"role": "assistant", "content": "Hi"}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
}
@ -356,7 +314,7 @@ def test_top_level_shared_affinity_kwarg_sets_header_without_body_leak():
raw_response.json = lambda: reply
captured: dict = {}
client = MagicMock(spec=_HTTPHandler)
client = MagicMock(spec=HTTPHandler)
def post(*args, **kwargs):
captured["headers"] = dict(kwargs.get("headers") or {})
@ -374,10 +332,47 @@ def test_top_level_shared_affinity_kwarg_sets_header_without_body_leak():
)
headers = {k.lower(): v for k, v in captured["headers"].items()}
assert headers.get("x-session-affinity") == hashlib.sha256(b"fw-test-key").hexdigest()
assert headers.get("x-session-affinity") == shared_affinity_token("fw-test-key")
assert "fireworks_shared_session_affinity" not in captured["body"]
def test_shared_affinity_kwarg_is_absent_from_the_request_body():
"""A run with the flag off must not carry the field anywhere in the request body, and a
run with it on must reach the same body, so the flag is never serialized to Fireworks."""
model = "accounts/fireworks/models/llama-v3p1-8b-instruct"
reply = {
"id": "c1",
"object": "chat.completion",
"created": 1,
"model": model,
"choices": [{"index": 0, "message": {"role": "assistant", "content": "Hi"}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
}
raw_response = MagicMock()
raw_response.status_code = 200
raw_response.headers = {}
raw_response.text = json.dumps(reply)
raw_response.json = lambda: reply
bodies: list = []
def run(**extra):
client = MagicMock(spec=HTTPHandler)
client.post.side_effect = lambda *a, **kw: (bodies.append(json.loads(kw.get("data") or "{}")), raw_response)[1]
litellm.completion(
model=f"fireworks_ai/{model}",
messages=[{"role": "user", "content": "hi"}],
api_key="fw-test-key",
client=client,
**extra,
)
run()
run(fireworks_shared_session_affinity=True)
assert bodies[0] == bodies[1]
def test_handle_message_content_with_tool_calls():
config = FireworksAIConfig()
message = Message(
@ -410,25 +405,18 @@ def test_handle_message_content_with_tool_calls():
},
}
]
updated_message = config._handle_message_content_with_tool_calls(
message, tool_calls
)
updated_message = config._handle_message_content_with_tool_calls(message, tool_calls)
assert updated_message.tool_calls is not None
assert len(updated_message.tool_calls) == 1
assert updated_message.tool_calls[0].function.name == "get_current_weather"
assert (
updated_message.tool_calls[0].function.arguments
== expected_tool_call.function.arguments
)
assert updated_message.tool_calls[0].function.arguments == expected_tool_call.function.arguments
def test_get_supported_openai_params_reasoning_effort():
"""Test that reasoning_effort is only included in supported params for models that support it."""
config = FireworksAIConfig()
supported_params = config.get_supported_openai_params(
"fireworks_ai/accounts/fireworks/models/glm-5p1"
)
supported_params = config.get_supported_openai_params("fireworks_ai/accounts/fireworks/models/glm-5p1")
assert "reasoning_effort" in supported_params
assert "thinking" in supported_params
@ -443,9 +431,7 @@ def test_get_supported_openai_params_parallel_tool_calls():
"""Test that parallel_tool_calls is included for models that support function calling."""
config = FireworksAIConfig()
supported_params = config.get_supported_openai_params(
"fireworks_ai/accounts/fireworks/models/glm-5p1"
)
supported_params = config.get_supported_openai_params("fireworks_ai/accounts/fireworks/models/glm-5p1")
assert "parallel_tool_calls" in supported_params
assert "tools" in supported_params
assert "tool_choice" in supported_params
@ -459,9 +445,7 @@ def test_get_supported_openai_params_parallel_tool_calls():
def test_get_supported_openai_params_short_model_name_resolves_account_prefixed_entry():
config = FireworksAIConfig()
supported_params = config.get_supported_openai_params(
"fireworks_ai/deepseek-v4-pro-0813"
)
supported_params = config.get_supported_openai_params("fireworks_ai/deepseek-v4-pro-0813")
assert "tool_choice" in supported_params
assert "reasoning_effort" in supported_params
@ -470,9 +454,7 @@ def test_get_supported_openai_params_short_model_name_resolves_account_prefixed_
def test_get_supported_openai_params_preserves_generic_reasoning_fallback():
config = FireworksAIConfig()
supported_params = config.get_supported_openai_params(
"fireworks_ai/accounts/fireworks/models/glm-5p3-flash"
)
supported_params = config.get_supported_openai_params("fireworks_ai/accounts/fireworks/models/glm-5p3-flash")
assert "reasoning_effort" in supported_params
@ -539,14 +521,10 @@ def test_get_models_url_no_double_v1(api_base, expected_url_prefix):
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.json.return_value = {
"models": [{"name": "accounts/fireworks/models/llama-v3-70b"}]
}
mock_response.json.return_value = {"models": [{"name": "accounts/fireworks/models/llama-v3-70b"}]}
with (
patch(
"litellm.module_level_client.get", return_value=mock_response
) as mock_get,
patch("litellm.module_level_client.get", return_value=mock_response) as mock_get,
patch(
"litellm.llms.fireworks_ai.chat.transformation.get_secret_str",
side_effect=lambda key: {
@ -558,13 +536,9 @@ def test_get_models_url_no_double_v1(api_base, expected_url_prefix):
):
result = config.get_models(api_key="test-key", api_base=api_base)
called_url = mock_get.call_args.kwargs.get("url") or mock_get.call_args[1].get(
"url", ""
)
called_url = mock_get.call_args.kwargs.get("url") or mock_get.call_args[1].get("url", "")
assert "/v1/v1/" not in called_url, f"Double /v1/ detected in URL: {called_url}"
assert called_url.startswith(
expected_url_prefix
), f"URL {called_url} does not start with {expected_url_prefix}"
assert called_url.startswith(expected_url_prefix), f"URL {called_url} does not start with {expected_url_prefix}"
assert result == ["fireworks_ai/accounts/fireworks/models/llama-v3-70b"]
@ -592,9 +566,7 @@ def test_transform_messages_helper_removes_provider_specific_fields():
},
]
# Call helper
out = config._transform_messages_helper(
messages, model="fireworks/test", litellm_params={}
)
out = config._transform_messages_helper(messages, model="fireworks/test", litellm_params={})
for msg in out:
assert "provider_specific_fields" not in msg
@ -615,15 +587,11 @@ def test_transform_messages_helper_strips_thinking_blocks_but_keeps_reasoning_co
{
"role": "assistant",
"content": "I can help.",
"thinking_blocks": [
{"type": "thinking", "thinking": "internal", "signature": ""}
],
"thinking_blocks": [{"type": "thinking", "thinking": "internal", "signature": ""}],
"reasoning_content": "internal",
},
]
out = config._transform_messages_helper(
messages, model="accounts/fireworks/models/glm-5p1", litellm_params={}
)
out = config._transform_messages_helper(messages, model="accounts/fireworks/models/glm-5p1", litellm_params={})
assert "thinking_blocks" not in out[1]
assert out[1]["reasoning_content"] == "internal"
assert out[1]["content"] == "I can help."
@ -1101,9 +1069,7 @@ def test_transform_messages_helper_rejects_file_blocks():
litellm.BadRequestError,
match="Fireworks AI chat completions does not support file content blocks",
):
config._transform_messages_helper(
messages, model="accounts/fireworks/models/kimi-k2p6", litellm_params={}
)
config._transform_messages_helper(messages, model="accounts/fireworks/models/kimi-k2p6", litellm_params={})
def test_transform_messages_helper_rejects_non_vision_image_inputs():
@ -1115,18 +1081,14 @@ def test_transform_messages_helper_rejects_non_vision_image_inputs():
{"type": "text", "text": "Describe this"},
{
"type": "image_url",
"image_url": {
"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE="
},
"image_url": {"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE="},
},
],
}
]
with pytest.raises(litellm.BadRequestError, match="does not support image inputs"):
config._transform_messages_helper(
messages, model="accounts/fireworks/models/glm-5p2", litellm_params={}
)
config._transform_messages_helper(messages, model="accounts/fireworks/models/glm-5p2", litellm_params={})
def test_transform_messages_helper_allows_vision_image_inputs():
@ -1138,9 +1100,7 @@ def test_transform_messages_helper_allows_vision_image_inputs():
{"type": "text", "text": "Describe this"},
{
"type": "image_url",
"image_url": {
"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE="
},
"image_url": {"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE="},
},
],
}
@ -1164,9 +1124,7 @@ def test_image_inputs_not_rejected_for_fuzzy_non_vision_match():
custom_model = "accounts/myorg/models/custom-glm-5p2"
assert config._get_model_cost_capability(custom_model, "supports_vision") is False
assert (
config._get_model_cost_capability_exact(custom_model, "supports_vision") is None
)
assert config._get_model_cost_capability_exact(custom_model, "supports_vision") is None
messages = [
{
@ -1174,16 +1132,12 @@ def test_image_inputs_not_rejected_for_fuzzy_non_vision_match():
"content": [
{
"type": "image_url",
"image_url": {
"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE="
},
"image_url": {"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE="},
},
],
}
]
out = config._transform_messages_helper(
messages, model=custom_model, litellm_params={}
)
out = config._transform_messages_helper(messages, model=custom_model, litellm_params={})
assert out == messages
@ -1196,9 +1150,7 @@ def test_transform_messages_helper_skips_non_dict_content():
}
]
out = config._transform_messages_helper(
messages, model="accounts/fireworks/models/glm-5p2", litellm_params={}
)
out = config._transform_messages_helper(messages, model="accounts/fireworks/models/glm-5p2", litellm_params={})
assert out == messages
@ -1438,9 +1390,7 @@ def test_streaming_surfaces_fireworks_response_fields():
surfaced: dict = {}
for chunk in stream:
fields = getattr(chunk, "provider_specific_fields", None) or {}
surfaced.update(
{k: v for k, v in fields.items() if k.startswith("fireworks_")}
)
surfaced.update({k: v for k, v in fields.items() if k.startswith("fireworks_")})
assert surfaced["fireworks_token_ids"] == [[123]]
assert surfaced["fireworks_raw_outputs"] == [raw_output]
@ -1493,9 +1443,7 @@ def test_transform_request_direct_route_passthrough():
def test_map_extra_body_params_translates_truncate_prompt_tokens():
config = FireworksAIConfig()
result = config.map_extra_body_params(
{"extra_body": {"truncate_prompt_tokens": 4096}}, _REASONING_MODEL
)
result = config.map_extra_body_params({"extra_body": {"truncate_prompt_tokens": 4096}}, _REASONING_MODEL)
assert result == {"prompt_truncate_len": 4096}
@ -1654,9 +1602,7 @@ def test_map_extra_body_params_non_dict_chat_template_kwargs_dropped():
def test_map_extra_body_params_guided_json():
config = FireworksAIConfig()
schema = {"type": "object", "properties": {"x": {"type": "string"}}}
result = config.map_extra_body_params(
{"extra_body": {"guided_json": schema}}, _REASONING_MODEL
)
result = config.map_extra_body_params({"extra_body": {"guided_json": schema}}, _REASONING_MODEL)
assert result == {
"response_format": {
"type": "json_schema",
@ -1667,16 +1613,10 @@ def test_map_extra_body_params_guided_json():
def test_map_extra_body_params_guided_grammar_and_choice():
config = FireworksAIConfig()
grammar = config.map_extra_body_params(
{"extra_body": {"guided_grammar": "root ::= 'hello'"}}, _REASONING_MODEL
)
assert grammar == {
"response_format": {"type": "grammar", "grammar": "root ::= 'hello'"}
}
grammar = config.map_extra_body_params({"extra_body": {"guided_grammar": "root ::= 'hello'"}}, _REASONING_MODEL)
assert grammar == {"response_format": {"type": "grammar", "grammar": "root ::= 'hello'"}}
choice = config.map_extra_body_params(
{"extra_body": {"guided_choice": ["yes", "no"]}}, _REASONING_MODEL
)
choice = config.map_extra_body_params({"extra_body": {"guided_choice": ["yes", "no"]}}, _REASONING_MODEL)
assert choice == {
"response_format": {
"type": "json_schema",
@ -1762,9 +1702,7 @@ def test_map_extra_body_params_strips_unsupported_nim_vllm_params(param, value,
config = FireworksAIConfig()
with caplog.at_level(logging.DEBUG):
result = config.map_extra_body_params(
{"extra_body": {param: value}}, _REASONING_MODEL
)
result = config.map_extra_body_params({"extra_body": {param: value}}, _REASONING_MODEL)
assert result == {}
assert param in caplog.text
@ -1836,15 +1774,10 @@ def test_nim_vllm_extras_translated_end_to_end_in_request_body():
assert request_body["top_k"] == 40
def test_streaming_preserves_selected_model_for_private_accounting():
from litellm.llms.custom_httpx.http_handler import HTTPHandler
requested_route = (
"accounts/fireworks/routers/firerouter/"
"kimi-k3/deepseek-v4-pro-0813/deepseek-v4-flash-0731"
)
requested_route = "accounts/fireworks/routers/firerouter/kimi-k3/deepseek-v4-pro-0813/deepseek-v4-flash-0731"
selected_model = "deepseek-v4-flash-0731"
sse_lines = [
"data: "
@ -1898,19 +1831,14 @@ def test_streaming_preserves_selected_model_for_private_accounting():
assert chunks
assert {chunk.model for chunk in chunks} == {requested_route}
assert {
chunk._hidden_params.get("provider_response_model") for chunk in chunks
} == {selected_model}
assert {chunk._hidden_params.get("provider_response_model") for chunk in chunks} == {selected_model}
assembled = litellm.stream_chunk_builder(chunks=chunks)
assert assembled is not None
assert assembled.model == requested_route
assert assembled._hidden_params["provider_response_model"] == selected_model
selected_model_info = litellm.model_cost[f"fireworks_ai/{selected_model}"]
expected_cost = (
5 * selected_model_info["input_cost_per_token"]
+ selected_model_info["output_cost_per_token"]
)
expected_cost = 5 * selected_model_info["input_cost_per_token"] + selected_model_info["output_cost_per_token"]
assert litellm.completion_cost(
completion_response=assembled,
custom_llm_provider="fireworks_ai",

View file

@ -85,6 +85,7 @@ CONNECTION_NAMES: Final = (
"litellm_credential_name",
"configurable_clientside_auth_params",
"use_xai_oauth",
"fireworks_shared_session_affinity",
"aws_batch_role_arn",
"s3_bucket_name",
"s3_region_name",