mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-22 10:40:46 +00:00
fix(fireworks_ai): make shared session affinity an owned kwarg, not a body field
The flag was routed into optional_params and stripped by provider code, which broke on three fronts: it mutated function parameters, put Fireworks-specific logic in main.py, and leaked into the text-completion request body. Register it as a LiteLLM-owned connection kwarg instead, the same way provider_affinity_header is plumbed, so the shared kwarg filter drops it from every provider request body before a provider ever sees it. Restore the one-argument get_fireworks_session_id signature the existing proxy tests call, hash the credential with usedforsecurity=False, and suppress the CodeQL finding the way the OCI and HIBP cases are. The user-key branch now hashes user_api_key_hash too: the proxy leaves opaque custom-auth keys and OAuth bearer tokens unhashed, so forwarding that value would send a raw proxy credential to Fireworks in a request header.
This commit is contained in:
parent
cf624b5276
commit
dbf293c015
9 changed files with 170 additions and 273 deletions
10
.github/workflows/codeql.yml
vendored
10
.github/workflows/codeql.yml
vendored
|
|
@ -72,13 +72,21 @@ jobs:
|
|||
# a lookup key into the haveibeenpwned range API (the protocol mandates
|
||||
# SHA-1) and the digest itself never leaves the proxy beyond its first 5
|
||||
# characters.
|
||||
- name: Filter SARIF (OCI sha256, HIBP sha1)
|
||||
# Finally it fires on the Fireworks shared-session affinity token in
|
||||
# litellm/llms/fireworks_ai/common_utils.py, where a credential is hashed so
|
||||
# concurrent requests on one Fireworks key map to a stable x-session-affinity
|
||||
# value and their shared prompt prefix hits the replica cache. Nothing is
|
||||
# authenticated by that digest; it is a cache-partition key, and SHA-256 is
|
||||
# kept because two credentials must not collide onto one token. CodeQL flags
|
||||
# it because the flow starts at a lookup key literally named `api_key`.
|
||||
- name: Filter SARIF (OCI sha256, HIBP sha1, Fireworks affinity token)
|
||||
if: matrix.language == 'python'
|
||||
uses: advanced-security/filter-sarif@2da736ff05ef065cb2894ac6892e47b5eac2c3c0 # v1.1
|
||||
with:
|
||||
patterns: |
|
||||
-litellm/llms/oci/common_utils.py:py/weak-sensitive-data-hashing
|
||||
-litellm/proxy/auth/password_policy.py:py/weak-sensitive-data-hashing
|
||||
-litellm/llms/fireworks_ai/common_utils.py:py/weak-sensitive-data-hashing
|
||||
input: sarif-results/python.sarif
|
||||
output: sarif-results/python.sarif
|
||||
|
||||
|
|
|
|||
|
|
@ -67,6 +67,7 @@ OPTIONAL_KWARGS_KEYS: Final = (
|
|||
"otpm",
|
||||
"use_xai_oauth",
|
||||
PROVIDER_AFFINITY_HEADER_KWARG_KEY,
|
||||
"fireworks_shared_session_affinity",
|
||||
}
|
||||
)
|
||||
| AWS_CREDENTIAL_KWARGS_KEYS
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
import json
|
||||
from collections.abc import AsyncIterator, Iterator, Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, Literal, cast
|
||||
|
||||
import httpx
|
||||
|
|
@ -44,7 +43,6 @@ from ..common_utils import (
|
|||
FIREROUTER,
|
||||
FireworksAIException,
|
||||
FireworksAIMixin,
|
||||
absorb_shared_affinity_param,
|
||||
resolve_fireworks_resource_name,
|
||||
)
|
||||
|
||||
|
|
@ -224,7 +222,6 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
|
|||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
)
|
||||
absorb_shared_affinity_param(litellm_params, optional_params)
|
||||
return self._add_session_affinity_header(validated_headers, litellm_params)
|
||||
|
||||
def get_supported_openai_params(self, model: str):
|
||||
|
|
@ -267,7 +264,6 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
|
|||
"prediction",
|
||||
"stream_options",
|
||||
"sampling_mask",
|
||||
"fireworks_shared_session_affinity",
|
||||
]
|
||||
|
||||
# Only add tools for models that support function calling
|
||||
|
|
@ -654,29 +650,6 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
|
|||
|
||||
return provider_specific_model_info
|
||||
|
||||
def transform_extra_body(
|
||||
self,
|
||||
extra_body: Mapping[str, object],
|
||||
request: Mapping[str, object],
|
||||
model: str,
|
||||
litellm_params: Mapping[str, object],
|
||||
) -> Mapping[str, object]:
|
||||
"""
|
||||
Pull `fireworks_shared_session_affinity` out of extra_body before the handler
|
||||
merges extra_body into the request body. The flag is a LiteLLM routing hint, not
|
||||
a Fireworks API field ("Extra inputs are not permitted"); it must only reach
|
||||
Fireworks as the x-session-affinity header, which the affinity logic derives
|
||||
from litellm_params.metadata.
|
||||
"""
|
||||
filtered: Final = {k: v for k, v in extra_body.items() if k != "fireworks_shared_session_affinity"}
|
||||
if "fireworks_shared_session_affinity" not in extra_body:
|
||||
return super().transform_extra_body(
|
||||
extra_body=extra_body, request=request, model=model, litellm_params=litellm_params
|
||||
)
|
||||
if filtered:
|
||||
return MappingProxyType(filtered)
|
||||
return MappingProxyType({})
|
||||
|
||||
def transform_request(
|
||||
self,
|
||||
model: str,
|
||||
|
|
@ -689,11 +662,6 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
|
|||
messages = self._transform_messages_helper(
|
||||
messages=messages, model=resolved_model, litellm_params=litellm_params
|
||||
)
|
||||
# `fireworks_shared_session_affinity` is a LiteLLM routing hint, not a Fireworks API
|
||||
# field: it must not reach the request body ("Extra inputs are not permitted") —
|
||||
# stash it in litellm_params metadata where the affinity-header logic reads it.
|
||||
absorb_shared_affinity_param(litellm_params, optional_params)
|
||||
optional_params.pop("fireworks_shared_session_affinity", None)
|
||||
if "tools" in optional_params and optional_params["tools"] is not None:
|
||||
tools: Final = self._transform_tools(tools=optional_params["tools"])
|
||||
optional_params["tools"] = tools
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import hashlib
|
||||
from collections.abc import Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
from typing import Final, cast
|
||||
|
||||
from httpx import Headers
|
||||
|
||||
|
|
@ -16,16 +16,24 @@ class FireworksAIException(BaseLLMException):
|
|||
pass
|
||||
|
||||
|
||||
def get_fireworks_session_id(
|
||||
litellm_params: Mapping[str, object], headers: Mapping[str, str]
|
||||
) -> str | None:
|
||||
SHARED_SESSION_AFFINITY_PARAM: Final = "fireworks_shared_session_affinity"
|
||||
|
||||
|
||||
def _lookup(mapping: object, key: str) -> object | None:
|
||||
if isinstance(mapping, Mapping):
|
||||
return cast("Mapping[str, object]", mapping).get(key)
|
||||
return None
|
||||
|
||||
|
||||
def get_fireworks_session_id(litellm_params: Mapping[str, object]) -> str | None:
|
||||
"""
|
||||
Session id to send as `x-session-affinity`, or None when the caller gave none.
|
||||
|
||||
Priority:
|
||||
1. Explicit `litellm_session_id` or `session_id` in litellm_params
|
||||
2. Explicit `session_id` in metadata
|
||||
3. Shared affinity (if enabled): hash of the API key or LiteLLM user key
|
||||
3. Shared affinity, when `fireworks_shared_session_affinity` is set: a stable digest of
|
||||
the caller's LiteLLM user key (proxy) or Fireworks credential (direct SDK)
|
||||
|
||||
Deliberately does not fall back to `litellm_trace_id`, and ignores session ids the
|
||||
proxy generated for a request that had none: both are per request, so using them
|
||||
|
|
@ -34,68 +42,50 @@ def get_fireworks_session_id(
|
|||
params: Final = litellm_params
|
||||
metadata: Final = params.get("metadata")
|
||||
|
||||
# 1 & 2. Explicit session IDs
|
||||
has_generated_session_id: Final = isinstance(metadata, Mapping) and metadata.get(
|
||||
SESSION_ID_GENERATED_METADATA_KEY
|
||||
)
|
||||
has_generated_session_id: Final = _lookup(metadata, SESSION_ID_GENERATED_METADATA_KEY)
|
||||
if not has_generated_session_id:
|
||||
for key in ("litellm_session_id", "session_id"):
|
||||
value = params.get(key)
|
||||
if value:
|
||||
return str(value)
|
||||
if isinstance(metadata, Mapping):
|
||||
value = metadata.get("session_id")
|
||||
if value:
|
||||
return str(value)
|
||||
session_from_metadata: Final = _lookup(metadata, "session_id")
|
||||
if session_from_metadata:
|
||||
return str(session_from_metadata)
|
||||
|
||||
# 3. Shared affinity
|
||||
shared_affinity: Final = params.get("fireworks_shared_session_affinity") or (
|
||||
metadata.get("fireworks_shared_session_affinity")
|
||||
if isinstance(metadata, Mapping)
|
||||
else None
|
||||
shared_affinity: Final = params.get(SHARED_SESSION_AFFINITY_PARAM) or _lookup(
|
||||
metadata, SHARED_SESSION_AFFINITY_PARAM
|
||||
)
|
||||
if shared_affinity:
|
||||
# Try LiteLLM user API key hash first (proxy-level isolation)
|
||||
user_key_hash = (
|
||||
metadata.get("user_api_key_hash") if isinstance(metadata, Mapping) else None
|
||||
)
|
||||
if user_key_hash:
|
||||
return f"litellm-user-{user_key_hash}"
|
||||
if not shared_affinity:
|
||||
return None
|
||||
|
||||
# Fall back to hashing the Fireworks API key (account-level pooling)
|
||||
auth = headers.get("Authorization", "")
|
||||
if auth.startswith("Bearer "):
|
||||
api_key = auth.removeprefix("Bearer ")
|
||||
if api_key:
|
||||
return hashlib.sha256(api_key.encode()).hexdigest()
|
||||
user_key_hash: Final = _lookup(metadata, "user_api_key_hash")
|
||||
if isinstance(user_key_hash, str) and user_key_hash:
|
||||
# `user_api_key_hash` is not guaranteed to be a digest: the proxy leaves opaque
|
||||
# custom-auth keys and OAuth bearer tokens unhashed, so hashing it here is what
|
||||
# keeps a raw proxy credential from riding to Fireworks in a request header.
|
||||
return f"litellm-user-{shared_affinity_token(user_key_hash)}"
|
||||
|
||||
return None
|
||||
configured: Final = params.get("api_key")
|
||||
credential: Final = configured if isinstance(configured, str) and configured else resolve_fireworks_api_key(None)
|
||||
if credential is None:
|
||||
return None
|
||||
return shared_affinity_token(credential)
|
||||
|
||||
|
||||
def absorb_shared_affinity_param(litellm_params: dict, optional_params: Mapping[str, object]) -> None:
|
||||
def shared_affinity_token(credential: str) -> str:
|
||||
"""
|
||||
Move a top-level `fireworks_shared_session_affinity` kwarg into litellm_params metadata.
|
||||
Opaque, stable routing token for one credential.
|
||||
|
||||
The flag is a LiteLLM routing hint, not a Fireworks API field: left in optional_params
|
||||
(or in optional_params.extra_body, where openai-compatible param handling stashes
|
||||
unknown kwargs) it would be serialized into the request body and rejected by Fireworks.
|
||||
Reads both the top-level param and its extra_body-nested form; strips it from both.
|
||||
This is a cache-isolation identifier, not a password hash: it is sent to Fireworks as
|
||||
`x-session-affinity` so concurrent requests on one credential land on one replica and
|
||||
their shared prompt prefix hits the cache. It is never used to authenticate, and it
|
||||
leaks nothing beyond a digest that Fireworks could compute itself from its own key.
|
||||
Collision resistance is however load bearing: two distinct credentials must not share
|
||||
a token, or one account's cache becomes readable by another, so the digest must stay a
|
||||
strong hash. `usedforsecurity=False` declares that no authentication property rests on
|
||||
it (and keeps it available under FIPS builds).
|
||||
"""
|
||||
value: object = optional_params.get("fireworks_shared_session_affinity")
|
||||
extra_body: Final = optional_params.get("extra_body")
|
||||
if value is None and isinstance(extra_body, Mapping):
|
||||
value = extra_body.get("fireworks_shared_session_affinity")
|
||||
if value is None:
|
||||
return
|
||||
if isinstance(extra_body, dict) and "fireworks_shared_session_affinity" in extra_body:
|
||||
extra_body.pop("fireworks_shared_session_affinity", None)
|
||||
if "fireworks_shared_session_affinity" in optional_params:
|
||||
optional_params.pop("fireworks_shared_session_affinity", None)
|
||||
metadata: Final = litellm_params.get("metadata")
|
||||
if isinstance(metadata, dict):
|
||||
metadata.setdefault("fireworks_shared_session_affinity", value)
|
||||
else:
|
||||
litellm_params["metadata"] = {"fireworks_shared_session_affinity": value}
|
||||
return hashlib.sha256(credential.encode(), usedforsecurity=False).hexdigest()
|
||||
|
||||
|
||||
def with_fireworks_session_affinity(
|
||||
|
|
@ -103,7 +93,7 @@ def with_fireworks_session_affinity(
|
|||
) -> Mapping[str, str]:
|
||||
if any(key.lower() == "x-session-affinity" for key in headers):
|
||||
return headers
|
||||
session_id: Final = get_fireworks_session_id(litellm_params, headers)
|
||||
session_id: Final = get_fireworks_session_id(litellm_params)
|
||||
if not session_id:
|
||||
return headers
|
||||
return MappingProxyType({**headers, "x-session-affinity": session_id})
|
||||
|
|
@ -167,7 +157,6 @@ class FireworksAIMixin:
|
|||
content_type_header: Final = (
|
||||
{} if any(key.lower() == "content-type" for key in auth_headers) else {"Content-Type": "application/json"}
|
||||
)
|
||||
absorb_shared_affinity_param(litellm_params, optional_params)
|
||||
return self._add_session_affinity_header({**auth_headers, **content_type_header}, litellm_params)
|
||||
|
||||
def _add_session_affinity_header(self, headers: dict, litellm_params: dict) -> dict:
|
||||
|
|
|
|||
|
|
@ -39,7 +39,6 @@ class FireworksAITextCompletionConfig(FireworksAIMixin, BaseTextCompletionConfig
|
|||
"response_format",
|
||||
"stream",
|
||||
"user",
|
||||
"fireworks_shared_session_affinity",
|
||||
]
|
||||
|
||||
def map_openai_params(
|
||||
|
|
|
|||
|
|
@ -119,7 +119,6 @@ from litellm.llms.base_llm.base_model_iterator import (
|
|||
from litellm.llms.bedrock.common_utils import BedrockModelInfo
|
||||
from litellm.llms.cohere.common_utils import CohereModelInfo
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler, http2_enabled
|
||||
from litellm.llms.fireworks_ai.common_utils import absorb_shared_affinity_param
|
||||
from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config
|
||||
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
|
||||
from litellm.llms.vertex_ai.common_utils import (
|
||||
|
|
@ -1865,7 +1864,6 @@ def _complete_fireworks_ai(
|
|||
shared_session: Final = ctx.shared_session
|
||||
stream: Final = ctx.stream
|
||||
timeout: Final = ctx.timeout
|
||||
absorb_shared_affinity_param(litellm_params, ctx.optional_params)
|
||||
optional_params: Final = (
|
||||
provider_config.map_extra_body_params(optional_params=ctx.optional_params, model=model)
|
||||
if isinstance(provider_config, litellm.FireworksAIConfig)
|
||||
|
|
@ -5704,7 +5702,11 @@ def completion(
|
|||
gigachat_access_token=kwargs.get("gigachat_access_token"),
|
||||
**{
|
||||
key: kwargs[key]
|
||||
for key in (*AWS_CREDENTIAL_KWARGS_KEYS, PROVIDER_AFFINITY_HEADER_KWARG_KEY)
|
||||
for key in (
|
||||
*AWS_CREDENTIAL_KWARGS_KEYS,
|
||||
PROVIDER_AFFINITY_HEADER_KWARG_KEY,
|
||||
"fireworks_shared_session_affinity",
|
||||
)
|
||||
if key in kwargs
|
||||
},
|
||||
)
|
||||
|
|
|
|||
|
|
@ -87,6 +87,7 @@ class ProviderConnection:
|
|||
litellm_credential_name: str | None = None
|
||||
configurable_clientside_auth_params: "Sequence[str | ConfigurableClientsideParamsCustomAuth] | None" = None
|
||||
use_xai_oauth: bool | None = None
|
||||
fireworks_shared_session_affinity: bool | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
|
|
|
|||
|
|
@ -9,8 +9,7 @@ import litellm
|
|||
from litellm.constants import SESSION_ID_GENERATED_METADATA_KEY
|
||||
from litellm.llms.custom_httpx.http_handler import HTTPHandler
|
||||
from litellm.llms.fireworks_ai.chat.transformation import FireworksAIConfig
|
||||
from litellm.llms.fireworks_ai.common_utils import get_fireworks_session_id
|
||||
from litellm.llms.fireworks_ai.common_utils import absorb_shared_affinity_param
|
||||
from litellm.llms.fireworks_ai.common_utils import get_fireworks_session_id, shared_affinity_token
|
||||
from litellm.types.utils import (
|
||||
ChatCompletionMessageToolCall,
|
||||
Function,
|
||||
|
|
@ -219,10 +218,7 @@ def test_validate_environment_raises_without_api_key(monkeypatch):
|
|||
|
||||
def test_get_fireworks_session_id_prefers_litellm_session_id_over_trace_id():
|
||||
assert (
|
||||
get_fireworks_session_id(
|
||||
{"litellm_session_id": "session-123", "litellm_trace_id": "trace-123"},
|
||||
{},
|
||||
)
|
||||
get_fireworks_session_id({"litellm_session_id": "session-123", "litellm_trace_id": "trace-123"})
|
||||
== "session-123"
|
||||
)
|
||||
|
||||
|
|
@ -235,117 +231,79 @@ def test_get_fireworks_session_id_ignores_proxy_generated_session_id_without_sha
|
|||
"litellm_session_id": "generated-1",
|
||||
"litellm_trace_id": "generated-1",
|
||||
"metadata": {"session_id": "generated-1", SESSION_ID_GENERATED_METADATA_KEY: True},
|
||||
},
|
||||
{},
|
||||
}
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
def test_get_fireworks_session_id_uses_shared_affinity_even_with_generated_id():
|
||||
"""With shared affinity, we use the stable hash even if the proxy generated a fresh ID."""
|
||||
"""With shared affinity, the stable token wins even when the proxy generated a fresh id."""
|
||||
litellm_params = {
|
||||
"litellm_session_id": "generated-1",
|
||||
"fireworks_shared_session_affinity": True,
|
||||
"metadata": {
|
||||
"session_id": "generated-1",
|
||||
SESSION_ID_GENERATED_METADATA_KEY: True,
|
||||
"fireworks_shared_session_affinity": True,
|
||||
"user_api_key_hash": "stable-user-hash",
|
||||
},
|
||||
}
|
||||
assert get_fireworks_session_id(litellm_params, {}) == "litellm-user-stable-user-hash"
|
||||
assert get_fireworks_session_id(litellm_params) == f"litellm-user-{shared_affinity_token('stable-user-hash')}"
|
||||
|
||||
|
||||
def test_get_fireworks_session_id_falls_back_to_api_key_hash():
|
||||
litellm_params = {
|
||||
"metadata": {
|
||||
"fireworks_shared_session_affinity": True,
|
||||
},
|
||||
}
|
||||
headers = {"Authorization": "Bearer fw-api-key"}
|
||||
import hashlib
|
||||
|
||||
expected = hashlib.sha256(b"fw-api-key").hexdigest()
|
||||
assert get_fireworks_session_id(litellm_params, headers) == expected
|
||||
|
||||
|
||||
def test_get_fireworks_session_id_uses_top_level_shared_affinity():
|
||||
def test_get_fireworks_session_id_falls_back_to_credential_token():
|
||||
litellm_params = {
|
||||
"fireworks_shared_session_affinity": True,
|
||||
"metadata": {
|
||||
"user_api_key_hash": "stable-user-hash",
|
||||
},
|
||||
"api_key": "fw-api-key",
|
||||
"metadata": {},
|
||||
}
|
||||
assert get_fireworks_session_id(litellm_params, {}) == "litellm-user-stable-user-hash"
|
||||
assert get_fireworks_session_id(litellm_params) == shared_affinity_token("fw-api-key")
|
||||
|
||||
|
||||
def test_get_fireworks_session_id_uses_metadata_nested_shared_affinity():
|
||||
litellm_params = {"metadata": {"fireworks_shared_session_affinity": True, "user_api_key_hash": "stable-user-hash"}}
|
||||
assert get_fireworks_session_id(litellm_params) == f"litellm-user-{shared_affinity_token('stable-user-hash')}"
|
||||
|
||||
|
||||
def test_get_fireworks_session_id_never_emits_a_raw_proxy_credential():
|
||||
"""A proxy user key is not always stored hashed: opaque custom-auth keys and OAuth
|
||||
bearer tokens pass through `_safe_hash_litellm_api_key` unchanged, so the affinity
|
||||
token must hash the value again rather than forwarding it verbatim."""
|
||||
raw_token = "opaque-custom-auth-token-abc123"
|
||||
litellm_params = {"metadata": {"fireworks_shared_session_affinity": True, "user_api_key_hash": raw_token}}
|
||||
token = get_fireworks_session_id(litellm_params)
|
||||
assert token is not None
|
||||
assert raw_token not in token
|
||||
assert token == f"litellm-user-{shared_affinity_token(raw_token)}"
|
||||
|
||||
|
||||
def test_get_fireworks_session_id_explicit_overrides_shared():
|
||||
litellm_params = {
|
||||
"litellm_session_id": "explicit-session",
|
||||
"metadata": {
|
||||
"fireworks_shared_session_affinity": True,
|
||||
"user_api_key_hash": "stable-user-hash",
|
||||
},
|
||||
"metadata": {"fireworks_shared_session_affinity": True, "user_api_key_hash": "stable-user-hash"},
|
||||
}
|
||||
assert get_fireworks_session_id(litellm_params, {}) == "explicit-session"
|
||||
assert get_fireworks_session_id(litellm_params) == "explicit-session"
|
||||
|
||||
|
||||
def test_absorb_shared_affinity_param_moves_top_level_flag_into_metadata():
|
||||
"""A top-level `fireworks_shared_session_affinity` kwarg is a LiteLLM routing hint,
|
||||
not a Fireworks API field: if left in optional_params it would be serialized into the
|
||||
request body and rejected by Fireworks ("Extra inputs are not permitted")."""
|
||||
litellm_params: dict = {"metadata": {"user_api_key_hash": "stable-user-hash"}}
|
||||
optional_params: dict = {"fireworks_shared_session_affinity": True, "temperature": 0.5}
|
||||
|
||||
absorb_shared_affinity_param(litellm_params, optional_params)
|
||||
|
||||
assert litellm_params["metadata"]["fireworks_shared_session_affinity"] is True
|
||||
assert "fireworks_shared_session_affinity" not in optional_params
|
||||
assert optional_params["temperature"] == 0.5 # untouched
|
||||
def test_get_fireworks_session_id_is_none_without_the_flag_or_a_credential():
|
||||
assert get_fireworks_session_id({"metadata": {"user_api_key_hash": "stable-user-hash"}}) is None
|
||||
|
||||
|
||||
def test_absorb_shared_affinity_param_creates_metadata_when_missing():
|
||||
litellm_params: dict = {}
|
||||
optional_params: dict = {"fireworks_shared_session_affinity": True}
|
||||
|
||||
absorb_shared_affinity_param(litellm_params, optional_params)
|
||||
|
||||
assert litellm_params["metadata"] == {"fireworks_shared_session_affinity": True}
|
||||
assert "fireworks_shared_session_affinity" not in optional_params
|
||||
|
||||
|
||||
def test_absorb_shared_affinity_param_noop_when_absent():
|
||||
litellm_params: dict = {"metadata": {"session_id": "s1"}}
|
||||
optional_params: dict = {"temperature": 0.5}
|
||||
|
||||
absorb_shared_affinity_param(litellm_params, optional_params)
|
||||
|
||||
assert litellm_params["metadata"] == {"session_id": "s1"}
|
||||
|
||||
|
||||
def test_absorb_shared_affinity_param_reads_extra_body_nested_flag():
|
||||
"""OpenAI-compatible param handling stashes unknown top-level kwargs inside
|
||||
optional_params.extra_body; the absorb helper must read and strip it there too."""
|
||||
litellm_params: dict = {"metadata": {"user_api_key_hash": "stable-user-hash"}}
|
||||
optional_params: dict = {"extra_body": {"fireworks_shared_session_affinity": True}}
|
||||
|
||||
absorb_shared_affinity_param(litellm_params, optional_params)
|
||||
|
||||
assert litellm_params["metadata"]["fireworks_shared_session_affinity"] is True
|
||||
assert optional_params["extra_body"] == {}
|
||||
def test_shared_affinity_token_is_stable_and_credential_specific():
|
||||
assert shared_affinity_token("fw-a") == shared_affinity_token("fw-a")
|
||||
assert shared_affinity_token("fw-a") != shared_affinity_token("fw-b")
|
||||
|
||||
|
||||
def test_top_level_shared_affinity_kwarg_sets_header_without_body_leak():
|
||||
"""Full pipeline: litellm.completion(fireworks_shared_session_affinity=True) must set
|
||||
x-session-affinity (hashed off the Fireworks credential) and must NOT put the flag
|
||||
into the request body — Fireworks rejects unknown body fields."""
|
||||
import hashlib
|
||||
|
||||
from litellm.llms.custom_httpx.http_handler import HTTPHandler as _HTTPHandler
|
||||
|
||||
into the request body, which Fireworks rejects."""
|
||||
model = "accounts/fireworks/models/llama-v3p1-8b-instruct"
|
||||
reply = {
|
||||
"id": "c1", "object": "chat.completion", "created": 1, "model": model,
|
||||
"id": "c1",
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": model,
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "Hi"}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
|
||||
}
|
||||
|
|
@ -356,7 +314,7 @@ def test_top_level_shared_affinity_kwarg_sets_header_without_body_leak():
|
|||
raw_response.json = lambda: reply
|
||||
|
||||
captured: dict = {}
|
||||
client = MagicMock(spec=_HTTPHandler)
|
||||
client = MagicMock(spec=HTTPHandler)
|
||||
|
||||
def post(*args, **kwargs):
|
||||
captured["headers"] = dict(kwargs.get("headers") or {})
|
||||
|
|
@ -374,10 +332,47 @@ def test_top_level_shared_affinity_kwarg_sets_header_without_body_leak():
|
|||
)
|
||||
|
||||
headers = {k.lower(): v for k, v in captured["headers"].items()}
|
||||
assert headers.get("x-session-affinity") == hashlib.sha256(b"fw-test-key").hexdigest()
|
||||
assert headers.get("x-session-affinity") == shared_affinity_token("fw-test-key")
|
||||
assert "fireworks_shared_session_affinity" not in captured["body"]
|
||||
|
||||
|
||||
def test_shared_affinity_kwarg_is_absent_from_the_request_body():
|
||||
"""A run with the flag off must not carry the field anywhere in the request body, and a
|
||||
run with it on must reach the same body, so the flag is never serialized to Fireworks."""
|
||||
model = "accounts/fireworks/models/llama-v3p1-8b-instruct"
|
||||
reply = {
|
||||
"id": "c1",
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": model,
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "Hi"}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
|
||||
}
|
||||
raw_response = MagicMock()
|
||||
raw_response.status_code = 200
|
||||
raw_response.headers = {}
|
||||
raw_response.text = json.dumps(reply)
|
||||
raw_response.json = lambda: reply
|
||||
|
||||
bodies: list = []
|
||||
|
||||
def run(**extra):
|
||||
client = MagicMock(spec=HTTPHandler)
|
||||
client.post.side_effect = lambda *a, **kw: (bodies.append(json.loads(kw.get("data") or "{}")), raw_response)[1]
|
||||
litellm.completion(
|
||||
model=f"fireworks_ai/{model}",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
api_key="fw-test-key",
|
||||
client=client,
|
||||
**extra,
|
||||
)
|
||||
|
||||
run()
|
||||
run(fireworks_shared_session_affinity=True)
|
||||
|
||||
assert bodies[0] == bodies[1]
|
||||
|
||||
|
||||
def test_handle_message_content_with_tool_calls():
|
||||
config = FireworksAIConfig()
|
||||
message = Message(
|
||||
|
|
@ -410,25 +405,18 @@ def test_handle_message_content_with_tool_calls():
|
|||
},
|
||||
}
|
||||
]
|
||||
updated_message = config._handle_message_content_with_tool_calls(
|
||||
message, tool_calls
|
||||
)
|
||||
updated_message = config._handle_message_content_with_tool_calls(message, tool_calls)
|
||||
assert updated_message.tool_calls is not None
|
||||
assert len(updated_message.tool_calls) == 1
|
||||
assert updated_message.tool_calls[0].function.name == "get_current_weather"
|
||||
assert (
|
||||
updated_message.tool_calls[0].function.arguments
|
||||
== expected_tool_call.function.arguments
|
||||
)
|
||||
assert updated_message.tool_calls[0].function.arguments == expected_tool_call.function.arguments
|
||||
|
||||
|
||||
def test_get_supported_openai_params_reasoning_effort():
|
||||
"""Test that reasoning_effort is only included in supported params for models that support it."""
|
||||
config = FireworksAIConfig()
|
||||
|
||||
supported_params = config.get_supported_openai_params(
|
||||
"fireworks_ai/accounts/fireworks/models/glm-5p1"
|
||||
)
|
||||
supported_params = config.get_supported_openai_params("fireworks_ai/accounts/fireworks/models/glm-5p1")
|
||||
assert "reasoning_effort" in supported_params
|
||||
assert "thinking" in supported_params
|
||||
|
||||
|
|
@ -443,9 +431,7 @@ def test_get_supported_openai_params_parallel_tool_calls():
|
|||
"""Test that parallel_tool_calls is included for models that support function calling."""
|
||||
config = FireworksAIConfig()
|
||||
|
||||
supported_params = config.get_supported_openai_params(
|
||||
"fireworks_ai/accounts/fireworks/models/glm-5p1"
|
||||
)
|
||||
supported_params = config.get_supported_openai_params("fireworks_ai/accounts/fireworks/models/glm-5p1")
|
||||
assert "parallel_tool_calls" in supported_params
|
||||
assert "tools" in supported_params
|
||||
assert "tool_choice" in supported_params
|
||||
|
|
@ -459,9 +445,7 @@ def test_get_supported_openai_params_parallel_tool_calls():
|
|||
def test_get_supported_openai_params_short_model_name_resolves_account_prefixed_entry():
|
||||
config = FireworksAIConfig()
|
||||
|
||||
supported_params = config.get_supported_openai_params(
|
||||
"fireworks_ai/deepseek-v4-pro-0813"
|
||||
)
|
||||
supported_params = config.get_supported_openai_params("fireworks_ai/deepseek-v4-pro-0813")
|
||||
|
||||
assert "tool_choice" in supported_params
|
||||
assert "reasoning_effort" in supported_params
|
||||
|
|
@ -470,9 +454,7 @@ def test_get_supported_openai_params_short_model_name_resolves_account_prefixed_
|
|||
def test_get_supported_openai_params_preserves_generic_reasoning_fallback():
|
||||
config = FireworksAIConfig()
|
||||
|
||||
supported_params = config.get_supported_openai_params(
|
||||
"fireworks_ai/accounts/fireworks/models/glm-5p3-flash"
|
||||
)
|
||||
supported_params = config.get_supported_openai_params("fireworks_ai/accounts/fireworks/models/glm-5p3-flash")
|
||||
|
||||
assert "reasoning_effort" in supported_params
|
||||
|
||||
|
|
@ -539,14 +521,10 @@ def test_get_models_url_no_double_v1(api_base, expected_url_prefix):
|
|||
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 200
|
||||
mock_response.json.return_value = {
|
||||
"models": [{"name": "accounts/fireworks/models/llama-v3-70b"}]
|
||||
}
|
||||
mock_response.json.return_value = {"models": [{"name": "accounts/fireworks/models/llama-v3-70b"}]}
|
||||
|
||||
with (
|
||||
patch(
|
||||
"litellm.module_level_client.get", return_value=mock_response
|
||||
) as mock_get,
|
||||
patch("litellm.module_level_client.get", return_value=mock_response) as mock_get,
|
||||
patch(
|
||||
"litellm.llms.fireworks_ai.chat.transformation.get_secret_str",
|
||||
side_effect=lambda key: {
|
||||
|
|
@ -558,13 +536,9 @@ def test_get_models_url_no_double_v1(api_base, expected_url_prefix):
|
|||
):
|
||||
result = config.get_models(api_key="test-key", api_base=api_base)
|
||||
|
||||
called_url = mock_get.call_args.kwargs.get("url") or mock_get.call_args[1].get(
|
||||
"url", ""
|
||||
)
|
||||
called_url = mock_get.call_args.kwargs.get("url") or mock_get.call_args[1].get("url", "")
|
||||
assert "/v1/v1/" not in called_url, f"Double /v1/ detected in URL: {called_url}"
|
||||
assert called_url.startswith(
|
||||
expected_url_prefix
|
||||
), f"URL {called_url} does not start with {expected_url_prefix}"
|
||||
assert called_url.startswith(expected_url_prefix), f"URL {called_url} does not start with {expected_url_prefix}"
|
||||
assert result == ["fireworks_ai/accounts/fireworks/models/llama-v3-70b"]
|
||||
|
||||
|
||||
|
|
@ -592,9 +566,7 @@ def test_transform_messages_helper_removes_provider_specific_fields():
|
|||
},
|
||||
]
|
||||
# Call helper
|
||||
out = config._transform_messages_helper(
|
||||
messages, model="fireworks/test", litellm_params={}
|
||||
)
|
||||
out = config._transform_messages_helper(messages, model="fireworks/test", litellm_params={})
|
||||
for msg in out:
|
||||
assert "provider_specific_fields" not in msg
|
||||
|
||||
|
|
@ -615,15 +587,11 @@ def test_transform_messages_helper_strips_thinking_blocks_but_keeps_reasoning_co
|
|||
{
|
||||
"role": "assistant",
|
||||
"content": "I can help.",
|
||||
"thinking_blocks": [
|
||||
{"type": "thinking", "thinking": "internal", "signature": ""}
|
||||
],
|
||||
"thinking_blocks": [{"type": "thinking", "thinking": "internal", "signature": ""}],
|
||||
"reasoning_content": "internal",
|
||||
},
|
||||
]
|
||||
out = config._transform_messages_helper(
|
||||
messages, model="accounts/fireworks/models/glm-5p1", litellm_params={}
|
||||
)
|
||||
out = config._transform_messages_helper(messages, model="accounts/fireworks/models/glm-5p1", litellm_params={})
|
||||
assert "thinking_blocks" not in out[1]
|
||||
assert out[1]["reasoning_content"] == "internal"
|
||||
assert out[1]["content"] == "I can help."
|
||||
|
|
@ -1101,9 +1069,7 @@ def test_transform_messages_helper_rejects_file_blocks():
|
|||
litellm.BadRequestError,
|
||||
match="Fireworks AI chat completions does not support file content blocks",
|
||||
):
|
||||
config._transform_messages_helper(
|
||||
messages, model="accounts/fireworks/models/kimi-k2p6", litellm_params={}
|
||||
)
|
||||
config._transform_messages_helper(messages, model="accounts/fireworks/models/kimi-k2p6", litellm_params={})
|
||||
|
||||
|
||||
def test_transform_messages_helper_rejects_non_vision_image_inputs():
|
||||
|
|
@ -1115,18 +1081,14 @@ def test_transform_messages_helper_rejects_non_vision_image_inputs():
|
|||
{"type": "text", "text": "Describe this"},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE="
|
||||
},
|
||||
"image_url": {"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE="},
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
with pytest.raises(litellm.BadRequestError, match="does not support image inputs"):
|
||||
config._transform_messages_helper(
|
||||
messages, model="accounts/fireworks/models/glm-5p2", litellm_params={}
|
||||
)
|
||||
config._transform_messages_helper(messages, model="accounts/fireworks/models/glm-5p2", litellm_params={})
|
||||
|
||||
|
||||
def test_transform_messages_helper_allows_vision_image_inputs():
|
||||
|
|
@ -1138,9 +1100,7 @@ def test_transform_messages_helper_allows_vision_image_inputs():
|
|||
{"type": "text", "text": "Describe this"},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE="
|
||||
},
|
||||
"image_url": {"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE="},
|
||||
},
|
||||
],
|
||||
}
|
||||
|
|
@ -1164,9 +1124,7 @@ def test_image_inputs_not_rejected_for_fuzzy_non_vision_match():
|
|||
custom_model = "accounts/myorg/models/custom-glm-5p2"
|
||||
|
||||
assert config._get_model_cost_capability(custom_model, "supports_vision") is False
|
||||
assert (
|
||||
config._get_model_cost_capability_exact(custom_model, "supports_vision") is None
|
||||
)
|
||||
assert config._get_model_cost_capability_exact(custom_model, "supports_vision") is None
|
||||
|
||||
messages = [
|
||||
{
|
||||
|
|
@ -1174,16 +1132,12 @@ def test_image_inputs_not_rejected_for_fuzzy_non_vision_match():
|
|||
"content": [
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE="
|
||||
},
|
||||
"image_url": {"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE="},
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
out = config._transform_messages_helper(
|
||||
messages, model=custom_model, litellm_params={}
|
||||
)
|
||||
out = config._transform_messages_helper(messages, model=custom_model, litellm_params={})
|
||||
assert out == messages
|
||||
|
||||
|
||||
|
|
@ -1196,9 +1150,7 @@ def test_transform_messages_helper_skips_non_dict_content():
|
|||
}
|
||||
]
|
||||
|
||||
out = config._transform_messages_helper(
|
||||
messages, model="accounts/fireworks/models/glm-5p2", litellm_params={}
|
||||
)
|
||||
out = config._transform_messages_helper(messages, model="accounts/fireworks/models/glm-5p2", litellm_params={})
|
||||
assert out == messages
|
||||
|
||||
|
||||
|
|
@ -1438,9 +1390,7 @@ def test_streaming_surfaces_fireworks_response_fields():
|
|||
surfaced: dict = {}
|
||||
for chunk in stream:
|
||||
fields = getattr(chunk, "provider_specific_fields", None) or {}
|
||||
surfaced.update(
|
||||
{k: v for k, v in fields.items() if k.startswith("fireworks_")}
|
||||
)
|
||||
surfaced.update({k: v for k, v in fields.items() if k.startswith("fireworks_")})
|
||||
|
||||
assert surfaced["fireworks_token_ids"] == [[123]]
|
||||
assert surfaced["fireworks_raw_outputs"] == [raw_output]
|
||||
|
|
@ -1493,9 +1443,7 @@ def test_transform_request_direct_route_passthrough():
|
|||
|
||||
def test_map_extra_body_params_translates_truncate_prompt_tokens():
|
||||
config = FireworksAIConfig()
|
||||
result = config.map_extra_body_params(
|
||||
{"extra_body": {"truncate_prompt_tokens": 4096}}, _REASONING_MODEL
|
||||
)
|
||||
result = config.map_extra_body_params({"extra_body": {"truncate_prompt_tokens": 4096}}, _REASONING_MODEL)
|
||||
assert result == {"prompt_truncate_len": 4096}
|
||||
|
||||
|
||||
|
|
@ -1654,9 +1602,7 @@ def test_map_extra_body_params_non_dict_chat_template_kwargs_dropped():
|
|||
def test_map_extra_body_params_guided_json():
|
||||
config = FireworksAIConfig()
|
||||
schema = {"type": "object", "properties": {"x": {"type": "string"}}}
|
||||
result = config.map_extra_body_params(
|
||||
{"extra_body": {"guided_json": schema}}, _REASONING_MODEL
|
||||
)
|
||||
result = config.map_extra_body_params({"extra_body": {"guided_json": schema}}, _REASONING_MODEL)
|
||||
assert result == {
|
||||
"response_format": {
|
||||
"type": "json_schema",
|
||||
|
|
@ -1667,16 +1613,10 @@ def test_map_extra_body_params_guided_json():
|
|||
|
||||
def test_map_extra_body_params_guided_grammar_and_choice():
|
||||
config = FireworksAIConfig()
|
||||
grammar = config.map_extra_body_params(
|
||||
{"extra_body": {"guided_grammar": "root ::= 'hello'"}}, _REASONING_MODEL
|
||||
)
|
||||
assert grammar == {
|
||||
"response_format": {"type": "grammar", "grammar": "root ::= 'hello'"}
|
||||
}
|
||||
grammar = config.map_extra_body_params({"extra_body": {"guided_grammar": "root ::= 'hello'"}}, _REASONING_MODEL)
|
||||
assert grammar == {"response_format": {"type": "grammar", "grammar": "root ::= 'hello'"}}
|
||||
|
||||
choice = config.map_extra_body_params(
|
||||
{"extra_body": {"guided_choice": ["yes", "no"]}}, _REASONING_MODEL
|
||||
)
|
||||
choice = config.map_extra_body_params({"extra_body": {"guided_choice": ["yes", "no"]}}, _REASONING_MODEL)
|
||||
assert choice == {
|
||||
"response_format": {
|
||||
"type": "json_schema",
|
||||
|
|
@ -1762,9 +1702,7 @@ def test_map_extra_body_params_strips_unsupported_nim_vllm_params(param, value,
|
|||
|
||||
config = FireworksAIConfig()
|
||||
with caplog.at_level(logging.DEBUG):
|
||||
result = config.map_extra_body_params(
|
||||
{"extra_body": {param: value}}, _REASONING_MODEL
|
||||
)
|
||||
result = config.map_extra_body_params({"extra_body": {param: value}}, _REASONING_MODEL)
|
||||
assert result == {}
|
||||
assert param in caplog.text
|
||||
|
||||
|
|
@ -1836,15 +1774,10 @@ def test_nim_vllm_extras_translated_end_to_end_in_request_body():
|
|||
assert request_body["top_k"] == 40
|
||||
|
||||
|
||||
|
||||
|
||||
def test_streaming_preserves_selected_model_for_private_accounting():
|
||||
from litellm.llms.custom_httpx.http_handler import HTTPHandler
|
||||
|
||||
requested_route = (
|
||||
"accounts/fireworks/routers/firerouter/"
|
||||
"kimi-k3/deepseek-v4-pro-0813/deepseek-v4-flash-0731"
|
||||
)
|
||||
requested_route = "accounts/fireworks/routers/firerouter/kimi-k3/deepseek-v4-pro-0813/deepseek-v4-flash-0731"
|
||||
selected_model = "deepseek-v4-flash-0731"
|
||||
sse_lines = [
|
||||
"data: "
|
||||
|
|
@ -1898,19 +1831,14 @@ def test_streaming_preserves_selected_model_for_private_accounting():
|
|||
|
||||
assert chunks
|
||||
assert {chunk.model for chunk in chunks} == {requested_route}
|
||||
assert {
|
||||
chunk._hidden_params.get("provider_response_model") for chunk in chunks
|
||||
} == {selected_model}
|
||||
assert {chunk._hidden_params.get("provider_response_model") for chunk in chunks} == {selected_model}
|
||||
|
||||
assembled = litellm.stream_chunk_builder(chunks=chunks)
|
||||
assert assembled is not None
|
||||
assert assembled.model == requested_route
|
||||
assert assembled._hidden_params["provider_response_model"] == selected_model
|
||||
selected_model_info = litellm.model_cost[f"fireworks_ai/{selected_model}"]
|
||||
expected_cost = (
|
||||
5 * selected_model_info["input_cost_per_token"]
|
||||
+ selected_model_info["output_cost_per_token"]
|
||||
)
|
||||
expected_cost = 5 * selected_model_info["input_cost_per_token"] + selected_model_info["output_cost_per_token"]
|
||||
assert litellm.completion_cost(
|
||||
completion_response=assembled,
|
||||
custom_llm_provider="fireworks_ai",
|
||||
|
|
|
|||
|
|
@ -85,6 +85,7 @@ CONNECTION_NAMES: Final = (
|
|||
"litellm_credential_name",
|
||||
"configurable_clientside_auth_params",
|
||||
"use_xai_oauth",
|
||||
"fireworks_shared_session_affinity",
|
||||
"aws_batch_role_arn",
|
||||
"s3_bucket_name",
|
||||
"s3_region_name",
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue