This commit is contained in:
andyh-viv 2026-10-24 08:31:13 +00:00 • committed by GitHub
commit 0216df7cc2
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 245 additions and 117 deletions

View file

@ -71,13 +71,21 @@ jobs:
# a lookup key into the haveibeenpwned range API (the protocol mandates
# SHA-1) and the digest itself never leaves the proxy beyond its first 5
# characters.
- name: Filter SARIF (OCI sha256, HIBP sha1)
# Finally it fires on the Fireworks shared-session affinity token in
# litellm/llms/fireworks_ai/common_utils.py, where a credential is hashed so
# concurrent requests on one Fireworks key map to a stable x-session-affinity
# value and their shared prompt prefix hits the replica cache. Nothing is
# authenticated by that digest; it is a cache-partition key, and SHA-256 is
# kept because two credentials must not collide onto one token. CodeQL flags
# it because the flow starts at a lookup key literally named `api_key`.
- name: Filter SARIF (OCI sha256, HIBP sha1, Fireworks affinity token)
if: matrix.language == 'python'
uses: advanced-security/filter-sarif@2da736ff05ef065cb2894ac6892e47b5eac2c3c0 # v1.1
with:
patterns: |
-litellm/llms/oci/common_utils.py:py/weak-sensitive-data-hashing
-litellm/proxy/auth/password_policy.py:py/weak-sensitive-data-hashing
-litellm/llms/fireworks_ai/common_utils.py:py/weak-sensitive-data-hashing
input: sarif-results/python.sarif
output: sarif-results/python.sarif

View file

@ -82,6 +82,7 @@ OPTIONAL_KWARGS_KEYS: Final = (
"fireworks_forward_user_id",
"responses_api_version",
PROVIDER_AFFINITY_HEADER_KWARG_KEY,
"fireworks_shared_session_affinity",
}
)
| AWS_CREDENTIAL_KWARGS_KEYS

View file

@ -1,6 +1,7 @@
import hashlib
from collections.abc import Mapping
from types import MappingProxyType
from typing import Final
from typing import Final, cast
from httpx import Headers
@ -15,27 +16,76 @@ class FireworksAIException(BaseLLMException):
pass
SHARED_SESSION_AFFINITY_PARAM: Final = "fireworks_shared_session_affinity"
def _lookup(mapping: object, key: str) -> object | None:
if isinstance(mapping, Mapping):
return cast("Mapping[str, object]", mapping).get(key)
return None
def get_fireworks_session_id(litellm_params: Mapping[str, object]) -> str | None:
"""
Session id to send as `x-session-affinity`, or None when the caller gave none.
Priority:
1. Explicit `litellm_session_id` or `session_id` in litellm_params
2. Explicit `session_id` in metadata
3. Shared affinity, when `fireworks_shared_session_affinity` is set: a stable digest of
the caller's LiteLLM user key (proxy) or Fireworks credential (direct SDK)
Deliberately does not fall back to `litellm_trace_id`, and ignores session ids the
proxy generated for a request that had none: both are per request, so using them
pins every request to a different Fireworks node and prompt caching never hits.
"""
params: Final = litellm_params
metadata: Final = params.get("metadata")
if isinstance(metadata, Mapping) and metadata.get(SESSION_ID_GENERATED_METADATA_KEY):
has_generated_session_id: Final = _lookup(metadata, SESSION_ID_GENERATED_METADATA_KEY)
if not has_generated_session_id:
for key in ("litellm_session_id", "session_id"):
value = params.get(key)
if value:
return str(value)
session_from_metadata: Final = _lookup(metadata, "session_id")
if session_from_metadata:
return str(session_from_metadata)
shared_affinity: Final = params.get(SHARED_SESSION_AFFINITY_PARAM) or _lookup(
metadata, SHARED_SESSION_AFFINITY_PARAM
)
if not shared_affinity:
return None
for key in ("litellm_session_id", "session_id"):
value = params.get(key)
if value:
return str(value)
if isinstance(metadata, Mapping):
value = metadata.get("session_id")
if value:
return str(value)
return None
user_key_hash: Final = _lookup(metadata, "user_api_key_hash")
if isinstance(user_key_hash, str) and user_key_hash:
# `user_api_key_hash` is not guaranteed to be a digest: the proxy leaves opaque
# custom-auth keys and OAuth bearer tokens unhashed, so hashing it here is what
# keeps a raw proxy credential from riding to Fireworks in a request header.
return f"litellm-user-{shared_affinity_token(user_key_hash)}"
configured: Final = params.get("api_key")
credential: Final = configured if isinstance(configured, str) and configured else resolve_fireworks_api_key(None)
if credential is None:
return None
return shared_affinity_token(credential)
def shared_affinity_token(credential: str) -> str:
"""
Opaque, stable routing token for one credential.
This is a cache-isolation identifier, not a password hash: it is sent to Fireworks as
`x-session-affinity` so concurrent requests on one credential land on one replica and
their shared prompt prefix hits the cache. It is never used to authenticate, and it
leaks nothing beyond a digest that Fireworks could compute itself from its own key.
Collision resistance is however load bearing: two distinct credentials must not share
a token, or one account's cache becomes readable by another, so the digest must stay a
strong hash. `usedforsecurity=False` declares that no authentication property rests on
it (and keeps it available under FIPS builds).
"""
return hashlib.sha256(credential.encode(), usedforsecurity=False).hexdigest()
def with_fireworks_session_affinity(

View file

@ -5978,6 +5978,7 @@ def completion(
key: kwargs[key]
for key in (
*AWS_CREDENTIAL_KWARGS_KEYS,
"fireworks_shared_session_affinity",
*ANTHROPIC_WIF_KWARGS_KEYS,
*OPENAI_WIF_KWARGS_KEYS,
*OAUTH_TOKEN_EXCHANGE_KWARGS_KEYS,

View file

@ -91,6 +91,7 @@ class ProviderConnection:
litellm_credential_name: str | None = None
configurable_clientside_auth_params: "Sequence[str | ConfigurableClientsideParamsCustomAuth] | None" = None
use_xai_oauth: bool | None = None
fireworks_shared_session_affinity: bool | None = None
github_copilot_auth_type: str | None = None
github_copilot_host: str | None = None
github_copilot_user_session: object | None = None

View file

@ -12,7 +12,7 @@ from litellm.constants import SESSION_ID_GENERATED_METADATA_KEY
from litellm.litellm_core_utils.get_supported_openai_params import get_supported_openai_params
from litellm.llms.custom_httpx.http_handler import HTTPHandler
from litellm.llms.fireworks_ai.chat.transformation import FireworksAIConfig
from litellm.llms.fireworks_ai.common_utils import get_fireworks_session_id
from litellm.llms.fireworks_ai.common_utils import get_fireworks_session_id, shared_affinity_token
from litellm.types.utils import (
ChatCompletionMessageToolCall,
Function,
@ -221,16 +221,13 @@ def test_validate_environment_raises_without_api_key(monkeypatch):
def test_get_fireworks_session_id_prefers_litellm_session_id_over_trace_id():
assert (
get_fireworks_session_id(
{"litellm_session_id": "session-123", "litellm_trace_id": "trace-123"}
)
get_fireworks_session_id({"litellm_session_id": "session-123", "litellm_trace_id": "trace-123"})
== "session-123"
)
def test_get_fireworks_session_id_ignores_proxy_generated_session_id():
"""general_settings.missing_session_id: generate stamps a fresh id per request; sending it
as x-session-affinity would pin every request to a different node."""
def test_get_fireworks_session_id_ignores_proxy_generated_session_id_without_shared_affinity():
"""Without shared affinity, a generated session id must be ignored."""
assert (
get_fireworks_session_id(
{
@ -243,6 +240,142 @@ def test_get_fireworks_session_id_ignores_proxy_generated_session_id():
)
def test_get_fireworks_session_id_uses_shared_affinity_even_with_generated_id():
"""With shared affinity, the stable token wins even when the proxy generated a fresh id."""
litellm_params = {
"litellm_session_id": "generated-1",
"fireworks_shared_session_affinity": True,
"metadata": {
"session_id": "generated-1",
SESSION_ID_GENERATED_METADATA_KEY: True,
"user_api_key_hash": "stable-user-hash",
},
}
assert get_fireworks_session_id(litellm_params) == f"litellm-user-{shared_affinity_token('stable-user-hash')}"
def test_get_fireworks_session_id_falls_back_to_credential_token():
litellm_params = {
"fireworks_shared_session_affinity": True,
"api_key": "fw-api-key",
"metadata": {},
}
assert get_fireworks_session_id(litellm_params) == shared_affinity_token("fw-api-key")
def test_get_fireworks_session_id_uses_metadata_nested_shared_affinity():
litellm_params = {"metadata": {"fireworks_shared_session_affinity": True, "user_api_key_hash": "stable-user-hash"}}
assert get_fireworks_session_id(litellm_params) == f"litellm-user-{shared_affinity_token('stable-user-hash')}"
def test_get_fireworks_session_id_never_emits_a_raw_proxy_credential():
"""A proxy user key is not always stored hashed: opaque custom-auth keys and OAuth
bearer tokens pass through `_safe_hash_litellm_api_key` unchanged, so the affinity
token must hash the value again rather than forwarding it verbatim."""
raw_token = "opaque-custom-auth-token-abc123"
litellm_params = {"metadata": {"fireworks_shared_session_affinity": True, "user_api_key_hash": raw_token}}
token = get_fireworks_session_id(litellm_params)
assert token is not None
assert raw_token not in token
assert token == f"litellm-user-{shared_affinity_token(raw_token)}"
def test_get_fireworks_session_id_explicit_overrides_shared():
litellm_params = {
"litellm_session_id": "explicit-session",
"metadata": {"fireworks_shared_session_affinity": True, "user_api_key_hash": "stable-user-hash"},
}
assert get_fireworks_session_id(litellm_params) == "explicit-session"
def test_get_fireworks_session_id_is_none_without_the_flag_or_a_credential():
assert get_fireworks_session_id({"metadata": {"user_api_key_hash": "stable-user-hash"}}) is None
def test_shared_affinity_token_is_stable_and_credential_specific():
assert shared_affinity_token("fw-a") == shared_affinity_token("fw-a")
assert shared_affinity_token("fw-a") != shared_affinity_token("fw-b")
def test_top_level_shared_affinity_kwarg_sets_header_without_body_leak():
"""Full pipeline: litellm.completion(fireworks_shared_session_affinity=True) must set
x-session-affinity (hashed off the Fireworks credential) and must NOT put the flag
into the request body, which Fireworks rejects."""
model = "accounts/fireworks/models/llama-v3p1-8b-instruct"
reply = {
"id": "c1",
"object": "chat.completion",
"created": 1,
"model": model,
"choices": [{"index": 0, "message": {"role": "assistant", "content": "Hi"}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
}
raw_response = MagicMock()
raw_response.status_code = 200
raw_response.headers = {}
raw_response.text = json.dumps(reply)
raw_response.json = lambda: reply
captured: dict = {}
client = MagicMock(spec=HTTPHandler)
def post(*args, **kwargs):
captured["headers"] = dict(kwargs.get("headers") or {})
captured["body"] = json.loads(kwargs.get("data") or "{}")
return raw_response
client.post.side_effect = post
litellm.completion(
model=f"fireworks_ai/{model}",
messages=[{"role": "user", "content": "hi"}],
api_key="fw-test-key",
client=client,
fireworks_shared_session_affinity=True,
)
headers = {k.lower(): v for k, v in captured["headers"].items()}
assert headers.get("x-session-affinity") == shared_affinity_token("fw-test-key")
assert "fireworks_shared_session_affinity" not in captured["body"]
def test_shared_affinity_kwarg_is_absent_from_the_request_body():
"""A run with the flag off must not carry the field anywhere in the request body, and a
run with it on must reach the same body, so the flag is never serialized to Fireworks."""
model = "accounts/fireworks/models/llama-v3p1-8b-instruct"
reply = {
"id": "c1",
"object": "chat.completion",
"created": 1,
"model": model,
"choices": [{"index": 0, "message": {"role": "assistant", "content": "Hi"}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
}
raw_response = MagicMock()
raw_response.status_code = 200
raw_response.headers = {}
raw_response.text = json.dumps(reply)
raw_response.json = lambda: reply
bodies: list = []
def run(**extra):
client = MagicMock(spec=HTTPHandler)
client.post.side_effect = lambda *a, **kw: (bodies.append(json.loads(kw.get("data") or "{}")), raw_response)[1]
litellm.completion(
model=f"fireworks_ai/{model}",
messages=[{"role": "user", "content": "hi"}],
api_key="fw-test-key",
client=client,
**extra,
)
run()
run(fireworks_shared_session_affinity=True)
assert bodies[0] == bodies[1]
def test_handle_message_content_with_tool_calls():
config = FireworksAIConfig()
message = Message(
@ -275,25 +408,18 @@ def test_handle_message_content_with_tool_calls():
},
}
]
updated_message = config._handle_message_content_with_tool_calls(
message, tool_calls
)
updated_message = config._handle_message_content_with_tool_calls(message, tool_calls)
assert updated_message.tool_calls is not None
assert len(updated_message.tool_calls) == 1
assert updated_message.tool_calls[0].function.name == "get_current_weather"
assert (
updated_message.tool_calls[0].function.arguments
== expected_tool_call.function.arguments
)
assert updated_message.tool_calls[0].function.arguments == expected_tool_call.function.arguments
def test_get_supported_openai_params_reasoning_effort():
"""Test that reasoning_effort is only included in supported params for models that support it."""
config = FireworksAIConfig()
supported_params = config.get_supported_openai_params(
"fireworks_ai/accounts/fireworks/models/glm-5p1"
)
supported_params = config.get_supported_openai_params("fireworks_ai/accounts/fireworks/models/glm-5p1")
assert "reasoning_effort" in supported_params
assert "thinking" in supported_params
@ -308,9 +434,7 @@ def test_get_supported_openai_params_parallel_tool_calls():
"""Test that parallel_tool_calls is included for models that support function calling."""
config = FireworksAIConfig()
supported_params = config.get_supported_openai_params(
"fireworks_ai/accounts/fireworks/models/glm-5p1"
)
supported_params = config.get_supported_openai_params("fireworks_ai/accounts/fireworks/models/glm-5p1")
assert "parallel_tool_calls" in supported_params
assert "tools" in supported_params
assert "tool_choice" in supported_params
@ -324,9 +448,7 @@ def test_get_supported_openai_params_parallel_tool_calls():
def test_get_supported_openai_params_short_model_name_resolves_account_prefixed_entry():
config = FireworksAIConfig()
supported_params = config.get_supported_openai_params(
"fireworks_ai/deepseek-v4-pro-0813"
)
supported_params = config.get_supported_openai_params("fireworks_ai/deepseek-v4-pro-0813")
assert "tool_choice" in supported_params
assert "reasoning_effort" in supported_params
@ -335,9 +457,7 @@ def test_get_supported_openai_params_short_model_name_resolves_account_prefixed_
def test_get_supported_openai_params_preserves_generic_reasoning_fallback():
config = FireworksAIConfig()
supported_params = config.get_supported_openai_params(
"fireworks_ai/accounts/fireworks/models/glm-5p3-flash"
)
supported_params = config.get_supported_openai_params("fireworks_ai/accounts/fireworks/models/glm-5p3-flash")
assert "reasoning_effort" in supported_params
@ -404,14 +524,10 @@ def test_get_models_url_no_double_v1(api_base, expected_url_prefix):
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.json.return_value = {
"models": [{"name": "accounts/fireworks/models/llama-v3-70b"}]
}
mock_response.json.return_value = {"models": [{"name": "accounts/fireworks/models/llama-v3-70b"}]}
with (
patch(
"litellm.module_level_client.get", return_value=mock_response
) as mock_get,
patch("litellm.module_level_client.get", return_value=mock_response) as mock_get,
patch(
"litellm.llms.fireworks_ai.chat.transformation.get_secret_str",
side_effect=lambda key: {
@ -423,13 +539,9 @@ def test_get_models_url_no_double_v1(api_base, expected_url_prefix):
):
result = config.get_models(api_key="test-key", api_base=api_base)
called_url = mock_get.call_args.kwargs.get("url") or mock_get.call_args[1].get(
"url", ""
)
called_url = mock_get.call_args.kwargs.get("url") or mock_get.call_args[1].get("url", "")
assert "/v1/v1/" not in called_url, f"Double /v1/ detected in URL: {called_url}"
assert called_url.startswith(
expected_url_prefix
), f"URL {called_url} does not start with {expected_url_prefix}"
assert called_url.startswith(expected_url_prefix), f"URL {called_url} does not start with {expected_url_prefix}"
assert result == ["fireworks_ai/accounts/fireworks/models/llama-v3-70b"]
@ -457,9 +569,7 @@ def test_transform_messages_helper_removes_provider_specific_fields():
},
]
# Call helper
out = config._transform_messages_helper(
messages, model="fireworks/test", litellm_params={}
)
out = config._transform_messages_helper(messages, model="fireworks/test", litellm_params={})
for msg in out:
assert "provider_specific_fields" not in msg
@ -480,15 +590,11 @@ def test_transform_messages_helper_strips_thinking_blocks_but_keeps_reasoning_co
{
"role": "assistant",
"content": "I can help.",
"thinking_blocks": [
{"type": "thinking", "thinking": "internal", "signature": ""}
],
"thinking_blocks": [{"type": "thinking", "thinking": "internal", "signature": ""}],
"reasoning_content": "internal",
},
]
out = config._transform_messages_helper(
messages, model="accounts/fireworks/models/glm-5p1", litellm_params={}
)
out = config._transform_messages_helper(messages, model="accounts/fireworks/models/glm-5p1", litellm_params={})
assert "thinking_blocks" not in out[1]
assert out[1]["reasoning_content"] == "internal"
assert out[1]["content"] == "I can help."
@ -1006,9 +1112,7 @@ def test_transform_messages_helper_rejects_file_blocks():
litellm.BadRequestError,
match="Fireworks AI chat completions does not support file content blocks",
):
config._transform_messages_helper(
messages, model="accounts/fireworks/models/kimi-k2p6", litellm_params={}
)
config._transform_messages_helper(messages, model="accounts/fireworks/models/kimi-k2p6", litellm_params={})
def test_transform_messages_helper_rejects_non_vision_image_inputs():
@ -1020,18 +1124,14 @@ def test_transform_messages_helper_rejects_non_vision_image_inputs():
{"type": "text", "text": "Describe this"},
{
"type": "image_url",
"image_url": {
"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE="
},
"image_url": {"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE="},
},
],
}
]
with pytest.raises(litellm.BadRequestError, match="does not support image inputs"):
config._transform_messages_helper(
messages, model="accounts/fireworks/models/glm-5p2", litellm_params={}
)
config._transform_messages_helper(messages, model="accounts/fireworks/models/glm-5p2", litellm_params={})
def test_transform_messages_helper_allows_vision_image_inputs():
@ -1043,9 +1143,7 @@ def test_transform_messages_helper_allows_vision_image_inputs():
{"type": "text", "text": "Describe this"},
{
"type": "image_url",
"image_url": {
"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE="
},
"image_url": {"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE="},
},
],
}
@ -1069,9 +1167,7 @@ def test_image_inputs_not_rejected_for_fuzzy_non_vision_match():
custom_model = "accounts/myorg/models/custom-glm-5p2"
assert config._get_model_cost_capability(custom_model, "supports_vision") is False
assert (
config._get_model_cost_capability_exact(custom_model, "supports_vision") is None
)
assert config._get_model_cost_capability_exact(custom_model, "supports_vision") is None
messages = [
{
@ -1079,16 +1175,12 @@ def test_image_inputs_not_rejected_for_fuzzy_non_vision_match():
"content": [
{
"type": "image_url",
"image_url": {
"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE="
},
"image_url": {"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE="},
},
],
}
]
out = config._transform_messages_helper(
messages, model=custom_model, litellm_params={}
)
out = config._transform_messages_helper(messages, model=custom_model, litellm_params={})
assert out == messages
@ -1101,9 +1193,7 @@ def test_transform_messages_helper_skips_non_dict_content():
}
]
out = config._transform_messages_helper(
messages, model="accounts/fireworks/models/glm-5p2", litellm_params={}
)
out = config._transform_messages_helper(messages, model="accounts/fireworks/models/glm-5p2", litellm_params={})
assert out == messages
@ -1343,9 +1433,7 @@ def test_streaming_surfaces_fireworks_response_fields():
surfaced: dict = {}
for chunk in stream:
fields = getattr(chunk, "provider_specific_fields", None) or {}
surfaced.update(
{k: v for k, v in fields.items() if k.startswith("fireworks_")}
)
surfaced.update({k: v for k, v in fields.items() if k.startswith("fireworks_")})
assert surfaced["fireworks_token_ids"] == [[123]]
assert surfaced["fireworks_raw_outputs"] == [raw_output]
@ -1398,9 +1486,7 @@ def test_transform_request_direct_route_passthrough():
def test_map_extra_body_params_translates_truncate_prompt_tokens():
config = FireworksAIConfig()
result = config.map_extra_body_params(
{"extra_body": {"truncate_prompt_tokens": 4096}}, _REASONING_MODEL
)
result = config.map_extra_body_params({"extra_body": {"truncate_prompt_tokens": 4096}}, _REASONING_MODEL)
assert result == {"prompt_truncate_len": 4096}
@ -1559,9 +1645,7 @@ def test_map_extra_body_params_non_dict_chat_template_kwargs_dropped():
def test_map_extra_body_params_guided_json():
config = FireworksAIConfig()
schema = {"type": "object", "properties": {"x": {"type": "string"}}}
result = config.map_extra_body_params(
{"extra_body": {"guided_json": schema}}, _REASONING_MODEL
)
result = config.map_extra_body_params({"extra_body": {"guided_json": schema}}, _REASONING_MODEL)
assert result == {
"response_format": {
"type": "json_schema",
@ -1572,16 +1656,10 @@ def test_map_extra_body_params_guided_json():
def test_map_extra_body_params_guided_grammar_and_choice():
config = FireworksAIConfig()
grammar = config.map_extra_body_params(
{"extra_body": {"guided_grammar": "root ::= 'hello'"}}, _REASONING_MODEL
)
assert grammar == {
"response_format": {"type": "grammar", "grammar": "root ::= 'hello'"}
}
grammar = config.map_extra_body_params({"extra_body": {"guided_grammar": "root ::= 'hello'"}}, _REASONING_MODEL)
assert grammar == {"response_format": {"type": "grammar", "grammar": "root ::= 'hello'"}}
choice = config.map_extra_body_params(
{"extra_body": {"guided_choice": ["yes", "no"]}}, _REASONING_MODEL
)
choice = config.map_extra_body_params({"extra_body": {"guided_choice": ["yes", "no"]}}, _REASONING_MODEL)
assert choice == {
"response_format": {
"type": "json_schema",
@ -1667,9 +1745,7 @@ def test_map_extra_body_params_strips_unsupported_nim_vllm_params(param, value,
config = FireworksAIConfig()
with caplog.at_level(logging.DEBUG):
result = config.map_extra_body_params(
{"extra_body": {param: value}}, _REASONING_MODEL
)
result = config.map_extra_body_params({"extra_body": {param: value}}, _REASONING_MODEL)
assert result == {}
assert param in caplog.text
@ -1741,15 +1817,10 @@ def test_nim_vllm_extras_translated_end_to_end_in_request_body():
assert request_body["top_k"] == 40
def test_streaming_preserves_selected_model_for_private_accounting():
from litellm.llms.custom_httpx.http_handler import HTTPHandler
requested_route = (
"accounts/fireworks/routers/firerouter/"
"kimi-k3/deepseek-v4-pro-0813/deepseek-v4-flash-0731"
)
requested_route = "accounts/fireworks/routers/firerouter/kimi-k3/deepseek-v4-pro-0813/deepseek-v4-flash-0731"
selected_model = "deepseek-v4-flash-0731"
sse_lines = [
"data: "
@ -1803,19 +1874,14 @@ def test_streaming_preserves_selected_model_for_private_accounting():
assert chunks
assert {chunk.model for chunk in chunks} == {requested_route}
assert {
chunk._hidden_params.get("provider_response_model") for chunk in chunks
} == {selected_model}
assert {chunk._hidden_params.get("provider_response_model") for chunk in chunks} == {selected_model}
assembled = litellm.stream_chunk_builder(chunks=chunks)
assert assembled is not None
assert assembled.model == requested_route
assert assembled._hidden_params["provider_response_model"] == selected_model
selected_model_info = litellm.model_cost[f"fireworks_ai/{selected_model}"]
expected_cost = (
5 * selected_model_info["input_cost_per_token"]
+ selected_model_info["output_cost_per_token"]
)
expected_cost = 5 * selected_model_info["input_cost_per_token"] + selected_model_info["output_cost_per_token"]
assert litellm.completion_cost(
completion_response=assembled,
custom_llm_provider="fireworks_ai",

View file

@ -94,6 +94,7 @@ CONNECTION_NAMES: Final = (
"github_copilot_host",
"configurable_clientside_auth_params",
"use_xai_oauth",
"fireworks_shared_session_affinity",
"fireworks_forward_user_id",
"aws_batch_role_arn",
"aws_bedrock_control_plane_endpoint",