mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
Merge 91f0c81bd1 into e04e89c701
This commit is contained in:
commit
0216df7cc2
7 changed files with 245 additions and 117 deletions
10
.github/workflows/codeql.yml
vendored
10
.github/workflows/codeql.yml
vendored
|
|
@ -71,13 +71,21 @@ jobs:
|
|||
# a lookup key into the haveibeenpwned range API (the protocol mandates
|
||||
# SHA-1) and the digest itself never leaves the proxy beyond its first 5
|
||||
# characters.
|
||||
- name: Filter SARIF (OCI sha256, HIBP sha1)
|
||||
# Finally it fires on the Fireworks shared-session affinity token in
|
||||
# litellm/llms/fireworks_ai/common_utils.py, where a credential is hashed so
|
||||
# concurrent requests on one Fireworks key map to a stable x-session-affinity
|
||||
# value and their shared prompt prefix hits the replica cache. Nothing is
|
||||
# authenticated by that digest; it is a cache-partition key, and SHA-256 is
|
||||
# kept because two credentials must not collide onto one token. CodeQL flags
|
||||
# it because the flow starts at a lookup key literally named `api_key`.
|
||||
- name: Filter SARIF (OCI sha256, HIBP sha1, Fireworks affinity token)
|
||||
if: matrix.language == 'python'
|
||||
uses: advanced-security/filter-sarif@2da736ff05ef065cb2894ac6892e47b5eac2c3c0 # v1.1
|
||||
with:
|
||||
patterns: |
|
||||
-litellm/llms/oci/common_utils.py:py/weak-sensitive-data-hashing
|
||||
-litellm/proxy/auth/password_policy.py:py/weak-sensitive-data-hashing
|
||||
-litellm/llms/fireworks_ai/common_utils.py:py/weak-sensitive-data-hashing
|
||||
input: sarif-results/python.sarif
|
||||
output: sarif-results/python.sarif
|
||||
|
||||
|
|
|
|||
|
|
@ -82,6 +82,7 @@ OPTIONAL_KWARGS_KEYS: Final = (
|
|||
"fireworks_forward_user_id",
|
||||
"responses_api_version",
|
||||
PROVIDER_AFFINITY_HEADER_KWARG_KEY,
|
||||
"fireworks_shared_session_affinity",
|
||||
}
|
||||
)
|
||||
| AWS_CREDENTIAL_KWARGS_KEYS
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import hashlib
|
||||
from collections.abc import Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
from typing import Final, cast
|
||||
|
||||
from httpx import Headers
|
||||
|
||||
|
|
@ -15,27 +16,76 @@ class FireworksAIException(BaseLLMException):
|
|||
pass
|
||||
|
||||
|
||||
SHARED_SESSION_AFFINITY_PARAM: Final = "fireworks_shared_session_affinity"
|
||||
|
||||
|
||||
def _lookup(mapping: object, key: str) -> object | None:
|
||||
if isinstance(mapping, Mapping):
|
||||
return cast("Mapping[str, object]", mapping).get(key)
|
||||
return None
|
||||
|
||||
|
||||
def get_fireworks_session_id(litellm_params: Mapping[str, object]) -> str | None:
|
||||
"""
|
||||
Session id to send as `x-session-affinity`, or None when the caller gave none.
|
||||
|
||||
Priority:
|
||||
1. Explicit `litellm_session_id` or `session_id` in litellm_params
|
||||
2. Explicit `session_id` in metadata
|
||||
3. Shared affinity, when `fireworks_shared_session_affinity` is set: a stable digest of
|
||||
the caller's LiteLLM user key (proxy) or Fireworks credential (direct SDK)
|
||||
|
||||
Deliberately does not fall back to `litellm_trace_id`, and ignores session ids the
|
||||
proxy generated for a request that had none: both are per request, so using them
|
||||
pins every request to a different Fireworks node and prompt caching never hits.
|
||||
"""
|
||||
params: Final = litellm_params
|
||||
metadata: Final = params.get("metadata")
|
||||
if isinstance(metadata, Mapping) and metadata.get(SESSION_ID_GENERATED_METADATA_KEY):
|
||||
|
||||
has_generated_session_id: Final = _lookup(metadata, SESSION_ID_GENERATED_METADATA_KEY)
|
||||
if not has_generated_session_id:
|
||||
for key in ("litellm_session_id", "session_id"):
|
||||
value = params.get(key)
|
||||
if value:
|
||||
return str(value)
|
||||
session_from_metadata: Final = _lookup(metadata, "session_id")
|
||||
if session_from_metadata:
|
||||
return str(session_from_metadata)
|
||||
|
||||
shared_affinity: Final = params.get(SHARED_SESSION_AFFINITY_PARAM) or _lookup(
|
||||
metadata, SHARED_SESSION_AFFINITY_PARAM
|
||||
)
|
||||
if not shared_affinity:
|
||||
return None
|
||||
for key in ("litellm_session_id", "session_id"):
|
||||
value = params.get(key)
|
||||
if value:
|
||||
return str(value)
|
||||
if isinstance(metadata, Mapping):
|
||||
value = metadata.get("session_id")
|
||||
if value:
|
||||
return str(value)
|
||||
return None
|
||||
|
||||
user_key_hash: Final = _lookup(metadata, "user_api_key_hash")
|
||||
if isinstance(user_key_hash, str) and user_key_hash:
|
||||
# `user_api_key_hash` is not guaranteed to be a digest: the proxy leaves opaque
|
||||
# custom-auth keys and OAuth bearer tokens unhashed, so hashing it here is what
|
||||
# keeps a raw proxy credential from riding to Fireworks in a request header.
|
||||
return f"litellm-user-{shared_affinity_token(user_key_hash)}"
|
||||
|
||||
configured: Final = params.get("api_key")
|
||||
credential: Final = configured if isinstance(configured, str) and configured else resolve_fireworks_api_key(None)
|
||||
if credential is None:
|
||||
return None
|
||||
return shared_affinity_token(credential)
|
||||
|
||||
|
||||
def shared_affinity_token(credential: str) -> str:
|
||||
"""
|
||||
Opaque, stable routing token for one credential.
|
||||
|
||||
This is a cache-isolation identifier, not a password hash: it is sent to Fireworks as
|
||||
`x-session-affinity` so concurrent requests on one credential land on one replica and
|
||||
their shared prompt prefix hits the cache. It is never used to authenticate, and it
|
||||
leaks nothing beyond a digest that Fireworks could compute itself from its own key.
|
||||
Collision resistance is however load bearing: two distinct credentials must not share
|
||||
a token, or one account's cache becomes readable by another, so the digest must stay a
|
||||
strong hash. `usedforsecurity=False` declares that no authentication property rests on
|
||||
it (and keeps it available under FIPS builds).
|
||||
"""
|
||||
return hashlib.sha256(credential.encode(), usedforsecurity=False).hexdigest()
|
||||
|
||||
|
||||
def with_fireworks_session_affinity(
|
||||
|
|
|
|||
|
|
@ -5978,6 +5978,7 @@ def completion(
|
|||
key: kwargs[key]
|
||||
for key in (
|
||||
*AWS_CREDENTIAL_KWARGS_KEYS,
|
||||
"fireworks_shared_session_affinity",
|
||||
*ANTHROPIC_WIF_KWARGS_KEYS,
|
||||
*OPENAI_WIF_KWARGS_KEYS,
|
||||
*OAUTH_TOKEN_EXCHANGE_KWARGS_KEYS,
|
||||
|
|
|
|||
|
|
@ -91,6 +91,7 @@ class ProviderConnection:
|
|||
litellm_credential_name: str | None = None
|
||||
configurable_clientside_auth_params: "Sequence[str | ConfigurableClientsideParamsCustomAuth] | None" = None
|
||||
use_xai_oauth: bool | None = None
|
||||
fireworks_shared_session_affinity: bool | None = None
|
||||
github_copilot_auth_type: str | None = None
|
||||
github_copilot_host: str | None = None
|
||||
github_copilot_user_session: object | None = None
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ from litellm.constants import SESSION_ID_GENERATED_METADATA_KEY
|
|||
from litellm.litellm_core_utils.get_supported_openai_params import get_supported_openai_params
|
||||
from litellm.llms.custom_httpx.http_handler import HTTPHandler
|
||||
from litellm.llms.fireworks_ai.chat.transformation import FireworksAIConfig
|
||||
from litellm.llms.fireworks_ai.common_utils import get_fireworks_session_id
|
||||
from litellm.llms.fireworks_ai.common_utils import get_fireworks_session_id, shared_affinity_token
|
||||
from litellm.types.utils import (
|
||||
ChatCompletionMessageToolCall,
|
||||
Function,
|
||||
|
|
@ -221,16 +221,13 @@ def test_validate_environment_raises_without_api_key(monkeypatch):
|
|||
|
||||
def test_get_fireworks_session_id_prefers_litellm_session_id_over_trace_id():
|
||||
assert (
|
||||
get_fireworks_session_id(
|
||||
{"litellm_session_id": "session-123", "litellm_trace_id": "trace-123"}
|
||||
)
|
||||
get_fireworks_session_id({"litellm_session_id": "session-123", "litellm_trace_id": "trace-123"})
|
||||
== "session-123"
|
||||
)
|
||||
|
||||
|
||||
def test_get_fireworks_session_id_ignores_proxy_generated_session_id():
|
||||
"""general_settings.missing_session_id: generate stamps a fresh id per request; sending it
|
||||
as x-session-affinity would pin every request to a different node."""
|
||||
def test_get_fireworks_session_id_ignores_proxy_generated_session_id_without_shared_affinity():
|
||||
"""Without shared affinity, a generated session id must be ignored."""
|
||||
assert (
|
||||
get_fireworks_session_id(
|
||||
{
|
||||
|
|
@ -243,6 +240,142 @@ def test_get_fireworks_session_id_ignores_proxy_generated_session_id():
|
|||
)
|
||||
|
||||
|
||||
def test_get_fireworks_session_id_uses_shared_affinity_even_with_generated_id():
|
||||
"""With shared affinity, the stable token wins even when the proxy generated a fresh id."""
|
||||
litellm_params = {
|
||||
"litellm_session_id": "generated-1",
|
||||
"fireworks_shared_session_affinity": True,
|
||||
"metadata": {
|
||||
"session_id": "generated-1",
|
||||
SESSION_ID_GENERATED_METADATA_KEY: True,
|
||||
"user_api_key_hash": "stable-user-hash",
|
||||
},
|
||||
}
|
||||
assert get_fireworks_session_id(litellm_params) == f"litellm-user-{shared_affinity_token('stable-user-hash')}"
|
||||
|
||||
|
||||
def test_get_fireworks_session_id_falls_back_to_credential_token():
|
||||
litellm_params = {
|
||||
"fireworks_shared_session_affinity": True,
|
||||
"api_key": "fw-api-key",
|
||||
"metadata": {},
|
||||
}
|
||||
assert get_fireworks_session_id(litellm_params) == shared_affinity_token("fw-api-key")
|
||||
|
||||
|
||||
def test_get_fireworks_session_id_uses_metadata_nested_shared_affinity():
|
||||
litellm_params = {"metadata": {"fireworks_shared_session_affinity": True, "user_api_key_hash": "stable-user-hash"}}
|
||||
assert get_fireworks_session_id(litellm_params) == f"litellm-user-{shared_affinity_token('stable-user-hash')}"
|
||||
|
||||
|
||||
def test_get_fireworks_session_id_never_emits_a_raw_proxy_credential():
|
||||
"""A proxy user key is not always stored hashed: opaque custom-auth keys and OAuth
|
||||
bearer tokens pass through `_safe_hash_litellm_api_key` unchanged, so the affinity
|
||||
token must hash the value again rather than forwarding it verbatim."""
|
||||
raw_token = "opaque-custom-auth-token-abc123"
|
||||
litellm_params = {"metadata": {"fireworks_shared_session_affinity": True, "user_api_key_hash": raw_token}}
|
||||
token = get_fireworks_session_id(litellm_params)
|
||||
assert token is not None
|
||||
assert raw_token not in token
|
||||
assert token == f"litellm-user-{shared_affinity_token(raw_token)}"
|
||||
|
||||
|
||||
def test_get_fireworks_session_id_explicit_overrides_shared():
|
||||
litellm_params = {
|
||||
"litellm_session_id": "explicit-session",
|
||||
"metadata": {"fireworks_shared_session_affinity": True, "user_api_key_hash": "stable-user-hash"},
|
||||
}
|
||||
assert get_fireworks_session_id(litellm_params) == "explicit-session"
|
||||
|
||||
|
||||
def test_get_fireworks_session_id_is_none_without_the_flag_or_a_credential():
|
||||
assert get_fireworks_session_id({"metadata": {"user_api_key_hash": "stable-user-hash"}}) is None
|
||||
|
||||
|
||||
def test_shared_affinity_token_is_stable_and_credential_specific():
|
||||
assert shared_affinity_token("fw-a") == shared_affinity_token("fw-a")
|
||||
assert shared_affinity_token("fw-a") != shared_affinity_token("fw-b")
|
||||
|
||||
|
||||
def test_top_level_shared_affinity_kwarg_sets_header_without_body_leak():
|
||||
"""Full pipeline: litellm.completion(fireworks_shared_session_affinity=True) must set
|
||||
x-session-affinity (hashed off the Fireworks credential) and must NOT put the flag
|
||||
into the request body, which Fireworks rejects."""
|
||||
model = "accounts/fireworks/models/llama-v3p1-8b-instruct"
|
||||
reply = {
|
||||
"id": "c1",
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": model,
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "Hi"}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
|
||||
}
|
||||
raw_response = MagicMock()
|
||||
raw_response.status_code = 200
|
||||
raw_response.headers = {}
|
||||
raw_response.text = json.dumps(reply)
|
||||
raw_response.json = lambda: reply
|
||||
|
||||
captured: dict = {}
|
||||
client = MagicMock(spec=HTTPHandler)
|
||||
|
||||
def post(*args, **kwargs):
|
||||
captured["headers"] = dict(kwargs.get("headers") or {})
|
||||
captured["body"] = json.loads(kwargs.get("data") or "{}")
|
||||
return raw_response
|
||||
|
||||
client.post.side_effect = post
|
||||
|
||||
litellm.completion(
|
||||
model=f"fireworks_ai/{model}",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
api_key="fw-test-key",
|
||||
client=client,
|
||||
fireworks_shared_session_affinity=True,
|
||||
)
|
||||
|
||||
headers = {k.lower(): v for k, v in captured["headers"].items()}
|
||||
assert headers.get("x-session-affinity") == shared_affinity_token("fw-test-key")
|
||||
assert "fireworks_shared_session_affinity" not in captured["body"]
|
||||
|
||||
|
||||
def test_shared_affinity_kwarg_is_absent_from_the_request_body():
|
||||
"""A run with the flag off must not carry the field anywhere in the request body, and a
|
||||
run with it on must reach the same body, so the flag is never serialized to Fireworks."""
|
||||
model = "accounts/fireworks/models/llama-v3p1-8b-instruct"
|
||||
reply = {
|
||||
"id": "c1",
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": model,
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "Hi"}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
|
||||
}
|
||||
raw_response = MagicMock()
|
||||
raw_response.status_code = 200
|
||||
raw_response.headers = {}
|
||||
raw_response.text = json.dumps(reply)
|
||||
raw_response.json = lambda: reply
|
||||
|
||||
bodies: list = []
|
||||
|
||||
def run(**extra):
|
||||
client = MagicMock(spec=HTTPHandler)
|
||||
client.post.side_effect = lambda *a, **kw: (bodies.append(json.loads(kw.get("data") or "{}")), raw_response)[1]
|
||||
litellm.completion(
|
||||
model=f"fireworks_ai/{model}",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
api_key="fw-test-key",
|
||||
client=client,
|
||||
**extra,
|
||||
)
|
||||
|
||||
run()
|
||||
run(fireworks_shared_session_affinity=True)
|
||||
|
||||
assert bodies[0] == bodies[1]
|
||||
|
||||
|
||||
def test_handle_message_content_with_tool_calls():
|
||||
config = FireworksAIConfig()
|
||||
message = Message(
|
||||
|
|
@ -275,25 +408,18 @@ def test_handle_message_content_with_tool_calls():
|
|||
},
|
||||
}
|
||||
]
|
||||
updated_message = config._handle_message_content_with_tool_calls(
|
||||
message, tool_calls
|
||||
)
|
||||
updated_message = config._handle_message_content_with_tool_calls(message, tool_calls)
|
||||
assert updated_message.tool_calls is not None
|
||||
assert len(updated_message.tool_calls) == 1
|
||||
assert updated_message.tool_calls[0].function.name == "get_current_weather"
|
||||
assert (
|
||||
updated_message.tool_calls[0].function.arguments
|
||||
== expected_tool_call.function.arguments
|
||||
)
|
||||
assert updated_message.tool_calls[0].function.arguments == expected_tool_call.function.arguments
|
||||
|
||||
|
||||
def test_get_supported_openai_params_reasoning_effort():
|
||||
"""Test that reasoning_effort is only included in supported params for models that support it."""
|
||||
config = FireworksAIConfig()
|
||||
|
||||
supported_params = config.get_supported_openai_params(
|
||||
"fireworks_ai/accounts/fireworks/models/glm-5p1"
|
||||
)
|
||||
supported_params = config.get_supported_openai_params("fireworks_ai/accounts/fireworks/models/glm-5p1")
|
||||
assert "reasoning_effort" in supported_params
|
||||
assert "thinking" in supported_params
|
||||
|
||||
|
|
@ -308,9 +434,7 @@ def test_get_supported_openai_params_parallel_tool_calls():
|
|||
"""Test that parallel_tool_calls is included for models that support function calling."""
|
||||
config = FireworksAIConfig()
|
||||
|
||||
supported_params = config.get_supported_openai_params(
|
||||
"fireworks_ai/accounts/fireworks/models/glm-5p1"
|
||||
)
|
||||
supported_params = config.get_supported_openai_params("fireworks_ai/accounts/fireworks/models/glm-5p1")
|
||||
assert "parallel_tool_calls" in supported_params
|
||||
assert "tools" in supported_params
|
||||
assert "tool_choice" in supported_params
|
||||
|
|
@ -324,9 +448,7 @@ def test_get_supported_openai_params_parallel_tool_calls():
|
|||
def test_get_supported_openai_params_short_model_name_resolves_account_prefixed_entry():
|
||||
config = FireworksAIConfig()
|
||||
|
||||
supported_params = config.get_supported_openai_params(
|
||||
"fireworks_ai/deepseek-v4-pro-0813"
|
||||
)
|
||||
supported_params = config.get_supported_openai_params("fireworks_ai/deepseek-v4-pro-0813")
|
||||
|
||||
assert "tool_choice" in supported_params
|
||||
assert "reasoning_effort" in supported_params
|
||||
|
|
@ -335,9 +457,7 @@ def test_get_supported_openai_params_short_model_name_resolves_account_prefixed_
|
|||
def test_get_supported_openai_params_preserves_generic_reasoning_fallback():
|
||||
config = FireworksAIConfig()
|
||||
|
||||
supported_params = config.get_supported_openai_params(
|
||||
"fireworks_ai/accounts/fireworks/models/glm-5p3-flash"
|
||||
)
|
||||
supported_params = config.get_supported_openai_params("fireworks_ai/accounts/fireworks/models/glm-5p3-flash")
|
||||
|
||||
assert "reasoning_effort" in supported_params
|
||||
|
||||
|
|
@ -404,14 +524,10 @@ def test_get_models_url_no_double_v1(api_base, expected_url_prefix):
|
|||
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 200
|
||||
mock_response.json.return_value = {
|
||||
"models": [{"name": "accounts/fireworks/models/llama-v3-70b"}]
|
||||
}
|
||||
mock_response.json.return_value = {"models": [{"name": "accounts/fireworks/models/llama-v3-70b"}]}
|
||||
|
||||
with (
|
||||
patch(
|
||||
"litellm.module_level_client.get", return_value=mock_response
|
||||
) as mock_get,
|
||||
patch("litellm.module_level_client.get", return_value=mock_response) as mock_get,
|
||||
patch(
|
||||
"litellm.llms.fireworks_ai.chat.transformation.get_secret_str",
|
||||
side_effect=lambda key: {
|
||||
|
|
@ -423,13 +539,9 @@ def test_get_models_url_no_double_v1(api_base, expected_url_prefix):
|
|||
):
|
||||
result = config.get_models(api_key="test-key", api_base=api_base)
|
||||
|
||||
called_url = mock_get.call_args.kwargs.get("url") or mock_get.call_args[1].get(
|
||||
"url", ""
|
||||
)
|
||||
called_url = mock_get.call_args.kwargs.get("url") or mock_get.call_args[1].get("url", "")
|
||||
assert "/v1/v1/" not in called_url, f"Double /v1/ detected in URL: {called_url}"
|
||||
assert called_url.startswith(
|
||||
expected_url_prefix
|
||||
), f"URL {called_url} does not start with {expected_url_prefix}"
|
||||
assert called_url.startswith(expected_url_prefix), f"URL {called_url} does not start with {expected_url_prefix}"
|
||||
assert result == ["fireworks_ai/accounts/fireworks/models/llama-v3-70b"]
|
||||
|
||||
|
||||
|
|
@ -457,9 +569,7 @@ def test_transform_messages_helper_removes_provider_specific_fields():
|
|||
},
|
||||
]
|
||||
# Call helper
|
||||
out = config._transform_messages_helper(
|
||||
messages, model="fireworks/test", litellm_params={}
|
||||
)
|
||||
out = config._transform_messages_helper(messages, model="fireworks/test", litellm_params={})
|
||||
for msg in out:
|
||||
assert "provider_specific_fields" not in msg
|
||||
|
||||
|
|
@ -480,15 +590,11 @@ def test_transform_messages_helper_strips_thinking_blocks_but_keeps_reasoning_co
|
|||
{
|
||||
"role": "assistant",
|
||||
"content": "I can help.",
|
||||
"thinking_blocks": [
|
||||
{"type": "thinking", "thinking": "internal", "signature": ""}
|
||||
],
|
||||
"thinking_blocks": [{"type": "thinking", "thinking": "internal", "signature": ""}],
|
||||
"reasoning_content": "internal",
|
||||
},
|
||||
]
|
||||
out = config._transform_messages_helper(
|
||||
messages, model="accounts/fireworks/models/glm-5p1", litellm_params={}
|
||||
)
|
||||
out = config._transform_messages_helper(messages, model="accounts/fireworks/models/glm-5p1", litellm_params={})
|
||||
assert "thinking_blocks" not in out[1]
|
||||
assert out[1]["reasoning_content"] == "internal"
|
||||
assert out[1]["content"] == "I can help."
|
||||
|
|
@ -1006,9 +1112,7 @@ def test_transform_messages_helper_rejects_file_blocks():
|
|||
litellm.BadRequestError,
|
||||
match="Fireworks AI chat completions does not support file content blocks",
|
||||
):
|
||||
config._transform_messages_helper(
|
||||
messages, model="accounts/fireworks/models/kimi-k2p6", litellm_params={}
|
||||
)
|
||||
config._transform_messages_helper(messages, model="accounts/fireworks/models/kimi-k2p6", litellm_params={})
|
||||
|
||||
|
||||
def test_transform_messages_helper_rejects_non_vision_image_inputs():
|
||||
|
|
@ -1020,18 +1124,14 @@ def test_transform_messages_helper_rejects_non_vision_image_inputs():
|
|||
{"type": "text", "text": "Describe this"},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE="
|
||||
},
|
||||
"image_url": {"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE="},
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
with pytest.raises(litellm.BadRequestError, match="does not support image inputs"):
|
||||
config._transform_messages_helper(
|
||||
messages, model="accounts/fireworks/models/glm-5p2", litellm_params={}
|
||||
)
|
||||
config._transform_messages_helper(messages, model="accounts/fireworks/models/glm-5p2", litellm_params={})
|
||||
|
||||
|
||||
def test_transform_messages_helper_allows_vision_image_inputs():
|
||||
|
|
@ -1043,9 +1143,7 @@ def test_transform_messages_helper_allows_vision_image_inputs():
|
|||
{"type": "text", "text": "Describe this"},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE="
|
||||
},
|
||||
"image_url": {"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE="},
|
||||
},
|
||||
],
|
||||
}
|
||||
|
|
@ -1069,9 +1167,7 @@ def test_image_inputs_not_rejected_for_fuzzy_non_vision_match():
|
|||
custom_model = "accounts/myorg/models/custom-glm-5p2"
|
||||
|
||||
assert config._get_model_cost_capability(custom_model, "supports_vision") is False
|
||||
assert (
|
||||
config._get_model_cost_capability_exact(custom_model, "supports_vision") is None
|
||||
)
|
||||
assert config._get_model_cost_capability_exact(custom_model, "supports_vision") is None
|
||||
|
||||
messages = [
|
||||
{
|
||||
|
|
@ -1079,16 +1175,12 @@ def test_image_inputs_not_rejected_for_fuzzy_non_vision_match():
|
|||
"content": [
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE="
|
||||
},
|
||||
"image_url": {"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE="},
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
out = config._transform_messages_helper(
|
||||
messages, model=custom_model, litellm_params={}
|
||||
)
|
||||
out = config._transform_messages_helper(messages, model=custom_model, litellm_params={})
|
||||
assert out == messages
|
||||
|
||||
|
||||
|
|
@ -1101,9 +1193,7 @@ def test_transform_messages_helper_skips_non_dict_content():
|
|||
}
|
||||
]
|
||||
|
||||
out = config._transform_messages_helper(
|
||||
messages, model="accounts/fireworks/models/glm-5p2", litellm_params={}
|
||||
)
|
||||
out = config._transform_messages_helper(messages, model="accounts/fireworks/models/glm-5p2", litellm_params={})
|
||||
assert out == messages
|
||||
|
||||
|
||||
|
|
@ -1343,9 +1433,7 @@ def test_streaming_surfaces_fireworks_response_fields():
|
|||
surfaced: dict = {}
|
||||
for chunk in stream:
|
||||
fields = getattr(chunk, "provider_specific_fields", None) or {}
|
||||
surfaced.update(
|
||||
{k: v for k, v in fields.items() if k.startswith("fireworks_")}
|
||||
)
|
||||
surfaced.update({k: v for k, v in fields.items() if k.startswith("fireworks_")})
|
||||
|
||||
assert surfaced["fireworks_token_ids"] == [[123]]
|
||||
assert surfaced["fireworks_raw_outputs"] == [raw_output]
|
||||
|
|
@ -1398,9 +1486,7 @@ def test_transform_request_direct_route_passthrough():
|
|||
|
||||
def test_map_extra_body_params_translates_truncate_prompt_tokens():
|
||||
config = FireworksAIConfig()
|
||||
result = config.map_extra_body_params(
|
||||
{"extra_body": {"truncate_prompt_tokens": 4096}}, _REASONING_MODEL
|
||||
)
|
||||
result = config.map_extra_body_params({"extra_body": {"truncate_prompt_tokens": 4096}}, _REASONING_MODEL)
|
||||
assert result == {"prompt_truncate_len": 4096}
|
||||
|
||||
|
||||
|
|
@ -1559,9 +1645,7 @@ def test_map_extra_body_params_non_dict_chat_template_kwargs_dropped():
|
|||
def test_map_extra_body_params_guided_json():
|
||||
config = FireworksAIConfig()
|
||||
schema = {"type": "object", "properties": {"x": {"type": "string"}}}
|
||||
result = config.map_extra_body_params(
|
||||
{"extra_body": {"guided_json": schema}}, _REASONING_MODEL
|
||||
)
|
||||
result = config.map_extra_body_params({"extra_body": {"guided_json": schema}}, _REASONING_MODEL)
|
||||
assert result == {
|
||||
"response_format": {
|
||||
"type": "json_schema",
|
||||
|
|
@ -1572,16 +1656,10 @@ def test_map_extra_body_params_guided_json():
|
|||
|
||||
def test_map_extra_body_params_guided_grammar_and_choice():
|
||||
config = FireworksAIConfig()
|
||||
grammar = config.map_extra_body_params(
|
||||
{"extra_body": {"guided_grammar": "root ::= 'hello'"}}, _REASONING_MODEL
|
||||
)
|
||||
assert grammar == {
|
||||
"response_format": {"type": "grammar", "grammar": "root ::= 'hello'"}
|
||||
}
|
||||
grammar = config.map_extra_body_params({"extra_body": {"guided_grammar": "root ::= 'hello'"}}, _REASONING_MODEL)
|
||||
assert grammar == {"response_format": {"type": "grammar", "grammar": "root ::= 'hello'"}}
|
||||
|
||||
choice = config.map_extra_body_params(
|
||||
{"extra_body": {"guided_choice": ["yes", "no"]}}, _REASONING_MODEL
|
||||
)
|
||||
choice = config.map_extra_body_params({"extra_body": {"guided_choice": ["yes", "no"]}}, _REASONING_MODEL)
|
||||
assert choice == {
|
||||
"response_format": {
|
||||
"type": "json_schema",
|
||||
|
|
@ -1667,9 +1745,7 @@ def test_map_extra_body_params_strips_unsupported_nim_vllm_params(param, value,
|
|||
|
||||
config = FireworksAIConfig()
|
||||
with caplog.at_level(logging.DEBUG):
|
||||
result = config.map_extra_body_params(
|
||||
{"extra_body": {param: value}}, _REASONING_MODEL
|
||||
)
|
||||
result = config.map_extra_body_params({"extra_body": {param: value}}, _REASONING_MODEL)
|
||||
assert result == {}
|
||||
assert param in caplog.text
|
||||
|
||||
|
|
@ -1741,15 +1817,10 @@ def test_nim_vllm_extras_translated_end_to_end_in_request_body():
|
|||
assert request_body["top_k"] == 40
|
||||
|
||||
|
||||
|
||||
|
||||
def test_streaming_preserves_selected_model_for_private_accounting():
|
||||
from litellm.llms.custom_httpx.http_handler import HTTPHandler
|
||||
|
||||
requested_route = (
|
||||
"accounts/fireworks/routers/firerouter/"
|
||||
"kimi-k3/deepseek-v4-pro-0813/deepseek-v4-flash-0731"
|
||||
)
|
||||
requested_route = "accounts/fireworks/routers/firerouter/kimi-k3/deepseek-v4-pro-0813/deepseek-v4-flash-0731"
|
||||
selected_model = "deepseek-v4-flash-0731"
|
||||
sse_lines = [
|
||||
"data: "
|
||||
|
|
@ -1803,19 +1874,14 @@ def test_streaming_preserves_selected_model_for_private_accounting():
|
|||
|
||||
assert chunks
|
||||
assert {chunk.model for chunk in chunks} == {requested_route}
|
||||
assert {
|
||||
chunk._hidden_params.get("provider_response_model") for chunk in chunks
|
||||
} == {selected_model}
|
||||
assert {chunk._hidden_params.get("provider_response_model") for chunk in chunks} == {selected_model}
|
||||
|
||||
assembled = litellm.stream_chunk_builder(chunks=chunks)
|
||||
assert assembled is not None
|
||||
assert assembled.model == requested_route
|
||||
assert assembled._hidden_params["provider_response_model"] == selected_model
|
||||
selected_model_info = litellm.model_cost[f"fireworks_ai/{selected_model}"]
|
||||
expected_cost = (
|
||||
5 * selected_model_info["input_cost_per_token"]
|
||||
+ selected_model_info["output_cost_per_token"]
|
||||
)
|
||||
expected_cost = 5 * selected_model_info["input_cost_per_token"] + selected_model_info["output_cost_per_token"]
|
||||
assert litellm.completion_cost(
|
||||
completion_response=assembled,
|
||||
custom_llm_provider="fireworks_ai",
|
||||
|
|
|
|||
|
|
@ -94,6 +94,7 @@ CONNECTION_NAMES: Final = (
|
|||
"github_copilot_host",
|
||||
"configurable_clientside_auth_params",
|
||||
"use_xai_oauth",
|
||||
"fireworks_shared_session_affinity",
|
||||
"fireworks_forward_user_id",
|
||||
"aws_batch_role_arn",
|
||||
"aws_bedrock_control_plane_endpoint",
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue