mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(streaming): keep only the logprobs keys upstream sent and type refusal logprobs
Some checks failed
Cost map pins recheck / rerun-cost-map-guard (push) Has been cancelled
LiteLLM Rust DB / db-queries (push) Has been cancelled
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled
Terraform Modules / fmt, validate, test (aws) (push) Has been cancelled
Terraform Modules / fmt, validate, test (gcp) (push) Has been cancelled
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled
Some checks failed
Cost map pins recheck / rerun-cost-map-guard (push) Has been cancelled
LiteLLM Rust DB / db-queries (push) Has been cancelled
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled
Terraform Modules / fmt, validate, test (aws) (push) Has been cancelled
Terraform Modules / fmt, validate, test (gcp) (push) Has been cancelled
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled
This commit is contained in:
parent
0bf4140737
commit
f38dfce30e
4 changed files with 26 additions and 8 deletions
|
|
@ -1202,7 +1202,10 @@ class CustomStreamWrapper:
|
|||
choice_json.pop(
|
||||
"finish_reason", None
|
||||
) # for mistral etc. which return a value in their last chunk (not-openai compatible).
|
||||
choices.append(StreamingChoices(**choice_json))
|
||||
choice_json.pop("logprobs", None)
|
||||
choices.append(
|
||||
StreamingChoices(**choice_json, logprobs=getattr(choice, "logprobs", None))
|
||||
)
|
||||
except Exception:
|
||||
choices.append(StreamingChoices())
|
||||
setattr(model_response, "choices", choices)
|
||||
|
|
|
|||
|
|
@ -1200,7 +1200,7 @@ class ChatCompletionTokenLogprob(OpenAIObject):
|
|||
unlikely.
|
||||
"""
|
||||
|
||||
top_logprobs: list[TopLogprob]
|
||||
top_logprobs: list[TopLogprob] = Field(default_factory=list)
|
||||
"""List of the most likely tokens and their log probability, at this token
|
||||
position.
|
||||
|
||||
|
|
@ -1242,6 +1242,8 @@ class ChoiceLogprobs(OpenAIObject):
|
|||
content: list[ChatCompletionTokenLogprob] | None = None
|
||||
"""A list of message content tokens with log probability information."""
|
||||
|
||||
refusal: Sequence[ChatCompletionTokenLogprob] | None = None
|
||||
|
||||
def __contains__(self, key) -> bool:
|
||||
# Define custom behavior for the 'in' operator
|
||||
return hasattr(self, key)
|
||||
|
|
@ -1258,7 +1260,7 @@ class ChoiceLogprobs(OpenAIObject):
|
|||
def _sdk_logprobs_as_dict(logprobs: object) -> object:
|
||||
if isinstance(logprobs, ChoiceLogprobs) or not isinstance(logprobs, BaseModel):
|
||||
return logprobs
|
||||
return logprobs.model_dump()
|
||||
return logprobs.model_dump(exclude_unset=True)
|
||||
|
||||
|
||||
CHOICE_LOGPROBS_OR_JSON: Final[TypeAdapter[ChoiceLogprobs | JsonValue]] = TypeAdapter(
|
||||
|
|
@ -2149,7 +2151,7 @@ class StreamingChoices(OpenAIObject):
|
|||
if enhancements is not None:
|
||||
self.enhancements = enhancements
|
||||
|
||||
if logprobs is not None and isinstance(logprobs, dict):
|
||||
if isinstance(logprobs, (dict, BaseModel)):
|
||||
self.logprobs = CHOICE_LOGPROBS_OR_JSON.validate_python(logprobs)
|
||||
else:
|
||||
self.logprobs = logprobs
|
||||
|
|
|
|||
|
|
@ -1815,6 +1815,13 @@ async def _sdk_stream_chunks(
|
|||
[],
|
||||
id="null-logprob",
|
||||
),
|
||||
pytest.param(
|
||||
{"content": [{"token": "<|return|>", "logprob": None}]},
|
||||
{},
|
||||
None,
|
||||
[],
|
||||
id="null-logprob-keeps-only-the-keys-upstream-sent",
|
||||
),
|
||||
pytest.param(-0.25, {}, None, [], id="scalar-logprobs"),
|
||||
pytest.param(
|
||||
{"content": [{"token": "<|return|>", "bytes": None, "logprob": None, "top_logprobs": None}], "refusal": None},
|
||||
|
|
@ -1868,8 +1875,9 @@ async def test_stream_with_nonstandard_logprobs_passes_them_through_and_finishes
|
|||
|
||||
|
||||
@pytest.mark.parametrize("async_stream", [False, True])
|
||||
@pytest.mark.parametrize("logprobs_field", ["content", "refusal"])
|
||||
async def test_stream_logprobs_without_top_logprobs_reach_caller_as_litellm_logprobs(
|
||||
async_stream: bool, logging_obj: Logging
|
||||
async_stream: bool, logprobs_field: Literal["content", "refusal"], logging_obj: Logging
|
||||
) -> None:
|
||||
chunks: Final = await _sdk_stream_chunks(
|
||||
tuple(
|
||||
|
|
@ -1878,7 +1886,7 @@ async def test_stream_logprobs_without_top_logprobs_reach_caller_as_litellm_logp
|
|||
{
|
||||
"index": 0,
|
||||
"delta": delta,
|
||||
"logprobs": {"content": [{"token": token, "logprob": -0.5}]},
|
||||
"logprobs": {logprobs_field: [{"token": token, "logprob": -0.5}]},
|
||||
"finish_reason": finish_reason,
|
||||
}
|
||||
]
|
||||
|
|
@ -1898,7 +1906,10 @@ async def test_stream_logprobs_without_top_logprobs_reach_caller_as_litellm_logp
|
|||
if choice.logprobs is not None
|
||||
)
|
||||
streamed_entries: Final = tuple(
|
||||
itertools.chain.from_iterable(logprobs.content or () for logprobs in streamed_logprobs)
|
||||
itertools.chain.from_iterable(
|
||||
(logprobs.content if logprobs_field == "content" else logprobs.refusal) or ()
|
||||
for logprobs in streamed_logprobs
|
||||
)
|
||||
)
|
||||
|
||||
assert [(entry.token, entry.logprob, entry.top_logprobs) for entry in streamed_entries] == [
|
||||
|
|
|
|||
4
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
4
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -29012,7 +29012,7 @@ export interface components {
|
|||
/** Token */
|
||||
token: string;
|
||||
/** Top Logprobs */
|
||||
top_logprobs: components["schemas"]["TopLogprob"][];
|
||||
top_logprobs?: components["schemas"]["TopLogprob"][];
|
||||
} & {
|
||||
[key: string]: unknown;
|
||||
};
|
||||
|
|
@ -29143,6 +29143,8 @@ export interface components {
|
|||
ChoiceLogprobs: {
|
||||
/** Content */
|
||||
content?: components["schemas"]["ChatCompletionTokenLogprob"][] | null;
|
||||
/** Refusal */
|
||||
refusal?: components["schemas"]["ChatCompletionTokenLogprob"][] | null;
|
||||
} & {
|
||||
[key: string]: unknown;
|
||||
};
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue