scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-05 07:37:50 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 233

fix(gemini): keep the placeholder resend inside the cache rebuild so either rejection order ends in an accepted body

Browse source 
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
mateo 2026-10-27 20:05:45 +00:00
parent 307c86c489
commit 74ebc06069
2 changed files with 17 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
litellm/llms/vertex_ai/gemini/rejected_thought_signature_retry.py
View file

@ -64,27 +64,29 @@ async def post_with_accepted_thought_signatures_async(
body: RequestBody,
post: Callable[[RequestBody], Awaitable[Sent]],
log_request: Callable[[RequestBody], None],
) -> Sent:
) -> tuple[RequestBody, Sent]:
"""Innermost retry around the raw post, so a cache rebuild (which re-transforms the messages and brings
the rejected signatures back) is itself wrapped by it. Returns the body the backend accepted."""
try:
return await post(body)
return body, await post(body)
except (httpx.HTTPStatusError, BaseLLMException) as error:
resend_body: Final = _resend_body(body, error)
if resend_body is None:
raise
log_request(resend_body)
return await post(resend_body)
return resend_body, await post(resend_body)
def post_with_accepted_thought_signatures_sync(
body: RequestBody,
post: Callable[[RequestBody], Sent],
log_request: Callable[[RequestBody], None],
) -> Sent:
) -> tuple[RequestBody, Sent]:
try:
return post(body)
return body, post(body)
except (httpx.HTTPStatusError, BaseLLMException) as error:
resend_body: Final = _resend_body(body, error)
if resend_body is None:
raise
log_request(resend_body)
return post(resend_body)
return resend_body, post(resend_body)

28
litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py
View file

@ -3476,12 +3476,8 @@ class VertexLLM(VertexBase):
async def post(body: RequestBody) -> ModelResponseIterator:
return await open_stream(client=client, data=json.dumps(body), request_body=body)
post_with_fresh_cache: Final = partial(
post_with_fresh_cache_async, rebuild=rebuild, post=post, log_request=log_request
)
return (
await post_with_accepted_thought_signatures_async(request_body, post_with_fresh_cache, log_request)
)[1]
accepted: Final = partial(post_with_accepted_thought_signatures_async, post=post, log_request=log_request)
return (await post_with_fresh_cache_async(request_body, rebuild, accepted, log_request))[1][1]
streaming_response: Final = CustomStreamWrapper(
completion_stream=None,
@ -3605,12 +3601,8 @@ class VertexLLM(VertexBase):
)
return response
post_with_fresh_cache: Final = partial(
post_with_fresh_cache_async, rebuild=rebuild, post=post, log_request=log_request
)
sent_body, first_response = await post_with_accepted_thought_signatures_async(
request_body, post_with_fresh_cache, log_request
)
accepted: Final = partial(post_with_accepted_thought_signatures_async, post=post, log_request=log_request)
_, (sent_body, first_response) = await post_with_fresh_cache_async(request_body, rebuild, accepted, log_request)
return _transform_long_decode_hops(
model=model,
hops=await collect_hops_async(parse_body(first_response), sent_body, post),
@ -3786,10 +3778,10 @@ class VertexLLM(VertexBase):
def post(body: RequestBody) -> ModelResponseIterator:
return open_stream(client=client, data=json.dumps(body), request_body=body)
post_with_fresh_cache: Final = partial(
post_with_fresh_cache_sync, rebuild=rebuild, post=post, log_request=log_request
accepted: Final = partial(
post_with_accepted_thought_signatures_sync, post=post, log_request=log_request
)
return post_with_accepted_thought_signatures_sync(data, post_with_fresh_cache, log_request)[1]
return post_with_fresh_cache_sync(data, rebuild, accepted, log_request)[1][1]
streaming_response: Final = CustomStreamWrapper(
completion_stream=None,
@ -3833,10 +3825,8 @@ class VertexLLM(VertexBase):
)
return response
post_with_fresh_cache: Final = partial(
post_with_fresh_cache_sync, rebuild=rebuild, post=post, log_request=log_request
)
sent_body, first_response = post_with_accepted_thought_signatures_sync(data, post_with_fresh_cache, log_request)
accepted: Final = partial(post_with_accepted_thought_signatures_sync, post=post, log_request=log_request)
_, (sent_body, first_response) = post_with_fresh_cache_sync(data, rebuild, accepted, log_request)
return _transform_long_decode_hops(
model=model,
hops=collect_hops_sync(parse_body(first_response), sent_body, post),