mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-03 02:22:24 +00:00
fix(github_copilot): normalize per-event item_id in /responses streaming
GitHub Copilot's native /v1/responses stream assigns a different item_id to every event of a single output item (output_item.added, the part.added / delta / done events, and output_item.done). Spec-strict clients like the Vercel AI SDK key streaming parts by item_id and abort with "reasoning part <id> not found" / "text part <id> not found" when a delta references an unregistered id. Override transform_streaming_response in GithubCopilotResponsesAPIConfig to anchor every event of an output item to the id from its output_item.added. Copilot accepts that id paired with the final encrypted_content on the next turn, so multi-turn replay is unaffected. Fixes #30071
This commit is contained in:
codgician
parent
51ba6e39cd
commit
c9f1920430
2 changed files with 274 additions and 0 deletions
|
|
@ -101,6 +101,7 @@ class GithubCopilotResponsesAPIConfig(OpenAIResponsesAPIConfig):
|
|||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.authenticator = Authenticator()
|
||||
self._stream_item_ids_by_output_index: Dict[int, str] = {}
|
||||
|
||||
@property
|
||||
def custom_llm_provider(self) -> LlmProviders:
|
||||
|
|
@ -129,6 +130,61 @@ class GithubCopilotResponsesAPIConfig(OpenAIResponsesAPIConfig):
|
|||
"""
|
||||
return dict(response_api_optional_params)
|
||||
|
||||
def transform_streaming_response(
|
||||
self,
|
||||
model: str,
|
||||
parsed_chunk: dict,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
) -> Any:
|
||||
parsed_chunk = self._normalize_stream_item_id(parsed_chunk)
|
||||
return super().transform_streaming_response(
|
||||
model=model,
|
||||
parsed_chunk=parsed_chunk,
|
||||
logging_obj=logging_obj,
|
||||
)
|
||||
|
||||
def _normalize_stream_item_id(self, parsed_chunk: dict) -> dict:
|
||||
"""Rewrite streamed item ids to one stable id per output_index.
|
||||
|
||||
GitHub Copilot tags each event of a single output item with a different
|
||||
item id, so clients that key streaming state by item id (e.g. the Vercel
|
||||
AI SDK) crash with "reasoning part <id> not found" / "text part <id> not
|
||||
found". Every sub-event carries a top-level ``item_id`` (whatever the
|
||||
item type), so its presence is the rewrite signal; output_item.added /
|
||||
.done instead nest the id under ``item``. The anchor is keyed by
|
||||
output_index and taken from output_item.added, which the protocol always
|
||||
emits first, so it is written before any sub-event reads it. Copilot
|
||||
accepts that id paired with the final encrypted_content next turn, so
|
||||
multi-turn replay is unaffected.
|
||||
|
||||
State is keyed by output_index on this config, which
|
||||
ProviderConfigManager builds fresh per request, so it is stream-scoped.
|
||||
"""
|
||||
output_index = parsed_chunk.get("output_index")
|
||||
if not isinstance(output_index, int):
|
||||
return parsed_chunk
|
||||
|
||||
if parsed_chunk.get("type") == "response.output_item.added":
|
||||
item = parsed_chunk.get("item")
|
||||
if isinstance(item, dict) and isinstance(item.get("id"), str):
|
||||
self._stream_item_ids_by_output_index[output_index] = item["id"]
|
||||
return parsed_chunk
|
||||
|
||||
stable_id = self._stream_item_ids_by_output_index.get(output_index)
|
||||
if stable_id is None:
|
||||
return parsed_chunk
|
||||
|
||||
if isinstance(parsed_chunk.get("item_id"), str):
|
||||
parsed_chunk = dict(parsed_chunk)
|
||||
parsed_chunk["item_id"] = stable_id
|
||||
elif parsed_chunk.get("type") == "response.output_item.done":
|
||||
item = parsed_chunk.get("item")
|
||||
if isinstance(item, dict):
|
||||
parsed_chunk = dict(parsed_chunk)
|
||||
parsed_chunk["item"] = {**item, "id": stable_id}
|
||||
|
||||
return parsed_chunk
|
||||
|
||||
def validate_environment(
|
||||
self,
|
||||
headers: dict,
|
||||
|
|
|
|||
|
|
@ -585,3 +585,221 @@ class TestGithubCopilotResponsesAPIRouting:
|
|||
provider=LlmProviders.GITHUB_COPILOT,
|
||||
)
|
||||
assert isinstance(config, GithubCopilotResponsesAPIConfig)
|
||||
|
||||
|
||||
class TestGithubCopilotReasoningStreamItemIdNormalization:
|
||||
"""GitHub Copilot's native /responses stream tags every reasoning-summary
|
||||
event with a different item_id (and the reasoning output_item.added /
|
||||
output_item.done ids also differ). Strict clients (Vercel ai-sdk) key
|
||||
reasoning state by item_id and crash when a summary delta references an
|
||||
unregistered id. The config normalizes every reasoning event in an
|
||||
output_index group to the id from its output_item.added."""
|
||||
|
||||
def _config(self):
|
||||
with patch(
|
||||
"litellm.llms.github_copilot.responses.transformation.Authenticator"
|
||||
):
|
||||
return GithubCopilotResponsesAPIConfig()
|
||||
|
||||
def _transform(self, config, chunk):
|
||||
return config.transform_streaming_response(
|
||||
model="github_copilot/gpt-5.5",
|
||||
parsed_chunk=chunk,
|
||||
logging_obj=MagicMock(),
|
||||
)
|
||||
|
||||
def test_summary_events_normalized_to_output_item_added_id(self):
|
||||
config = self._config()
|
||||
|
||||
self._transform(
|
||||
config,
|
||||
{
|
||||
"type": "response.output_item.added",
|
||||
"output_index": 0,
|
||||
"item": {"id": "stable_rs_id", "type": "reasoning"},
|
||||
},
|
||||
)
|
||||
|
||||
summary_chunks = [
|
||||
{
|
||||
"type": "response.reasoning_summary_part.added",
|
||||
"output_index": 0,
|
||||
"summary_index": 0,
|
||||
"item_id": "bad_part_added",
|
||||
"part": {"type": "summary_text", "text": ""},
|
||||
},
|
||||
{
|
||||
"type": "response.reasoning_summary_text.delta",
|
||||
"output_index": 0,
|
||||
"summary_index": 0,
|
||||
"item_id": "bad_delta_1",
|
||||
"delta": "Hello",
|
||||
},
|
||||
{
|
||||
"type": "response.reasoning_summary_text.delta",
|
||||
"output_index": 0,
|
||||
"summary_index": 0,
|
||||
"item_id": "bad_delta_2",
|
||||
"delta": " world",
|
||||
},
|
||||
{
|
||||
"type": "response.reasoning_summary_text.done",
|
||||
"output_index": 0,
|
||||
"summary_index": 0,
|
||||
"item_id": "bad_text_done",
|
||||
"text": "Hello world",
|
||||
},
|
||||
{
|
||||
"type": "response.reasoning_summary_part.done",
|
||||
"output_index": 0,
|
||||
"summary_index": 0,
|
||||
"item_id": "bad_part_done",
|
||||
"part": {"type": "summary_text", "text": "Hello world"},
|
||||
},
|
||||
]
|
||||
for chunk in summary_chunks:
|
||||
event = self._transform(config, chunk)
|
||||
assert event.item_id == "stable_rs_id"
|
||||
|
||||
def test_reasoning_output_item_done_normalized_to_added_id(self):
|
||||
config = self._config()
|
||||
self._transform(
|
||||
config,
|
||||
{
|
||||
"type": "response.output_item.added",
|
||||
"output_index": 0,
|
||||
"item": {"id": "stable_rs_id", "type": "reasoning"},
|
||||
},
|
||||
)
|
||||
event = self._transform(
|
||||
config,
|
||||
{
|
||||
"type": "response.output_item.done",
|
||||
"output_index": 0,
|
||||
"item": {
|
||||
"id": "different_done_id",
|
||||
"type": "reasoning",
|
||||
"encrypted_content": "ENC",
|
||||
},
|
||||
},
|
||||
)
|
||||
assert event.item.id == "stable_rs_id"
|
||||
|
||||
def test_interleaved_message_item_does_not_corrupt_mapping(self):
|
||||
config = self._config()
|
||||
self._transform(
|
||||
config,
|
||||
{
|
||||
"type": "response.output_item.added",
|
||||
"output_index": 0,
|
||||
"item": {"id": "stable_rs_id", "type": "reasoning"},
|
||||
},
|
||||
)
|
||||
self._transform(
|
||||
config,
|
||||
{
|
||||
"type": "response.output_item.added",
|
||||
"output_index": 1,
|
||||
"item": {"id": "msg_id", "type": "message"},
|
||||
},
|
||||
)
|
||||
event = self._transform(
|
||||
config,
|
||||
{
|
||||
"type": "response.reasoning_summary_text.delta",
|
||||
"output_index": 0,
|
||||
"summary_index": 0,
|
||||
"item_id": "bad_delta",
|
||||
"delta": "x",
|
||||
},
|
||||
)
|
||||
assert event.item_id == "stable_rs_id"
|
||||
|
||||
def test_event_without_registered_item_passes_through_unchanged(self):
|
||||
config = self._config()
|
||||
event = self._transform(
|
||||
config,
|
||||
{
|
||||
"type": "response.output_text.delta",
|
||||
"output_index": 0,
|
||||
"item_id": "msg_native_id",
|
||||
"delta": "hi",
|
||||
},
|
||||
)
|
||||
assert event.item_id == "msg_native_id"
|
||||
|
||||
def test_message_text_events_normalized_to_output_item_added_id(self):
|
||||
config = self._config()
|
||||
self._transform(
|
||||
config,
|
||||
{
|
||||
"type": "response.output_item.added",
|
||||
"output_index": 0,
|
||||
"item": {"id": "stable_msg_id", "type": "message"},
|
||||
},
|
||||
)
|
||||
for chunk in [
|
||||
{
|
||||
"type": "response.content_part.added",
|
||||
"output_index": 0,
|
||||
"content_index": 0,
|
||||
"item_id": "bad_cp_added",
|
||||
"part": {"type": "output_text", "text": ""},
|
||||
},
|
||||
{
|
||||
"type": "response.output_text.delta",
|
||||
"output_index": 0,
|
||||
"content_index": 0,
|
||||
"item_id": "bad_text_delta",
|
||||
"delta": "Paris",
|
||||
},
|
||||
{
|
||||
"type": "response.output_text.done",
|
||||
"output_index": 0,
|
||||
"content_index": 0,
|
||||
"item_id": "bad_text_done",
|
||||
"text": "Paris",
|
||||
},
|
||||
]:
|
||||
event = self._transform(config, chunk)
|
||||
assert event.item_id == "stable_msg_id"
|
||||
|
||||
def test_event_without_output_index_passes_through_unchanged(self):
|
||||
config = self._config()
|
||||
event = self._transform(
|
||||
config,
|
||||
{
|
||||
"type": "response.completed",
|
||||
"response": {"id": "resp_1", "status": "completed", "output": []},
|
||||
},
|
||||
)
|
||||
assert event.type == "response.completed"
|
||||
|
||||
def test_normalization_continues_after_a_terminal_event(self):
|
||||
config = self._config()
|
||||
self._transform(
|
||||
config,
|
||||
{
|
||||
"type": "response.output_item.added",
|
||||
"output_index": 0,
|
||||
"item": {"id": "stable_rs_id", "type": "reasoning"},
|
||||
},
|
||||
)
|
||||
self._transform(
|
||||
config,
|
||||
{
|
||||
"type": "response.completed",
|
||||
"response": {"id": "resp_1", "status": "completed", "output": []},
|
||||
},
|
||||
)
|
||||
event = self._transform(
|
||||
config,
|
||||
{
|
||||
"type": "response.reasoning_summary_text.delta",
|
||||
"output_index": 0,
|
||||
"summary_index": 0,
|
||||
"item_id": "bad_delta",
|
||||
"delta": "x",
|
||||
},
|
||||
)
|
||||
assert event.item_id == "stable_rs_id"
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue