mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-25 05:54:53 +00:00
test(translation): add sonnet 5.5 converse and gemini direct cross-model mid rows (#47328)
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
devin-ai-integration[bot]
GitHub
parent
c4d24ec2f1
commit
da80552897
4 changed files with 1227 additions and 2 deletions
|
|
@ -8,7 +8,7 @@ Every row sends `CODING_AGENT_REQUEST` to `/v1/chat/completions`, changing only
|
|||
| Mid | Opus 5 and 5.5, Sonnet 5.5, Gemini 3.8 Flash and 3.1 Pro Preview, GPT 5.6 Sol, 6 Sol and 6.1 Sol, GPT 5.6 Terra |
|
||||
| High | Fable 5 and 5.1, Gemini 4 Argon, GPT 6 Astra |
|
||||
|
||||
The 48 rows cover Anthropic direct, OpenAI direct, Vertex, Azure OpenAI, Azure AI and Bedrock Converse where accessible. Vertex Haiku 5.5 was unavailable during capture and is not represented
|
||||
The 51 rows cover Anthropic direct, OpenAI direct, Vertex, Gemini direct, Azure OpenAI, Azure AI and Bedrock Converse where accessible. Vertex Haiku 5.5 was unavailable during capture and is not represented
|
||||
|
||||
The request contains system instructions, conversation history, previous assistant tool calls and results, four function definitions, automatic parallel tools, low reasoning effort and an output limit. Each row exercises a tool-call response, raw streaming, SDK stream assembly and a follow-up that replays the returned assistant message and tool-call IDs
|
||||
|
||||
|
|
@ -33,7 +33,7 @@ Streaming cases store `expected_litellm_message` and `expected_litellm_usage`. M
|
|||
|
||||
Usage requires nonnegative integer counts, positive prompt and completion counts, and `total_tokens == prompt_tokens + completion_tokens`. The complete normalized usage must equal the independently expected counters, including cached and reasoning tokens and derived text/cache details. Missing, changed or unexpected counters fail. Only null optional fields, the explicitly listed provider metadata/duplicate counters and computed `cost` are excluded. Thus a wrong prompt count with an adjusted total still fails
|
||||
|
||||
The raw stream check assembles tool names and argument fragments by index. The SDK check separately uses `OpenAI.chat.completions.stream()` without removing reasoning fields. All 48 SDK checks run, including the seven Haiku rows. [LIT-9653](https://linear.app/litellm-ai/issue/LIT-9653) was fixed in [#46611](https://github.com/BerriAI/litellm/pull/46611), so there are no Haiku skips
|
||||
The raw stream check assembles tool names and argument fragments by index. The SDK check separately uses `OpenAI.chat.completions.stream()` without removing reasoning fields. All 51 SDK checks run, including the seven Haiku rows. [LIT-9653](https://linear.app/litellm-ai/issue/LIT-9653) was fixed in [#46611](https://github.com/BerriAI/litellm/pull/46611), so there are no Haiku skips
|
||||
|
||||
## Recorded upstreams
|
||||
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ from integration.translation.chat_completions.bases.bedrock_converse import (
|
|||
CLAUDE_HAIKU_5_5_TEST_CASE,
|
||||
CLAUDE_OPUS_5_5_TEST_CASE,
|
||||
CLAUDE_OPUS_5_TEST_CASE,
|
||||
CLAUDE_SONNET_5_5_TEST_CASE,
|
||||
GPT_5_6_LUNA_TEST_CASE,
|
||||
GPT_5_6_SOL_TEST_CASE,
|
||||
GPT_5_6_TERRA_TEST_CASE,
|
||||
|
|
@ -3192,6 +3193,445 @@ CLAUDE_OPUS_5_5_AGENT_STREAMING_TEST_CASE: Final = StreamingTranslationTestCase(
|
|||
)
|
||||
|
||||
|
||||
BEDROCK_CONVERSE_AGENT_PROVIDER_REQUEST: Final = {
|
||||
**BEDROCK_CONVERSE_AGENT_REQUEST,
|
||||
"additionalModelRequestFields": {
|
||||
"thinking": {"type": "adaptive", "display": "summarized"},
|
||||
"tool_choice": {"disable_parallel_tool_use": False},
|
||||
"output_config": {"effort": "low"},
|
||||
},
|
||||
"additionalModelResponseFieldPaths": ["/usage/output_tokens_details"],
|
||||
"toolConfig": {
|
||||
"tools": [
|
||||
{
|
||||
"toolSpec": {
|
||||
"inputSchema": {
|
||||
"json": {
|
||||
"type": "object",
|
||||
"properties": {"path": {"type": "string"}},
|
||||
"required": ["path"],
|
||||
}
|
||||
},
|
||||
"name": "read_file",
|
||||
"description": "Read a UTF-8 file in the repository.",
|
||||
}
|
||||
},
|
||||
{
|
||||
"toolSpec": {
|
||||
"inputSchema": {
|
||||
"json": {
|
||||
"type": "object",
|
||||
"properties": {"query": {"type": "string"}},
|
||||
"required": ["query"],
|
||||
}
|
||||
},
|
||||
"name": "search_code",
|
||||
"description": "Search repository source for a literal query.",
|
||||
}
|
||||
},
|
||||
{
|
||||
"toolSpec": {
|
||||
"inputSchema": {
|
||||
"json": {
|
||||
"type": "object",
|
||||
"properties": {"path": {"type": "string"}},
|
||||
"required": ["path"],
|
||||
}
|
||||
},
|
||||
"name": "run_tests",
|
||||
"description": "Run a specific pytest test path and return the exit code and output.",
|
||||
}
|
||||
},
|
||||
{
|
||||
"toolSpec": {
|
||||
"inputSchema": {
|
||||
"json": {
|
||||
"type": "object",
|
||||
"properties": {"patch": {"type": "string"}},
|
||||
"required": ["patch"],
|
||||
}
|
||||
},
|
||||
"name": "apply_patch",
|
||||
"description": "Apply a unified diff to repository files.",
|
||||
}
|
||||
},
|
||||
],
|
||||
"toolChoice": {"auto": {}},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
CLAUDE_SONNET_5_5_AGENT_TEST_CASE: Final = replace(
|
||||
CLAUDE_SONNET_5_5_TEST_CASE,
|
||||
scenario="agent_tools",
|
||||
litellm_request={**CODING_AGENT_REQUEST, "model": "bedrock/converse/us.anthropic.claude-sonnet-5-5"},
|
||||
expected_provider_endpoint="/model/us.anthropic.claude-sonnet-5-5/converse",
|
||||
expected_provider_headers={
|
||||
"authorization": "Bearer synthetic-bedrock-key",
|
||||
"content-type": "application/json",
|
||||
},
|
||||
expected_provider_request=BEDROCK_CONVERSE_AGENT_PROVIDER_REQUEST,
|
||||
mock_provider_response={
|
||||
"additionalModelResponseFields": {"usage": {"output_tokens_details": {"thinking_tokens": 0}}},
|
||||
"metrics": {"latencyMs": 1294},
|
||||
"output": {
|
||||
"message": {
|
||||
"content": [
|
||||
{
|
||||
"toolUse": {
|
||||
"input": {"query": "retries_remaining"},
|
||||
"name": "search_code",
|
||||
"toolUseId": "tooluse_2Z6A2nogMa1RGHgnwZXH6G",
|
||||
"type": "tool_use",
|
||||
}
|
||||
},
|
||||
{
|
||||
"toolUse": {
|
||||
"input": {"path": "tests/test_retry.py"},
|
||||
"name": "run_tests",
|
||||
"toolUseId": "tooluse_mwNqPITfkD3zc6JHkg5GJa",
|
||||
"type": "tool_use",
|
||||
}
|
||||
},
|
||||
],
|
||||
"role": "assistant",
|
||||
}
|
||||
},
|
||||
"stopReason": "tool_use",
|
||||
"usage": {
|
||||
"cacheReadInputTokenCount": 0,
|
||||
"cacheReadInputTokens": 0,
|
||||
"cacheWriteInputTokenCount": 0,
|
||||
"cacheWriteInputTokens": 0,
|
||||
"inputTokens": 1144,
|
||||
"outputTokens": 105,
|
||||
"serverToolUsage": {},
|
||||
"totalTokens": 1249,
|
||||
},
|
||||
},
|
||||
expected_litellm_response={
|
||||
**CODING_AGENT_RESPONSE,
|
||||
"model": "bedrock/converse/us.anthropic.claude-sonnet-5-5",
|
||||
"choices": [
|
||||
{
|
||||
"finish_reason": "tool_calls",
|
||||
"index": 0,
|
||||
"message": {
|
||||
"content": "",
|
||||
"role": "assistant",
|
||||
"tool_calls": [
|
||||
{
|
||||
"index": 0,
|
||||
"function": {"arguments": '{"query": "retries_remaining"}', "name": "search_code"},
|
||||
"id": "tooluse_2Z6A2nogMa1RGHgnwZXH6G",
|
||||
"type": "function",
|
||||
},
|
||||
{
|
||||
"index": 1,
|
||||
"function": {"arguments": '{"path": "tests/test_retry.py"}', "name": "run_tests"},
|
||||
"id": "tooluse_mwNqPITfkD3zc6JHkg5GJa",
|
||||
"type": "function",
|
||||
},
|
||||
],
|
||||
},
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
"completion_tokens": 105,
|
||||
"prompt_tokens": 1144,
|
||||
"total_tokens": 1249,
|
||||
"completion_tokens_details": {"reasoning_tokens": 0, "text_tokens": 105},
|
||||
"prompt_tokens_details": {
|
||||
"cached_tokens": 0,
|
||||
"text_tokens": 1144,
|
||||
"cache_write_tokens": 0,
|
||||
"cache_creation_tokens": 0,
|
||||
},
|
||||
"cache_creation_input_tokens": 0,
|
||||
"cache_read_input_tokens": 0,
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
CLAUDE_SONNET_5_5_AGENT_STREAMING_TEST_CASE: Final = StreamingTranslationTestCase(
|
||||
scenario="agent_streaming",
|
||||
litellm_endpoint=CLAUDE_SONNET_5_5_AGENT_TEST_CASE.litellm_endpoint,
|
||||
litellm_request={
|
||||
**CLAUDE_SONNET_5_5_AGENT_TEST_CASE.litellm_request,
|
||||
"stream": True,
|
||||
"stream_options": {"include_usage": True},
|
||||
},
|
||||
expected_provider_headers=CLAUDE_SONNET_5_5_AGENT_TEST_CASE.expected_provider_headers,
|
||||
expected_provider_endpoint="/model/us.anthropic.claude-sonnet-5-5/converse-stream",
|
||||
expected_provider_request=BEDROCK_CONVERSE_AGENT_PROVIDER_REQUEST,
|
||||
mock_provider_frames=(
|
||||
EventStreamFrame(
|
||||
data={"p": "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWX", "role": "assistant"},
|
||||
headers={":event-type": "messageStart", ":content-type": "application/json", ":message-type": "event"},
|
||||
),
|
||||
EventStreamFrame(
|
||||
data={
|
||||
"contentBlockIndex": 0,
|
||||
"p": "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXY",
|
||||
"start": {
|
||||
"toolUse": {
|
||||
"name": "search_code",
|
||||
"toolUseId": "tooluse_6cLTZNBlXnKXY9x3Rp4wNo",
|
||||
"type": "tool_use",
|
||||
}
|
||||
},
|
||||
},
|
||||
headers={":event-type": "contentBlockStart", ":content-type": "application/json", ":message-type": "event"},
|
||||
),
|
||||
EventStreamFrame(
|
||||
data={"contentBlockIndex": 0, "delta": {"toolUse": {"input": ""}}, "p": "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQR"},
|
||||
headers={":event-type": "contentBlockDelta", ":content-type": "application/json", ":message-type": "event"},
|
||||
),
|
||||
EventStreamFrame(
|
||||
data={"contentBlockIndex": 0, "delta": {"toolUse": {"input": '{"qu'}}, "p": "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQR"},
|
||||
headers={":event-type": "contentBlockDelta", ":content-type": "application/json", ":message-type": "event"},
|
||||
),
|
||||
EventStreamFrame(
|
||||
data={"contentBlockIndex": 0, "delta": {"toolUse": {"input": 'ery": '}}, "p": "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"},
|
||||
headers={":event-type": "contentBlockDelta", ":content-type": "application/json", ":message-type": "event"},
|
||||
),
|
||||
EventStreamFrame(
|
||||
data={"contentBlockIndex": 0, "delta": {"toolUse": {"input": '"retries_r'}}, "p": "abcdefghijk"},
|
||||
headers={":event-type": "contentBlockDelta", ":content-type": "application/json", ":message-type": "event"},
|
||||
),
|
||||
EventStreamFrame(
|
||||
data={"contentBlockIndex": 0, "delta": {"toolUse": {"input": 'emaining"}'}}, "p": "abcdef"},
|
||||
headers={":event-type": "contentBlockDelta", ":content-type": "application/json", ":message-type": "event"},
|
||||
),
|
||||
EventStreamFrame(
|
||||
data={"contentBlockIndex": 0, "p": "abcdefghijklmnopqrstu"},
|
||||
headers={":event-type": "contentBlockStop", ":content-type": "application/json", ":message-type": "event"},
|
||||
),
|
||||
EventStreamFrame(
|
||||
data={
|
||||
"contentBlockIndex": 1,
|
||||
"p": "abcdefghijklmnopqrstuvwxyz",
|
||||
"start": {
|
||||
"toolUse": {
|
||||
"name": "run_tests",
|
||||
"toolUseId": "tooluse_uoozKiVjrkiFbYLgq52Lh1",
|
||||
"type": "tool_use",
|
||||
}
|
||||
},
|
||||
},
|
||||
headers={":event-type": "contentBlockStart", ":content-type": "application/json", ":message-type": "event"},
|
||||
),
|
||||
EventStreamFrame(
|
||||
data={"contentBlockIndex": 1, "delta": {"toolUse": {"input": ""}}, "p": "abcdefghijklmnopqrstuvwxyzA"},
|
||||
headers={":event-type": "contentBlockDelta", ":content-type": "application/json", ":message-type": "event"},
|
||||
),
|
||||
EventStreamFrame(
|
||||
data={"contentBlockIndex": 1, "delta": {"toolUse": {"input": '{"p'}}, "p": "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ012345"},
|
||||
headers={":event-type": "contentBlockDelta", ":content-type": "application/json", ":message-type": "event"},
|
||||
),
|
||||
EventStreamFrame(
|
||||
data={"contentBlockIndex": 1, "delta": {"toolUse": {"input": 'ath": "tests'}}, "p": "abcdefghijklmnopqrstuvwxyzABCDEFGHIJK"},
|
||||
headers={":event-type": "contentBlockDelta", ":content-type": "application/json", ":message-type": "event"},
|
||||
),
|
||||
EventStreamFrame(
|
||||
data={"contentBlockIndex": 1, "delta": {"toolUse": {"input": "/tes"}}, "p": "abcdefghijklmnopqrstuvw"},
|
||||
headers={":event-type": "contentBlockDelta", ":content-type": "application/json", ":message-type": "event"},
|
||||
),
|
||||
EventStreamFrame(
|
||||
data={
|
||||
"contentBlockIndex": 1,
|
||||
"delta": {"toolUse": {"input": "t_retry."}},
|
||||
"p": "abcdefghijklmnopqrstuvwxyzAB",
|
||||
},
|
||||
headers={":event-type": "contentBlockDelta", ":content-type": "application/json", ":message-type": "event"},
|
||||
),
|
||||
EventStreamFrame(
|
||||
data={"contentBlockIndex": 1, "delta": {"toolUse": {"input": 'py"}'}}, "p": "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRST"},
|
||||
headers={":event-type": "contentBlockDelta", ":content-type": "application/json", ":message-type": "event"},
|
||||
),
|
||||
EventStreamFrame(
|
||||
data={"contentBlockIndex": 1, "p": "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTU"},
|
||||
headers={":event-type": "contentBlockStop", ":content-type": "application/json", ":message-type": "event"},
|
||||
),
|
||||
EventStreamFrame(
|
||||
data={
|
||||
"additionalModelResponseFields": {"usage": {"output_tokens_details": {"thinking_tokens": 0}}},
|
||||
"p": "abcdefghijklmnopqr",
|
||||
"stopReason": "tool_use",
|
||||
},
|
||||
headers={":event-type": "messageStop", ":content-type": "application/json", ":message-type": "event"},
|
||||
),
|
||||
EventStreamFrame(
|
||||
data={
|
||||
"metrics": {"latencyMs": 1258},
|
||||
"p": "abcde",
|
||||
"usage": {"inputTokens": 1144, "outputTokens": 105, "serverToolUsage": {}, "totalTokens": 1249},
|
||||
},
|
||||
headers={":event-type": "metadata", ":content-type": "application/json", ":message-type": "event"},
|
||||
),
|
||||
),
|
||||
expected_litellm_message=coding_agent_stream_message(
|
||||
("tooluse_6cLTZNBlXnKXY9x3Rp4wNo", "tooluse_uoozKiVjrkiFbYLgq52Lh1"),
|
||||
),
|
||||
expected_litellm_usage={
|
||||
"prompt_tokens": 1144,
|
||||
"completion_tokens": 105,
|
||||
"total_tokens": 1249,
|
||||
"completion_tokens_details": {"reasoning_tokens": 0, "text_tokens": 105},
|
||||
"prompt_tokens_details": {
|
||||
"cached_tokens": 0,
|
||||
"text_tokens": 1144,
|
||||
"cache_write_tokens": 0,
|
||||
"cache_creation_tokens": 0,
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
CLAUDE_SONNET_5_5_AGENT_FOLLOWUP_PROVIDER_REQUEST: Final = {
|
||||
**BEDROCK_CONVERSE_AGENT_PROVIDER_REQUEST,
|
||||
"messages": [
|
||||
*list_value(BEDROCK_CONVERSE_AGENT_PROVIDER_REQUEST["messages"]),
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{
|
||||
"toolUse": {
|
||||
"input": {"query": "retries_remaining"},
|
||||
"name": "search_code",
|
||||
"toolUseId": "tooluse_2Z6A2nogMa1RGHgnwZXH6G",
|
||||
}
|
||||
},
|
||||
{
|
||||
"toolUse": {
|
||||
"input": {"path": "tests/test_retry.py"},
|
||||
"name": "run_tests",
|
||||
"toolUseId": "tooluse_mwNqPITfkD3zc6JHkg5GJa",
|
||||
}
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"toolResult": {
|
||||
"content": [
|
||||
{
|
||||
"text": (
|
||||
'{"matches": [{"path": "src/worker.py", "line": 12, "text": '
|
||||
'"remaining = retries_remaining(3, attempts)"}]}'
|
||||
)
|
||||
}
|
||||
],
|
||||
"toolUseId": "tooluse_2Z6A2nogMa1RGHgnwZXH6G",
|
||||
}
|
||||
},
|
||||
{
|
||||
"toolResult": {
|
||||
"content": [
|
||||
{
|
||||
"text": (
|
||||
'{"exit_code": 1, "output": "FAILED tests/test_retry.py::'
|
||||
'test_retries_remaining - assert 4 == 2"}'
|
||||
)
|
||||
}
|
||||
],
|
||||
"toolUseId": "tooluse_mwNqPITfkD3zc6JHkg5GJa",
|
||||
}
|
||||
},
|
||||
{
|
||||
"text": (
|
||||
"Stop after these diagnostics. Do not call more tools or change files. "
|
||||
"Summarize the bug, the caller found and the failed test in two short sentences."
|
||||
)
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
CLAUDE_SONNET_5_5_AGENT_FOLLOWUP_TEST_CASE: Final = replace(
|
||||
CLAUDE_SONNET_5_5_AGENT_TEST_CASE,
|
||||
scenario="agent_followup",
|
||||
litellm_request=followup_request(
|
||||
CLAUDE_SONNET_5_5_AGENT_TEST_CASE.litellm_request,
|
||||
CLAUDE_SONNET_5_5_AGENT_TEST_CASE.expected_litellm_response,
|
||||
),
|
||||
expected_provider_endpoint="/model/us.anthropic.claude-sonnet-5-5/converse",
|
||||
expected_provider_request=CLAUDE_SONNET_5_5_AGENT_FOLLOWUP_PROVIDER_REQUEST,
|
||||
mock_provider_response={
|
||||
"additionalModelResponseFields": {"usage": {"output_tokens_details": {"thinking_tokens": 0}}},
|
||||
"metrics": {"latencyMs": 1976},
|
||||
"output": {
|
||||
"message": {
|
||||
"content": [
|
||||
{
|
||||
"text": (
|
||||
"The bug is that `retries_remaining` in `src/retry.py` returns `limit + attempts` "
|
||||
"instead of `max(0, limit - attempts)`. That gives 4 for `(3, 1)` where the test "
|
||||
"expects 2, and it can't be clamped to zero. The only other caller is "
|
||||
"`src/worker.py:12` (`retries_remaining(3, attempts)`), and "
|
||||
"`tests/test_retry.py::test_retries_remaining` fails with `assert 4 == 2`. "
|
||||
"I haven't changed any files."
|
||||
)
|
||||
}
|
||||
],
|
||||
"role": "assistant",
|
||||
}
|
||||
},
|
||||
"stopReason": "end_turn",
|
||||
"usage": {
|
||||
"cacheReadInputTokenCount": 0,
|
||||
"cacheReadInputTokens": 0,
|
||||
"cacheWriteInputTokenCount": 0,
|
||||
"cacheWriteInputTokens": 0,
|
||||
"inputTokens": 1452,
|
||||
"outputTokens": 163,
|
||||
"serverToolUsage": {},
|
||||
"totalTokens": 1615,
|
||||
},
|
||||
},
|
||||
expected_litellm_response={
|
||||
**CODING_AGENT_RESPONSE,
|
||||
"model": "bedrock/converse/us.anthropic.claude-sonnet-5-5",
|
||||
"choices": [
|
||||
{
|
||||
"finish_reason": "stop",
|
||||
"index": 0,
|
||||
"message": {
|
||||
"content": (
|
||||
"The bug is that `retries_remaining` in `src/retry.py` returns `limit + attempts` "
|
||||
"instead of `max(0, limit - attempts)`. That gives 4 for `(3, 1)` where the test "
|
||||
"expects 2, and it can't be clamped to zero. The only other caller is "
|
||||
"`src/worker.py:12` (`retries_remaining(3, attempts)`), and "
|
||||
"`tests/test_retry.py::test_retries_remaining` fails with `assert 4 == 2`. "
|
||||
"I haven't changed any files."
|
||||
),
|
||||
"role": "assistant",
|
||||
},
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
"completion_tokens": 163,
|
||||
"prompt_tokens": 1452,
|
||||
"total_tokens": 1615,
|
||||
"completion_tokens_details": {"reasoning_tokens": 0, "text_tokens": 163},
|
||||
"prompt_tokens_details": {
|
||||
"cached_tokens": 0,
|
||||
"text_tokens": 1452,
|
||||
"cache_write_tokens": 0,
|
||||
"cache_creation_tokens": 0,
|
||||
},
|
||||
"cache_creation_input_tokens": 0,
|
||||
"cache_read_input_tokens": 0,
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
CLAUDE_OPUS_5_AGENT_FOLLOWUP_TEST_CASE: Final = replace(
|
||||
CLAUDE_OPUS_5_TEST_CASE,
|
||||
scenario="agent_followup",
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ from integration.translation.chat_completions.cross_model import (
|
|||
azure,
|
||||
azure_ai,
|
||||
bedrock_converse,
|
||||
gemini,
|
||||
openai,
|
||||
vertex_ai,
|
||||
)
|
||||
|
|
@ -126,6 +127,11 @@ MID: Final[tuple[AgentCases, ...]] = (
|
|||
bedrock_converse.CLAUDE_OPUS_5_5_AGENT_FOLLOWUP_TEST_CASE,
|
||||
bedrock_converse.CLAUDE_OPUS_5_5_AGENT_STREAMING_TEST_CASE,
|
||||
),
|
||||
(
|
||||
bedrock_converse.CLAUDE_SONNET_5_5_AGENT_TEST_CASE,
|
||||
bedrock_converse.CLAUDE_SONNET_5_5_AGENT_FOLLOWUP_TEST_CASE,
|
||||
bedrock_converse.CLAUDE_SONNET_5_5_AGENT_STREAMING_TEST_CASE,
|
||||
),
|
||||
(
|
||||
bedrock_converse.CLAUDE_OPUS_5_TEST_CASE,
|
||||
bedrock_converse.CLAUDE_OPUS_5_AGENT_FOLLOWUP_TEST_CASE,
|
||||
|
|
@ -187,11 +193,21 @@ MID: Final[tuple[AgentCases, ...]] = (
|
|||
vertex_ai.GEMINI_3_8_FLASH_AGENT_FOLLOWUP_TEST_CASE,
|
||||
vertex_ai.GEMINI_3_8_FLASH_AGENT_STREAMING_TEST_CASE,
|
||||
),
|
||||
(
|
||||
gemini.GEMINI_3_8_FLASH_AGENT_TEST_CASE,
|
||||
gemini.GEMINI_3_8_FLASH_AGENT_FOLLOWUP_TEST_CASE,
|
||||
gemini.GEMINI_3_8_FLASH_AGENT_STREAMING_TEST_CASE,
|
||||
),
|
||||
(
|
||||
vertex_ai.GEMINI_3_1_PRO_PREVIEW_AGENT_TEST_CASE,
|
||||
vertex_ai.GEMINI_3_1_PRO_PREVIEW_AGENT_FOLLOWUP_TEST_CASE,
|
||||
vertex_ai.GEMINI_3_1_PRO_PREVIEW_AGENT_STREAMING_TEST_CASE,
|
||||
),
|
||||
(
|
||||
gemini.GEMINI_3_1_PRO_PREVIEW_AGENT_TEST_CASE,
|
||||
gemini.GEMINI_3_1_PRO_PREVIEW_AGENT_FOLLOWUP_TEST_CASE,
|
||||
gemini.GEMINI_3_1_PRO_PREVIEW_AGENT_STREAMING_TEST_CASE,
|
||||
),
|
||||
)
|
||||
|
||||
HIGH: Final[tuple[AgentCases, ...]] = (
|
||||
|
|
|
|||
|
|
@ -0,0 +1,769 @@
|
|||
from dataclasses import replace
|
||||
from typing import Final
|
||||
|
||||
from integration._support.client import list_value
|
||||
from integration.translation.case import SSEFrame, StreamingTranslationTestCase
|
||||
from integration.translation.chat_completions.bases.gemini import (
|
||||
GEMINI_3_1_PRO_PREVIEW_TEST_CASE,
|
||||
GEMINI_3_8_FLASH_TEST_CASE,
|
||||
)
|
||||
from integration.translation.chat_completions.cross_model.contracts import followup_request
|
||||
from integration.translation.chat_completions.templates import (
|
||||
CODING_AGENT_REQUEST,
|
||||
CODING_AGENT_RESPONSE,
|
||||
coding_agent_stream_message,
|
||||
)
|
||||
from integration.translation.templates.vertex_ai import VERTEX_AI_AGENT_REQUEST
|
||||
|
||||
GEMINI_SKIP_VALIDATION_SIGNATURE: Final = "c2tpcF90aG91Z2h0X3NpZ25hdHVyZV92YWxpZGF0b3I="
|
||||
|
||||
GEMINI_3_1_PRO_PREVIEW_THOUGHT: Final = (
|
||||
"Okay, here's my summary, framed as the internal thought process:\n\n"
|
||||
"**Concurrent Debugging of `retries_remaining` Usage**\n\n"
|
||||
"Alright, the objective is clear: I need to track down the root cause. My gut tells me this "
|
||||
"`retries_remaining` issue isn't isolated. My first move will be to perform a comprehensive search "
|
||||
"for all instances where `retries_remaining` is referenced within the codebase. We're talking "
|
||||
"about a full, concurrent search. I don't want to waste time waiting.\n\n"
|
||||
"Once I have a list, the plan is to launch the relevant test suite for each of those callers, "
|
||||
"*concurrently*. That's the only way to reproduce and identify the specific failure scenario. "
|
||||
"Speed and thoroughness are key here. I'll need to instrument each test run to capture as much "
|
||||
"diagnostic information as possible - logging, stack traces, maybe even memory dumps if the issue "
|
||||
"seems complex. I need a clear picture of what's happening within those retry loops to figure out "
|
||||
"what's causing the problem.\n\n\n"
|
||||
)
|
||||
|
||||
GEMINI_3_8_FLASH_FOLLOWUP_TEXT: Final = (
|
||||
"The function `retries_remaining` erroneously adds `limit` and `attempts` instead of subtracting "
|
||||
"them with a minimum of zero, causing `test_retries_remaining` to fail when "
|
||||
"`retries_remaining(3, 1)` returns `4` instead of `2`. In addition to the tests, `src/worker.py` "
|
||||
"was found as a caller invoking `retries_remaining(3, attempts)`."
|
||||
)
|
||||
|
||||
GEMINI_3_1_PRO_PREVIEW_FOLLOWUP_THOUGHT: Final = (
|
||||
"**My Analysis of the Bug Report**\n\n"
|
||||
"Okay, let's break this down efficiently. The key is to avoid unnecessary verbosity and, "
|
||||
"importantly, not invoke any tooling or editing operations. This is about summarizing the defect "
|
||||
"itself. I see the caller is in `src/worker.py` and the failing test is "
|
||||
"`tests/test_retry.py::test_retries_remaining`.\n\n"
|
||||
"The core issue appears to be an incorrect calculation within the `retries_remaining` function. "
|
||||
"It looks like the implementation adds attempts to the limit instead of subtracting them, leading "
|
||||
"to a logical error. This directly impacts the `test_retries_remaining` test and, I presume, causes "
|
||||
"issues in the `src/worker.py` module. To fix this, the return value needs to be changed to "
|
||||
"`max(0, limit - attempts)` to properly calculate the remaining retries. This correction should "
|
||||
"resolve the test failure and ensure the correct behavior of the retry logic called from the worker.\n\n\n"
|
||||
)
|
||||
|
||||
GEMINI_3_1_PRO_PREVIEW_FOLLOWUP_TEXT: Final = (
|
||||
"The bug is that `retries_remaining` incorrectly adds `attempts` to `limit` instead of subtracting "
|
||||
"it, causing `test_retries_remaining` to fail by returning 4 instead of 2. Fixing the implementation "
|
||||
"to return `max(0, limit - attempts)` will resolve the test failure and correct the behavior for its "
|
||||
"caller in `src/worker.py`."
|
||||
)
|
||||
|
||||
GEMINI_3_8_FLASH_AGENT_PROVIDER_RESPONSE: Final = {
|
||||
"candidates": [
|
||||
{
|
||||
"content": {
|
||||
"parts": [
|
||||
{
|
||||
"functionCall": {
|
||||
"name": "search_code",
|
||||
"args": {"query": "retries_remaining"},
|
||||
"id": "call_4323393",
|
||||
},
|
||||
"thoughtSignature": GEMINI_SKIP_VALIDATION_SIGNATURE,
|
||||
},
|
||||
{
|
||||
"functionCall": {
|
||||
"name": "run_tests",
|
||||
"args": {"path": "tests/test_retry.py"},
|
||||
"id": "call_4323394",
|
||||
}
|
||||
},
|
||||
],
|
||||
"role": "model",
|
||||
},
|
||||
"finishReason": "STOP",
|
||||
"index": 0,
|
||||
"finishMessage": "Model generated function call(s).",
|
||||
}
|
||||
],
|
||||
"usageMetadata": {
|
||||
"promptTokenCount": 516,
|
||||
"candidatesTokenCount": 41,
|
||||
"totalTokenCount": 557,
|
||||
"promptTokensDetails": [{"modality": "TEXT", "tokenCount": 516}],
|
||||
"serviceTier": "standard",
|
||||
},
|
||||
"modelVersion": "gemini-3.8-flash",
|
||||
"responseId": "yyHWapj7Br7sz7IPte2L2QU",
|
||||
}
|
||||
|
||||
GEMINI_3_1_PRO_PREVIEW_AGENT_PROVIDER_RESPONSE: Final = {
|
||||
"candidates": [
|
||||
{
|
||||
"content": {
|
||||
"parts": [
|
||||
{"text": GEMINI_3_1_PRO_PREVIEW_THOUGHT, "thought": True},
|
||||
{
|
||||
"functionCall": {
|
||||
"name": "search_code",
|
||||
"args": {"query": "retries_remaining"},
|
||||
"id": "call_4206648",
|
||||
},
|
||||
"thoughtSignature": GEMINI_SKIP_VALIDATION_SIGNATURE,
|
||||
},
|
||||
{
|
||||
"functionCall": {
|
||||
"name": "run_tests",
|
||||
"args": {"path": "tests/test_retry.py"},
|
||||
"id": "call_4206649",
|
||||
}
|
||||
},
|
||||
],
|
||||
"role": "model",
|
||||
},
|
||||
"finishReason": "STOP",
|
||||
"index": 0,
|
||||
"finishMessage": "Model generated function call(s).",
|
||||
}
|
||||
],
|
||||
"usageMetadata": {
|
||||
"promptTokenCount": 516,
|
||||
"candidatesTokenCount": 41,
|
||||
"totalTokenCount": 602,
|
||||
"promptTokensDetails": [{"modality": "TEXT", "tokenCount": 516}],
|
||||
"thoughtsTokenCount": 45,
|
||||
"serviceTier": "standard",
|
||||
},
|
||||
"modelVersion": "gemini-3.1-pro-preview",
|
||||
"responseId": "0CHWas7iDZLqz7IPsLnekAs",
|
||||
}
|
||||
|
||||
GEMINI_3_8_FLASH_AGENT_TEST_CASE: Final = replace(
|
||||
GEMINI_3_8_FLASH_TEST_CASE,
|
||||
scenario="agent_tools",
|
||||
litellm_request={**CODING_AGENT_REQUEST, "model": "gemini/gemini-3.8-flash"},
|
||||
expected_provider_endpoint="/models/gemini-3.8-flash:generateContent",
|
||||
expected_provider_request=VERTEX_AI_AGENT_REQUEST,
|
||||
mock_provider_response=GEMINI_3_8_FLASH_AGENT_PROVIDER_RESPONSE,
|
||||
expected_litellm_response={
|
||||
**CODING_AGENT_RESPONSE,
|
||||
"model": "gemini/gemini-3.8-flash",
|
||||
"choices": [
|
||||
{
|
||||
"finish_reason": "tool_calls",
|
||||
"index": 0,
|
||||
"message": {
|
||||
"content": None,
|
||||
"role": "assistant",
|
||||
"tool_calls": [
|
||||
{
|
||||
"index": 0,
|
||||
"provider_specific_fields": {
|
||||
"thought_signature": GEMINI_SKIP_VALIDATION_SIGNATURE,
|
||||
},
|
||||
"function": {
|
||||
"arguments": '{"query": "retries_remaining"}',
|
||||
"name": "search_code",
|
||||
},
|
||||
"id": "call_4323393__thought__" + GEMINI_SKIP_VALIDATION_SIGNATURE,
|
||||
"type": "function",
|
||||
},
|
||||
{
|
||||
"index": 1,
|
||||
"function": {
|
||||
"arguments": '{"path": "tests/test_retry.py"}',
|
||||
"name": "run_tests",
|
||||
},
|
||||
"id": "call_4323394",
|
||||
"type": "function",
|
||||
},
|
||||
],
|
||||
"images": [],
|
||||
"thinking_blocks": [],
|
||||
"provider_specific_fields": {
|
||||
"thought_signatures": [GEMINI_SKIP_VALIDATION_SIGNATURE],
|
||||
},
|
||||
},
|
||||
"provider_specific_fields": {"native_finish_reason": "STOP"},
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
"completion_tokens": 41,
|
||||
"prompt_tokens": 516,
|
||||
"total_tokens": 557,
|
||||
"completion_tokens_details": {"text_tokens": 41},
|
||||
"prompt_tokens_details": {"text_tokens": 516},
|
||||
},
|
||||
"vertex_ai_grounding_metadata": [],
|
||||
"vertex_ai_url_context_metadata": [],
|
||||
"vertex_ai_safety_results": [],
|
||||
"vertex_ai_citation_metadata": [],
|
||||
},
|
||||
)
|
||||
|
||||
GEMINI_3_1_PRO_PREVIEW_AGENT_TEST_CASE: Final = replace(
|
||||
GEMINI_3_1_PRO_PREVIEW_TEST_CASE,
|
||||
scenario="agent_tools",
|
||||
litellm_request={**CODING_AGENT_REQUEST, "model": "gemini/gemini-3.1-pro-preview"},
|
||||
expected_provider_endpoint="/models/gemini-3.1-pro-preview:generateContent",
|
||||
expected_provider_request=VERTEX_AI_AGENT_REQUEST,
|
||||
mock_provider_response=GEMINI_3_1_PRO_PREVIEW_AGENT_PROVIDER_RESPONSE,
|
||||
expected_litellm_response={
|
||||
**CODING_AGENT_RESPONSE,
|
||||
"model": "gemini/gemini-3.1-pro-preview",
|
||||
"choices": [
|
||||
{
|
||||
"finish_reason": "tool_calls",
|
||||
"index": 0,
|
||||
"message": {
|
||||
"content": None,
|
||||
"role": "assistant",
|
||||
"tool_calls": [
|
||||
{
|
||||
"index": 0,
|
||||
"provider_specific_fields": {
|
||||
"thought_signature": GEMINI_SKIP_VALIDATION_SIGNATURE,
|
||||
},
|
||||
"function": {
|
||||
"arguments": '{"query": "retries_remaining"}',
|
||||
"name": "search_code",
|
||||
},
|
||||
"id": "call_4206648__thought__" + GEMINI_SKIP_VALIDATION_SIGNATURE,
|
||||
"type": "function",
|
||||
},
|
||||
{
|
||||
"index": 1,
|
||||
"function": {
|
||||
"arguments": '{"path": "tests/test_retry.py"}',
|
||||
"name": "run_tests",
|
||||
},
|
||||
"id": "call_4206649",
|
||||
"type": "function",
|
||||
},
|
||||
],
|
||||
"images": [],
|
||||
"reasoning_content": GEMINI_3_1_PRO_PREVIEW_THOUGHT,
|
||||
"thinking_blocks": [
|
||||
{
|
||||
"type": "thinking",
|
||||
"thinking": GEMINI_3_1_PRO_PREVIEW_THOUGHT,
|
||||
}
|
||||
],
|
||||
"provider_specific_fields": {
|
||||
"thought_signatures": [GEMINI_SKIP_VALIDATION_SIGNATURE],
|
||||
},
|
||||
},
|
||||
"provider_specific_fields": {"native_finish_reason": "STOP"},
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
"completion_tokens": 86,
|
||||
"prompt_tokens": 516,
|
||||
"total_tokens": 602,
|
||||
"completion_tokens_details": {"reasoning_tokens": 45, "text_tokens": 41},
|
||||
"prompt_tokens_details": {"text_tokens": 516},
|
||||
},
|
||||
"vertex_ai_grounding_metadata": [],
|
||||
"vertex_ai_url_context_metadata": [],
|
||||
"vertex_ai_safety_results": [],
|
||||
"vertex_ai_citation_metadata": [],
|
||||
},
|
||||
)
|
||||
|
||||
GEMINI_3_8_FLASH_AGENT_FOLLOWUP_PROVIDER_REQUEST: Final = {
|
||||
**VERTEX_AI_AGENT_REQUEST,
|
||||
"contents": [
|
||||
*list_value(VERTEX_AI_AGENT_REQUEST["contents"]),
|
||||
{
|
||||
"role": "model",
|
||||
"parts": [
|
||||
{
|
||||
"function_call": {
|
||||
"name": "search_code",
|
||||
"args": {"query": "retries_remaining"},
|
||||
"id": "call_4323393",
|
||||
},
|
||||
"thoughtSignature": GEMINI_SKIP_VALIDATION_SIGNATURE,
|
||||
},
|
||||
{
|
||||
"function_call": {
|
||||
"name": "run_tests",
|
||||
"args": {"path": "tests/test_retry.py"},
|
||||
"id": "call_4323394",
|
||||
}
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"parts": [
|
||||
{
|
||||
"function_response": {
|
||||
"name": "search_code",
|
||||
"response": {
|
||||
"matches": [
|
||||
{
|
||||
"path": "src/worker.py",
|
||||
"line": 12,
|
||||
"text": "remaining = retries_remaining(3, attempts)",
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": "call_4323393",
|
||||
}
|
||||
},
|
||||
{
|
||||
"function_response": {
|
||||
"name": "run_tests",
|
||||
"response": {
|
||||
"exit_code": 1,
|
||||
"output": "FAILED tests/test_retry.py::test_retries_remaining - assert 4 == 2",
|
||||
},
|
||||
"id": "call_4323394",
|
||||
}
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"parts": [
|
||||
{
|
||||
"text": (
|
||||
"Stop after these diagnostics. Do not call more tools or change files. "
|
||||
"Summarize the bug, the caller found and the failed test in two short sentences."
|
||||
)
|
||||
}
|
||||
],
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
GEMINI_3_1_PRO_PREVIEW_AGENT_FOLLOWUP_PROVIDER_REQUEST: Final = {
|
||||
**GEMINI_3_8_FLASH_AGENT_FOLLOWUP_PROVIDER_REQUEST,
|
||||
"contents": [
|
||||
*list_value(VERTEX_AI_AGENT_REQUEST["contents"]),
|
||||
{
|
||||
"role": "model",
|
||||
"parts": [
|
||||
{"text": GEMINI_3_1_PRO_PREVIEW_THOUGHT, "thought": True},
|
||||
{
|
||||
"function_call": {
|
||||
"name": "search_code",
|
||||
"args": {"query": "retries_remaining"},
|
||||
"id": "call_4206648",
|
||||
},
|
||||
"thoughtSignature": GEMINI_SKIP_VALIDATION_SIGNATURE,
|
||||
},
|
||||
{
|
||||
"function_call": {
|
||||
"name": "run_tests",
|
||||
"args": {"path": "tests/test_retry.py"},
|
||||
"id": "call_4206649",
|
||||
}
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"parts": [
|
||||
{
|
||||
"function_response": {
|
||||
"name": "search_code",
|
||||
"response": {
|
||||
"matches": [
|
||||
{
|
||||
"path": "src/worker.py",
|
||||
"line": 12,
|
||||
"text": "remaining = retries_remaining(3, attempts)",
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": "call_4206648",
|
||||
}
|
||||
},
|
||||
{
|
||||
"function_response": {
|
||||
"name": "run_tests",
|
||||
"response": {
|
||||
"exit_code": 1,
|
||||
"output": "FAILED tests/test_retry.py::test_retries_remaining - assert 4 == 2",
|
||||
},
|
||||
"id": "call_4206649",
|
||||
}
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"parts": [
|
||||
{
|
||||
"text": (
|
||||
"Stop after these diagnostics. Do not call more tools or change files. "
|
||||
"Summarize the bug, the caller found and the failed test in two short sentences."
|
||||
)
|
||||
}
|
||||
],
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
GEMINI_3_8_FLASH_AGENT_FOLLOWUP_TEST_CASE: Final = replace(
|
||||
GEMINI_3_8_FLASH_AGENT_TEST_CASE,
|
||||
scenario="agent_followup",
|
||||
litellm_request=followup_request(
|
||||
GEMINI_3_8_FLASH_AGENT_TEST_CASE.litellm_request,
|
||||
GEMINI_3_8_FLASH_AGENT_TEST_CASE.expected_litellm_response,
|
||||
),
|
||||
expected_provider_endpoint="/models/gemini-3.8-flash:generateContent",
|
||||
expected_provider_request=GEMINI_3_8_FLASH_AGENT_FOLLOWUP_PROVIDER_REQUEST,
|
||||
mock_provider_response={
|
||||
"candidates": [
|
||||
{
|
||||
"content": {
|
||||
"parts": [
|
||||
{
|
||||
"text": GEMINI_3_8_FLASH_FOLLOWUP_TEXT,
|
||||
"thoughtSignature": GEMINI_SKIP_VALIDATION_SIGNATURE,
|
||||
}
|
||||
],
|
||||
"role": "model",
|
||||
},
|
||||
"finishReason": "STOP",
|
||||
"index": 0,
|
||||
}
|
||||
],
|
||||
"usageMetadata": {
|
||||
"promptTokenCount": 670,
|
||||
"candidatesTokenCount": 89,
|
||||
"totalTokenCount": 759,
|
||||
"promptTokensDetails": [{"modality": "TEXT", "tokenCount": 670}],
|
||||
"serviceTier": "standard",
|
||||
},
|
||||
"modelVersion": "gemini-3.8-flash",
|
||||
"responseId": "nyLWatOHEKHoz7IPnICMuAs",
|
||||
},
|
||||
expected_litellm_response={
|
||||
**CODING_AGENT_RESPONSE,
|
||||
"model": "gemini/gemini-3.8-flash",
|
||||
"choices": [
|
||||
{
|
||||
"finish_reason": "stop",
|
||||
"index": 0,
|
||||
"message": {
|
||||
"content": GEMINI_3_8_FLASH_FOLLOWUP_TEXT,
|
||||
"role": "assistant",
|
||||
"images": [],
|
||||
"thinking_blocks": [],
|
||||
"provider_specific_fields": {
|
||||
"thought_signatures": [GEMINI_SKIP_VALIDATION_SIGNATURE],
|
||||
},
|
||||
},
|
||||
"provider_specific_fields": {"native_finish_reason": "STOP"},
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
"completion_tokens": 89,
|
||||
"prompt_tokens": 670,
|
||||
"total_tokens": 759,
|
||||
"completion_tokens_details": {"text_tokens": 89},
|
||||
"prompt_tokens_details": {"text_tokens": 670},
|
||||
},
|
||||
"vertex_ai_grounding_metadata": [],
|
||||
"vertex_ai_url_context_metadata": [],
|
||||
"vertex_ai_safety_results": [],
|
||||
"vertex_ai_citation_metadata": [],
|
||||
},
|
||||
)
|
||||
|
||||
GEMINI_3_1_PRO_PREVIEW_AGENT_FOLLOWUP_TEST_CASE: Final = replace(
|
||||
GEMINI_3_1_PRO_PREVIEW_AGENT_TEST_CASE,
|
||||
scenario="agent_followup",
|
||||
litellm_request=followup_request(
|
||||
GEMINI_3_1_PRO_PREVIEW_AGENT_TEST_CASE.litellm_request,
|
||||
GEMINI_3_1_PRO_PREVIEW_AGENT_TEST_CASE.expected_litellm_response,
|
||||
),
|
||||
expected_provider_endpoint="/models/gemini-3.1-pro-preview:generateContent",
|
||||
expected_provider_request=GEMINI_3_1_PRO_PREVIEW_AGENT_FOLLOWUP_PROVIDER_REQUEST,
|
||||
mock_provider_response={
|
||||
"candidates": [
|
||||
{
|
||||
"content": {
|
||||
"parts": [
|
||||
{"text": GEMINI_3_1_PRO_PREVIEW_FOLLOWUP_THOUGHT, "thought": True},
|
||||
{
|
||||
"text": GEMINI_3_1_PRO_PREVIEW_FOLLOWUP_TEXT,
|
||||
"thoughtSignature": GEMINI_SKIP_VALIDATION_SIGNATURE,
|
||||
},
|
||||
],
|
||||
"role": "model",
|
||||
},
|
||||
"finishReason": "STOP",
|
||||
"index": 0,
|
||||
}
|
||||
],
|
||||
"usageMetadata": {
|
||||
"promptTokenCount": 887,
|
||||
"candidatesTokenCount": 78,
|
||||
"totalTokenCount": 1329,
|
||||
"promptTokensDetails": [{"modality": "TEXT", "tokenCount": 887}],
|
||||
"thoughtsTokenCount": 364,
|
||||
"serviceTier": "standard",
|
||||
},
|
||||
"modelVersion": "gemini-3.1-pro-preview",
|
||||
"responseId": "oiLWauy3Kr2Ez7IPse_RwAs",
|
||||
},
|
||||
expected_litellm_response={
|
||||
**CODING_AGENT_RESPONSE,
|
||||
"model": "gemini/gemini-3.1-pro-preview",
|
||||
"choices": [
|
||||
{
|
||||
"finish_reason": "stop",
|
||||
"index": 0,
|
||||
"message": {
|
||||
"content": GEMINI_3_1_PRO_PREVIEW_FOLLOWUP_TEXT,
|
||||
"role": "assistant",
|
||||
"images": [],
|
||||
"reasoning_content": GEMINI_3_1_PRO_PREVIEW_FOLLOWUP_THOUGHT,
|
||||
"thinking_blocks": [
|
||||
{
|
||||
"type": "thinking",
|
||||
"thinking": GEMINI_3_1_PRO_PREVIEW_FOLLOWUP_THOUGHT,
|
||||
}
|
||||
],
|
||||
"provider_specific_fields": {
|
||||
"thought_signatures": [GEMINI_SKIP_VALIDATION_SIGNATURE],
|
||||
},
|
||||
},
|
||||
"provider_specific_fields": {"native_finish_reason": "STOP"},
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
"completion_tokens": 442,
|
||||
"prompt_tokens": 887,
|
||||
"total_tokens": 1329,
|
||||
"completion_tokens_details": {"reasoning_tokens": 364, "text_tokens": 78},
|
||||
"prompt_tokens_details": {"text_tokens": 887},
|
||||
},
|
||||
"vertex_ai_grounding_metadata": [],
|
||||
"vertex_ai_url_context_metadata": [],
|
||||
"vertex_ai_safety_results": [],
|
||||
"vertex_ai_citation_metadata": [],
|
||||
},
|
||||
)
|
||||
|
||||
GEMINI_3_8_FLASH_AGENT_STREAMING_TEST_CASE: Final = StreamingTranslationTestCase(
|
||||
scenario="agent_streaming",
|
||||
litellm_endpoint=GEMINI_3_8_FLASH_AGENT_TEST_CASE.litellm_endpoint,
|
||||
litellm_request={
|
||||
**GEMINI_3_8_FLASH_AGENT_TEST_CASE.litellm_request,
|
||||
"stream": True,
|
||||
"stream_options": {"include_usage": True},
|
||||
},
|
||||
expected_provider_headers=GEMINI_3_8_FLASH_AGENT_TEST_CASE.expected_provider_headers,
|
||||
expected_provider_endpoint="/models/gemini-3.8-flash:streamGenerateContent?alt=sse",
|
||||
expected_provider_request=VERTEX_AI_AGENT_REQUEST,
|
||||
mock_provider_frames=(
|
||||
SSEFrame(
|
||||
data={
|
||||
"candidates": [
|
||||
{
|
||||
"content": {
|
||||
"parts": [
|
||||
{
|
||||
"functionCall": {
|
||||
"name": "search_code",
|
||||
"args": {"query": "retries_remaining"},
|
||||
"id": "call_5122749",
|
||||
},
|
||||
"thoughtSignature": GEMINI_SKIP_VALIDATION_SIGNATURE,
|
||||
}
|
||||
],
|
||||
"role": "model",
|
||||
},
|
||||
"index": 0,
|
||||
}
|
||||
],
|
||||
"usageMetadata": {
|
||||
"promptTokenCount": 516,
|
||||
"candidatesTokenCount": 19,
|
||||
"totalTokenCount": 535,
|
||||
"promptTokensDetails": [{"modality": "TEXT", "tokenCount": 516}],
|
||||
"serviceTier": "standard",
|
||||
},
|
||||
"modelVersion": "gemini-3.8-flash",
|
||||
"responseId": "zSHWaoPJKvPrz7IP7dqb-AM",
|
||||
}
|
||||
),
|
||||
SSEFrame(
|
||||
data={
|
||||
"candidates": [
|
||||
{
|
||||
"content": {
|
||||
"parts": [
|
||||
{
|
||||
"functionCall": {
|
||||
"name": "run_tests",
|
||||
"args": {"path": "tests/test_retry.py"},
|
||||
"id": "call_5122752",
|
||||
}
|
||||
}
|
||||
],
|
||||
"role": "model",
|
||||
},
|
||||
"index": 0,
|
||||
}
|
||||
],
|
||||
"usageMetadata": {
|
||||
"promptTokenCount": 516,
|
||||
"candidatesTokenCount": 41,
|
||||
"totalTokenCount": 557,
|
||||
"promptTokensDetails": [{"modality": "TEXT", "tokenCount": 516}],
|
||||
"serviceTier": "standard",
|
||||
},
|
||||
"modelVersion": "gemini-3.8-flash",
|
||||
"responseId": "zSHWaoPJKvPrz7IP7dqb-AM",
|
||||
}
|
||||
),
|
||||
SSEFrame(
|
||||
data={
|
||||
"candidates": [
|
||||
{
|
||||
"content": {"parts": [{"text": ""}], "role": "model"},
|
||||
"finishReason": "STOP",
|
||||
"index": 0,
|
||||
}
|
||||
],
|
||||
"usageMetadata": {
|
||||
"promptTokenCount": 516,
|
||||
"candidatesTokenCount": 41,
|
||||
"totalTokenCount": 557,
|
||||
"promptTokensDetails": [{"modality": "TEXT", "tokenCount": 516}],
|
||||
"serviceTier": "standard",
|
||||
},
|
||||
"modelVersion": "gemini-3.8-flash",
|
||||
"responseId": "zSHWaoPJKvPrz7IP7dqb-AM",
|
||||
}
|
||||
),
|
||||
),
|
||||
expected_litellm_message=coding_agent_stream_message(
|
||||
("call_5122749__thought__" + GEMINI_SKIP_VALIDATION_SIGNATURE, "call_5122752"),
|
||||
),
|
||||
expected_litellm_usage={
|
||||
"prompt_tokens": 516,
|
||||
"completion_tokens": 41,
|
||||
"total_tokens": 557,
|
||||
"prompt_tokens_details": {"text_tokens": 516},
|
||||
"completion_tokens_details": {"text_tokens": 41},
|
||||
},
|
||||
)
|
||||
|
||||
GEMINI_3_1_PRO_PREVIEW_AGENT_STREAMING_TEST_CASE: Final = StreamingTranslationTestCase(
|
||||
scenario="agent_streaming",
|
||||
litellm_endpoint=GEMINI_3_1_PRO_PREVIEW_AGENT_TEST_CASE.litellm_endpoint,
|
||||
litellm_request={
|
||||
**GEMINI_3_1_PRO_PREVIEW_AGENT_TEST_CASE.litellm_request,
|
||||
"stream": True,
|
||||
"stream_options": {"include_usage": True},
|
||||
},
|
||||
expected_provider_headers=GEMINI_3_1_PRO_PREVIEW_AGENT_TEST_CASE.expected_provider_headers,
|
||||
expected_provider_endpoint="/models/gemini-3.1-pro-preview:streamGenerateContent?alt=sse",
|
||||
expected_provider_request=VERTEX_AI_AGENT_REQUEST,
|
||||
mock_provider_frames=(
|
||||
SSEFrame(
|
||||
data={
|
||||
"candidates": [
|
||||
{
|
||||
"content": {
|
||||
"parts": [
|
||||
{
|
||||
"functionCall": {
|
||||
"name": "search_code",
|
||||
"args": {"query": "retries_remaining"},
|
||||
"id": "call_6112941",
|
||||
},
|
||||
"thoughtSignature": GEMINI_SKIP_VALIDATION_SIGNATURE,
|
||||
}
|
||||
],
|
||||
"role": "model",
|
||||
},
|
||||
"index": 0,
|
||||
}
|
||||
],
|
||||
"usageMetadata": {
|
||||
"promptTokenCount": 516,
|
||||
"candidatesTokenCount": 19,
|
||||
"totalTokenCount": 599,
|
||||
"promptTokensDetails": [{"modality": "TEXT", "tokenCount": 516}],
|
||||
"thoughtsTokenCount": 64,
|
||||
"serviceTier": "standard",
|
||||
},
|
||||
"modelVersion": "gemini-3.1-pro-preview",
|
||||
"responseId": "1SHWaoD1Ode1qtsPo6rR4AI",
|
||||
}
|
||||
),
|
||||
SSEFrame(
|
||||
data={
|
||||
"candidates": [
|
||||
{
|
||||
"content": {
|
||||
"parts": [
|
||||
{
|
||||
"functionCall": {
|
||||
"name": "run_tests",
|
||||
"args": {"path": "tests/test_retry.py"},
|
||||
"id": "call_6112947",
|
||||
}
|
||||
}
|
||||
],
|
||||
"role": "model",
|
||||
},
|
||||
"index": 0,
|
||||
}
|
||||
],
|
||||
"usageMetadata": {
|
||||
"promptTokenCount": 516,
|
||||
"candidatesTokenCount": 41,
|
||||
"totalTokenCount": 621,
|
||||
"promptTokensDetails": [{"modality": "TEXT", "tokenCount": 516}],
|
||||
"thoughtsTokenCount": 64,
|
||||
"serviceTier": "standard",
|
||||
},
|
||||
"modelVersion": "gemini-3.1-pro-preview",
|
||||
"responseId": "1SHWaoD1Ode1qtsPo6rR4AI",
|
||||
}
|
||||
),
|
||||
SSEFrame(
|
||||
data={
|
||||
"candidates": [
|
||||
{
|
||||
"content": {"parts": [{"text": ""}], "role": "model"},
|
||||
"finishReason": "STOP",
|
||||
"index": 0,
|
||||
}
|
||||
],
|
||||
"usageMetadata": {
|
||||
"promptTokenCount": 516,
|
||||
"candidatesTokenCount": 41,
|
||||
"totalTokenCount": 621,
|
||||
"promptTokensDetails": [{"modality": "TEXT", "tokenCount": 516}],
|
||||
"thoughtsTokenCount": 64,
|
||||
"serviceTier": "standard",
|
||||
},
|
||||
"modelVersion": "gemini-3.1-pro-preview",
|
||||
"responseId": "1SHWaoD1Ode1qtsPo6rR4AI",
|
||||
}
|
||||
),
|
||||
),
|
||||
expected_litellm_message=coding_agent_stream_message(
|
||||
("call_6112941__thought__" + GEMINI_SKIP_VALIDATION_SIGNATURE, "call_6112947"),
|
||||
),
|
||||
expected_litellm_usage={
|
||||
"prompt_tokens": 516,
|
||||
"completion_tokens": 105,
|
||||
"total_tokens": 621,
|
||||
"prompt_tokens_details": {"text_tokens": 516},
|
||||
"completion_tokens_details": {"reasoning_tokens": 64, "text_tokens": 41},
|
||||
},
|
||||
)
|
||||
Loading…
Add table
Reference in a new issue