scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-06 02:48:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 103

test(cost): pin OpenAI reported web search count with mixed actions (#44414)

Browse source 
Co-authored-by: kerry <kerry@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
devin-ai-integration[bot] 2026-10-03 11:31:45 -07:00 • committed by GitHub
parent 6c32384d8c
commit a849093d89
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 251 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

251
tests/integration/cost_calculation/cost_tracking_cases.json
View file

@ -27855,6 +27855,257 @@
"tool_usage_cost": 0.0125
}
},
{
"name": "gpt-5.6-responses_web_search_reported_count_mixed_actions",
"covers": "quota_management.spend_tracking.cost_matrix.logs_cost",
"model": "gpt-5.6",
"endpoint": "/v1/responses",
"request": {
"model": "$MODEL",
"input": "search this text",
"tools": [
{
"type": "web_search_preview",
"search_context_size": "medium"
}
]
},
"response": {
"content_type": "application/json",
"body": {
"id": "resp_$REQUEST_ID",
"object": "response",
"created_at": 1700000000,
"status": "completed",
"model": "gpt-5.6",
"output": [
{
"type": "web_search_call",
"id": "ws_0_$REQUEST_ID",
"status": "completed",
"action": {
"type": "search",
"query": "scripted query"
}
},
{
"type": "web_search_call",
"id": "ws_1_$REQUEST_ID",
"status": "completed",
"action": {
"type": "open_page",
"url": "https://scripted.example/a"
}
},
{
"type": "web_search_call",
"id": "ws_2_$REQUEST_ID",
"status": "completed",
"action": {
"type": "open_page",
"url": "https://scripted.example/b"
}
},
{
"type": "web_search_call",
"id": "ws_3_$REQUEST_ID",
"status": "completed",
"action": {
"type": "find_in_page",
"pattern": "release",
"url": "https://scripted.example/a"
}
},
{
"type": "message",
"id": "msg_$REQUEST_ID",
"status": "completed",
"role": "assistant",
"content": [
{
"type": "output_text",
"text": "scripted response",
"annotations": []
}
]
}
],
"usage": {
"input_tokens": 1840,
"output_tokens": 412,
"total_tokens": 2252,
"input_tokens_details": {
"cached_tokens": 0
},
"output_tokens_details": {
"reasoning_tokens": 0
}
},
"tool_usage": {
"image_gen": {
"input_tokens": 0,
"input_tokens_details": {
"image_tokens": 0,
"text_tokens": 0
},
"output_tokens": 0,
"output_tokens_details": {
"image_tokens": 0,
"text_tokens": 0
},
"total_tokens": 0
},
"web_search": {
"num_requests": 1
}
}
}
},
"expected": {
"spend": 0.021488,
"input_cost": 0.00322,
"output_cost": 0.005768,
"prompt_tokens": 1840,
"completion_tokens": 412,
"tool_usage_cost": 0.0125
}
},
{
"name": "gpt-5.6-responses_web_search_reported_count_open_page_only",
"covers": "quota_management.spend_tracking.cost_matrix.logs_cost",
"model": "gpt-5.6",
"endpoint": "/v1/responses",
"request": {
"model": "$MODEL",
"input": "search this text",
"tools": [
{
"type": "web_search_preview",
"search_context_size": "medium"
}
]
},
"response": {
"content_type": "application/json",
"body": {
"id": "resp_$REQUEST_ID",
"object": "response",
"created_at": 1700000000,
"status": "completed",
"model": "gpt-5.6",
"output": [
{
"type": "web_search_call",
"id": "ws_0_$REQUEST_ID",
"status": "completed",
"action": {
"type": "open_page",
"url": "https://scripted.example/a"
}
},
{
"type": "web_search_call",
"id": "ws_1_$REQUEST_ID",
"status": "completed",
"action": {
"type": "find_in_page",
"pattern": "release",
"url": "https://scripted.example/a"
}
},
{
"type": "message",
"id": "msg_$REQUEST_ID",
"status": "completed",
"role": "assistant",
"content": [
{
"type": "output_text",
"text": "scripted response",
"annotations": []
}
]
}
],
"usage": {
"input_tokens": 1840,
"output_tokens": 412,
"total_tokens": 2252,
"input_tokens_details": {
"cached_tokens": 0
},
"output_tokens_details": {
"reasoning_tokens": 0
}
},
"tool_usage": {
"image_gen": {
"input_tokens": 0,
"input_tokens_details": {
"image_tokens": 0,
"text_tokens": 0
},
"output_tokens": 0,
"output_tokens_details": {
"image_tokens": 0,
"text_tokens": 0
},
"total_tokens": 0
},
"web_search": {
"num_requests": 0
}
}
}
},
"expected": {
"spend": 0.008988,
"input_cost": 0.00322,
"output_cost": 0.005768,
"prompt_tokens": 1840,
"completion_tokens": 412,
"tool_usage_cost": 0.0
}
},
{
"name": "gpt-5.6-responses_web_search_reported_count_mixed_actions_stream",
"covers": "quota_management.spend_tracking.cost_matrix.logs_cost",
"model": "gpt-5.6",
"endpoint": "/v1/responses",
"request": {
"model": "$MODEL",
"input": "search this text",
"tools": [
{
"type": "web_search_preview",
"search_context_size": "medium"
}
],
"stream": true
},
"response": {
"content_type": "text/event-stream",
"frames": [
"event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_$REQUEST_ID\",\"object\":\"response\",\"created_at\":1700000000,\"status\":\"in_progress\",\"model\":\"gpt-5.6\",\"output\":[{\"type\":\"message\",\"id\":\"msg_$REQUEST_ID\",\"status\":\"completed\",\"role\":\"assistant\",\"content\":[{\"type\":\"output_text\",\"text\":\"scripted response\",\"annotations\":[]}]}],\"usage\":null}}",
"event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"output_index\":0,\"item\":{\"type\":\"message\",\"id\":\"msg_$REQUEST_ID\",\"status\":\"in_progress\",\"role\":\"assistant\",\"content\":[]}}",
"event: response.content_part.added\ndata: {\"type\":\"response.content_part.added\",\"item_id\":\"msg_$REQUEST_ID\",\"output_index\":0,\"content_index\":0,\"part\":{\"type\":\"content_part\",\"text\":\"\"}}",
"event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"item_id\":\"msg_$REQUEST_ID\",\"output_index\":0,\"content_index\":0,\"delta\":\"scripted \"}",
"event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"item_id\":\"msg_$REQUEST_ID\",\"output_index\":0,\"content_index\":0,\"delta\":\"response\"}",
"event: response.output_text.done\ndata: {\"type\":\"response.output_text.done\",\"item_id\":\"msg_$REQUEST_ID\",\"output_index\":0,\"content_index\":0,\"text\":\"scripted response\"}",
"event: response.content_part.done\ndata: {\"type\":\"response.content_part.done\",\"item_id\":\"msg_$REQUEST_ID\",\"output_index\":0,\"content_index\":0,\"part\":{\"type\":\"output_text\",\"text\":\"scripted response\",\"annotations\":[]}}",
"event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"output_index\":0,\"item\":{\"type\":\"message\",\"id\":\"msg_$REQUEST_ID\",\"status\":\"completed\",\"role\":\"assistant\",\"content\":[{\"type\":\"output_text\",\"text\":\"scripted response\",\"annotations\":[]}]}}",
"event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_$REQUEST_ID\",\"object\":\"response\",\"created_at\":1700000000,\"status\":\"completed\",\"model\":\"gpt-5.6\",\"output\":[{\"type\":\"web_search_call\",\"id\":\"ws_0_$REQUEST_ID\",\"status\":\"completed\",\"action\":{\"type\":\"search\",\"query\":\"scripted query\"}},{\"type\":\"web_search_call\",\"id\":\"ws_1_$REQUEST_ID\",\"status\":\"completed\",\"action\":{\"type\":\"open_page\",\"url\":\"https://scripted.example/a\"}},{\"type\":\"web_search_call\",\"id\":\"ws_2_$REQUEST_ID\",\"status\":\"completed\",\"action\":{\"type\":\"open_page\",\"url\":\"https://scripted.example/b\"}},{\"type\":\"web_search_call\",\"id\":\"ws_3_$REQUEST_ID\",\"status\":\"completed\",\"action\":{\"type\":\"find_in_page\",\"pattern\":\"release\",\"url\":\"https://scripted.example/a\"}},{\"type\":\"message\",\"id\":\"msg_$REQUEST_ID\",\"status\":\"completed\",\"role\":\"assistant\",\"content\":[{\"type\":\"output_text\",\"text\":\"scripted response\",\"annotations\":[]}] }],\"usage\":{\"input_tokens\":1840,\"output_tokens\":412,\"total_tokens\":2252,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens_details\":{\"reasoning_tokens\":0}},\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":1}}}}"
]
},
"expected": {
"spend": 0.021488,
"input_cost": 0.00322,
"output_cost": 0.005768,
"prompt_tokens": 1840,
"completion_tokens": 412,
"tool_usage_cost": 0.0125
}
},
{
"name": "gpt-5.3-codex-responses_file_search",
"covers": "quota_management.spend_tracking.cost_matrix.logs_cost",