mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-14 09:03:16 +00:00
Merge 7780aed7f7 into e9b1c11098
This commit is contained in:
Aryan Deshmukh
GitHub
commit
71ad5d780d
9 changed files with 657 additions and 92 deletions
|
|
@ -1754,13 +1754,70 @@
|
|||
"title": "litellm_spend_capture_rate",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"description": "Tool calls the model asked for in its responses, by tool name; counted from the response, so a call the client never runs is included",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"lineWidth": 1,
|
||||
"showPoints": "never",
|
||||
"spanNulls": false
|
||||
},
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 115
|
||||
},
|
||||
"id": 117,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(litellm_model_tool_calls_total[$__rate_interval])) by (tool_name)",
|
||||
"legendFormat": "{{tool_name}}",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "litellm_model_tool_calls rate",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 115
|
||||
"y": 123
|
||||
},
|
||||
"id": 30,
|
||||
"panels": [],
|
||||
|
|
@ -1793,7 +1850,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 116
|
||||
"y": 124
|
||||
},
|
||||
"id": 31,
|
||||
"options": {
|
||||
|
|
@ -1850,7 +1907,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 116
|
||||
"y": 124
|
||||
},
|
||||
"id": 32,
|
||||
"options": {
|
||||
|
|
@ -1907,7 +1964,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 124
|
||||
"y": 132
|
||||
},
|
||||
"id": 33,
|
||||
"options": {
|
||||
|
|
@ -1964,7 +2021,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 124
|
||||
"y": 132
|
||||
},
|
||||
"id": 34,
|
||||
"options": {
|
||||
|
|
@ -2021,7 +2078,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 132
|
||||
"y": 140
|
||||
},
|
||||
"id": 35,
|
||||
"options": {
|
||||
|
|
@ -2058,7 +2115,7 @@
|
|||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 140
|
||||
"y": 148
|
||||
},
|
||||
"id": 36,
|
||||
"panels": [],
|
||||
|
|
@ -2091,7 +2148,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 141
|
||||
"y": 149
|
||||
},
|
||||
"id": 37,
|
||||
"options": {
|
||||
|
|
@ -2148,7 +2205,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 141
|
||||
"y": 149
|
||||
},
|
||||
"id": 38,
|
||||
"options": {
|
||||
|
|
@ -2205,7 +2262,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 149
|
||||
"y": 157
|
||||
},
|
||||
"id": 39,
|
||||
"options": {
|
||||
|
|
@ -2262,7 +2319,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 149
|
||||
"y": 157
|
||||
},
|
||||
"id": 40,
|
||||
"options": {
|
||||
|
|
@ -2319,7 +2376,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 157
|
||||
"y": 165
|
||||
},
|
||||
"id": 41,
|
||||
"options": {
|
||||
|
|
@ -2376,7 +2433,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 157
|
||||
"y": 165
|
||||
},
|
||||
"id": 42,
|
||||
"options": {
|
||||
|
|
@ -2433,7 +2490,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 165
|
||||
"y": 173
|
||||
},
|
||||
"id": 43,
|
||||
"options": {
|
||||
|
|
@ -2490,7 +2547,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 165
|
||||
"y": 173
|
||||
},
|
||||
"id": 44,
|
||||
"options": {
|
||||
|
|
@ -2547,7 +2604,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 173
|
||||
"y": 181
|
||||
},
|
||||
"id": 45,
|
||||
"options": {
|
||||
|
|
@ -2604,7 +2661,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 173
|
||||
"y": 181
|
||||
},
|
||||
"id": 46,
|
||||
"options": {
|
||||
|
|
@ -2661,7 +2718,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 181
|
||||
"y": 189
|
||||
},
|
||||
"id": 47,
|
||||
"options": {
|
||||
|
|
@ -2698,7 +2755,7 @@
|
|||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 189
|
||||
"y": 197
|
||||
},
|
||||
"id": 48,
|
||||
"panels": [],
|
||||
|
|
@ -2731,7 +2788,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 190
|
||||
"y": 198
|
||||
},
|
||||
"id": 49,
|
||||
"options": {
|
||||
|
|
@ -2788,7 +2845,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 190
|
||||
"y": 198
|
||||
},
|
||||
"id": 50,
|
||||
"options": {
|
||||
|
|
@ -2845,7 +2902,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 198
|
||||
"y": 206
|
||||
},
|
||||
"id": 51,
|
||||
"options": {
|
||||
|
|
@ -2902,7 +2959,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 198
|
||||
"y": 206
|
||||
},
|
||||
"id": 52,
|
||||
"options": {
|
||||
|
|
@ -2959,7 +3016,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 206
|
||||
"y": 214
|
||||
},
|
||||
"id": 53,
|
||||
"options": {
|
||||
|
|
@ -3016,7 +3073,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 206
|
||||
"y": 214
|
||||
},
|
||||
"id": 54,
|
||||
"options": {
|
||||
|
|
@ -3053,7 +3110,7 @@
|
|||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 214
|
||||
"y": 222
|
||||
},
|
||||
"id": 55,
|
||||
"panels": [],
|
||||
|
|
@ -3086,7 +3143,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 215
|
||||
"y": 223
|
||||
},
|
||||
"id": 56,
|
||||
"options": {
|
||||
|
|
@ -3143,7 +3200,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 215
|
||||
"y": 223
|
||||
},
|
||||
"id": 57,
|
||||
"options": {
|
||||
|
|
@ -3200,7 +3257,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 223
|
||||
"y": 231
|
||||
},
|
||||
"id": 58,
|
||||
"options": {
|
||||
|
|
@ -3257,7 +3314,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 223
|
||||
"y": 231
|
||||
},
|
||||
"id": 59,
|
||||
"options": {
|
||||
|
|
@ -3314,7 +3371,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 231
|
||||
"y": 239
|
||||
},
|
||||
"id": 60,
|
||||
"options": {
|
||||
|
|
@ -3371,7 +3428,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 231
|
||||
"y": 239
|
||||
},
|
||||
"id": 61,
|
||||
"options": {
|
||||
|
|
@ -3428,7 +3485,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 239
|
||||
"y": 247
|
||||
},
|
||||
"id": 62,
|
||||
"options": {
|
||||
|
|
@ -3485,7 +3542,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 239
|
||||
"y": 247
|
||||
},
|
||||
"id": 63,
|
||||
"options": {
|
||||
|
|
@ -3542,7 +3599,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 247
|
||||
"y": 255
|
||||
},
|
||||
"id": 64,
|
||||
"options": {
|
||||
|
|
@ -3599,7 +3656,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 247
|
||||
"y": 255
|
||||
},
|
||||
"id": 65,
|
||||
"options": {
|
||||
|
|
@ -3656,7 +3713,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 255
|
||||
"y": 263
|
||||
},
|
||||
"id": 66,
|
||||
"options": {
|
||||
|
|
@ -3713,7 +3770,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 255
|
||||
"y": 263
|
||||
},
|
||||
"id": 67,
|
||||
"options": {
|
||||
|
|
@ -3770,7 +3827,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 263
|
||||
"y": 271
|
||||
},
|
||||
"id": 68,
|
||||
"options": {
|
||||
|
|
@ -3827,7 +3884,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 263
|
||||
"y": 271
|
||||
},
|
||||
"id": 69,
|
||||
"options": {
|
||||
|
|
@ -3884,7 +3941,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 271
|
||||
"y": 279
|
||||
},
|
||||
"id": 70,
|
||||
"options": {
|
||||
|
|
@ -3941,7 +3998,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 271
|
||||
"y": 279
|
||||
},
|
||||
"id": 71,
|
||||
"options": {
|
||||
|
|
@ -3978,7 +4035,7 @@
|
|||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 279
|
||||
"y": 287
|
||||
},
|
||||
"id": 72,
|
||||
"panels": [],
|
||||
|
|
@ -4011,7 +4068,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 280
|
||||
"y": 288
|
||||
},
|
||||
"id": 73,
|
||||
"options": {
|
||||
|
|
@ -4068,7 +4125,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 280
|
||||
"y": 288
|
||||
},
|
||||
"id": 74,
|
||||
"options": {
|
||||
|
|
@ -4125,7 +4182,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 288
|
||||
"y": 296
|
||||
},
|
||||
"id": 75,
|
||||
"options": {
|
||||
|
|
@ -4184,7 +4241,7 @@
|
|||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 296
|
||||
"y": 304
|
||||
},
|
||||
"id": 76,
|
||||
"panels": [],
|
||||
|
|
@ -4217,7 +4274,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 297
|
||||
"y": 305
|
||||
},
|
||||
"id": 77,
|
||||
"options": {
|
||||
|
|
@ -4274,7 +4331,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 297
|
||||
"y": 305
|
||||
},
|
||||
"id": 78,
|
||||
"options": {
|
||||
|
|
@ -4311,7 +4368,7 @@
|
|||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 305
|
||||
"y": 313
|
||||
},
|
||||
"id": 79,
|
||||
"panels": [],
|
||||
|
|
@ -4344,7 +4401,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 306
|
||||
"y": 314
|
||||
},
|
||||
"id": 80,
|
||||
"options": {
|
||||
|
|
@ -4401,7 +4458,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 306
|
||||
"y": 314
|
||||
},
|
||||
"id": 81,
|
||||
"options": {
|
||||
|
|
@ -4458,7 +4515,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 314
|
||||
"y": 322
|
||||
},
|
||||
"id": 82,
|
||||
"options": {
|
||||
|
|
@ -4515,7 +4572,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 314
|
||||
"y": 322
|
||||
},
|
||||
"id": 83,
|
||||
"options": {
|
||||
|
|
@ -4572,7 +4629,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 322
|
||||
"y": 330
|
||||
},
|
||||
"id": 84,
|
||||
"options": {
|
||||
|
|
@ -4651,7 +4708,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 322
|
||||
"y": 330
|
||||
},
|
||||
"id": 85,
|
||||
"options": {
|
||||
|
|
@ -4708,7 +4765,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 330
|
||||
"y": 338
|
||||
},
|
||||
"id": 86,
|
||||
"options": {
|
||||
|
|
@ -4765,7 +4822,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 330
|
||||
"y": 338
|
||||
},
|
||||
"id": 87,
|
||||
"options": {
|
||||
|
|
@ -4822,7 +4879,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 338
|
||||
"y": 346
|
||||
},
|
||||
"id": 88,
|
||||
"options": {
|
||||
|
|
@ -4859,7 +4916,7 @@
|
|||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 346
|
||||
"y": 354
|
||||
},
|
||||
"id": 89,
|
||||
"panels": [],
|
||||
|
|
@ -4892,7 +4949,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 347
|
||||
"y": 355
|
||||
},
|
||||
"id": 90,
|
||||
"options": {
|
||||
|
|
@ -4949,7 +5006,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 347
|
||||
"y": 355
|
||||
},
|
||||
"id": 91,
|
||||
"options": {
|
||||
|
|
@ -5006,7 +5063,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 355
|
||||
"y": 363
|
||||
},
|
||||
"id": 92,
|
||||
"options": {
|
||||
|
|
@ -5063,7 +5120,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 355
|
||||
"y": 363
|
||||
},
|
||||
"id": 93,
|
||||
"options": {
|
||||
|
|
@ -5120,7 +5177,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 363
|
||||
"y": 371
|
||||
},
|
||||
"id": 94,
|
||||
"options": {
|
||||
|
|
@ -5157,7 +5214,7 @@
|
|||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 371
|
||||
"y": 379
|
||||
},
|
||||
"id": 95,
|
||||
"panels": [],
|
||||
|
|
@ -5190,7 +5247,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 372
|
||||
"y": 380
|
||||
},
|
||||
"id": 96,
|
||||
"options": {
|
||||
|
|
@ -5247,7 +5304,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 372
|
||||
"y": 380
|
||||
},
|
||||
"id": 97,
|
||||
"options": {
|
||||
|
|
@ -5304,7 +5361,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 380
|
||||
"y": 388
|
||||
},
|
||||
"id": 98,
|
||||
"options": {
|
||||
|
|
@ -5341,7 +5398,7 @@
|
|||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 388
|
||||
"y": 396
|
||||
},
|
||||
"id": 99,
|
||||
"panels": [],
|
||||
|
|
@ -5374,7 +5431,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 389
|
||||
"y": 397
|
||||
},
|
||||
"id": 100,
|
||||
"options": {
|
||||
|
|
@ -5431,7 +5488,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 389
|
||||
"y": 397
|
||||
},
|
||||
"id": 101,
|
||||
"options": {
|
||||
|
|
@ -5488,7 +5545,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 397
|
||||
"y": 405
|
||||
},
|
||||
"id": 102,
|
||||
"options": {
|
||||
|
|
@ -5545,7 +5602,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 397
|
||||
"y": 405
|
||||
},
|
||||
"id": 103,
|
||||
"options": {
|
||||
|
|
@ -5624,7 +5681,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 405
|
||||
"y": 413
|
||||
},
|
||||
"id": 104,
|
||||
"options": {
|
||||
|
|
@ -5661,7 +5718,7 @@
|
|||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 413
|
||||
"y": 421
|
||||
},
|
||||
"id": 105,
|
||||
"panels": [],
|
||||
|
|
@ -5694,7 +5751,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 414
|
||||
"y": 422
|
||||
},
|
||||
"id": 106,
|
||||
"options": {
|
||||
|
|
@ -5883,7 +5940,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 414
|
||||
"y": 422
|
||||
},
|
||||
"id": 107,
|
||||
"options": {
|
||||
|
|
@ -6083,7 +6140,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 422
|
||||
"y": 430
|
||||
},
|
||||
"id": 108,
|
||||
"options": {
|
||||
|
|
@ -6283,7 +6340,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 422
|
||||
"y": 430
|
||||
},
|
||||
"id": 109,
|
||||
"options": {
|
||||
|
|
@ -6406,7 +6463,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 430
|
||||
"y": 438
|
||||
},
|
||||
"id": 110,
|
||||
"options": {
|
||||
|
|
@ -6443,7 +6500,7 @@
|
|||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 438
|
||||
"y": 446
|
||||
},
|
||||
"id": 112,
|
||||
"panels": [],
|
||||
|
|
@ -6476,7 +6533,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 439
|
||||
"y": 447
|
||||
},
|
||||
"id": 113,
|
||||
"options": {
|
||||
|
|
@ -6533,7 +6590,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 439
|
||||
"y": 447
|
||||
},
|
||||
"id": 114,
|
||||
"options": {
|
||||
|
|
@ -6570,7 +6627,7 @@
|
|||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 447
|
||||
"y": 455
|
||||
},
|
||||
"id": 115,
|
||||
"panels": [],
|
||||
|
|
@ -6603,7 +6660,7 @@
|
|||
"h": 8,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 448
|
||||
"y": 456
|
||||
},
|
||||
"id": 116,
|
||||
"options": {
|
||||
|
|
|
|||
|
|
@ -488,6 +488,10 @@ prometheus_end_user_metrics_max_series_per_metric: Optional[int] = 10000
|
|||
prometheus_end_user_metrics_ttl_seconds: Optional[float] = 3600.0
|
||||
prometheus_end_user_metrics_cleanup_interval_seconds: Optional[float] = 60.0
|
||||
prometheus_metrics_max_series_per_metric: Optional[int] = None
|
||||
# Distinct ``tool_name`` values ``litellm_model_tool_calls_total`` gives their own series, per worker
|
||||
# process; later names count under ``other`` until restart (a real tool named ``other`` shares that series).
|
||||
# It bounds names, not series: each name still splits by the metric's other labels. None disables the cap.
|
||||
prometheus_model_tool_name_max_values: Optional[int] = 1000
|
||||
prometheus_metrics_ttl_seconds: Optional[float] = None
|
||||
prometheus_metrics_cleanup_interval_seconds: Optional[float] = 60.0
|
||||
disable_add_prefix_to_prompt: bool = False # used by anthropic, to disable adding prefix to prompt
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import asyncio
|
|||
import math
|
||||
import os
|
||||
import sys
|
||||
from collections import Counter
|
||||
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
||||
from dataclasses import replace
|
||||
from datetime import datetime, timedelta
|
||||
|
|
@ -117,6 +118,7 @@ _TableRowT: Final = TypeVar("_TableRowT", bound=BaseModel)
|
|||
_DEFAULT_BUDGET_METRICS_PER_REQUEST_TIMEOUT: Final = 5.0
|
||||
|
||||
UNRECOGNIZED_REQUESTED_MODEL_LABEL: Final = "other"
|
||||
OVERFLOW_MODEL_TOOL_NAME_LABEL: Final = "other"
|
||||
|
||||
_NON_ENUM_METRIC_LABELS: Final[frozenset[str]] = frozenset(
|
||||
(
|
||||
|
|
@ -540,6 +542,7 @@ class PrometheusLogger(CustomLogger):
|
|||
_custom_buckets: Final = litellm.prometheus_latency_buckets
|
||||
self.latency_buckets = tuple(_custom_buckets) if _custom_buckets is not None else LATENCY_BUCKETS
|
||||
self._bounded_prometheus_series_tracker = BoundedPrometheusSeriesTracker()
|
||||
self._admitted_model_tool_names: set[str] = set() # mutable-ok: names admitted under the tool_name cap
|
||||
_multiproc_dir: Final = os.environ.get("PROMETHEUS_MULTIPROC_DIR")
|
||||
self._series_cap_tracker = (
|
||||
BoundedPrometheusSeriesTracker()
|
||||
|
|
@ -671,6 +674,12 @@ class PrometheusLogger(CustomLogger):
|
|||
labelnames=self.get_labels_for_metric("litellm_images_generated_metric"),
|
||||
)
|
||||
|
||||
self.litellm_model_tool_calls_total = self._counter_factory(
|
||||
"litellm_model_tool_calls_total",
|
||||
"Tool calls the model asked for in its response, by tool name",
|
||||
labelnames=self.get_labels_for_metric("litellm_model_tool_calls_total"),
|
||||
)
|
||||
|
||||
# Remaining Budget for Team
|
||||
self.litellm_remaining_team_budget_metric = self._gauge_factory(
|
||||
"litellm_remaining_team_budget_metric",
|
||||
|
|
@ -1835,6 +1844,11 @@ class PrometheusLogger(CustomLogger):
|
|||
label_context=label_context,
|
||||
)
|
||||
|
||||
try:
|
||||
self._increment_model_tool_call_metrics(response_obj=response_obj, enum_values=enum_values)
|
||||
except Exception as e: # noqa: BLE001 # counting tool calls must never stop the rest of success logging
|
||||
verbose_logger.exception("Prometheus: failed to count the model's tool calls: %s", e)
|
||||
|
||||
# MCP tool call metrics
|
||||
self._increment_mcp_tool_call_metrics(
|
||||
standard_logging_payload=standard_logging_payload,
|
||||
|
|
@ -2154,6 +2168,38 @@ class PrometheusLogger(CustomLogger):
|
|||
return value
|
||||
return None
|
||||
|
||||
def _increment_model_tool_call_metrics(self, response_obj: object, enum_values: UserAPIKeyLabelValues) -> None:
|
||||
from litellm.proxy.db.spend_log_tool_index import response_tool_call_names
|
||||
|
||||
tool_names: Final = tuple(
|
||||
self._bounded_model_tool_name(name) for name in response_tool_call_names(response_obj)
|
||||
)
|
||||
for tool_name, amount in Counter(tool_names).items():
|
||||
self._inc_model_tool_call_counter(replace(enum_values, tool_name=tool_name), amount=amount)
|
||||
|
||||
def _bounded_model_tool_name(self, tool_name: str) -> str:
|
||||
"""The model picks tool names, from tools the caller declared or not, so
|
||||
only the first ``prometheus_model_tool_name_max_values`` distinct names
|
||||
get their own series; later names count under ``other``. Concurrent
|
||||
callbacks may admit a few names past the cap, which still bounds it."""
|
||||
max_values: Final = litellm.prometheus_model_tool_name_max_values
|
||||
if max_values is None or tool_name in self._admitted_model_tool_names:
|
||||
return tool_name
|
||||
if len(self._admitted_model_tool_names) >= max_values:
|
||||
return OVERFLOW_MODEL_TOOL_NAME_LABEL
|
||||
self._admitted_model_tool_names.add(tool_name)
|
||||
return tool_name
|
||||
|
||||
def _inc_model_tool_call_counter(self, tool_enum_values: UserAPIKeyLabelValues, amount: int) -> None:
|
||||
PrometheusLogger._inc_labeled_counter(
|
||||
self,
|
||||
self.litellm_model_tool_calls_total,
|
||||
"litellm_model_tool_calls_total",
|
||||
tool_enum_values,
|
||||
label_context=PrometheusLabelFactoryContext(tool_enum_values),
|
||||
amount=amount,
|
||||
)
|
||||
|
||||
def _increment_mcp_tool_call_metrics(
|
||||
self,
|
||||
standard_logging_payload: StandardLoggingPayload,
|
||||
|
|
|
|||
|
|
@ -179,6 +179,9 @@ from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import (
|
|||
from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import (
|
||||
split_concatenated_json_objects as split_concatenated_json_objects, # noqa: PLC0414 # public compatibility re-export
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import (
|
||||
tool_call_argument_count as tool_call_argument_count, # noqa: PLC0414 # public compatibility re-export
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.tpm_media_utils import (
|
||||
_image_part_url, # pyright: ignore[reportPrivateUsage] # Shared implementation for as_openai_image_part
|
||||
)
|
||||
|
|
|
|||
|
|
@ -76,6 +76,7 @@ from .common_utils import (
|
|||
native_redacted_thinking_data,
|
||||
parse_tool_call_arguments,
|
||||
salvage_concatenated_tool_arguments,
|
||||
tool_call_argument_count,
|
||||
)
|
||||
from .image_handling import convert_url_to_base64, is_pdf_media_type
|
||||
|
||||
|
|
@ -6546,6 +6547,65 @@ def get_tool_calls_from_response(response: object, include_all_choices: bool = F
|
|||
return []
|
||||
|
||||
|
||||
def _tool_call_count(raw_arguments: object) -> int:
|
||||
# A missing ``arguments`` field parses as ``{}`` on the parse path: one call.
|
||||
if not isinstance(raw_arguments, str):
|
||||
return 1
|
||||
return tool_call_argument_count("{}" if raw_arguments == REDACTED_BY_LITELLM else raw_arguments)
|
||||
|
||||
|
||||
def _repeated_name(call: object) -> tuple[str | None, ...]:
|
||||
return (_optional_tool_name(message_field(call, "name")),) * _tool_call_count(message_field(call, "arguments"))
|
||||
|
||||
|
||||
def _choice_tool_call_names(choice: object) -> tuple[str | None, ...]:
|
||||
functions: Final = tuple(message_field(tool_call, "function") for tool_call in _choice_tool_calls(choice))
|
||||
return tuple(
|
||||
itertools.chain.from_iterable(_repeated_name(function) for function in functions if function is not None)
|
||||
)
|
||||
|
||||
|
||||
def _chat_tool_call_names(response: object, include_all_choices: bool) -> tuple[str | None, ...]:
|
||||
return tuple(
|
||||
itertools.chain.from_iterable(
|
||||
_choice_tool_call_names(choice) for choice in _selected_choices(response, include_all_choices)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _responses_api_tool_call_names(response: object) -> tuple[str | None, ...]:
|
||||
return tuple(itertools.chain.from_iterable(_repeated_name(item) for item in _response_function_calls(response)))
|
||||
|
||||
|
||||
def _messages_api_tool_call_names(response: object) -> tuple[str | None, ...]:
|
||||
content: Final = message_field(response, "content")
|
||||
blocks: Final = parts_of(content)
|
||||
if not isinstance(content, list):
|
||||
return ()
|
||||
return tuple(
|
||||
_optional_tool_name(message_field(block, "name"))
|
||||
for block in blocks
|
||||
if message_field(block, "type") == "tool_use"
|
||||
)
|
||||
|
||||
|
||||
def get_tool_call_names_from_response(response: object, include_all_choices: bool = False) -> tuple[str | None, ...]:
|
||||
"""
|
||||
The ``name`` of every call ``get_tool_calls_from_response`` returns, in the
|
||||
same order and with the same count, without parsing argument values into
|
||||
objects or logging them. Usage counters (Prometheus, the spend-log tool
|
||||
index) only need the names, and the parse path logs up to the full argument
|
||||
text of a malformed call, which may hold prompt data.
|
||||
"""
|
||||
chat_names: Final = _chat_tool_call_names(response, include_all_choices=include_all_choices)
|
||||
if chat_names:
|
||||
return chat_names
|
||||
responses_names: Final = _responses_api_tool_call_names(response)
|
||||
if responses_names:
|
||||
return responses_names
|
||||
return _messages_api_tool_call_names(response)
|
||||
|
||||
|
||||
def has_tool_with_name(tools: object, tool_name: str) -> bool:
|
||||
"""
|
||||
Check whether a tools list (as sent to an LLM) includes a tool with the
|
||||
|
|
|
|||
|
|
@ -250,3 +250,21 @@ def salvage_concatenated_tool_arguments(raw: str) -> tuple[dict[str, object], ..
|
|||
if exceeded:
|
||||
return ()
|
||||
return kept
|
||||
|
||||
|
||||
def tool_call_argument_count(arguments: str) -> int:
|
||||
"""How many tool calls one raw argument string stands for, decided the way
|
||||
``get_tool_calls_from_response`` decides it: valid or repairable JSON is one
|
||||
call, a concatenation of JSON objects is one call per salvaged object, and
|
||||
anything else is one call with empty arguments. Unlike that path it never
|
||||
logs any part of the argument text, so usage counters can call it on every
|
||||
response."""
|
||||
if not arguments.strip():
|
||||
return 1
|
||||
try:
|
||||
json.loads(arguments)
|
||||
except json.JSONDecodeError:
|
||||
if _attempt_json_repair(arguments) is not None:
|
||||
return 1
|
||||
return len(salvage_concatenated_tool_arguments(arguments)) or 1
|
||||
return 1
|
||||
|
|
|
|||
|
|
@ -47,20 +47,17 @@ class ToolUsageTransaction:
|
|||
|
||||
def response_tool_call_names(completion_response: object) -> tuple[str, ...]:
|
||||
"""Tool names invoked in a completion response, in call order, for any response
|
||||
surface get_tool_calls_from_response understands (chat completions, Responses
|
||||
surface get_tool_call_names_from_response understands (chat completions, Responses
|
||||
API output items, Anthropic Messages tool_use blocks), plus the MCP tools an
|
||||
auto-executed Responses API call ran. Reads every choice of an ``n>1`` chat
|
||||
response: each choice cost money and its tool calls ran."""
|
||||
if completion_response is None or isinstance(completion_response, Exception):
|
||||
return ()
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import (
|
||||
get_tool_calls_from_response,
|
||||
get_tool_call_names_from_response,
|
||||
)
|
||||
|
||||
called_names: Final = (
|
||||
tool_call.get("name")
|
||||
for tool_call in get_tool_calls_from_response(completion_response, include_all_choices=True)
|
||||
)
|
||||
called_names: Final = get_tool_call_names_from_response(completion_response, include_all_choices=True)
|
||||
return tuple(
|
||||
stripped
|
||||
for name in chain(
|
||||
|
|
|
|||
|
|
@ -229,6 +229,7 @@ class UserAPIKeyLabelNames(Enum):
|
|||
ORG_ALIAS = "org_alias"
|
||||
MCP_TOOL_NAME = "mcp_tool_name"
|
||||
MCP_SERVER_NAME = "mcp_server_name"
|
||||
MODEL_TOOL_NAME = "tool_name"
|
||||
SERVICE_TIER = "service_tier"
|
||||
INPUT_SEQUENCE_LENGTH = "input_sequence_length"
|
||||
|
||||
|
|
@ -256,6 +257,7 @@ DEFINED_PROMETHEUS_METRICS = Literal[
|
|||
"litellm_output_audio_tokens_metric",
|
||||
"litellm_video_duration_seconds_metric",
|
||||
"litellm_images_generated_metric",
|
||||
"litellm_model_tool_calls_total",
|
||||
"litellm_deployment_successful_fallbacks",
|
||||
"litellm_deployment_failed_fallbacks",
|
||||
"litellm_remaining_team_budget_metric",
|
||||
|
|
@ -676,6 +678,7 @@ class PrometheusMetricLabels:
|
|||
|
||||
litellm_video_duration_seconds_metric = litellm_output_tokens_metric
|
||||
litellm_images_generated_metric = litellm_output_tokens_metric
|
||||
litellm_model_tool_calls_total = litellm_output_tokens_metric + [UserAPIKeyLabelNames.MODEL_TOOL_NAME.value]
|
||||
|
||||
litellm_deployment_state = [
|
||||
UserAPIKeyLabelNames.v2_LITELLM_MODEL_NAME.value,
|
||||
|
|
@ -924,6 +927,7 @@ class PrometheusMetricLabels:
|
|||
"litellm_output_tokens_metric",
|
||||
"litellm_video_duration_seconds_metric",
|
||||
"litellm_images_generated_metric",
|
||||
"litellm_model_tool_calls_total",
|
||||
}
|
||||
)
|
||||
_input_sequence_length_metrics: ClassVar[frozenset[str]] = frozenset(
|
||||
|
|
@ -1105,6 +1109,7 @@ class UserAPIKeyLabelValues:
|
|||
org_alias: str | None = None
|
||||
mcp_tool_name: str | None = None
|
||||
mcp_server_name: str | None = None
|
||||
tool_name: str | None = None
|
||||
service_tier: str | None = None
|
||||
input_sequence_length: str | None = None
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,375 @@
|
|||
import datetime
|
||||
import logging
|
||||
from collections.abc import Mapping
|
||||
from typing import Final, cast
|
||||
|
||||
import pytest
|
||||
from prometheus_client import REGISTRY
|
||||
from prometheus_client.samples import Sample
|
||||
|
||||
import litellm
|
||||
from litellm.constants import REDACTED_BY_LITELLM
|
||||
from litellm.integrations.prometheus import PrometheusLogger
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import (
|
||||
get_tool_call_names_from_response,
|
||||
get_tool_calls_from_response,
|
||||
)
|
||||
from litellm.types.utils import StandardLoggingPayload
|
||||
|
||||
NOW: Final = datetime.datetime(2026, 11, 3, 12, 0, 0)
|
||||
|
||||
|
||||
def _clear_prometheus_registry() -> None:
|
||||
for collector in tuple(REGISTRY._collector_to_names): # pyright: ignore[reportPrivateUsage] # test registry reset
|
||||
REGISTRY.unregister(collector)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def isolated_registry():
|
||||
_clear_prometheus_registry()
|
||||
yield
|
||||
_clear_prometheus_registry()
|
||||
|
||||
|
||||
def _success_kwargs() -> dict[str, object]:
|
||||
payload: Final = cast(
|
||||
StandardLoggingPayload,
|
||||
{
|
||||
"id": "t",
|
||||
"call_type": "acompletion",
|
||||
"response_cost": 0.001,
|
||||
"status": "success",
|
||||
"total_tokens": 30,
|
||||
"prompt_tokens": 10,
|
||||
"completion_tokens": 20,
|
||||
"startTime": NOW - datetime.timedelta(seconds=3),
|
||||
"endTime": NOW,
|
||||
"completionStartTime": NOW - datetime.timedelta(seconds=1),
|
||||
"model": "gpt-4.1",
|
||||
"model_id": "model-123",
|
||||
"model_group": "gpt-4.1-group",
|
||||
"api_base": "https://api.openai.com",
|
||||
"custom_llm_provider": "openai",
|
||||
"request_tags": [],
|
||||
"stream": False,
|
||||
"metadata": {
|
||||
"user_api_key_hash": "key-hash",
|
||||
"user_api_key_alias": "key-alias",
|
||||
"user_api_key_team_id": "team-1",
|
||||
"user_api_key_team_alias": "team-alias",
|
||||
"user_api_key_user_id": "user-1",
|
||||
"user_api_key_user_email": None,
|
||||
"user_api_key_org_id": None,
|
||||
"user_api_key_org_alias": None,
|
||||
"requester_metadata": None,
|
||||
"user_api_key_end_user_id": None,
|
||||
"usage_object": None,
|
||||
},
|
||||
"hidden_params": {"litellm_overhead_time_ms": None, "additional_headers": None},
|
||||
},
|
||||
)
|
||||
return {
|
||||
"model": "gpt-4.1",
|
||||
"litellm_params": {"metadata": {}},
|
||||
"standard_logging_object": payload,
|
||||
"stream": False,
|
||||
"start_time": NOW - datetime.timedelta(seconds=3),
|
||||
"api_call_start_time": NOW - datetime.timedelta(seconds=2),
|
||||
"completion_start_time": NOW - datetime.timedelta(seconds=1),
|
||||
"end_time": NOW,
|
||||
}
|
||||
|
||||
|
||||
def _samples(name: str) -> tuple[Sample, ...]:
|
||||
return tuple(sample for metric in REGISTRY.collect() for sample in metric.samples if sample.name == f"{name}_total")
|
||||
|
||||
|
||||
def _tool_call_counts() -> Mapping[str, float]:
|
||||
return {sample.labels["tool_name"]: sample.value for sample in _samples("litellm_model_tool_calls")}
|
||||
|
||||
|
||||
def _chat_tool_call(call_id: str, name: str) -> Mapping[str, object]:
|
||||
return {"id": call_id, "type": "function", "function": {"name": name, "arguments": "{}"}}
|
||||
|
||||
|
||||
def _chat_choice(index: int, *tool_calls: Mapping[str, object]) -> Mapping[str, object]:
|
||||
return {
|
||||
"index": index,
|
||||
"finish_reason": "tool_calls",
|
||||
"message": {"role": "assistant", "content": None, "tool_calls": list(tool_calls)},
|
||||
}
|
||||
|
||||
|
||||
CHAT_RESPONSE: Final = litellm.ModelResponse(
|
||||
choices=[
|
||||
_chat_choice(0, _chat_tool_call("a", "Read"), _chat_tool_call("b", "Read"), _chat_tool_call("c", "Bash")),
|
||||
_chat_choice(1, _chat_tool_call("d", "Bash")),
|
||||
]
|
||||
)
|
||||
RESPONSES_API_RESPONSE: Final = {
|
||||
"id": "resp_1",
|
||||
"output": [
|
||||
{"type": "message", "content": [{"type": "output_text", "text": "checking"}]},
|
||||
{"type": "function_call", "call_id": "call_1", "name": "get_weather", "arguments": "{}"},
|
||||
],
|
||||
}
|
||||
MESSAGES_TOOL_USE_RESPONSE: Final = {
|
||||
"id": "msg_1",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{"type": "text", "text": "Let me look"},
|
||||
{"type": "tool_use", "id": "toolu_1", "name": "Grep", "input": {"pattern": "x"}},
|
||||
{"type": "tool_use", "id": "toolu_2", "name": "Grep", "input": {"pattern": "y"}},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("response", "expected"),
|
||||
(
|
||||
pytest.param(CHAT_RESPONSE, {"Read": 2, "Bash": 2}, id="chat-completions-every-choice"),
|
||||
pytest.param(RESPONSES_API_RESPONSE, {"get_weather": 1}, id="responses-api"),
|
||||
pytest.param(MESSAGES_TOOL_USE_RESPONSE, {"Grep": 2}, id="messages-tool-use"),
|
||||
),
|
||||
)
|
||||
async def test_counts_each_tool_the_model_called(response: object, expected: Mapping[str, float]):
|
||||
logger: Final = PrometheusLogger()
|
||||
|
||||
await logger.async_log_success_event(_success_kwargs(), response, NOW, NOW)
|
||||
|
||||
assert _tool_call_counts() == expected
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_counts_add_up_across_requests():
|
||||
logger: Final = PrometheusLogger()
|
||||
|
||||
await logger.async_log_success_event(_success_kwargs(), MESSAGES_TOOL_USE_RESPONSE, NOW, NOW)
|
||||
await logger.async_log_success_event(_success_kwargs(), MESSAGES_TOOL_USE_RESPONSE, NOW, NOW)
|
||||
|
||||
assert _tool_call_counts() == {"Grep": 4}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"response",
|
||||
(
|
||||
pytest.param(litellm.ModelResponse(), id="chat-without-tool-calls"),
|
||||
pytest.param({"id": "msg_2", "content": [{"type": "text", "text": "hi"}]}, id="messages-text-only"),
|
||||
pytest.param(None, id="no-response"),
|
||||
),
|
||||
)
|
||||
async def test_emits_nothing_when_the_model_called_no_tool(response: object):
|
||||
logger: Final = PrometheusLogger()
|
||||
|
||||
await logger.async_log_success_event(_success_kwargs(), response, NOW, NOW)
|
||||
|
||||
assert _tool_call_counts() == {}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_failure_while_counting_tool_calls_does_not_stop_the_other_metrics(monkeypatch: pytest.MonkeyPatch):
|
||||
def explode(_response: object) -> list[str]:
|
||||
raise RuntimeError("unreadable response")
|
||||
|
||||
monkeypatch.setattr("litellm.proxy.db.spend_log_tool_index.response_tool_call_names", explode)
|
||||
logger: Final = PrometheusLogger()
|
||||
|
||||
await logger.async_log_success_event(_success_kwargs(), MESSAGES_TOOL_USE_RESPONSE, NOW, NOW)
|
||||
|
||||
assert _tool_call_counts() == {}
|
||||
assert _samples("litellm_output_tokens_metric")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_series_carry_the_output_token_labels_plus_tool_name():
|
||||
logger: Final = PrometheusLogger()
|
||||
|
||||
await logger.async_log_success_event(_success_kwargs(), MESSAGES_TOOL_USE_RESPONSE, NOW, NOW)
|
||||
|
||||
token_labels: Final = {**_samples("litellm_output_tokens_metric")[0].labels}
|
||||
(tool_sample,) = _samples("litellm_model_tool_calls")
|
||||
assert tool_sample.labels == {**token_labels, "tool_name": "Grep"}
|
||||
|
||||
|
||||
SECRET: Final = "card 4111-1111-1111-1111"
|
||||
|
||||
|
||||
def _chat_response_with_arguments(*arguments: str) -> litellm.ModelResponse:
|
||||
calls: Final = tuple(
|
||||
{"id": f"call_{n}", "type": "function", "function": {"name": "pay", "arguments": raw}}
|
||||
for n, raw in enumerate(arguments)
|
||||
)
|
||||
return litellm.ModelResponse(choices=[_chat_choice(0, *calls)])
|
||||
|
||||
|
||||
class _Records(logging.Handler):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(level=logging.DEBUG)
|
||||
self.messages: list[str] = []
|
||||
|
||||
def emit(self, record: logging.LogRecord) -> None:
|
||||
self.messages.append(record.getMessage())
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def litellm_log_records():
|
||||
handler: Final = _Records()
|
||||
lg: Final = logging.getLogger("LiteLLM")
|
||||
previous_level: Final = lg.level
|
||||
lg.addHandler(handler)
|
||||
lg.setLevel(logging.DEBUG)
|
||||
yield handler.messages
|
||||
lg.removeHandler(handler)
|
||||
lg.setLevel(previous_level)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"arguments",
|
||||
(
|
||||
pytest.param('{"note": "' + SECRET, id="truncated-repairable"),
|
||||
pytest.param('{"note": "' + SECRET + '" oops', id="unparseable"),
|
||||
),
|
||||
)
|
||||
async def test_counting_never_logs_the_tool_arguments(arguments: str, litellm_log_records: list[str]):
|
||||
logger: Final = PrometheusLogger()
|
||||
|
||||
await logger.async_log_success_event(_success_kwargs(), _chat_response_with_arguments(arguments), NOW, NOW)
|
||||
|
||||
assert _tool_call_counts() == {"pay": 1}
|
||||
assert not [message for message in litellm_log_records if "4111" in message]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"arguments",
|
||||
(
|
||||
pytest.param(('{"a": 1}',), id="valid"),
|
||||
pytest.param(('{"a": 1',), id="truncated"),
|
||||
pytest.param(('{"a": 1}{"a": 2}',), id="concatenated"),
|
||||
pytest.param(('{"a": 1}{"a": 1}',), id="concatenated-identical"),
|
||||
pytest.param(("".join(f'{{"a": {n}}}' for n in range(12)),), id="concatenated-over-cap"),
|
||||
pytest.param(("not json",), id="garbage"),
|
||||
pytest.param(("",), id="empty"),
|
||||
pytest.param((REDACTED_BY_LITELLM,), id="redacted"),
|
||||
pytest.param(('{"a": 1}', '{"b": 2}{"b": 3}'), id="two-calls"),
|
||||
),
|
||||
)
|
||||
def test_name_only_extraction_counts_like_the_parse_path(arguments: tuple[str, ...]):
|
||||
response: Final = _chat_response_with_arguments(*arguments)
|
||||
|
||||
parsed: Final = tuple(call["name"] for call in get_tool_calls_from_response(response, include_all_choices=True))
|
||||
|
||||
assert get_tool_call_names_from_response(response, include_all_choices=True) == parsed
|
||||
|
||||
|
||||
def _dict_chat_response(*choices: Mapping[str, object]) -> Mapping[str, object]:
|
||||
return {"id": "chat_1", "choices": list(choices)}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"response",
|
||||
(
|
||||
pytest.param(
|
||||
_dict_chat_response(
|
||||
{"message": {"tool_calls": [{"id": "a", "function": {"name": "f", "arguments": '{"x": 1}{"x": 2}'}}]}}
|
||||
),
|
||||
id="plain-dict-chat",
|
||||
),
|
||||
pytest.param(
|
||||
_dict_chat_response({"message": {"tool_calls": [{"id": "a", "function": {"name": "f"}}]}}),
|
||||
id="missing-arguments",
|
||||
),
|
||||
pytest.param(
|
||||
_dict_chat_response(
|
||||
{"message": {"tool_calls": [{"id": "a", "function": {"name": "f", "arguments": None}}]}}
|
||||
),
|
||||
id="none-arguments",
|
||||
),
|
||||
pytest.param(
|
||||
_dict_chat_response(
|
||||
{"message": {"tool_calls": [{"id": "a", "function": {"name": "f", "arguments": {"x": 1}}}]}}
|
||||
),
|
||||
id="parsed-dict-arguments",
|
||||
),
|
||||
pytest.param(
|
||||
_dict_chat_response(
|
||||
{"message": {"tool_calls": [{"id": "a", "function": None}, {"id": "b", "function": {"name": "g"}}]}}
|
||||
),
|
||||
id="function-none",
|
||||
),
|
||||
pytest.param(
|
||||
_dict_chat_response(
|
||||
{"message": {"content": "no tools"}},
|
||||
{"message": {"tool_calls": [{"id": "b", "function": {"name": "g", "arguments": '{"y": 1'}}]}},
|
||||
),
|
||||
id="second-choice-only",
|
||||
),
|
||||
pytest.param(
|
||||
{"output": [{"type": "function_call", "call_id": "c", "name": "h", "arguments": '{"z": 1}{"z": 2}'}]},
|
||||
id="responses-api-concatenated",
|
||||
),
|
||||
pytest.param(
|
||||
{"output": [{"type": "function_call", "call_id": "c", "name": "h"}]}, id="responses-api-no-arguments"
|
||||
),
|
||||
pytest.param(MESSAGES_TOOL_USE_RESPONSE, id="messages-tool-use"),
|
||||
pytest.param(RESPONSES_API_RESPONSE, id="responses-api"),
|
||||
),
|
||||
)
|
||||
def test_name_only_extraction_matches_the_parse_path_on_every_surface(response: object):
|
||||
parsed: Final = tuple(call["name"] for call in get_tool_calls_from_response(response, include_all_choices=True))
|
||||
|
||||
assert get_tool_call_names_from_response(response, include_all_choices=True) == parsed
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_api_counting_never_logs_the_tool_arguments(litellm_log_records: list[str]):
|
||||
logger: Final = PrometheusLogger()
|
||||
response: Final = {
|
||||
"id": "resp_2",
|
||||
"output": [
|
||||
{"type": "function_call", "call_id": "c", "name": "pay", "arguments": '{"note": "' + SECRET + '" x'}
|
||||
],
|
||||
}
|
||||
|
||||
await logger.async_log_success_event(_success_kwargs(), response, NOW, NOW)
|
||||
|
||||
assert _tool_call_counts() == {"pay": 1}
|
||||
assert not [message for message in litellm_log_records if "4111" in message]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_cap_of_zero_counts_every_tool_as_other(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setattr(litellm, "prometheus_model_tool_name_max_values", 0)
|
||||
logger: Final = PrometheusLogger()
|
||||
|
||||
await logger.async_log_success_event(_success_kwargs(), MESSAGES_TOOL_USE_RESPONSE, NOW, NOW)
|
||||
|
||||
assert _tool_call_counts() == {"other": 2}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tool_names_past_the_cap_count_as_other(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setattr(litellm, "prometheus_model_tool_name_max_values", 2)
|
||||
logger: Final = PrometheusLogger()
|
||||
response: Final = litellm.ModelResponse(
|
||||
choices=[_chat_choice(0, *(_chat_tool_call(name, name) for name in ("Read", "Bash", "Grep", "Glob", "Read")))]
|
||||
)
|
||||
|
||||
await logger.async_log_success_event(_success_kwargs(), response, NOW, NOW)
|
||||
|
||||
assert _tool_call_counts() == {"Read": 2, "Bash": 1, "other": 2}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_cap_when_the_setting_is_none(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setattr(litellm, "prometheus_model_tool_name_max_values", None)
|
||||
logger: Final = PrometheusLogger()
|
||||
names: Final = tuple(f"tool_{n}" for n in range(1200))
|
||||
response: Final = litellm.ModelResponse(choices=[_chat_choice(0, *(_chat_tool_call(n, n) for n in names))])
|
||||
|
||||
await logger.async_log_success_event(_success_kwargs(), response, NOW, NOW)
|
||||
|
||||
assert len(_tool_call_counts()) == len(names)
|
||||
Loading…
Add table
Reference in a new issue