mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-14 23:21:35 +00:00
fix(mavvrik_focus): guard against malformed Tags JSON in token merge
json.loads on the existing Tags value had no error handling; a malformed value would raise JSONDecodeError and abort the entire export window instead of just skipping that row's token merge.
This commit is contained in:
parent
24dc17f1fa
commit
19de9894c2
2 changed files with 18 additions and 1 deletions
|
|
@ -60,7 +60,10 @@ def _with_token_tags(data: pl.DataFrame, normalized: pl.DataFrame) -> pl.DataFra
|
|||
has_both = "prompt_tokens" in available and "completion_tokens" in available
|
||||
|
||||
def _merge(tags_json: str, row: dict) -> str:
|
||||
tags = json.loads(tags_json) if tags_json else {}
|
||||
try:
|
||||
tags = json.loads(tags_json) if tags_json else {}
|
||||
except (TypeError, ValueError):
|
||||
tags = {}
|
||||
for key in available:
|
||||
value = row.get(key)
|
||||
if value is not None:
|
||||
|
|
|
|||
|
|
@ -88,6 +88,20 @@ def test_with_token_tags_merges_prompt_and_completion_tokens() -> None:
|
|||
}
|
||||
|
||||
|
||||
def test_with_token_tags_recovers_from_malformed_tags_json() -> None:
|
||||
data = pl.DataFrame({"prompt_tokens": [57], "completion_tokens": [753]})
|
||||
normalized = pl.DataFrame({"Tags": ["not-valid-json"]})
|
||||
|
||||
result = _with_token_tags(data, normalized)
|
||||
|
||||
tags = json.loads(result["Tags"][0])
|
||||
assert tags == {
|
||||
"prompt_tokens": "57",
|
||||
"completion_tokens": "753",
|
||||
"total_tokens": "810",
|
||||
}
|
||||
|
||||
|
||||
def test_with_token_tags_omits_total_when_only_one_token_column_present() -> None:
|
||||
data = pl.DataFrame({"prompt_tokens": [57]})
|
||||
normalized = pl.DataFrame({"Tags": [json.dumps({"model": "azure/gpt-4o-mini"})]})
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue