mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-14 23:21:35 +00:00
fix(mavvrik_focus): guard Tags column and non-dict parsed JSON
This commit is contained in:
parent
19de9894c2
commit
03bc06079e
2 changed files with 26 additions and 1 deletions
|
|
@ -53,7 +53,7 @@ def _with_token_tags(data: pl.DataFrame, normalized: pl.DataFrame) -> pl.DataFra
|
|||
adds/renames columns, it never filters or reorders rows.
|
||||
"""
|
||||
available = [k for k in _TOKEN_TAG_KEYS if k in data.columns]
|
||||
if not available or len(data) != len(normalized):
|
||||
if not available or len(data) != len(normalized) or "Tags" not in normalized.columns:
|
||||
return normalized
|
||||
|
||||
token_rows = data.select(available).to_dicts()
|
||||
|
|
@ -64,6 +64,8 @@ def _with_token_tags(data: pl.DataFrame, normalized: pl.DataFrame) -> pl.DataFra
|
|||
tags = json.loads(tags_json) if tags_json else {}
|
||||
except (TypeError, ValueError):
|
||||
tags = {}
|
||||
if not isinstance(tags, dict):
|
||||
tags = {}
|
||||
for key in available:
|
||||
value = row.get(key)
|
||||
if value is not None:
|
||||
|
|
|
|||
|
|
@ -102,6 +102,29 @@ def test_with_token_tags_recovers_from_malformed_tags_json() -> None:
|
|||
}
|
||||
|
||||
|
||||
def test_with_token_tags_recovers_from_non_dict_tags_json() -> None:
|
||||
data = pl.DataFrame({"prompt_tokens": [57], "completion_tokens": [753]})
|
||||
normalized = pl.DataFrame({"Tags": ["null"]})
|
||||
|
||||
result = _with_token_tags(data, normalized)
|
||||
|
||||
tags = json.loads(result["Tags"][0])
|
||||
assert tags == {
|
||||
"prompt_tokens": "57",
|
||||
"completion_tokens": "753",
|
||||
"total_tokens": "810",
|
||||
}
|
||||
|
||||
|
||||
def test_with_token_tags_noop_when_tags_column_absent() -> None:
|
||||
data = pl.DataFrame({"prompt_tokens": [57], "completion_tokens": [753]})
|
||||
normalized = pl.DataFrame({"OtherColumn": ["x"]})
|
||||
|
||||
result = _with_token_tags(data, normalized)
|
||||
|
||||
assert result is normalized
|
||||
|
||||
|
||||
def test_with_token_tags_omits_total_when_only_one_token_column_present() -> None:
|
||||
data = pl.DataFrame({"prompt_tokens": [57]})
|
||||
normalized = pl.DataFrame({"Tags": [json.dumps({"model": "azure/gpt-4o-mini"})]})
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue