mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-10 03:28:53 +00:00
test: add routing correctness test and eval harness for litellm.compress()
- test_simple_compression: parametrized test verifying BM25 routes the
right message based on query ("How to cook?" keeps cooking, "Fix auth"
keeps auth content)
- eval_compression.py: end-to-end eval harness comparing baseline vs
compressed model performance on HumanEval-style coding problems
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
f40aa6a0ea
commit
c3a3de8cc9
2 changed files with 1150 additions and 0 deletions
1125
tests/eval_compression.py
Normal file
1125
tests/eval_compression.py
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -265,3 +265,28 @@ def test_embedding_scorer():
|
|||
)
|
||||
assert result["compression_ratio"] > 0
|
||||
assert len(result["cache"]) > 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"final_user_message, expected_content",
|
||||
[
|
||||
("How to cook?", "Unrelated cooking recipes "),
|
||||
("Fix auth", "Authentication code "),
|
||||
],
|
||||
)
|
||||
def test_simple_compression(final_user_message, expected_content):
|
||||
messages = [
|
||||
{"role": "user", "content": "Authentication code " * 2000},
|
||||
{"role": "user", "content": "Unrelated cooking recipes " * 2000},
|
||||
{"role": "user", "content": final_user_message},
|
||||
]
|
||||
result = litellm.compress(messages, model="gpt-4o", compression_trigger=1000)
|
||||
print(result["messages"])
|
||||
if expected_content == "Unrelated cooking recipes ":
|
||||
assert "Unrelated cooking recipes " in result["messages"][1]["content"]
|
||||
assert "Authentication code " not in result["messages"][0]["content"]
|
||||
elif expected_content == "Authentication code ":
|
||||
assert "Authentication code " in result["messages"][0]["content"]
|
||||
assert "Unrelated cooking recipes " not in result["messages"][1]["content"]
|
||||
else:
|
||||
raise ValueError(f"Unexpected expected_content: {expected_content}")
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue