mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-16 04:29:31 +00:00
596c51a5fa
* feat(telemetry): add AggregatingSink, fixed-bucket histograms, Report and HttpExporter * fix(telemetry): add records to rows in place, guard concurrent flushes, cap restored rows and count bad histogram values Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(telemetry): give the shared row helper its lint reasons Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(telemetry): retry the same statuses as the proxy's should_retry (408, 409, 429, 5xx) Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(telemetry): cover populated block_types and header_keys in report JSON Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: nate <nate@berri.ai> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
95 lines
3.3 KiB
Python
95 lines
3.3 KiB
Python
from typing import Final
|
|
|
|
from litellm.telemetry.histogram import LATENCY_BOUNDS_MS
|
|
from litellm.telemetry.records import (
|
|
AttemptRecord,
|
|
BlockCounts,
|
|
BlockType,
|
|
InstanceInfo,
|
|
RequestRecord,
|
|
StatusClass,
|
|
TelemetryGroup,
|
|
UIAction,
|
|
UIEvent,
|
|
)
|
|
from litellm.telemetry.report import AttemptKey, AttemptMetrics, Report, RequestKey, RequestMetrics, report_to_json
|
|
|
|
|
|
def _metrics_of(*records: AttemptRecord) -> AttemptMetrics:
|
|
metrics: Final = AttemptMetrics.empty()
|
|
for record in records:
|
|
metrics.add(record)
|
|
return metrics
|
|
|
|
|
|
def test_attempt_and_ui_event_rows_serialize_with_merged_counts() -> None:
|
|
first: Final = AttemptRecord(
|
|
provider="bedrock",
|
|
provider_status=StatusClass.SERVER_ERROR,
|
|
stream=True,
|
|
deployment_hash="abc",
|
|
latency_to_first_token_ms=-1.0,
|
|
)
|
|
second: Final = AttemptRecord(
|
|
provider="bedrock",
|
|
provider_status=StatusClass.SERVER_ERROR,
|
|
stream=True,
|
|
deployment_hash="abc",
|
|
latency_to_first_token_ms=200.0,
|
|
)
|
|
report: Final = Report(
|
|
instance=InstanceInfo(instance_id="i", litellm_version="1.0.0", groups=frozenset(TelemetryGroup)),
|
|
window_start=0.0,
|
|
window_end=60.0,
|
|
attempts=((AttemptKey.of(first), _metrics_of(first, second)),),
|
|
ui_events=((UIEvent(page="models", action=UIAction.CLICK, target="add_model"), 3),),
|
|
)
|
|
|
|
json_report: Final = report_to_json(report)
|
|
assert isinstance(json_report, dict)
|
|
assert json_report["attempts"] == [
|
|
{
|
|
"provider": "bedrock",
|
|
"deployment_hash": "abc",
|
|
"provider_status": "5xx",
|
|
"stream": True,
|
|
"attempt_count": 2,
|
|
"latency_to_first_token_ms": {"counts": [0, 0, 1, *[0] * (len(LATENCY_BOUNDS_MS) - 2)], "invalid": 1},
|
|
}
|
|
]
|
|
assert json_report["ui_events"] == [{"page": "models", "action": "click", "target": "add_model", "count": 3}]
|
|
|
|
|
|
def test_block_types_and_header_keys_serialize_with_merged_counts() -> None:
|
|
first: Final = RequestRecord(
|
|
endpoint="/v1/messages",
|
|
stream=False,
|
|
litellm_status=StatusClass.SUCCESS,
|
|
blocks=BlockCounts(total=3, by_type=((BlockType.TEXT, 2), (BlockType.IMAGE, 1))),
|
|
header_keys=frozenset({"anthropic-beta", "x-custom"}),
|
|
)
|
|
second: Final = RequestRecord(
|
|
endpoint="/v1/messages",
|
|
stream=False,
|
|
litellm_status=StatusClass.SUCCESS,
|
|
blocks=BlockCounts(total=1, by_type=((BlockType.TEXT, 1),)),
|
|
header_keys=frozenset({"anthropic-beta"}),
|
|
)
|
|
metrics: Final = RequestMetrics.empty()
|
|
metrics.add(first)
|
|
metrics.add(second)
|
|
report: Final = Report(
|
|
instance=InstanceInfo(instance_id="i", litellm_version="1.0.0", groups=frozenset(TelemetryGroup)),
|
|
window_start=0.0,
|
|
window_end=60.0,
|
|
requests=((RequestKey.of(first), metrics),),
|
|
)
|
|
|
|
json_report: Final = report_to_json(report)
|
|
assert isinstance(json_report, dict)
|
|
(row,) = json_report["requests"]
|
|
assert {key: row[key] for key in ("block_count", "block_types", "header_keys")} == {
|
|
"block_count": {"counts": [1, 1, 0, 0, 0], "invalid": 0},
|
|
"block_types": {"image": 1, "text": 3},
|
|
"header_keys": {"anthropic-beta": 2, "x-custom": 1},
|
|
}
|