mirror of
https://github.com/abhigyanpatwari/GitNexus.git
synced 2026-11-17 09:33:02 +00:00
* feat(eval): add cost diagnostics and trials * style(eval): format retrieval case fixtures * fix(eval): address review and CI failures Refs #3773. Validate locked uv commands, model aliases and oversized budgets. Stop the fake MCP server before releasing its listener and replace the wall-clock staleness assertion with a concurrent result check. Note: pre-existing failure in test_oracle_assets.py reference-checkout type resolution not addressed by this PR. * feat(eval): publish detailed run measurements Collect per-sweep provider ledgers and publish sanitized measurements in the Actions summary and artifacts. Validate experiment tasks with the runner and withhold incomplete token totals. Refs #3773. Note: pre-existing failure in test_oracle_assets.py reference-checkout type resolution not addressed by this PR. * fix(eval): validate inputs and quoted commands Reject non-mapping task YAML before reading fields. Attribute literal quoted CLI queries while preserving Bash attribution for shell operators and expansions. Refs #3773. * fix(eval): count empty filtered query results * fix(eval): handle alternate event content --------- Co-authored-by: Gergo Magyar <gergomagyar0@gmail.com>
167 lines
7.8 KiB
Python
167 lines
7.8 KiB
Python
"""Append each upstream request's usage exactly as the provider reported it.
|
|
|
|
This runs INSIDE the LiteLLM proxy, on the far side of the translation that
|
|
turns an OpenAI response into the Anthropic shape Claude Code expects. That is
|
|
the only point that still knows which provider served the request, what model
|
|
actually answered, and what the native usage object said before its fields were
|
|
renamed into someone else's semantics.
|
|
|
|
Deliberately self-contained: the proxy loads this file by path from the config
|
|
directory, so it cannot assume ``workflow_bench`` is importable. Normalization
|
|
lives in workflow_bench.provider_usage and runs offline over what this writes -
|
|
the native object is the evidence, and deriving from it here would mean the
|
|
derivation could not be revisited without re-running a paid sweep.
|
|
|
|
Never raises. A cell that fails still spent money upstream, and losing the
|
|
accounting because the log write failed would be the worse outcome.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import math
|
|
import os
|
|
import threading
|
|
from typing import Any
|
|
|
|
from litellm.integrations.custom_logger import CustomLogger
|
|
|
|
# Literals, not imports. LiteLLM loads this file BY PATH from the config
|
|
# directory via spec_from_file_location, so it has no parent package and the
|
|
# directory is not on sys.path - a relative or sibling import raises
|
|
# ImportError and the proxy refuses to start. workflow_bench.provider_usage
|
|
# holds the canonical copies and a test asserts these agree with them, which
|
|
# catches drift without coupling at import time.
|
|
USAGE_LOG_ENV_VAR = "GITNEXUS_BENCH_PROVIDER_USAGE"
|
|
SWEEP_ID_ENV_VAR = "GITNEXUS_BENCH_SWEEP_ID"
|
|
|
|
|
|
def canonical_provider(label, call_type): # noqa: ANN001, ANN201
|
|
"""Adapter key for the usage shape, or None when it cannot be resolved.
|
|
|
|
Mirrors workflow_bench.provider_usage.canonical_provider; see the note
|
|
above for why this is a copy rather than an import.
|
|
"""
|
|
|
|
if label == "openai":
|
|
return "litellm-normalized"
|
|
if label == "anthropic":
|
|
return "anthropic"
|
|
return None
|
|
|
|
SCHEMA_VERSION = 1
|
|
_LOCK = threading.Lock()
|
|
|
|
|
|
def _plain(value: Any) -> Any:
|
|
"""Provider usage arrives as pydantic models; keep the shape, drop the class."""
|
|
|
|
for attr in ("model_dump", "dict"):
|
|
method = getattr(value, attr, None)
|
|
if callable(method):
|
|
try:
|
|
return method()
|
|
except Exception:
|
|
pass
|
|
if isinstance(value, dict):
|
|
return value
|
|
return None
|
|
|
|
|
|
def _request_metrics(kwargs: dict, start_time: Any, end_time: Any) -> dict:
|
|
"""Numeric envelope evidence only; never log request content or headers."""
|
|
optional = kwargs.get("optional_params")
|
|
optional = optional if isinstance(optional, dict) else {}
|
|
metrics = {}
|
|
for field in ("messages", "tools", "system"):
|
|
value = kwargs.get(field, optional.get(field))
|
|
metrics[f"{field}_bytes"] = None
|
|
if isinstance(value, (str, dict, list)):
|
|
try:
|
|
metrics[f"{field}_bytes"] = len(json.dumps(
|
|
value, ensure_ascii=False, separators=(",", ":"),
|
|
).encode(errors="backslashreplace"))
|
|
except (TypeError, ValueError, RecursionError):
|
|
# Keep size unknown when serialization fails; usage accounting
|
|
# must still be recorded even without this optional metric.
|
|
pass
|
|
if field != "system":
|
|
metrics[f"{field}_count"] = len(value) if isinstance(value, list) else None
|
|
try:
|
|
elapsed = end_time - start_time
|
|
elapsed = elapsed.total_seconds() if hasattr(elapsed, "total_seconds") else elapsed
|
|
except (TypeError, ValueError):
|
|
elapsed = None
|
|
metrics["duration_s"] = elapsed if type(elapsed) in (int, float) and math.isfinite(elapsed) and elapsed >= 0 else None
|
|
cost = kwargs.get("response_cost")
|
|
metrics["proxy_estimated_cost_usd"] = cost if type(cost) in (int, float) and math.isfinite(cost) and cost >= 0 else None
|
|
return metrics
|
|
|
|
|
|
class ProviderUsageLogger(CustomLogger):
|
|
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time) -> None: # noqa: ANN001
|
|
self._append("success", kwargs, response_obj, start_time, end_time)
|
|
|
|
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time) -> None: # noqa: ANN001
|
|
# Failed requests are billed too, and a sweep that only accounts for
|
|
# successes understates what it spent.
|
|
self._append("failure", kwargs, response_obj, start_time, end_time)
|
|
|
|
def log_success_event(self, kwargs, response_obj, start_time, end_time) -> None: # noqa: ANN001
|
|
self._append("success", kwargs, response_obj, start_time, end_time)
|
|
|
|
def log_failure_event(self, kwargs, response_obj, start_time, end_time) -> None: # noqa: ANN001
|
|
# The synchronous counterpart. Overriding only the success hook here
|
|
# recorded successes and let failures fall through to the base class,
|
|
# which accounts for nothing - and a failed request is still billed, so
|
|
# a sweep missing them understates what it spent.
|
|
self._append("failure", kwargs, response_obj, start_time, end_time)
|
|
|
|
def _append(self, status, kwargs, response_obj, start_time, end_time) -> None: # noqa: ANN001
|
|
path = os.environ.get(USAGE_LOG_ENV_VAR)
|
|
if not path:
|
|
return
|
|
try:
|
|
params = kwargs.get("litellm_params") or {}
|
|
call_type = kwargs.get("call_type")
|
|
provider_label = kwargs.get("custom_llm_provider") or params.get("custom_llm_provider")
|
|
metadata = params.get("metadata") or {}
|
|
event = {
|
|
"schema_version": SCHEMA_VERSION,
|
|
"status": status,
|
|
# Identity. The REQUESTED model is the caller's role name and the
|
|
# ACTUAL model is what answered; pricing must follow the second,
|
|
# because several roles map onto one upstream model here.
|
|
"requested_model": kwargs.get("model"),
|
|
"actual_model": getattr(response_obj, "model", None),
|
|
# Two fields, because they answer different questions. The raw
|
|
# label is what LiteLLM said; "provider" is the adapter key for
|
|
# the object actually in hand, which is always LiteLLM's own
|
|
# normalised shape here. An unrecognised label stays None so
|
|
# normalize_usage refuses rather than guessing token semantics.
|
|
"provider_label": provider_label,
|
|
"provider": canonical_provider(provider_label, call_type),
|
|
"response_id": getattr(response_obj, "id", None),
|
|
"call_type": call_type,
|
|
"sweep_id": os.environ.get(SWEEP_ID_ENV_VAR),
|
|
# The per-request half of identity, and the only thing that can
|
|
# attribute a request to a cell: one proxy serves the whole
|
|
# sweep, so anything read from the environment is the same for
|
|
# every event. Recorded even when absent, because knowing the
|
|
# attribution is unavailable is itself a fact about the run.
|
|
"session_id": metadata.get("litellm_session_id") or metadata.get("session_id"),
|
|
"started_at": str(start_time),
|
|
"completed_at": str(end_time),
|
|
# Verbatim. Not flattened, not renamed, not summed.
|
|
"native_usage": _plain(getattr(response_obj, "usage", None)),
|
|
"request_metrics": _request_metrics(kwargs, start_time, end_time),
|
|
}
|
|
line = json.dumps(event, default=str) + "\n"
|
|
with _LOCK, open(path, "a", encoding="utf-8") as handle:
|
|
handle.write(line)
|
|
except Exception:
|
|
# Accounting is evidence, not control flow: never take the sweep down.
|
|
return
|
|
|
|
|
|
handler = ProviderUsageLogger()
|