diff --git a/.github/assets/lens-datasets/add-to-dataset-dialog.png b/.github/assets/lens-datasets/add-to-dataset-dialog.png new file mode 100644 index 00000000000..8590ad48dd5 Binary files /dev/null and b/.github/assets/lens-datasets/add-to-dataset-dialog.png differ diff --git a/.github/assets/lens-datasets/dataset-case-panel.png b/.github/assets/lens-datasets/dataset-case-panel.png new file mode 100644 index 00000000000..dd579ac58af Binary files /dev/null and b/.github/assets/lens-datasets/dataset-case-panel.png differ diff --git a/.github/assets/lens-datasets/dataset-detail.png b/.github/assets/lens-datasets/dataset-detail.png new file mode 100644 index 00000000000..c68b9b6f73c Binary files /dev/null and b/.github/assets/lens-datasets/dataset-detail.png differ diff --git a/.github/assets/lens-datasets/datasets-list.png b/.github/assets/lens-datasets/datasets-list.png new file mode 100644 index 00000000000..14a6200aae0 Binary files /dev/null and b/.github/assets/lens-datasets/datasets-list.png differ diff --git a/.github/assets/lens-datasets/old-revision.png b/.github/assets/lens-datasets/old-revision.png new file mode 100644 index 00000000000..d6e369e2485 Binary files /dev/null and b/.github/assets/lens-datasets/old-revision.png differ diff --git a/.github/assets/lens-datasets/trace-header.png b/.github/assets/lens-datasets/trace-header.png new file mode 100644 index 00000000000..85bd759b5d0 Binary files /dev/null and b/.github/assets/lens-datasets/trace-header.png differ diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261006000100_lens_datasets/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261006000100_lens_datasets/migration.sql new file mode 100644 index 00000000000..80ce096d409 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261006000100_lens_datasets/migration.sql @@ -0,0 +1,7 @@ +CREATE TABLE IF NOT EXISTS "LiteLLM_LensDataset" ( + "id" TEXT NOT NULL, + "revision" INTEGER NOT NULL, + "created_at" TIMESTAMP(3) NOT NULL, + "data" JSONB NOT NULL, + CONSTRAINT "LiteLLM_LensDataset_pkey" PRIMARY KEY ("id", "revision") +); diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 4aabd56131b..514df905866 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -1965,3 +1965,12 @@ model LiteLLM_LensWorker { token_hash String @unique data Json } + +model LiteLLM_LensDataset { + id String + revision Int + created_at DateTime + data Json + + @@id([id, revision]) +} diff --git a/litellm-rust/crates/traces/src/ui.rs b/litellm-rust/crates/traces/src/ui.rs index eebeaab70ce..ef849d2b440 100644 --- a/litellm-rust/crates/traces/src/ui.rs +++ b/litellm-rust/crates/traces/src/ui.rs @@ -85,6 +85,32 @@ struct RawMessage { kwargs: Option>, } +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct AssistantSummary { + content: Option, + tool_names: Vec, +} + +impl AssistantSummary { + fn into_ui(self) -> UiMessage { + let calls: Vec = self + .tool_names + .into_iter() + .map(|name| UiToolCall { + name, + arguments: "{}".to_owned(), + }) + .collect(); + UiMessage { + role: ChatRole::Assistant, + content: self.content.unwrap_or_default(), + name: None, + tool_calls: (!calls.is_empty()).then_some(calls), + } + } +} + #[derive(Deserialize)] struct ContentBlock { #[serde(rename = "type", default)] @@ -211,6 +237,19 @@ fn messages(parsed: &Value) -> Option> { Some(unwrapped.into_iter().map(RawMessage::into_ui).collect()) } +fn assistant_summaries(parsed: &Value) -> Option> { + let summaries = Vec::::deserialize(parsed).ok()?; + if summaries.is_empty() { + return None; + } + Some( + summaries + .into_iter() + .map(AssistantSummary::into_ui) + .collect(), + ) +} + pub fn to_ui_content(raw: &str) -> UiContent { let text = || UiContent::Text { text: raw.to_owned(), @@ -223,7 +262,7 @@ pub fn to_ui_content(raw: &str) -> UiContent { Ok(parsed @ (Value::Array(_) | Value::Object(_))) => parsed, _ => return text(), }; - if let Some(messages) = messages(&parsed) { + if let Some(messages) = messages(&parsed).or_else(|| assistant_summaries(&parsed)) { return UiContent::Messages { messages }; } match parsed { @@ -356,6 +395,33 @@ mod tests { ); } + #[rstest] + fn assistant_summaries_become_assistant_messages() { + let raw = json!([ + {"content": "`/etc/hosts` has 11 lines.", "tool_names": []}, + {"content": null, "tool_names": ["terminal"]}, + ]); + assert_eq!( + to_ui_content(&raw.to_string()), + UiContent::Messages { + messages: vec![ + message("assistant", "`/etc/hosts` has 11 lines."), + UiMessage { + tool_calls: Some(vec![call("terminal", "{}")]), + ..message("assistant", "") + }, + ] + } + ); + } + + #[rstest] + #[case::extra_field(r#"[{"content": "x", "tool_names": [], "score": 1}]"#)] + #[case::missing_tool_names(r#"[{"content": "x"}]"#)] + fn near_summaries_stay_text(#[case] raw: &str) { + assert!(matches!(to_ui_content(raw), UiContent::Text { .. })); + } + #[rstest] fn plain_objects_become_fields_in_key_order() { let raw = r#"{"zeta": "plain", "alpha": {"nested": [1, 2]}, "count": 3, "missing": null}"#; diff --git a/litellm/constants.py b/litellm/constants.py index 938a3c85c29..95d05c6d819 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -60,6 +60,9 @@ TRACE_READ_RETRY_AFTER_SECONDS: Final = get_env_int("TRACE_READ_RETRY_AFTER_SECO OTLP_MAX_CONCURRENT_INGESTS: Final = get_env_int("OTLP_MAX_CONCURRENT_INGESTS", 2) AGENT_TRACING_INPUT_PREVIEW_CHARS: Final = get_env_int("AGENT_TRACING_INPUT_PREVIEW_CHARS", 240) AGENT_TRACING_LIST_PAGE_SIZE: Final = get_env_int("AGENT_TRACING_LIST_PAGE_SIZE", 50) +LENS_DATASET_MAX_CASES: Final = get_env_int("LENS_DATASET_MAX_CASES", 200) +LENS_DATASET_MAX_CASE_CHARS: Final = get_env_int("LENS_DATASET_MAX_CASE_CHARS", 20_000) +LENS_DATASET_TRACE_PAGE_SIZE: Final = 500 DEFAULT_S3_FLUSH_INTERVAL_SECONDS: Final = int(os.getenv("DEFAULT_S3_FLUSH_INTERVAL_SECONDS", 10)) DEFAULT_S3_BATCH_SIZE: Final = int(os.getenv("DEFAULT_S3_BATCH_SIZE", 512)) DEFAULT_S3_MAX_CONCURRENT_UPLOADS: Final = int(os.getenv("DEFAULT_S3_MAX_CONCURRENT_UPLOADS", "16")) @@ -1807,6 +1810,7 @@ SPEND_LOG_PARTITION_INTERVAL: Final = os.getenv("SPEND_LOG_PARTITION_INTERVAL", SPEND_LOG_PARTITION_PRECREATE_AHEAD: Final = int(os.getenv("SPEND_LOG_PARTITION_PRECREATE_AHEAD", 7)) SPEND_LOG_WRITE_BATCH_MAX_BYTES: Final = max(1, int(os.getenv("SPEND_LOG_WRITE_BATCH_MAX_BYTES", 2_000_000))) SPEND_LOG_WRITE_BATCH_MAX_ROWS: Final = max(1, int(os.getenv("SPEND_LOG_WRITE_BATCH_MAX_ROWS", "100"))) +SPEND_ROLLUP_LOCK_TIMEOUT_MS: Final = max(1, int(os.getenv("SPEND_ROLLUP_LOCK_TIMEOUT_MS", "5000"))) SPEND_LOG_QUEUE_SIZE_THRESHOLD: Final = int(os.getenv("SPEND_LOG_QUEUE_SIZE_THRESHOLD", 100)) SPEND_LOG_QUEUE_MAX_BYTES: Final = max(1, int(os.getenv("SPEND_LOG_QUEUE_MAX_BYTES", "64000000"))) SPEND_LOG_QUEUE_POLL_INTERVAL: Final = float(os.getenv("SPEND_LOG_QUEUE_POLL_INTERVAL", 2.0)) diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 290cba90fd7..6f16b124e6a 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -32,6 +32,7 @@ from litellm.integrations.otel.model.db_endpoint import db_span_attributes from litellm.integrations.otel.model.metadata import flatten_metadata from litellm.integrations.otel.model.semconv import LiteLLM, Metric from litellm.integrations.otel.plumbing.otlp_tls import resolve_otlp_http_tls +from litellm.integrations.otel.routing import routing_decision_attributes from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call_from_params from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.secret_redaction import redact_string @@ -2407,6 +2408,8 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): metadata: Final = standard_logging_payload["metadata"] for key, value in metadata.items(): self.safe_set_attribute(span=span, key=f"metadata.{key}", value=value) + decision: Final = metadata.get("routing_decision") + span.set_attributes(routing_decision_attributes(decision)) # get hidden params hidden_params: Final = getattr(standard_logging_payload, "hidden_params", None) or ( diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index ca8e507bff5..888c90b7661 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -54,6 +54,7 @@ from litellm.integrations.otel.model.spans import SpanRole, span_role_for_servic from litellm.integrations.otel.model.utils import to_ns from litellm.integrations.otel.plumbing.context import ( active_phase, + active_phase_span, is_recordable_span, mcp_message_transport_span, post_response_root, @@ -778,6 +779,11 @@ class OpenTelemetryV2(CustomLogger): if is_recordable_span(span): span.add_event(name, attributes) + def set_phase_attributes(self, attributes: Mapping[str, str | int | float | bool]) -> None: + span: Final = active_phase_span() + if span is not None and is_recordable_span(span): + span.set_attributes(attributes) + async def async_pre_call_hook( self, user_api_key_dict: "UserAPIKeyAuth", @@ -1042,6 +1048,12 @@ def phase_event(name: str, attributes: Mapping[str, str | int] | None = None) -> logger.add_phase_event(name, attributes) +def phase_attributes(attributes: Mapping[str, str | int | float | bool]) -> None: + logger: Final = _registered_v2_logger() + if logger is not None: + logger.set_phase_attributes(attributes) + + def build_otel_v2_logger( config: OpenTelemetryV2Config, callback_name: str | None = None, diff --git a/litellm/integrations/otel/mappers/genai.py b/litellm/integrations/otel/mappers/genai.py index 75a1098819d..0ac8b47e239 100644 --- a/litellm/integrations/otel/mappers/genai.py +++ b/litellm/integrations/otel/mappers/genai.py @@ -174,7 +174,7 @@ class GenAIMapper: return {} def _llm_call(self, data: LLMCallSpanData) -> AttributeMap: - attrs: Final = collect(self._LLM_CALL_ATTRS, data) + attrs: Final = {**collect(self._LLM_CALL_ATTRS, data), **data.routing_attributes} if data.tools: attrs[LiteLLM.TOOLS_DECLARED] = len(data.tools) attrs.update( diff --git a/litellm/integrations/otel/model/payloads.py b/litellm/integrations/otel/model/payloads.py index efd7f6c7dd9..ccec93920c9 100644 --- a/litellm/integrations/otel/model/payloads.py +++ b/litellm/integrations/otel/model/payloads.py @@ -30,6 +30,7 @@ from litellm.integrations.otel.model.utils import ( as_str_mapping, as_str_tuple, ) +from litellm.integrations.otel.routing import RoutingAttributeValue, routing_decision_attributes # ``RequestIdentity`` and the request-metadata translation now live in # :mod:`metadata`; re-exported here so existing ``model.payloads`` imports keep @@ -436,6 +437,7 @@ class LLMCallSpanData: trace: TraceControls = field(default_factory=TraceControls) session_id: str | None = None embedding_output: EmbeddingOutput | None = None + routing_attributes: Mapping[str, RoutingAttributeValue] = field(default_factory=lambda: MappingProxyType({})) @classmethod def from_standard_logging_payload( @@ -453,6 +455,8 @@ class LLMCallSpanData: # model split, the response model, api base, and identity all come from # here rather than being re-derived from the raw payload dicts. context: Final = RequestContext.from_standard_logging_payload(payload) + metadata: Final = payload.get("metadata") + decision: Final = metadata["routing_decision"] if metadata and "routing_decision" in metadata else None # Normalize ``response`` to a dict once so the content/id reads below are a # plain ``.get`` — no repeated ``isinstance`` guards. raw_response: Final = payload.get("response") @@ -496,6 +500,7 @@ class LLMCallSpanData: trace=trace or TraceControls(), session_id=session_id or None, embedding_output=embedding_output if capture_content else None, + routing_attributes=routing_decision_attributes(decision), ) diff --git a/litellm/integrations/otel/model/spans.py b/litellm/integrations/otel/model/spans.py index 3644ba60dd5..3b17aea3cd8 100644 --- a/litellm/integrations/otel/model/spans.py +++ b/litellm/integrations/otel/model/spans.py @@ -349,6 +349,7 @@ _PRISMA_MODELS: Final[frozenset[str]] = frozenset( "LiteLLM_WorkflowEvent", "LiteLLM_WorkflowMessage", "LiteLLM_Lens", + "LiteLLM_LensDataset", "LiteLLM_LensRun", "LiteLLM_LensReview", "LiteLLM_LensWorker", diff --git a/litellm/integrations/otel/routing.py b/litellm/integrations/otel/routing.py new file mode 100644 index 00000000000..82dd6237edc --- /dev/null +++ b/litellm/integrations/otel/routing.py @@ -0,0 +1,57 @@ +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final, TypeAlias + +RoutingAttributeValue: TypeAlias = str | int | float | bool + +_ROUTING_SCALAR_FIELDS: Final = frozenset( + { + "router_model_name", + "router_type", + "router_config_id", + "router_config_updated_at", + "router_config_fingerprint", + "routed_model", + "cause", + "tier", + "tier_label", + "request_type", + "score", + "classifier_model", + "classifier_cost", + "classifier_failure_reason", + "classifier_error_type", + "classifier_confidence", + "classifier_primary_rule", + "classifier_capability_boundary", + "classifier_p_solve", + "classifier_calibrated_p_solve", + "classifier_calibration_version", + "classifier_efficient_p_solve", + "classifier_capable_p_solve", + "classifier_calibrated_efficient_p_solve", + "classifier_calibrated_capable_p_solve", + "classifier_max_quality_gap", + "classifier_prompt_version", + "classifier_threshold", + "escalated", + "context_escalated", + "context_escalation_original_tier", + "reasoning_override_min_score", + "conversation_continuing", + "savings_baseline_model", + "savings_baseline_deployment_id", + } +) + + +def routing_decision_attributes(decision: Mapping[str, object] | None) -> Mapping[str, RoutingAttributeValue]: + if not isinstance(decision, Mapping): + return MappingProxyType({}) + return MappingProxyType( + { + f"litellm.routing.{key}": value + for key in _ROUTING_SCALAR_FIELDS + if isinstance(value := decision.get(key), (str, int, float, bool)) + } + ) diff --git a/litellm/integrations/otel/runtime.py b/litellm/integrations/otel/runtime.py index ff75bc4d800..89227770448 100644 --- a/litellm/integrations/otel/runtime.py +++ b/litellm/integrations/otel/runtime.py @@ -7,7 +7,9 @@ V2 is not the active logger — so a call site can wrap a request phase or seed identity unconditionally. """ -from collections.abc import Callable, Iterator, Mapping +from __future__ import annotations + +from collections.abc import Callable, Generator, Mapping from contextlib import AbstractContextManager, contextmanager from functools import cache from typing import TYPE_CHECKING, Final, TypeAlias @@ -16,11 +18,18 @@ if TYPE_CHECKING: from opentelemetry.trace import Span PhaseEventAttributes: TypeAlias = Mapping[str, str | int] +PhaseAttributes: TypeAlias = Mapping[str, str | int | float | bool] @cache def _otel_runtime() -> ( - "tuple[Callable[[str], AbstractContextManager[Span | None]], Callable[..., None], Callable[[str, PhaseEventAttributes | None], None]] | None" + tuple[ + Callable[[str], AbstractContextManager[Span | None]], + Callable[..., None], + Callable[[str, PhaseEventAttributes | None], None], + Callable[[PhaseAttributes], None], + ] + | None ): """Resolve the SDK-backed hooks once and cache the outcome, absence included. @@ -32,11 +41,11 @@ def _otel_runtime() -> ( from litellm.integrations.otel import logger except Exception: return None - return (logger.phase_span, logger.seed_request_identity, logger.phase_event) + return (logger.phase_span, logger.seed_request_identity, logger.phase_event, logger.phase_attributes) @contextmanager -def phase_span(name: str) -> "Iterator[Span | None]": +def phase_span(name: str) -> Generator[Span | None]: """Run a request phase inside a live active span so its DB/service calls nest. Yields ``None`` (a plain no-op) when the OTel SDK is unavailable or V2 is not @@ -64,3 +73,10 @@ def seed_request_identity(user_api_key_dict: object, model: object = None) -> No if runtime is None: return runtime[1](user_api_key_dict, model=model) + + +def phase_attributes(attributes: PhaseAttributes) -> None: + runtime: Final = _otel_runtime() + if runtime is None: + return + runtime[3](attributes) diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index 5fdde118fb9..7bbb0e63202 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -75,6 +75,7 @@ from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import ( ) from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler from litellm.proxy.db.model_usage_rollup import build_model_usage_transaction +from litellm.proxy.db.rollup_lock_timeout import apply_rollup_lock_timeout from litellm.proxy.route_llm_request import ROUTE_ENDPOINT_MAPPING from litellm.proxy.spend_tracking.compression_savings import ( extract_compression_saved_tokens, @@ -292,6 +293,7 @@ async def _spend_update_tx( ) -> AsyncGenerator[_SpendTransaction]: tx: Final[_SpendTransactionManager] = prisma_client.db.tx(timeout=timedelta(seconds=60)) async with db_span(call_type, table), tx as transaction: + await apply_rollup_lock_timeout(transaction) yield transaction @@ -2032,11 +2034,17 @@ class DBSpendUpdateWriter: start_time: float, proxy_logging_obj: ProxyLogging, ) -> None: - """Retry a failed spend-update transaction on connection errors or deadlocks, else re-raise.""" + """Retry a failed spend-update transaction on connection errors, deadlocks or a rollup + ``lock_timeout`` (55P03), else re-raise. All three roll the transaction back before any + increment applied, so re-sending the same batch cannot double-count.""" from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler from litellm.proxy.utils import _raise_failed_update_spend_exception - is_retryable = isinstance(e, DB_RETRY_SAFE_ERROR_TYPES) or PrismaDBExceptionHandler.is_deadlock_error(e) + is_retryable = ( + isinstance(e, DB_RETRY_SAFE_ERROR_TYPES) + or PrismaDBExceptionHandler.is_deadlock_error(e) + or PrismaDBExceptionHandler.is_lock_timeout_error(e) + ) if not is_retryable or attempt >= n_retry_times: _raise_failed_update_spend_exception(e=e, start_time=start_time, proxy_logging_obj=proxy_logging_obj) verbose_proxy_logger.warning( diff --git a/litellm/proxy/db/exception_handler.py b/litellm/proxy/db/exception_handler.py index c3526548942..379e0012ea8 100644 --- a/litellm/proxy/db/exception_handler.py +++ b/litellm/proxy/db/exception_handler.py @@ -31,6 +31,8 @@ _CONNECTION_CAPACITY_PHRASES: Final = ( "remaining connection slots are reserved", ) _PRISMA_POOL_TIMEOUT_CODE: Final = "P2024" +_LOCK_TIMEOUT_SQLSTATE: Final = "55P03" +_LOCK_TIMEOUT_PHRASE: Final = "canceling statement due to lock timeout" def _exception_chain(e: BaseException) -> Iterator[BaseException]: @@ -266,6 +268,21 @@ class PrismaDBExceptionHandler: error_message: Final = str(e).lower() return any(phrase in error_message for phrase in _CONNECTION_CAPACITY_PHRASES) + @staticmethod + def is_lock_timeout_error(e: Exception) -> bool: + """True iff Postgres cancelled a statement for exceeding ``lock_timeout`` + (SQLSTATE 55P03). The statement never acquired its lock, so it never + applied and the transaction rolled back: the rows it carried are safe to + re-send, and the pooled connection is free again rather than pinned + behind the holder.""" + import prisma + + if not isinstance(e, _exception_types(prisma.errors.PrismaError)): + return False + if PrismaDBExceptionHandler.postgres_sqlstate(e) == _LOCK_TIMEOUT_SQLSTATE: + return True + return _LOCK_TIMEOUT_PHRASE in str(e).lower() + @staticmethod def postgres_sqlstate(e: Exception) -> str | None: """The SQLSTATE Postgres attached to a failed statement, as prisma surfaces it, or None.""" diff --git a/litellm/proxy/db/model_usage_rollup.py b/litellm/proxy/db/model_usage_rollup.py index 98d53573f30..d83500d66e8 100644 --- a/litellm/proxy/db/model_usage_rollup.py +++ b/litellm/proxy/db/model_usage_rollup.py @@ -9,6 +9,7 @@ from itertools import groupby from typing import TYPE_CHECKING, Final, Protocol from pydantic import TypeAdapter, ValidationError +from typing_extensions import LiteralString from litellm.constants import ( INTERNAL_CALL_ORIGIN_METADATA_KEY, @@ -17,6 +18,7 @@ from litellm.constants import ( ) from litellm.proxy._types import DB_RETRY_SAFE_ERROR_TYPES, SpendLogsPayload from litellm.proxy.db.model_insights_tasks import load_model_insight_tasks +from litellm.proxy.db.rollup_lock_timeout import ROLLUP_LOCK_TIMEOUT_SQL, rollup_lock_timeout_setting if TYPE_CHECKING: from litellm.proxy.utils import PrismaClient @@ -32,6 +34,8 @@ class _UpsertTable(Protocol): class _ModelUsageBatch(Protocol): litellm_dailymodelusage: _UpsertTable + def execute_raw(self, query: LiteralString, *args: object) -> None: ... + class _ModelUsageBatchManager(Protocol): async def __aenter__(self) -> _ModelUsageBatch: ... @@ -132,6 +136,7 @@ async def flush_model_usage_transactions( for attempt in range(n_retry_times + 1): try: async with _model_usage_batch(prisma_client) as batcher: + batcher.execute_raw(ROLLUP_LOCK_TIMEOUT_SQL, rollup_lock_timeout_setting()) for key, grouped in groupby(ordered, key=lambda transaction: transaction.key): entries = tuple(grouped) spend = sum(entry.spend for entry in entries) diff --git a/litellm/proxy/db/prisma_query_span.py b/litellm/proxy/db/prisma_query_span.py index 70c8e2ec3f9..d99f4dcc768 100644 --- a/litellm/proxy/db/prisma_query_span.py +++ b/litellm/proxy/db/prisma_query_span.py @@ -39,6 +39,7 @@ _ROOT_FIELD: Final = re.compile(r"result:\s*(\w+)") _RAW_SQL: Final = re.compile(r'query:\s*"((?:[^"\\]|\\.)*)"') _LEADING_KEYWORD: Final = re.compile(r"(?:\\[nrt]|\s|\()*(\w+)") _SETTING: Final = re.compile(r"(?:\\[nrt]|\s)*SET\s+(?:LOCAL\s+|SESSION\s+)?([A-Za-z_.]+)", re.IGNORECASE) +_SET_CONFIG: Final = re.compile(r"(?:\\[nrt]|\s)*SELECT\s+set_config\(\s*'([A-Za-z_.]+)'", re.IGNORECASE) _CATALOG: Final = re.compile(r"\bpg_\w+|\bto_regclass\b|\binformation_schema\b|\bcurrent_setting\s*\(|^\s*SHOW\b") _PROBE: Final = re.compile(r"(?:\\[nrt]|\s)*SELECT\s+\d+\s*;?(?:\\[nrt]|\s)*$", re.IGNORECASE) _CTE_WRITE: Final = re.compile(r"\b(UPDATE|INSERT|DELETE)\s+(?:INTO\s+|FROM\s+)?(?:\\?\")", re.IGNORECASE) @@ -90,9 +91,13 @@ def sql_relation(sql: str) -> str | None: def sql_operation(sql: str) -> tuple[str | None, str | None]: """``(verb, target)`` for a raw statement: the SQL verb from its leading keyword and the - relation it names, or for ``SET`` the setting it changes.""" + relation it names, or for ``SET`` (and its parameterizable twin ``SELECT set_config``) + the setting it changes.""" if _PROBE.match(sql): return "ping", None + set_config: Final = _SET_CONFIG.match(sql) + if set_config is not None: + return "set", set_config.group(1).lower() keyword: Final = _LEADING_KEYWORD.match(sql) leading: Final = keyword.group(1).upper() if keyword is not None else "" cte_write: Final = _CTE_WRITE.search(sql) if leading == "WITH" else None diff --git a/litellm/proxy/db/rollup_lock_timeout.py b/litellm/proxy/db/rollup_lock_timeout.py new file mode 100644 index 00000000000..f16847e9c96 --- /dev/null +++ b/litellm/proxy/db/rollup_lock_timeout.py @@ -0,0 +1,34 @@ +"""The row-lock budget every spend rollup transaction runs under. + +A rollup upsert that waits on a row another pod holds keeps its pooled +connection for as long as it waits. ``lock_timeout`` bounds that wait, so a +long holder costs the fleet one aborted statement per waiter instead of a +connection pinned until the chain drains. The aborted statement never applied, +so the caller requeues its rows (SQLSTATE 55P03, see +``PrismaDBExceptionHandler.is_lock_timeout_error``). + +``set_config(..., true)`` is ``SET LOCAL``: it lasts for the enclosing +transaction only, and takes the value as a bind parameter, which ``SET`` +cannot. Issued as the first statement of every rollup transaction. +""" + +from typing import Final, Protocol + +from typing_extensions import LiteralString + +from litellm.constants import SPEND_ROLLUP_LOCK_TIMEOUT_MS + +ROLLUP_LOCK_TIMEOUT_SQL: Final = "SELECT set_config('lock_timeout', $1::text, true)" + + +def rollup_lock_timeout_setting() -> str: + return f"{SPEND_ROLLUP_LOCK_TIMEOUT_MS}ms" + + +class _RawExecutor(Protocol): + async def execute_raw(self, query: LiteralString, *args: object) -> int: ... + + +async def apply_rollup_lock_timeout(transaction: _RawExecutor) -> None: + """Bound the row-lock waits of the open ``transaction`` for the rest of its life.""" + _ = await transaction.execute_raw(ROLLUP_LOCK_TIMEOUT_SQL, rollup_lock_timeout_setting()) diff --git a/litellm/proxy/db/spend_log_tool_index.py b/litellm/proxy/db/spend_log_tool_index.py index b0d8bb9aba1..81338988c66 100644 --- a/litellm/proxy/db/spend_log_tool_index.py +++ b/litellm/proxy/db/spend_log_tool_index.py @@ -15,15 +15,18 @@ from __future__ import annotations import asyncio import random -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from dataclasses import dataclass from datetime import datetime, timezone from itertools import groupby -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Any, Final, Protocol + +from typing_extensions import LiteralString from litellm.constants import SPEND_LOG_WRITE_BATCH_MAX_BYTES, SPEND_LOG_WRITE_BATCH_MAX_ROWS from litellm.proxy._types import DB_RETRY_SAFE_ERROR_TYPES from litellm.proxy.db.db_span import db_span +from litellm.proxy.db.rollup_lock_timeout import ROLLUP_LOCK_TIMEOUT_SQL, rollup_lock_timeout_setting from litellm.proxy.db.spend_log_batching import spend_log_write_batches from litellm.repositories.table_repositories import SpendLogToolIndexRepository @@ -99,6 +102,27 @@ def build_tool_usage_transaction( ) +class _ToolRollupTable(Protocol): + def upsert(self, *, where: Mapping[str, object], data: Mapping[str, object]) -> None: ... + + +class _ToolRollupBatch(Protocol): + litellm_dailytoolspend: _ToolRollupTable + + def execute_raw(self, query: LiteralString, *args: object) -> None: ... + + +class _ToolRollupBatchManager(Protocol): + async def __aenter__(self) -> _ToolRollupBatch: ... + + async def __aexit__(self, exc_type: object, exc_value: object, traceback: object) -> bool | None: ... + + +def _tool_rollup_batch(prisma_client: PrismaClient) -> _ToolRollupBatchManager: + batch: Final[_ToolRollupBatchManager] = prisma_client.db.batch_() + return batch + + async def flush_tool_usage_transactions( prisma_client: PrismaClient, transactions: Sequence[ToolUsageTransaction], @@ -140,8 +164,9 @@ async def flush_tool_usage_transactions( await index_table.create_many(data=statement_rows, skip_duplicates=True) async with ( db_span("commit_daily_tool_spend", "LiteLLM_DailyToolSpend"), - prisma_client.db.batch_() as batcher, + _tool_rollup_batch(prisma_client) as batcher, ): + batcher.execute_raw(ROLLUP_LOCK_TIMEOUT_SQL, rollup_lock_timeout_setting()) for (date_key, tool_name), grouped in groupby(per_tool_day, key=lambda entry: (entry[0], entry[1])): entries = tuple(grouped) spend = sum(entry[2] for entry in entries) diff --git a/litellm/proxy/lens/dataset_endpoints.py b/litellm/proxy/lens/dataset_endpoints.py new file mode 100644 index 00000000000..1fc1019ab76 --- /dev/null +++ b/litellm/proxy/lens/dataset_endpoints.py @@ -0,0 +1,175 @@ +from datetime import datetime, timezone +from types import MappingProxyType +from typing import Annotated, Final, TypeAlias +from uuid import uuid4 + +from fastapi import APIRouter, Depends, HTTPException, Query, Response + +from litellm.constants import LENS_DATASET_TRACE_PAGE_SIZE +from litellm.proxy.db.routing_prisma_wrapper import writer_wrapper +from litellm.proxy.lens.dataset_repository import DatasetRepository, DatasetStore +from litellm.proxy.lens.datasets import build_cases, export_jsonl, included_cases, revision_cases, revision_problem +from litellm.proxy.lens.endpoints import Auth, repository, user_scope +from litellm.proxy.lens.models import ( + BuildRequest, + BuildResult, + Dataset, + DatasetCase, + DatasetCreate, + DatasetSummary, + EvalCases, + Finding, + RevisionSave, + Scope, +) +from litellm.proxy.lens.repository import LensRepository, WriterDatabase +from litellm.proxy.lens.state import can_access +from litellm.proxy.tracing_endpoints import read_failure +from litellm.proxy.tracing_runtime import provide_receiver, require_receiver +from litellm.rust_bridge.trace.errors import TraceChanged +from litellm.rust_bridge.trace.generated.types import SpanDetail, Trace, TraceScope +from litellm.tracing import TraceReceiver + +router: Final = APIRouter(prefix="/lens/datasets", tags=["Lens"]) +ALL_TRACES: Final = TraceScope(all_teams=1, user_id="", team_ids=()) + + +def dataset_store() -> DatasetStore: + from litellm.proxy.proxy_server import prisma_client + + if prisma_client is None: + raise HTTPException(503, "Lens needs a connected Postgres database") + return DatasetRepository(WriterDatabase(writer_wrapper(prisma_client.db))) + + +Datasets: TypeAlias = Annotated[DatasetStore, Depends(dataset_store)] +Lenses: TypeAlias = Annotated[LensRepository, Depends(repository)] +Receiver: TypeAlias = Annotated[TraceReceiver | None, Depends(provide_receiver)] +RevisionQuery: TypeAlias = Annotated[int | None, Query(ge=0)] + + +class ProxyDatasetReader: + def __init__(self, receiver: TraceReceiver | None, lenses: LensRepository, scope: Scope) -> None: + self.receiver: Final = receiver + self.lenses: Final = lenses + self.scope: Final = scope + + async def trace(self, trace_id: str, trace_ref: str) -> Trace | None: + try: + return await self._all_pages(trace_id, trace_ref) + except (TraceChanged, ValueError, OverflowError, RuntimeError) as error: + raise read_failure(error) from error + + async def _all_pages(self, trace_id: str, trace_ref: str) -> Trace | None: + receiver: Final = require_receiver(self.receiver) + first: Final = await receiver.get_trace(trace_id, ALL_TRACES, trace_ref, page_size=LENS_DATASET_TRACE_PAGE_SIZE) + if first is None: + return None + spans = first["spans"] # rebind-ok: accumulate spans across cursor pages + cursor = first.get("next_cursor") # rebind-ok: advance the trace cursor + while cursor: + page = await receiver.get_trace(trace_id, ALL_TRACES, trace_ref, cursor, LENS_DATASET_TRACE_PAGE_SIZE) + if page is None: + return None + spans = (*spans, *page["spans"]) + cursor = page.get("next_cursor") + return Trace(summary=first["summary"], agents=first["agents"], spans=spans, next_cursor=None) + + async def span(self, trace_id: str, span_id: str, trace_ref: str) -> SpanDetail | None: + try: + return await require_receiver(self.receiver).get_span(trace_id, span_id, ALL_TRACES, trace_ref) + except (TraceChanged, ValueError, OverflowError, RuntimeError) as error: + raise read_failure(error) from error + + async def findings(self, lens_id: str, ids: tuple[str, ...]) -> tuple[Finding, ...]: + lens: Final = await self.lenses.get(lens_id) + if lens is None or not can_access(self.scope, lens.scope): + raise HTTPException(404, "Lens not found") + wanted: Final = frozenset(ids) + return tuple(f for f in lens.findings if f.id in wanted) + + +async def get_dataset(datasets: DatasetStore, dataset_id: str, scope: Scope, revision: int | None = None) -> Dataset: + dataset: Final = await datasets.get(dataset_id, revision) + if dataset is None or not can_access(scope, Scope(team_id=dataset.team_id)): + raise HTTPException(404, "Dataset not found") + return dataset + + +@router.get("", response_model=tuple[DatasetSummary, ...]) +async def list_datasets(auth: Auth, datasets: Datasets) -> tuple[DatasetSummary, ...]: + scope: Final = user_scope(auth) + return tuple(s.summary for s in await datasets.summaries() if can_access(scope, Scope(team_id=s.team_id))) + + +@router.post("", response_model=Dataset) +async def create_dataset(body: DatasetCreate, auth: Auth, datasets: Datasets) -> Dataset: + user_scope(auth, write=True) + now: Final = datetime.now(timezone.utc) + dataset: Final = Dataset( + id=str(uuid4()), + name=body.name, + agent_name=body.agent_name, + team_id=auth.team_id or "", + created_at=now, + revision=0, + created_by=auth.user_id or "", + cases=(), + ) + if not await datasets.insert(dataset, now): + raise HTTPException(409, "Dataset already exists") + return dataset + + +@router.post("/build", response_model=BuildResult) +async def build_dataset_cases( + body: BuildRequest, auth: Auth, datasets: Datasets, lenses: Lenses, receiver: Receiver +) -> BuildResult: + scope: Final = user_scope(auth) + existing: Final[tuple[DatasetCase, ...]] = ( + (await get_dataset(datasets, body.dataset_id, scope)).cases if body.dataset_id else () + ) + return await build_cases(body, ProxyDatasetReader(receiver, lenses, scope), existing) + + +@router.get("/{dataset_id}", response_model=Dataset) +async def read_dataset(dataset_id: str, auth: Auth, datasets: Datasets, revision: RevisionQuery = None) -> Dataset: + return await get_dataset(datasets, dataset_id, user_scope(auth), revision) + + +@router.post("/{dataset_id}/revisions", response_model=Dataset) +async def save_revision(dataset_id: str, body: RevisionSave, auth: Auth, datasets: Datasets) -> Dataset: + latest: Final = await get_dataset(datasets, dataset_id, user_scope(auth, write=True)) + if body.base_revision != latest.revision: + raise HTTPException(409, "Dataset changed, reload") + cases: Final = revision_cases(body.cases) + if problem := revision_problem(cases): + raise HTTPException(422, problem) + saved: Final = latest.model_copy( + update=MappingProxyType({"revision": latest.revision + 1, "cases": cases, "created_by": auth.user_id or ""}) + ) + if not await datasets.insert(saved, datetime.now(timezone.utc)): + raise HTTPException(409, "Dataset changed, reload") + return saved + + +@router.get( + "/{dataset_id}/export", + response_class=Response, + responses={200: {"content": {"application/x-ndjson": {}}}}, +) +async def export_dataset(dataset_id: str, auth: Auth, datasets: Datasets, revision: RevisionQuery = None) -> Response: + dataset: Final = await get_dataset(datasets, dataset_id, user_scope(auth), revision) + return Response( + content=export_jsonl(dataset.cases), + media_type="application/x-ndjson", + headers=MappingProxyType( + {"Content-Disposition": f'attachment; filename="dataset-{dataset.id}-r{dataset.revision}.jsonl"'} + ), + ) + + +@router.get("/{dataset_id}/revisions/{revision}/cases", response_model=EvalCases) +async def eval_cases(dataset_id: str, revision: int, auth: Auth, datasets: Datasets) -> EvalCases: + dataset: Final = await get_dataset(datasets, dataset_id, user_scope(auth), revision) + return EvalCases(dataset_id=dataset.id, revision=dataset.revision, cases=included_cases(dataset.cases)) diff --git a/litellm/proxy/lens/dataset_repository.py b/litellm/proxy/lens/dataset_repository.py new file mode 100644 index 00000000000..50e5f3ee53a --- /dev/null +++ b/litellm/proxy/lens/dataset_repository.py @@ -0,0 +1,65 @@ +from datetime import datetime +from typing import Final, Protocol + +from pydantic import TypeAdapter + +from litellm.proxy.lens.models import Dataset, DatasetSummary, Record +from litellm.proxy.lens.repository import Database, Row + +_ROWS: Final = TypeAdapter(tuple[Row, ...]) + + +class StoredSummary(Record): + team_id: str + summary: DatasetSummary + + +class DatasetStore(Protocol): + async def summaries(self) -> tuple[StoredSummary, ...]: ... + async def get(self, dataset_id: str, revision: int | None = None) -> Dataset | None: ... + async def insert(self, dataset: Dataset, saved_at: datetime) -> bool: ... + + +class DatasetRepository: + def __init__(self, db: Database) -> None: + self.db: Final = db + + async def summaries(self) -> tuple[StoredSummary, ...]: + rows: Final = _ROWS.validate_python( + await self.db.query_raw( + """SELECT jsonb_build_object( + 'team_id', data->>'team_id', + 'summary', jsonb_build_object( + 'id', id, 'name', data->>'name', 'agent_name', data->>'agent_name', 'revision', revision, + 'case_count', jsonb_array_length(data->'cases'), + 'updated_at', created_at AT TIME ZONE 'UTC' + ) + ) AS data FROM ( + SELECT DISTINCT ON (id) id, revision, created_at, data FROM "LiteLLM_LensDataset" + ORDER BY id, revision DESC + ) AS latest ORDER BY created_at DESC""" + ) + ) + return tuple(StoredSummary.model_validate(row.data) for row in rows) + + async def get(self, dataset_id: str, revision: int | None = None) -> Dataset | None: + rows: Final = _ROWS.validate_python( + await self.db.query_raw( + """SELECT data FROM "LiteLLM_LensDataset" WHERE id=$1 AND ($2::int IS NULL OR revision=$2::int) + ORDER BY revision DESC LIMIT 1""", + dataset_id, + revision, + ) + ) + return Dataset.model_validate(rows[0].data) if rows else None + + async def insert(self, dataset: Dataset, saved_at: datetime) -> bool: + inserted: Final = await self.db.execute_raw( + """INSERT INTO "LiteLLM_LensDataset" (id, revision, created_at, data) + VALUES ($1, $2, $3::timestamptz AT TIME ZONE 'UTC', $4::jsonb) ON CONFLICT (id, revision) DO NOTHING""", + dataset.id, + dataset.revision, + saved_at.isoformat(), + dataset.model_dump_json(), + ) + return inserted == 1 diff --git a/litellm/proxy/lens/datasets.py b/litellm/proxy/lens/datasets.py new file mode 100644 index 00000000000..aef672833df --- /dev/null +++ b/litellm/proxy/lens/datasets.py @@ -0,0 +1,307 @@ +import hashlib +from collections.abc import Iterable +from dataclasses import dataclass +from functools import partial, reduce +from itertools import chain +from types import MappingProxyType +from typing import Final, Protocol, TypeAlias + +from pydantic import BaseModel, ConfigDict, ValidationError +from typing_extensions import assert_never + +from litellm.constants import LENS_DATASET_MAX_CASE_CHARS, LENS_DATASET_MAX_CASES +from litellm.proxy.lens.models import ( + BuildRequest, + BuildResult, + BuildSource, + CaseSource, + DatasetCase, + DatasetMessage, + DatasetToolCall, + Finding, + FindingSource, + Record, + SkippedCase, + TextSource, + TraceSource, +) +from litellm.proxy.lens.sources import parse_execution +from litellm.rust_bridge.trace.generated.types import SpanDetail, Trace, UIContent, UIMessage + +AGENT_VERSION_ATTRIBUTE: Final = "agent.version" + + +class DatasetReader(Protocol): + async def trace(self, trace_id: str, trace_ref: str) -> Trace | None: ... + async def span(self, trace_id: str, span_id: str, trace_ref: str) -> SpanDetail | None: ... + async def findings(self, lens_id: str, ids: tuple[str, ...]) -> tuple[Finding, ...]: ... + + +class _Content(Record): + messages: tuple[DatasetMessage, ...] + reply: str + tool_calls: tuple[DatasetToolCall, ...] + + +class _TextLine(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + messages: tuple[DatasetMessage, ...] + reply: str = "" + tool_calls: tuple[DatasetToolCall, ...] = () + expected: str = "" + source: CaseSource = CaseSource() + agent_version: str = "" + + +@dataclass(frozen=True, slots=True) +class _Reply: + text: str + tool_calls: tuple[DatasetToolCall, ...] + + +@dataclass(frozen=True, slots=True) +class _Admission: + seen: frozenset[str] + cases: tuple[DatasetCase, ...] = () + skipped: tuple[SkippedCase, ...] = () + + +Candidate: TypeAlias = DatasetCase | SkippedCase + + +def case_id(messages: tuple[DatasetMessage, ...], reply: str, tool_calls: tuple[DatasetToolCall, ...]) -> str: + content: Final = _Content(messages=messages, reply=reply, tool_calls=tool_calls) + return hashlib.sha256(content.model_dump_json().encode()).hexdigest() + + +def _tool_call_chars(tool_calls: tuple[DatasetToolCall, ...]) -> int: + return sum(len(t.name) + len(t.arguments) for t in tool_calls) + + +def case_chars(case: DatasetCase) -> int: + return ( + sum(len(m.content) + len(m.name) + _tool_call_chars(m.tool_calls) for m in case.messages) + + len(case.reply) + + _tool_call_chars(case.tool_calls) + + len(case.expected) + ) + + +def make_case( + messages: tuple[DatasetMessage, ...], + reply: str, + tool_calls: tuple[DatasetToolCall, ...], + source: CaseSource, + expected: str = "", + agent_version: str = "", +) -> Candidate: + if not messages and not reply.strip() and not tool_calls: + return SkippedCase(source=source, reason="no_content") + case: Final = DatasetCase( + id=case_id(messages, reply, tool_calls), + messages=messages, + reply=reply, + tool_calls=tool_calls, + expected=expected, + source=source, + agent_version=agent_version, + ) + if case_chars(case) > LENS_DATASET_MAX_CASE_CHARS: + return SkippedCase(source=source, reason="too_large") + return case + + +def _message(message: UIMessage) -> DatasetMessage: + return DatasetMessage( + role=message["role"], + content=message["content"], + name=message.get("name") or "", + tool_calls=_tool_calls((message,)), + ) + + +def _tool_calls(messages: Iterable[UIMessage]) -> tuple[DatasetToolCall, ...]: + calls: Final = chain.from_iterable(m.get("tool_calls", ()) for m in messages) + return tuple(DatasetToolCall(name=c["name"], arguments=c["arguments"]) for c in calls) + + +def _conversation(ui: UIContent, raw: str) -> tuple[DatasetMessage, ...]: + if ui["kind"] == "messages": + return tuple(_message(m) for m in ui["messages"]) + return (DatasetMessage(role="user", content=raw),) if raw.strip() else () + + +def _reply(ui: UIContent, raw: str) -> _Reply: + if ui["kind"] == "messages": + replies: Final = tuple(m for m in ui["messages"] if m["role"] == "assistant") + return _Reply("\n\n".join(m["content"] for m in replies if m["content"]), _tool_calls(replies)) + if ui["kind"] == "text": + return _Reply(ui["text"], ()) + return _Reply(raw, ()) + + +def case_from_span(detail: SpanDetail, source: CaseSource) -> Candidate: + reply: Final = _reply(detail["output_ui"], detail["output"]) + return make_case( + _conversation(detail["input_ui"], detail["input"]), + reply.text, + reply.tool_calls, + source.model_copy(update=MappingProxyType({"span_id": detail["span_id"]})), + agent_version=detail["attributes"].get(AGENT_VERSION_ATTRIBUTE, ""), + ) + + +async def _last_conversation(reader: DatasetReader, source: TraceSource, trace: Trace) -> SpanDetail | None: + llm_spans: Final = sorted( + (s for s in trace["spans"] if s["type"] == "llm"), key=lambda s: s["start_offset_ms"], reverse=True + ) + for span in llm_spans: + detail = await reader.span(source.trace_id, span["span_id"], source.trace_ref) + if detail is not None and detail["input_ui"]["kind"] == "messages": + return detail + return None + + +async def _trace_cases(reader: DatasetReader, source: TraceSource) -> tuple[Candidate, ...]: + origin: Final = CaseSource(trace_id=source.trace_id, trace_ref=source.trace_ref, span_id=source.span_id) + if source.span_id: + span: Final = await reader.span(source.trace_id, source.span_id, source.trace_ref) + return (case_from_span(span, origin) if span else SkippedCase(source=origin, reason="no_content"),) + trace: Final = await reader.trace(source.trace_id, source.trace_ref) + detail: Final = await _last_conversation(reader, source, trace) if trace else None + return (case_from_span(detail, origin) if detail else SkippedCase(source=origin, reason="no_content"),) + + +async def _evidence_case(reader: DatasetReader, origin: CaseSource, execution: str) -> Candidate: + try: + _, _, trace_id, trace_ref = parse_execution(execution) + except (ValueError, ValidationError): + return SkippedCase(source=origin, reason="no_content") + located: Final = origin.model_copy(update=MappingProxyType({"trace_id": trace_id, "trace_ref": trace_ref})) + span: Final = await reader.span(trace_id, origin.span_id, trace_ref) + return case_from_span(span, located) if span else SkippedCase(source=located, reason="no_content") + + +def _first_by_key(keys: tuple[str, ...]) -> frozenset[int]: + first: Final = MappingProxyType({key: index for index, key in reversed(tuple(enumerate(keys)))}) + return frozenset(first.values()) + + +def _evidence_spans(lens_id: str, findings: tuple[Finding, ...]) -> tuple[tuple[str, CaseSource], ...]: + pairs: Final = tuple(chain.from_iterable(((f.id, e) for e in f.evidence) for f in findings)) + kept: Final = _first_by_key(tuple(f"{e.execution_id}\0{e.span_id}" for _, e in pairs)) + return tuple( + (e.execution_id, CaseSource(span_id=e.span_id, finding_id=fid, lens_id=lens_id)) + for index, (fid, e) in enumerate(pairs) + if index in kept + ) + + +async def _finding_cases(reader: DatasetReader, source: FindingSource) -> tuple[Candidate, ...]: + findings: Final = await reader.findings(source.lens_id, source.finding_ids) + spans: Final = _evidence_spans(source.lens_id, findings) + return tuple([await _evidence_case(reader, origin, execution) for execution, origin in spans]) + + +def _looks_like_json_line(line: str) -> bool: + return line.lstrip().startswith("{") + + +def _text_line_case(line: str) -> Candidate: + try: + parsed: Final = _TextLine.model_validate_json(line) + except ValidationError: + return SkippedCase(source=CaseSource(), reason="invalid") + return make_case( + parsed.messages, + parsed.reply, + parsed.tool_calls, + parsed.source, + expected=parsed.expected, + agent_version=parsed.agent_version, + ) + + +def _text_cases(source: TextSource) -> tuple[Candidate, ...]: + lines: Final = tuple(line for line in source.text.splitlines() if line.strip()) + if lines and all(_looks_like_json_line(line) for line in lines): + return tuple(_text_line_case(line) for line in lines) + return (make_case((DatasetMessage(role="user", content=source.text),), "", (), CaseSource()),) + + +async def _source_cases(reader: DatasetReader, source: BuildSource) -> tuple[Candidate, ...]: + match source: + case TraceSource(): + return await _trace_cases(reader, source) + case FindingSource(): + return await _finding_cases(reader, source) + case TextSource(): + return _text_cases(source) + return assert_never(source) + + +def _skip(state: _Admission, skipped: SkippedCase) -> _Admission: + return _Admission(state.seen, state.cases, (*state.skipped, skipped)) + + +def _admit(existing_count: int, state: _Admission, candidate: Candidate) -> _Admission: + if isinstance(candidate, SkippedCase): + return _skip(state, candidate) + if candidate.id in state.seen: + return _skip(state, SkippedCase(source=candidate.source, reason="duplicate")) + if existing_count + len(state.cases) >= LENS_DATASET_MAX_CASES: + return _skip(state, SkippedCase(source=candidate.source, reason="over_limit")) + return _Admission(state.seen | {candidate.id}, (*state.cases, candidate), state.skipped) + + +def _unread_source(source: BuildSource) -> CaseSource: + match source: + case TraceSource(): + return CaseSource(trace_id=source.trace_id, trace_ref=source.trace_ref, span_id=source.span_id) + case FindingSource(): + return CaseSource(lens_id=source.lens_id, finding_id=source.finding_ids[0]) + case TextSource(): + return CaseSource() + return assert_never(source) + + +async def _admit_source( + reader: DatasetReader, existing_count: int, state: _Admission, source: BuildSource +) -> _Admission: + if existing_count + len(state.cases) >= LENS_DATASET_MAX_CASES: + return _skip(state, SkippedCase(source=_unread_source(source), reason="over_limit")) + return reduce(partial(_admit, existing_count), await _source_cases(reader, source), state) + + +async def build_cases(request: BuildRequest, reader: DatasetReader, existing: tuple[DatasetCase, ...]) -> BuildResult: + state = _Admission(seen=frozenset(c.id for c in existing)) # rebind-ok: sources are read in order until full + for source in request.sources: + state = await _admit_source(reader, len(existing), state, source) + return BuildResult(cases=state.cases, skipped=state.skipped) + + +def rehashed(case: DatasetCase) -> DatasetCase: + return case.model_copy(update=MappingProxyType({"id": case_id(case.messages, case.reply, case.tool_calls)})) + + +def revision_cases(cases: tuple[DatasetCase, ...]) -> tuple[DatasetCase, ...]: + hashed: Final = tuple(rehashed(c) for c in cases) + kept: Final = _first_by_key(tuple(c.id for c in hashed)) + return tuple(c for index, c in enumerate(hashed) if index in kept) + + +def revision_problem(cases: tuple[DatasetCase, ...]) -> str | None: + if len(cases) > LENS_DATASET_MAX_CASES: + return f"A dataset holds at most {LENS_DATASET_MAX_CASES} cases" + if any(case_chars(c) > LENS_DATASET_MAX_CASE_CHARS for c in cases): + return f"Each case must be at most {LENS_DATASET_MAX_CASE_CHARS} characters" + return None + + +def included_cases(cases: tuple[DatasetCase, ...]) -> tuple[DatasetCase, ...]: + return tuple(c for c in cases if c.included) + + +def export_jsonl(cases: tuple[DatasetCase, ...]) -> str: + return "".join(c.model_dump_json() + "\n" for c in included_cases(cases)) diff --git a/litellm/proxy/lens/models.py b/litellm/proxy/lens/models.py index b29809a440b..7d3f791e55a 100644 --- a/litellm/proxy/lens/models.py +++ b/litellm/proxy/lens/models.py @@ -500,3 +500,106 @@ class ModelResult(Record): cost: float context_exceeded: bool = False finish_reason: Literal["length", "content_filter"] | None = Field(default=None, exclude=True) + + +class DatasetToolCall(Record): + name: str + arguments: str + + +class DatasetMessage(Record): + role: Literal["system", "user", "assistant", "tool"] + content: str + name: str = "" + tool_calls: tuple[DatasetToolCall, ...] = () + + +class CaseSource(Record): + trace_id: str = "" + trace_ref: str = "" + span_id: str = "" + finding_id: str = "" + lens_id: str = "" + + +class DatasetCase(Record): + id: str + messages: tuple[DatasetMessage, ...] + reply: str = "" + tool_calls: tuple[DatasetToolCall, ...] = () + expected: str = "" + included: bool = True + source: CaseSource + agent_version: str = "" + + +class SkippedCase(Record): + source: CaseSource + reason: Literal["duplicate", "no_content", "too_large", "over_limit", "invalid"] + + +class TraceSource(Record): + kind: Literal["trace"] = "trace" + trace_id: str = Field(min_length=1) + trace_ref: str = "" + span_id: str = "" + + +class FindingSource(Record): + kind: Literal["finding"] = "finding" + lens_id: str = Field(min_length=1) + finding_ids: tuple[str, ...] = Field(min_length=1) + + +class TextSource(Record): + kind: Literal["text"] = "text" + text: str = Field(min_length=1) + + +BuildSource: TypeAlias = Annotated[TraceSource | FindingSource | TextSource, Field(discriminator="kind")] + + +class BuildRequest(Record): + sources: tuple[BuildSource, ...] = Field(min_length=1) + dataset_id: str = "" + + +class BuildResult(Record): + cases: tuple[DatasetCase, ...] + skipped: tuple[SkippedCase, ...] + + +class DatasetCreate(Record): + name: str = Field(min_length=1, max_length=120) + agent_name: str = "" + + +class Dataset(Record): + id: str + name: str + agent_name: str + team_id: str + created_at: datetime + revision: int + created_by: str + cases: tuple[DatasetCase, ...] + + +class DatasetSummary(Record): + id: str + name: str + agent_name: str + revision: int + case_count: int + updated_at: datetime + + +class RevisionSave(Record): + base_revision: int = Field(ge=0) + cases: tuple[DatasetCase, ...] + + +class EvalCases(Record): + dataset_id: str + revision: int + cases: tuple[DatasetCase, ...] diff --git a/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py b/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py index 7fd607fc4d0..3d1be882779 100644 --- a/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py +++ b/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py @@ -2,6 +2,8 @@ import json from collections.abc import Callable from typing import TYPE_CHECKING, Final +from fastapi import HTTPException + import litellm from litellm._logging import verbose_router_logger from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook import ( @@ -37,6 +39,24 @@ def _credential_identity(credentials: VERTEX_CREDENTIALS_TYPES | None) -> str | return credentials +# Network I/O prohibited; get_credentials() is synchronous inside async routes. +# oidc/env_path/ is excluded because it bypasses the oidc/file/ credential-directory allowlist. +_LOCAL_OIDC_PREFIXES: Final = ("oidc/file/", "oidc/env/") + + +def _resolve_oidc_reference(credential: str | None) -> str | None: + """Resolve local ``oidc/...`` credentials on every call to pick up rotated tokens.""" + if credential is None or not credential.startswith(_LOCAL_OIDC_PREFIXES): + return credential + try: + return get_secret_str(credential) + except (OSError, ValueError) as exc: + raise HTTPException( + status_code=401, + detail=f"pass-through credential {credential} could not be resolved: {exc}", + ) from exc + + class PassthroughEndpointRouter: """ Use this class to Get credentials for pass-through endpoints @@ -60,14 +80,14 @@ class PassthroughEndpointRouter: region_name=region_name, ) if deployment_api_key is not None: - return deployment_api_key + return _resolve_oidc_reference(deployment_api_key) verbose_router_logger.debug( "No pass-through deployment credentials found for %s, looking for env variable", custom_llm_provider ) _env_variable_name: Final = self._get_default_env_variable_name_passthrough_endpoint( custom_llm_provider=custom_llm_provider, ) - return get_secret_str(_env_variable_name) + return _resolve_oidc_reference(get_secret_str(_env_variable_name)) def _get_deployment_api_key( self, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 5d6ba0f8e13..1f3f6ea0adb 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -579,6 +579,7 @@ from litellm.proxy.hooks.prompt_injection_detection import ( ) from litellm.proxy.hooks.proxy_track_cost_callback import _ProxyDBLogger, run_spend_event from litellm.proxy.image_endpoints.endpoints import router as image_router +from litellm.proxy.lens.dataset_endpoints import router as lens_dataset_router from litellm.proxy.lens.endpoints import router as lens_router from litellm.proxy.list_api.common import ( ManagementProblem, @@ -20110,6 +20111,7 @@ app.include_router(auto_router_management_router) app.include_router(tag_management_router) app.include_router(workflow_management_router) app.include_router(memory_router) +app.include_router(lens_dataset_router) app.include_router(lens_router) app.include_router(plugin_router) app.include_router(cost_tracking_settings_router) diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 4aabd56131b..514df905866 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -1965,3 +1965,12 @@ model LiteLLM_LensWorker { token_hash String @unique data Json } + +model LiteLLM_LensDataset { + id String + revision Int + created_at DateTime + data Json + + @@id([id, revision]) +} diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index d90ff1581e2..a6e6209298b 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -7645,6 +7645,13 @@ async def update_daily_tag_spend( verbose_proxy_logger.error("Error updating daily tag spend: %s", e) +def _rollup_batch_never_applied(e: Exception) -> bool: + """True when a drained rollup batch provably never reached its rows: Postgres had no + connection for it (53300) or cancelled it under ``lock_timeout`` before it took the row + lock (55P03). Either way re-sending the batch cannot double-count, so it is requeued.""" + return PrismaDBExceptionHandler.is_database_capacity_error(e) or PrismaDBExceptionHandler.is_lock_timeout_error(e) + + async def update_spend_logs_job( prisma_client: PrismaClient, db_writer_client: AsyncHTTPHandler | None, @@ -7710,9 +7717,11 @@ async def _run_spend_logs_job( ) # Tool usage tracking: drain the request-time queue into the tool index and the - # LiteLLM_DailyToolSpend rollup. A batch Postgres had no connection for was never - # sent, so it is requeued and the job stops; any other failure is dropped because - # a replay could double-count the rollup. + # LiteLLM_DailyToolSpend rollup. A batch Postgres had no connection for, or whose + # rollup statement was cancelled by lock_timeout, never applied, so it is requeued; + # any other failure is dropped because a replay could double-count the rollup. Only + # a full server (53300) stops the job: a timed-out row belongs to this rollup alone, + # the writers below have their own rows and keep flushing. async with prisma_client._tool_usage_transactions_lock: tool_usage_to_process: Final = prisma_client.tool_usage_transactions[:MAX_LOGS_PER_INTERVAL] prisma_client.tool_usage_transactions = prisma_client.tool_usage_transactions[len(tool_usage_to_process) :] @@ -7724,19 +7733,20 @@ async def _run_spend_logs_job( transactions=tool_usage_to_process, ) except Exception as tool_tracking_err: - if PrismaDBExceptionHandler.is_database_capacity_error(tool_tracking_err): + if _rollup_batch_never_applied(tool_tracking_err): async with prisma_client._tool_usage_transactions_lock: prisma_client.tool_usage_transactions = [ *tool_usage_to_process, *prisma_client.tool_usage_transactions, ] verbose_proxy_logger.warning( - "Spend tracking - database out of connections during tool usage flush, " - "requeued %s tool usage transactions for the next flush: %s", + "Spend tracking - database out of connections or rollup row lock timed out during tool usage " + "flush, requeued %s tool usage transactions for the next flush: %s", len(tool_usage_to_process), tool_tracking_err, ) - raise + if PrismaDBExceptionHandler.is_database_capacity_error(tool_tracking_err): + raise else: verbose_proxy_logger.error( "Spend tracking - tool usage flush failed; %s tool usage transactions dropped: %s", @@ -7744,6 +7754,7 @@ async def _run_spend_logs_job( tool_tracking_err, ) + # Model usage rollup: same requeue and stop rules as the tool rollup above async with prisma_client._model_usage_transactions_lock: model_usage_to_process: Final = prisma_client.model_usage_transactions prisma_client.model_usage_transactions = [] @@ -7752,11 +7763,28 @@ async def _run_spend_logs_job( await flush_model_usage_transactions(prisma_client=prisma_client, transactions=model_usage_to_process) except Exception as model_usage_err: - verbose_proxy_logger.error( - "Spend tracking - model usage flush failed; %s model usage transactions dropped: %s", - len(model_usage_to_process), - model_usage_err, - ) + if _rollup_batch_never_applied(model_usage_err): + async with ( + prisma_client._model_usage_transactions_lock # pyright: ignore[reportPrivateUsage] # needed to requeue + ): + prisma_client.model_usage_transactions = [ + *model_usage_to_process, + *prisma_client.model_usage_transactions, + ] + verbose_proxy_logger.warning( + "Spend tracking - database out of connections or rollup row lock timed out during model usage " + "flush, requeued %s model usage transactions for the next flush: %s", + len(model_usage_to_process), + model_usage_err, + ) + if PrismaDBExceptionHandler.is_database_capacity_error(model_usage_err): + raise + else: + verbose_proxy_logger.error( + "Spend tracking - model usage flush failed; %s model usage transactions dropped: %s", + len(model_usage_to_process), + model_usage_err, + ) await flush_baseline_accounting(prisma_client) diff --git a/litellm/router.py b/litellm/router.py index 010e5946cb8..f93f12f2f2e 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -73,7 +73,8 @@ from litellm.constants import ( ) from litellm.integrations.custom_guardrail import is_guardrail_intervention from litellm.integrations.custom_logger import CustomLogger -from litellm.integrations.otel.runtime import phase_event, phase_span +from litellm.integrations.otel.routing import routing_decision_attributes +from litellm.integrations.otel.runtime import phase_attributes, phase_event, phase_span from litellm.litellm_core_utils.asyncify import run_async_function from litellm.litellm_core_utils.core_helpers import ( _get_parent_otel_span_from_kwargs, @@ -162,6 +163,7 @@ from litellm.router_utils.add_retry_fallback_headers import ( ) from litellm.router_utils.auto_router_model_naming import ( AUTO_ROUTER_MODEL_PREFIX, + STRATEGY_ROUTER_PARAM_FIELDS, GatedAutoRouterCapability, capability_limit_violation, claimed_capabilities, @@ -8724,6 +8726,19 @@ class Router: breadcrumbs: Final = (*kept_breadcrumbs, attempt_record) earlier: Final = request_metadata.get("request_retry_count") request_retry_count: Final = (earlier if type(earlier) is int and 0 <= earlier else 0) + 1 + decision: Final = request_metadata.get("routing_decision") + router_name: Final = decision.get("router_model_name") if isinstance(decision, Mapping) else None + if isinstance(router_name, str): + phase_event( + "litellm.routing.retry", + { + "litellm.routing.router_model_name": router_name, + "litellm.retry.count": request_retry_count, + "error.type": type(e).__name__, + **({"litellm.deployment.model_group": model_group} if isinstance(model_group, str) else {}), + **({"litellm.deployment.id": deployment_id} if isinstance(deployment_id, str) else {}), + }, + ) kwargs[_metadata_var]["previous_models"] = breadcrumbs # rebind-ok: the logging object already holds this dict kwargs[_metadata_var]["request_retry_count"] = request_retry_count # rebind-ok: same dict, read by the cap return kwargs @@ -9572,9 +9587,29 @@ class Router: ) registry[deployment.model_name] = [ *registry.get(deployment.model_name, []), - TaggedPreRoutingStrategy(tags=tags, strategy=strategy), + TaggedPreRoutingStrategy( + tags=tags, + strategy=strategy, + definition_fingerprint=self._routing_definition_fingerprint(deployment), + deployment=deployment, + ), ] + @staticmethod + def _routing_definition_fingerprint(deployment: Deployment) -> str | None: + """Identify configured routing fields, excluding external files and live strategy state.""" + try: + definition: Final = json.dumps( + deployment.litellm_params.model_dump( + include=STRATEGY_ROUTER_PARAM_FIELDS | {"model", "tags"}, exclude_none=True, warnings=False + ), + sort_keys=True, + default=Router._json_default_stable_id, + ) + except (TypeError, ValueError): + return None + return Router.generate_model_id(deployment.model_name, {"routing_definition": definition}) + @staticmethod def _unregister_pre_routing_strategy( registry: dict[str, list[TaggedPreRoutingStrategy[_PreRoutingStrategyT]]], @@ -9653,7 +9688,12 @@ class Router: if adaptive_router is not None: self.adaptive_routers[model_name] = [ *self.adaptive_routers.get(model_name, []), - TaggedPreRoutingStrategy(tags=tagged.tags, strategy=adaptive_router), + TaggedPreRoutingStrategy( + tags=tagged.tags, + strategy=adaptive_router, + definition_fingerprint=tagged.definition_fingerprint, + deployment=tagged.deployment, + ), ] self._sync_adaptive_router_hooks() @@ -14224,6 +14264,7 @@ class Router: spend metadata is stamped before routing and the response carries the tier group the strategy picked. """ + self._record_routing_decision(request_kwargs=request_kwargs, routing_decision=None) requested_registered_model_name: Final = self._get_model_from_alias(model=model) or model registered_model_name: Final = await self._resolve_claude_code_session_router( model=model, @@ -14247,7 +14288,6 @@ class Router: ) if selected_strategy is None: await arm_compaction(request_kwargs, None) - self._record_routing_decision(request_kwargs=request_kwargs, routing_decision=None) self._stamp_or_clear_metadata_key( request_kwargs=request_kwargs, key=SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, value=None ) @@ -14259,6 +14299,26 @@ class Router: from litellm.proxy.auth.auto_router_checks import authorize_member_auto_router_inference from litellm.router_strategy.complexity_router.complexity_router import ComplexityRouter + marker: Final = self._selected_strategy_marker_deployment( + model=registered_model_name, strategy_tags=selected_strategy.tags, request_kwargs=request_kwargs + ) + definition: Final = selected_strategy.deployment + config_id: Final = definition.model_info.id if definition else None + updated_at: Final = definition.model_info.updated_at if definition else None + router_kind: Final = classify_strategy_router_model(definition.litellm_params.model) if definition else None + provenance: Final[StandardLoggingRoutingDecision] = { + "router_model_name": definition.model_name if definition else registered_model_name, + **({"router_type": router_kind} if router_kind is not None else {}), + **({"router_config_id": config_id} if config_id is not None else {}), + **({"router_config_updated_at": updated_at.isoformat()} if updated_at is not None else {}), + **( + {"router_config_fingerprint": selected_strategy.definition_fingerprint} + if selected_strategy.definition_fingerprint is not None + else {} + ), + } + phase_attributes(routing_decision_attributes(provenance)) + reject_recursive_compactor(registered_model_name) await arm_compaction( request_kwargs, @@ -14282,11 +14342,7 @@ class Router: ) await authorize_member_auto_router_inference( - deployment=self._selected_strategy_marker_deployment( - model=registered_model_name, - strategy_tags=selected_strategy.tags, - request_kwargs=request_kwargs, - ), + deployment=marker, request_kwargs=request_kwargs, llm_router=self, ) @@ -14324,13 +14380,18 @@ class Router: input=input, specific_deployment=specific_deployment, ) + traced_routed: Final = ( + routed.model_copy(update={"routing_decision": {**provenance, **routed.routing_decision}}) + if routed is not None and routed.routing_decision is not None + else routed + ) # Routing-only compression must not leak into the response: the model call and # deployment-context filtering key off this field. Compared by value, since # pydantic rebuilds the list rather than keeping the object passed in. pre_routing_hook_response: Final = ( - routed.model_copy(update={"messages": messages}) - if routed is not None and routing_messages is not None and routed.messages == routing_messages - else routed + traced_routed.model_copy(update={"messages": messages}) + if traced_routed is not None and routing_messages is not None and traced_routed.messages == routing_messages + else traced_routed ) self._record_routing_decision( request_kwargs=request_kwargs, @@ -14481,6 +14542,7 @@ class Router: """ from litellm.types.router import BaselineRouteStamp + phase_attributes(routing_decision_attributes(routing_decision)) baseline_model: Final = routing_decision.get("savings_baseline_model") if routing_decision else None baseline_id: Final = routing_decision.get("savings_baseline_deployment_id") if routing_decision else None router_name: Final = routing_decision.get("router_model_name") if routing_decision else None diff --git a/litellm/router_strategy/auto_router/auto_router.py b/litellm/router_strategy/auto_router/auto_router.py index 6b97276a079..0b79cf8f17d 100644 --- a/litellm/router_strategy/auto_router/auto_router.py +++ b/litellm/router_strategy/auto_router/auto_router.py @@ -4,12 +4,13 @@ Auto-Routing Strategy that works with a Semantic Router Config import asyncio from collections.abc import Mapping, Sequence -from typing import TYPE_CHECKING, Any, Final, Optional +from typing import TYPE_CHECKING, Any, Final, NamedTuple, Optional from pydantic import ConfigDict from litellm._logging import verbose_router_logger from litellm.constants import DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS +from litellm.exceptions import Timeout as LiteLLMTimeout from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.internal_call_metadata import ( effective_turn_off_message_logging, @@ -17,7 +18,12 @@ from litellm.litellm_core_utils.internal_call_metadata import ( parent_session_kwargs, ) from litellm.types.llms.base import LiteLLMBaseModel -from litellm.types.utils import AUTOROUTER_CLASSIFIER_CALL_ORIGIN +from litellm.types.utils import ( + AUTOROUTER_CLASSIFIER_CALL_ORIGIN, + ClassifierFailureReason, + RoutingDecisionCause, + StandardLoggingRoutingDecision, +) if TYPE_CHECKING: from semantic_router.routers import SemanticRouter @@ -41,6 +47,12 @@ class _CallerMetadata(LiteLLMBaseModel): litellm_metadata: Mapping[str, object] | None = None +class _SemanticMatchOutcome(NamedTuple): + route_name: str | None = None + failure_reason: ClassifierFailureReason | None = None + error_type: str | None = None + + class AutoRouter(CustomLogger): DEFAULT_AUTO_SYNC_VALUE = "local" @@ -72,6 +84,7 @@ class AutoRouter(CustomLogger): from litellm.router_strategy.auto_router.litellm_encoder import LiteLLMRouterEncoder + self.model_name: Final = model_name self.auto_router_config_path: str | None = auto_router_config_path self.auto_router_config: str | None = auto_router_config self.auto_sync_value = self.DEFAULT_AUTO_SYNC_VALUE @@ -205,20 +218,38 @@ class AutoRouter(CustomLogger): if resolved_messages is None: return None - routelayer = await self._ensure_routelayer() + routelayer: Final = await self._ensure_routelayer() message_content: Final = self._extract_text_from_messages(resolved_messages) - route_name: Final = await self._matched_route_name(routelayer, message_content, request_kwargs) + outcome: Final = await self._match_route(routelayer, message_content, request_kwargs) + routed_model: Final = outcome.route_name or self.default_model + cause: Final[RoutingDecisionCause] = ( + "semantic_error" + if outcome.failure_reason is not None + else "semantic_match" + if outcome.route_name + else "semantic_no_match" + ) + decision: Final[StandardLoggingRoutingDecision] = { + "router_model_name": self.model_name, + "router_type": "semantic", + "routed_model": routed_model, + "cause": cause, + "classifier_model": self.embedding_model, + **({"classifier_failure_reason": outcome.failure_reason} if outcome.failure_reason is not None else {}), + **({"classifier_error_type": outcome.error_type} if outcome.error_type is not None else {}), + } return PreRoutingHookResponse( - model=route_name or self.default_model, + model=routed_model, messages=messages, + routing_decision=decision, ) - async def _matched_route_name( + async def _match_route( self, routelayer: "SemanticRouter", text: str, request_kwargs: Mapping[str, object] - ) -> str | None: - """Name of the route `text` matches, or None when nothing matched or the match failed. + ) -> _SemanticMatchOutcome: + """Matched route or the reason a default model will serve the request. `text` is embedded here rather than by `routelayer(text=...)` so the caller's metadata reaches `aembedding()` and the embedding's spend lands on the key/team that sent the request; @@ -247,10 +278,16 @@ class AutoRouter(CustomLogger): verbose_router_logger.warning( "AutoRouter: semantic routing failed (%s), falling back to default model %s", e, self.default_model ) - return None + failure: Final = e.__cause__ if isinstance(e, ValueError) and e.__cause__ is not None else e + return _SemanticMatchOutcome( + failure_reason="timeout" + if isinstance(failure, (TimeoutError, asyncio.TimeoutError, LiteLLMTimeout)) + else "classifier_error", + error_type=type(failure).__name__, + ) verbose_router_logger.debug("route_choice: %s", route_choice) if isinstance(route_choice, RouteChoice): - return route_choice.name + return _SemanticMatchOutcome(route_name=route_choice.name) if isinstance(route_choice, list) and route_choice: - return route_choice[0].name - return None + return _SemanticMatchOutcome(route_name=route_choice[0].name) + return _SemanticMatchOutcome() diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index bdb081bad9a..d7b88476f62 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -42,6 +42,7 @@ from litellm.constants import ( SESSION_ID_GENERATED_METADATA_KEY, ) from litellm.integrations.custom_logger import CustomLogger +from litellm.integrations.otel.runtime import phase_event from litellm.litellm_core_utils.classifier_logging import masked_originating_request from litellm.litellm_core_utils.core_helpers import ( _get_parent_otel_span_from_kwargs, @@ -77,6 +78,7 @@ from litellm.types.llms.openai import ( ) from litellm.types.utils import ( AUTOROUTER_CLASSIFIER_CALL_ORIGIN, + ClassifierFailureReason, ModelResponse, RoutingDecisionCause, StandardLoggingHeuristicV2Forecast, @@ -1085,10 +1087,21 @@ class ClassificationOutcome(NamedTuple): llm_v2_forecast: LLMV2Decision | None = None jev_verdict: JevVerdict | None = None heuristic_v2_forecast: StandardLoggingHeuristicV2Forecast | None = None + classifier_failure_reason: ClassifierFailureReason | None = None + classifier_error_type: str | None = None -def _with_signal(outcome: ClassificationOutcome, signal: str | None) -> ClassificationOutcome: - return outcome if signal is None else outcome._replace(signals=(*outcome.signals, signal)) +def _with_classifier_failure( + outcome: ClassificationOutcome, + failure_reason: ClassifierFailureReason, + error_type: str | None, + signal: str | None, +) -> ClassificationOutcome: + return outcome._replace( + signals=(*outcome.signals, signal) if signal is not None else outcome.signals, + classifier_failure_reason=failure_reason, + classifier_error_type=error_type, + ) def _with_llm_v2_forecast( @@ -1221,6 +1234,12 @@ def _is_classifier_timeout(exc: BaseException) -> bool: return isinstance(exc, LiteLLMTimeout) +def _classifier_failure_reason(exc: Exception) -> ClassifierFailureReason: + if _is_classifier_timeout(exc): + return "timeout" + return "invalid_response" if isinstance(exc, ValidationError) else "classifier_error" + + def _allowed(models: tuple[str, ...], fit_filter: frozenset[str] | None) -> tuple[str, ...]: return models if fit_filter is None else tuple(model for model in models if model in fit_filter) @@ -1835,6 +1854,8 @@ class ComplexityRouter(CustomLogger): escalated: bool = False, classifier_model: str | None = None, classifier_cost: float | None = None, + classifier_failure_reason: ClassifierFailureReason | None = None, + classifier_error_type: str | None = None, conversation_continuing: bool = True, tier_litellm_params: Mapping[str, object] | None = None, context_escalation_original_tier: ComplexityTier | str | None = None, @@ -1853,6 +1874,12 @@ class ComplexityRouter(CustomLogger): routed_model=routed_model, cause=cause, conversation_continuing=conversation_continuing, + **( + {"classifier_failure_reason": classifier_failure_reason} + if classifier_failure_reason is not None + else {} + ), + **({"classifier_error_type": classifier_error_type} if classifier_error_type is not None else {}), ) if (baseline := self.savings_baseline) is not None: decision["savings_baseline_model"] = baseline.model @@ -2094,7 +2121,9 @@ class ComplexityRouter(CustomLogger): permit: Final = breaker.acquire_permit() if breaker is not None else None if breaker is not None and permit is None: return self._capability_classifier_failure_outcome( - "capability classifier circuit is open", signal=_CLASSIFIER_CIRCUIT_OPEN_SIGNAL + "capability classifier circuit is open", + failure_reason="circuit_open", + signal=_CLASSIFIER_CIRCUIT_OPEN_SIGNAL, ) try: tier, classifier_cost, forecast = await self._classify_with_capability_llm(prompt, request_kwargs, messages) @@ -2118,9 +2147,20 @@ class ComplexityRouter(CustomLogger): except Exception as e: # noqa: BLE001 -- every unavailable or invalid judge verdict must fail closed if breaker is not None and permit is not None: breaker.record_failure(permit, is_timeout=_is_classifier_timeout(e)) - return self._capability_classifier_failure_outcome(f"capability classifier failed ({type(e).__name__})") + return self._capability_classifier_failure_outcome( + f"capability classifier failed ({type(e).__name__})", + failure_reason=_classifier_failure_reason(e), + error_type=type(e).__name__, + ) - def _capability_classifier_failure_outcome(self, reason: str, signal: str | None = None) -> ClassificationOutcome: + def _capability_classifier_failure_outcome( + self, + reason: str, + *, + failure_reason: ClassifierFailureReason, + error_type: str | None = None, + signal: str | None = None, + ) -> ClassificationOutcome: """Fail closed to the configured capable tier without consulting another taxonomy.""" capability: Final = self.config.capability_classifier_config if capability is None: @@ -2141,6 +2181,8 @@ class ComplexityRouter(CustomLogger): score=None, signals=signals, cause="capability_classifier_fallback", + classifier_failure_reason=failure_reason, + classifier_error_type=error_type, ) async def _llm_classifier_outcome( @@ -2164,6 +2206,7 @@ class ComplexityRouter(CustomLogger): prompt, system_prompt, scored, + failure_reason="circuit_open", signal=_CLASSIFIER_CIRCUIT_OPEN_SIGNAL, ) try: @@ -2193,7 +2236,12 @@ class ComplexityRouter(CustomLogger): if breaker is not None and permit is not None: breaker.record_failure(permit, is_timeout=_is_classifier_timeout(e)) return self._classifier_failure_outcome( - f"LLM classifier failed ({type(e).__name__})", prompt, system_prompt, scored + f"LLM classifier failed ({type(e).__name__})", + prompt, + system_prompt, + scored, + failure_reason=_classifier_failure_reason(e), + error_type=type(e).__name__, ) async def _jev_classifier_outcome( @@ -2206,11 +2254,16 @@ class ComplexityRouter(CustomLogger): config: Final = self.config.opensource_classifier_config client: Final = self._jev_client if config is None or client is None: - return self._classifier_failure_outcome("jev classifier is not configured", prompt, system_prompt) + return self._classifier_failure_outcome( + "jev classifier is not configured", prompt, system_prompt, failure_reason="not_configured" + ) marker_pairs: Final = self._reminder_markers_for_request(request_kwargs or EMPTY_MAPPING) if _encrypted_classifier_task(request_kwargs, marker_pairs) is not None: return self._classifier_failure_outcome( - "jev classifier does not support encrypted agent tasks", prompt, system_prompt + "jev classifier does not support encrypted agent tasks", + prompt, + system_prompt, + failure_reason="unsupported_input", ) breaker: Final = self._classifier_circuit_breaker permit: Final = breaker.acquire_permit() if breaker is not None else None @@ -2219,6 +2272,7 @@ class ComplexityRouter(CustomLogger): "jev classifier circuit is open", prompt, system_prompt, + failure_reason="circuit_open", signal=_CLASSIFIER_CIRCUIT_OPEN_SIGNAL, ) criteria: Final[Mapping[str, str]] = ( @@ -2288,7 +2342,11 @@ class ComplexityRouter(CustomLogger): if breaker is not None and permit is not None: breaker.record_failure(permit, is_timeout=_is_classifier_timeout(e)) return self._classifier_failure_outcome( - f"jev classifier failed ({type(e).__name__})", prompt, system_prompt + f"jev classifier failed ({type(e).__name__})", + prompt, + system_prompt, + failure_reason=_classifier_failure_reason(e), + error_type=type(e).__name__, ) def _classifier_failure_outcome( @@ -2297,6 +2355,9 @@ class ComplexityRouter(CustomLogger): prompt: str, system_prompt: str | None, scored: ClassificationOutcome | None = None, + *, + failure_reason: ClassifierFailureReason, + error_type: str | None = None, signal: str | None = None, ) -> ClassificationOutcome: """The outcome when the LLM classifier or classifier plugin produced no usable tier: @@ -2307,35 +2368,44 @@ class ComplexityRouter(CustomLogger): v2: Final = self.config.llm_v2_config if v2 is not None: verbose_router_logger.warning("ComplexityRouter: %s, routing to llm_v2 capable tier", reason) - return _with_signal( + return _with_classifier_failure( ClassificationOutcome( tier=ComplexityTier(v2.capable_tier), score=None, signals=("llm-v2:fallback-capable",), cause="llm_v2_fallback", ), + failure_reason, + error_type, signal, ) fallback_tier: Final = self.config.fallback_tier if fallback_tier is not None: verbose_router_logger.warning("ComplexityRouter: %s, routing to fallback_tier %s", reason, fallback_tier) - return _with_signal( + return _with_classifier_failure( ClassificationOutcome( tier=fallback_tier, score=None, signals=(f"classifier-fallback:{fallback_tier}",), cause="classifier_fallback", ), + failure_reason, + error_type, signal, ) verbose_router_logger.warning( "ComplexityRouter: %s, falling back to %s", reason, self.config.classifier_fallback ) if self.config.classifier_fallback == "default_model": - return _with_signal(self._default_model_fallback_outcome(), signal) + return _with_classifier_failure(self._default_model_fallback_outcome(), failure_reason, error_type, signal) if scored is not None: - return _with_signal(scored, signal) - return _with_signal(self._classify_locally(prompt, system_prompt), signal) + return _with_classifier_failure(scored, failure_reason, error_type, signal) + return _with_classifier_failure( + self._classify_locally(prompt, system_prompt), + failure_reason, + error_type, + signal, + ) async def _classify_with_plugin( self, @@ -2349,7 +2419,9 @@ class ComplexityRouter(CustomLogger): plugin: Final = self.config.classifier_plugin if plugin is None: - return self._classifier_failure_outcome("classifier_plugin is not set", prompt, system_prompt) + return self._classifier_failure_outcome( + "classifier_plugin is not set", prompt, system_prompt, failure_reason="not_configured" + ) kwargs: Final = request_kwargs if request_kwargs is not None else EMPTY_MAPPING pools: Final = self._tier_pools() try: @@ -2365,29 +2437,48 @@ class ComplexityRouter(CustomLogger): verdict: Final = await asyncio.wait_for( plugin.classify(context), timeout=self.config.classifier_plugin_timeout_ms / 1000 ) - except asyncio.TimeoutError: + except asyncio.TimeoutError as e: return self._classifier_failure_outcome( - f"classifier plugin timed out after {self.config.classifier_plugin_timeout_ms}ms", prompt, system_prompt + f"classifier plugin timed out after {self.config.classifier_plugin_timeout_ms}ms", + prompt, + system_prompt, + failure_reason="timeout", + error_type=type(e).__name__, ) except Exception as e: # noqa: BLE001 -- an operator hook can fail in arbitrary ways (network, bug); any failure must fall back rather than fail the request - return self._classifier_failure_outcome(f"classifier plugin failed ({e})", prompt, system_prompt) + return self._classifier_failure_outcome( + f"classifier plugin failed ({e})", + prompt, + system_prompt, + failure_reason=_classifier_failure_reason(e), + error_type=type(e).__name__, + ) if verdict is None: - return self._classifier_failure_outcome("classifier plugin declined to classify", prompt, system_prompt) + return self._classifier_failure_outcome( + "classifier plugin declined to classify", prompt, system_prompt, failure_reason="declined" + ) if not isinstance(verdict, str): return self._classifier_failure_outcome( f"classifier plugin returned a non-string verdict of type {type(verdict).__name__}", prompt, system_prompt, + failure_reason="invalid_response", ) tier: Final = self.config.resolve_classified_tier(verdict) if tier is None: return self._classifier_failure_outcome( - f"classifier plugin returned unknown tier {verdict!r}", prompt, system_prompt + f"classifier plugin returned unknown tier {verdict!r}", + prompt, + system_prompt, + failure_reason="invalid_response", ) tier_key: Final = _tier_name(tier) if not pools.get(tier_key): return self._classifier_failure_outcome( - f"classifier plugin returned tier {tier_key!r}, which has no models configured", prompt, system_prompt + f"classifier plugin returned tier {tier_key!r}, which has no models configured", + prompt, + system_prompt, + failure_reason="invalid_response", ) return ClassificationOutcome( tier=tier, @@ -2651,9 +2742,13 @@ class ComplexityRouter(CustomLogger): verdict: Final = LLMV2Verdict.model_validate_json(extract_classifier_json(content)) except ValidationError as error: _log_rejected_classifier_verdict(error, content, request_kwargs) - return self._classifier_failure_outcome("Invalid LLM V2 forecast", prompt, system_prompt)._replace( - classifier_cost=classifier_cost - ) + return self._classifier_failure_outcome( + "Invalid LLM V2 forecast", + prompt, + system_prompt, + failure_reason="invalid_response", + error_type=type(error).__name__, + )._replace(classifier_cost=classifier_cost) decision: Final = v2.classify(verdict) return ClassificationOutcome( tier=ComplexityTier(v2.efficient_tier if decision.use_efficient else v2.capable_tier), @@ -4094,6 +4189,10 @@ class ComplexityRouter(CustomLogger): try: semantic_tier: Final = await self._semantic_tier_override(user_message, request_kwargs) except Exception as e: # noqa: BLE001 -- embedding call can fail many ways (timeout, provider/network/parse error); any failure must fall back to scoring, never fail the request + phase_event( + "litellm.router.semantic_keyword_fallback", + {"litellm.router.failure_reason": _classifier_failure_reason(e), "error.type": type(e).__name__}, + ) verbose_router_logger.warning( "ComplexityRouter: semantic keyword matching failed (%s), falling back to complexity scoring", e ) @@ -4728,6 +4827,8 @@ class ComplexityRouter(CustomLogger): conversation_continuing=conversation_continuing, cause=outcome.cause, signals=outcome.signals, + classifier_failure_reason=outcome.classifier_failure_reason, + classifier_error_type=outcome.classifier_error_type, escalation_keyword=escalation_keyword, escalated=False, tier_litellm_params=fallback_tier_params, @@ -4840,6 +4941,8 @@ class ComplexityRouter(CustomLogger): escalated=escalated, classifier_model=classifier_model, classifier_cost=outcome.classifier_cost, + classifier_failure_reason=outcome.classifier_failure_reason, + classifier_error_type=outcome.classifier_error_type, tier_litellm_params=tier_litellm_params, context_escalation_original_tier=context_original_tier, ) diff --git a/litellm/types/router.py b/litellm/types/router.py index 8fd53079156..66a5b3540f9 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -1214,6 +1214,8 @@ class TaggedPreRoutingStrategy(Generic[_PreRoutingStrategyT_co]): tags: tuple[str, ...] strategy: _PreRoutingStrategyT_co + definition_fingerprint: str | None = None + deployment: Deployment | None = None @dataclass(frozen=True, slots=True) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 959e9182dd3..a0279bcadc7 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3061,6 +3061,14 @@ RoutingDecisionCause = Literal[ "keyword", "quality_tier", "bandit", + "semantic_match", + "semantic_no_match", + "semantic_error", +] + + +ClassifierFailureReason = Literal[ + "timeout", "circuit_open", "not_configured", "unsupported_input", "invalid_response", "declined", "classifier_error" ] @@ -3093,7 +3101,10 @@ class StandardLoggingRoutingDecision(TypedDict, total=False): """Per-request provenance for a pre-routing strategy (auto-router) decision.""" router_model_name: str - router_type: Literal["complexity", "adaptive", "quality"] + router_type: ReadOnly[Literal["complexity", "adaptive", "quality", "semantic"]] + router_config_id: ReadOnly[str] + router_config_updated_at: ReadOnly[str] + router_config_fingerprint: ReadOnly[str] routed_model: str cause: RoutingDecisionCause tier: str @@ -3105,6 +3116,8 @@ class StandardLoggingRoutingDecision(TypedDict, total=False): escalation_keyword: str classifier_model: str classifier_cost: float + classifier_failure_reason: ReadOnly[ClassifierFailureReason] + classifier_error_type: ReadOnly[str] classifier_probabilities: ReadOnly[Mapping[str, float]] classifier_confidence: ReadOnly[float] heuristic_v2_forecast: ReadOnly[StandardLoggingHeuristicV2Forecast] @@ -3143,6 +3156,9 @@ DERIVED_ROUTING_DECISION_FIELDS: Final[frozenset[str]] = frozenset( { "router_model_name", "router_type", + "router_config_id", + "router_config_updated_at", + "router_config_fingerprint", "routed_model", "cause", "tier", @@ -3151,6 +3167,8 @@ DERIVED_ROUTING_DECISION_FIELDS: Final[frozenset[str]] = frozenset( "score", "classifier_model", "classifier_cost", + "classifier_failure_reason", + "classifier_error_type", "classifier_probabilities", "classifier_confidence", "heuristic_v2_forecast", diff --git a/schema.prisma b/schema.prisma index 4aabd56131b..514df905866 100644 --- a/schema.prisma +++ b/schema.prisma @@ -1965,3 +1965,12 @@ model LiteLLM_LensWorker { token_hash String @unique data Json } + +model LiteLLM_LensDataset { + id String + revision Int + created_at DateTime + data Json + + @@id([id, revision]) +} diff --git a/tests/integration/database/test_lens_dataset_repository.py b/tests/integration/database/test_lens_dataset_repository.py new file mode 100644 index 00000000000..baeaa4d50b4 --- /dev/null +++ b/tests/integration/database/test_lens_dataset_repository.py @@ -0,0 +1,149 @@ +import os +from collections.abc import AsyncIterator +from datetime import datetime, timedelta, timezone +from typing import Final +from uuid import uuid4 + +import pytest +import pytest_asyncio +from prisma import Prisma +from prisma.types import DatasourceOverride + +from litellm.proxy.db.prisma_client import PrismaWrapper +from litellm.proxy.lens.dataset_repository import DatasetRepository, StoredSummary +from litellm.proxy.lens.models import CaseSource, Dataset, DatasetCase, DatasetMessage, DatasetSummary +from litellm.proxy.lens.repository import WriterDatabase + +SAVED_AT: Final = datetime(2026, 3, 1, 12, 0, 0, 123000, tzinfo=timezone.utc) + + +@pytest_asyncio.fixture(loop_scope="function") +async def lens_db() -> AsyncIterator[Prisma]: + async with Prisma(datasource=DatasourceOverride(url=os.environ["DATABASE_URL"])) as db: + yield db + + +@pytest_asyncio.fixture(loop_scope="function") +async def dataset_ids(lens_db: Prisma) -> AsyncIterator[tuple[str, ...]]: + ids: Final = tuple(uuid4().hex for _ in range(3)) + yield ids + await lens_db.execute_raw('DELETE FROM "LiteLLM_LensDataset" WHERE id = ANY($1::text[])', list(ids)) + + +def _repo(db: Prisma) -> DatasetRepository: + return DatasetRepository(WriterDatabase(PrismaWrapper(db))) + + +def _dataset(dataset_id: str, revision: int, case_count: int, team_id: str = "team-a") -> Dataset: + return Dataset( + id=dataset_id, + name=f"Dataset r{revision}", + agent_name="support-agent", + team_id=team_id, + created_at=SAVED_AT, + revision=revision, + created_by="user-1", + cases=tuple( + DatasetCase( + id=f"case-{revision}-{i}", + messages=(DatasetMessage(role="user", content=f"question {i}"),), + reply=f"answer {i}", + source=CaseSource(trace_id=f"trace-{i}"), + ) + for i in range(case_count) + ), + ) + + +@pytest.mark.asyncio +async def test_get_returns_requested_revision_and_defaults_to_latest( + lens_db: Prisma, dataset_ids: tuple[str, ...] +) -> None: + repo: Final = _repo(lens_db) + dataset_id: Final = dataset_ids[0] + revisions: Final = tuple(_dataset(dataset_id, revision, revision) for revision in (1, 3, 2)) + assert [await repo.insert(dataset, SAVED_AT) for dataset in revisions] == [True, True, True] + + assert await repo.get(dataset_id, revision=1) == revisions[0] + assert await repo.get(dataset_id, revision=2) == revisions[2] + assert await repo.get(dataset_id) == revisions[1] + + +@pytest.mark.asyncio +async def test_get_unknown_dataset_or_revision_returns_none(lens_db: Prisma, dataset_ids: tuple[str, ...]) -> None: + repo: Final = _repo(lens_db) + assert await repo.insert(_dataset(dataset_ids[0], 1, 1), SAVED_AT) + + assert await repo.get(dataset_ids[1]) is None + assert await repo.get(dataset_ids[1], revision=1) is None + assert await repo.get(dataset_ids[0], revision=2) is None + + +@pytest.mark.asyncio +async def test_inserting_an_existing_revision_is_rejected_and_keeps_the_first( + lens_db: Prisma, dataset_ids: tuple[str, ...] +) -> None: + repo: Final = _repo(lens_db) + original: Final = _dataset(dataset_ids[0], 1, 1) + overwrite: Final = _dataset(dataset_ids[0], 1, 4).model_copy(update={"name": "Overwritten"}) + + assert await repo.insert(original, SAVED_AT) is True + assert await repo.insert(overwrite, SAVED_AT + timedelta(days=1)) is False + + assert await repo.get(dataset_ids[0], revision=1) == original + summaries: Final = tuple(s for s in await repo.summaries() if s.summary.id == dataset_ids[0]) + assert tuple(s.summary.updated_at for s in summaries) == (SAVED_AT,) + + +@pytest.mark.asyncio +async def test_summaries_list_each_dataset_once_at_its_latest_revision_newest_first( + lens_db: Prisma, dataset_ids: tuple[str, ...] +) -> None: + repo: Final = _repo(lens_db) + older, newer, single = dataset_ids + writes: Final = ( + (_dataset(older, 1, 1, "team-a"), SAVED_AT), + (_dataset(older, 2, 3, "team-a"), SAVED_AT + timedelta(minutes=1)), + (_dataset(newer, 1, 5, "team-b"), SAVED_AT + timedelta(minutes=2)), + (_dataset(newer, 2, 2, "team-b"), SAVED_AT + timedelta(minutes=4)), + (_dataset(single, 1, 0, ""), SAVED_AT + timedelta(minutes=3)), + ) + assert [await repo.insert(dataset, saved_at) for dataset, saved_at in writes] == [True] * len(writes) + + summaries: Final = tuple(s for s in await repo.summaries() if s.summary.id in dataset_ids) + + assert summaries == ( + StoredSummary( + team_id="team-b", + summary=DatasetSummary( + id=newer, + name="Dataset r2", + agent_name="support-agent", + revision=2, + case_count=2, + updated_at=SAVED_AT + timedelta(minutes=4), + ), + ), + StoredSummary( + team_id="", + summary=DatasetSummary( + id=single, + name="Dataset r1", + agent_name="support-agent", + revision=1, + case_count=0, + updated_at=SAVED_AT + timedelta(minutes=3), + ), + ), + StoredSummary( + team_id="team-a", + summary=DatasetSummary( + id=older, + name="Dataset r2", + agent_name="support-agent", + revision=2, + case_count=3, + updated_at=SAVED_AT + timedelta(minutes=1), + ), + ), + ) diff --git a/tests/unit/integrations/otel/test_otel_v2_logger.py b/tests/unit/integrations/otel/test_otel_v2_logger.py index 6c99e8bf14d..3b1dd592d08 100644 --- a/tests/unit/integrations/otel/test_otel_v2_logger.py +++ b/tests/unit/integrations/otel/test_otel_v2_logger.py @@ -182,10 +182,63 @@ def test_async_log_success_event_emits_llm_call_span(): assert span.attributes[GenAI.OPERATION_NAME] == "chat" assert span.attributes[GenAI.REQUEST_MODEL] == "gpt-4o" assert span.attributes[LiteLLM.CALL_ID] == "call_1" + assert not any(key.startswith("litellm.routing.") for key in span.attributes) # Success leaves status UNSET (semconv default), not forced OK. assert span.status.status_code is StatusCode.UNSET +@pytest.mark.parametrize("fail", [False, True]) +@pytest.mark.parametrize("legacy_compat", [False, True]) +def test_routing_diagnostics_reach_llm_spans_without_promoting_prompt_text(fail: bool, legacy_compat: bool) -> None: + logger, exporter = _logger(legacy_compat=legacy_compat) + decision: Final = { + "router_model_name": "auto-router", + "router_type": "complexity", + "router_config_id": "definition-1", + "router_config_updated_at": "2026-10-06T00:00:00Z", + "router_config_fingerprint": "definition-hash", + "routed_model": "answer", + "cause": "heuristic", + "score": 0.25, + "classifier_cost": 0.001, + "classifier_failure_reason": "timeout", + "classifier_error_type": "TimeoutError", + "escalated": False, + } + payload: Final = _payload( + metadata={ + "routing_decision": { + **decision, + "signals": ["private prompt"], + "matched_keyword": "private prompt", + "escalation_keyword": "private prompt", + "classifier_crux": "private prompt", + "tier_litellm_params": {"api_key": "private key"}, + "classifier_probabilities": {"easy": 0.25}, + "unknown_field": "private config", + "tier_label": {"invalid": "scalar field"}, + } + }, + status="failure" if fail else "success", + ) + _emit_llm(logger, _kwargs(payload), fail=fail) + (span,) = exporter.get_finished_spans() + actual: Final = {key: value for key, value in span.attributes.items() if key.startswith("litellm.routing.")} + assert actual == {f"litellm.routing.{key}": value for key, value in decision.items()} + assert isinstance(actual["litellm.routing.score"], float) + assert actual["litellm.routing.escalated"] is False + + +@pytest.mark.parametrize("decision", ["custom", 42, ["custom"]]) +def test_malformed_routing_metadata_preserves_the_completed_llm_span(decision: object) -> None: + logger, exporter = _logger() + _emit_llm(logger, _kwargs(_payload(metadata={"routing_decision": decision}))) + (span,) = exporter.get_finished_spans() + assert span.attributes[GenAI.REQUEST_MODEL] == "gpt-4o" + assert span.attributes[LiteLLM.CALL_ID] == "call_1" + assert not any(key.startswith("litellm.routing.") for key in span.attributes) + + def test_llm_call_span_carries_the_callers_conversation_id(): logger, exporter = _logger() kwargs = {**_kwargs(), "litellm_params": {"litellm_session_id": "conv-42", "metadata": {}}} @@ -1103,6 +1156,29 @@ def test_phase_event_lands_on_root_span_even_inside_active_phase_span(): assert by_name["auth /chat/completions"].events == () +def test_routing_attributes_stay_on_the_active_phase_while_retry_events_stay_on_root() -> None: + logger, exporter = _logger() + root: Final = logger.tracer.start_span("request", kind=SpanKind.SERVER) + set_request_root_span(root) + with trace.use_span(root, end_on_exit=True): + logger.set_phase_attributes({"litellm.routing.cause": "outside phase"}) + with logger.start_phase_span("route auto-router"): + logger.set_phase_attributes({"litellm.routing.router_config_id": "definition-1"}) + with logger.tracer.start_as_current_span("classifier transport"): + logger.set_phase_attributes({"litellm.routing.cause": "heuristic"}) + logger.add_phase_event("litellm.request.retry", {"error.type": "TimeoutError"}) + logger.set_phase_attributes({"litellm.routing.cause": "after phase"}) + by_name: Final = {span.name: span for span in exporter.get_finished_spans()} + route: Final = by_name["route auto-router"] + assert route.attributes["litellm.routing.router_config_id"] == "definition-1" + assert route.attributes["litellm.routing.cause"] == "heuristic" + assert route.parent.span_id == root.get_span_context().span_id + assert route.events == () + assert "litellm.routing.cause" not in by_name["request"].attributes + assert "litellm.routing.cause" not in by_name["classifier transport"].attributes + assert by_name["request"].events[0].attributes["error.type"] == "TimeoutError" + + def test_live_llm_span_anchors_to_root_with_no_active_span(): """Bug 2 (pass-through), live path: even with no span active at ``pre_call``, the anchor is a recordable parent, so the span opens live under the server root diff --git a/tests/unit/integrations/otel/test_runtime.py b/tests/unit/integrations/otel/test_runtime.py index 5747be27656..8788c5619ee 100644 --- a/tests/unit/integrations/otel/test_runtime.py +++ b/tests/unit/integrations/otel/test_runtime.py @@ -10,6 +10,11 @@ import lock. These tests pin the import to a single resolution. import builtins import importlib.abc import sys +from collections.abc import Sequence +from types import ModuleType +from typing import Final + +import pytest import litellm.integrations.otel.runtime as runtime @@ -92,4 +97,28 @@ def test_phase_span_does_not_import_the_proxy_in_an_sdk_process(monkeypatch): with runtime.phase_span("route gpt-5-mini") as span: assert span is None + assert runtime.phase_attributes({"litellm.routing.score": 0.25}) is None assert proxy_imports == [] + + +def test_phase_attributes_no_op_when_sdk_import_is_unavailable(monkeypatch: pytest.MonkeyPatch) -> None: + runtime._otel_runtime.cache_clear() + real_import: Final = builtins.__import__ + + def without_sdk( + name: str, + globals: dict[str, object] | None = None, + locals: dict[str, object] | None = None, + fromlist: Sequence[str] = (), + level: int = 0, + ) -> ModuleType: + if name == "litellm.integrations.otel" and "logger" in fromlist: + raise ImportError("OpenTelemetry SDK is not installed") + return real_import(name, globals, locals, fromlist, level) + + monkeypatch.setattr(builtins, "__import__", without_sdk) + try: + assert runtime.phase_attributes({"litellm.routing.score": 0.25}) is None + assert runtime._otel_runtime() is None + finally: + runtime._otel_runtime.cache_clear() diff --git a/tests/unit/integrations/test_opentelemetry.py b/tests/unit/integrations/test_opentelemetry.py index 2ce3561d441..27242199cdb 100644 --- a/tests/unit/integrations/test_opentelemetry.py +++ b/tests/unit/integrations/test_opentelemetry.py @@ -310,6 +310,65 @@ class TestOpenTelemetryTeamAttributesOnChildSpans(unittest.TestCase): ) +def test_v1_routing_diagnostics_add_scalar_attributes_and_preserve_legacy_metadata() -> None: + exporter: Final = InMemorySpanExporter() + provider: Final = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + otel: Final = OpenTelemetry(tracer_provider=provider) + decision: Final = { + "router_model_name": "auto-router", + "routed_model": "answer", + "router_config_id": "definition-1", + "cause": "heuristic", + "classifier_failure_reason": "timeout", + "classifier_error_type": "TimeoutError", + "classifier_cost": 0.001, + } + with provider.get_tracer(__name__).start_as_current_span("inference") as span: + otel.set_attributes( + span, + { + "model": "answer", + "standard_logging_object": { + "id": "routing-call", + "call_type": "acompletion", + "metadata": {"routing_decision": decision}, + }, + }, + {"model": "answer", "choices": [], "usage": {}}, + ) + (finished,) = exporter.get_finished_spans() + actual: Final = { + key: value for key, value in finished.attributes.items() if key.startswith("litellm.routing.") + } + assert actual == {f"litellm.routing.{key}": value for key, value in decision.items()} + assert finished.attributes["metadata.routing_decision"] == str(decision) + + +@pytest.mark.parametrize("decision", ["custom", 42, ["custom"]]) +def test_v1_malformed_routing_metadata_preserves_inference_attributes(decision: object) -> None: + exporter: Final = InMemorySpanExporter() + provider: Final = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + otel: Final = OpenTelemetry(tracer_provider=provider) + with provider.get_tracer(__name__).start_as_current_span("inference") as span: + otel.set_attributes( + span, + { + "model": "answer", + "standard_logging_object": { + "id": "routing-call", + "call_type": "acompletion", + "metadata": {"routing_decision": decision}, + }, + }, + {"model": "answer", "choices": [], "usage": {}}, + ) + (finished,) = exporter.get_finished_spans() + assert finished.attributes["gen_ai.request.model"] == "answer" + assert not any(key.startswith("litellm.routing.") for key in finished.attributes) + + class TestOpenTelemetryCostBreakdown(unittest.TestCase): def test_cost_breakdown_emitted_to_otel_span(self): """ diff --git a/tests/unit/proxy/db/test_daily_spend_bulk_upsert.py b/tests/unit/proxy/db/test_daily_spend_bulk_upsert.py index 7893fb82281..5bce9137666 100644 --- a/tests/unit/proxy/db/test_daily_spend_bulk_upsert.py +++ b/tests/unit/proxy/db/test_daily_spend_bulk_upsert.py @@ -13,6 +13,7 @@ from litellm.proxy.db.daily_spend_bulk_upsert import ( merge_by_conflict_key, ) from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter +from litellm.proxy.db.rollup_lock_timeout import ROLLUP_LOCK_TIMEOUT_SQL TAG_TABLE = DAILY_SPEND_TABLES["tag"] USER_TABLE = DAILY_SPEND_TABLES["user"] @@ -146,8 +147,12 @@ def test_non_tag_tables_carry_no_request_id_column(): class _RecordingDb: def __init__(self) -> None: self.statements: list[tuple[str, tuple[object, ...]]] = [] + self.session_settings: list[tuple[str, tuple[object, ...]]] = [] async def execute_raw(self, query: str, *args: object) -> int: + if query == ROLLUP_LOCK_TIMEOUT_SQL: + self.session_settings.append((query, args)) + return 0 self.statements.append((query, args)) return len(args) diff --git a/tests/unit/proxy/db/test_db_spend_update_writer.py b/tests/unit/proxy/db/test_db_spend_update_writer.py index fe5e31d00b2..8822ecd5bd0 100644 --- a/tests/unit/proxy/db/test_db_spend_update_writer.py +++ b/tests/unit/proxy/db/test_db_spend_update_writer.py @@ -19,6 +19,7 @@ from redis.exceptions import DataError import litellm from litellm._logging import verbose_proxy_logger from litellm._service_logger import ServiceTypes +from litellm.constants import SPEND_ROLLUP_LOCK_TIMEOUT_MS from litellm.proxy._types import DailyTagSpendTransaction, Litellm_EntityType, SpendUpdateQueueItem from litellm.proxy.db.db_spend_update_writer import ( _TEAM_ADVISORY_LOCK_SQL, @@ -33,6 +34,7 @@ from litellm.proxy.db.db_transaction_queue.spend_update_queue import SpendUpdate from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import ( build_window_spend_transaction, ) +from litellm.proxy.db.rollup_lock_timeout import ROLLUP_LOCK_TIMEOUT_SQL from tests.unit.proxy.db.fake_prisma_engine import engine_call @@ -103,11 +105,21 @@ async def test_update_database_attributes_router_rejected_failure_to_model_group ) with ( - patch("litellm.proxy.proxy_server.disable_spend_logs", True), # test-quality-ok: update_database reads this proxy_server module global at call time; no injection seam - patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), # test-quality-ok: update_database reads this proxy_server module global at call time; no injection seam - patch("litellm.proxy.proxy_server.user_api_key_cache", MagicMock()), # test-quality-ok: update_database reads this proxy_server module global at call time; no injection seam - patch("litellm.proxy.proxy_server.litellm_proxy_budget_name", "test-budget"), # test-quality-ok: update_database reads this proxy_server module global at call time; no injection seam - patch("litellm.proxy.proxy_server.llm_router", llm_router), # test-quality-ok: get_llm_router reads this proxy_server module global at call time; no injection seam + patch( + "litellm.proxy.proxy_server.disable_spend_logs", True + ), # test-quality-ok: update_database reads this proxy_server module global at call time; no injection seam + patch( + "litellm.proxy.proxy_server.prisma_client", MagicMock() + ), # test-quality-ok: update_database reads this proxy_server module global at call time; no injection seam + patch( + "litellm.proxy.proxy_server.user_api_key_cache", MagicMock() + ), # test-quality-ok: update_database reads this proxy_server module global at call time; no injection seam + patch( + "litellm.proxy.proxy_server.litellm_proxy_budget_name", "test-budget" + ), # test-quality-ok: update_database reads this proxy_server module global at call time; no injection seam + patch( + "litellm.proxy.proxy_server.llm_router", llm_router + ), # test-quality-ok: get_llm_router reads this proxy_server module global at call time; no injection seam ): await db_writer.update_database( token="test-token", @@ -320,7 +332,9 @@ async def test_a_routed_request_reaches_the_auto_router_rollup_whether_or_not_sp } with ( - patch("litellm.proxy.proxy_server.disable_spend_logs", disable_spend_logs), # test-quality-ok: update_database reads this proxy_server module global at call time; no injection seam + patch( + "litellm.proxy.proxy_server.disable_spend_logs", disable_spend_logs + ), # test-quality-ok: update_database reads this proxy_server module global at call time; no injection seam patch("litellm.proxy.proxy_server.prisma_client", prisma), patch("litellm.proxy.proxy_server.litellm_proxy_budget_name", "test-budget"), patch( @@ -356,9 +370,13 @@ class _RecordingDb: def __init__(self, execute_raw: Callable[[], int] | None = None) -> None: self.statements: list[Statement] = [] + self.session_settings: list[Statement] = [] self._execute_raw = execute_raw async def execute_raw(self, query: str, *args: object) -> int: + if query == ROLLUP_LOCK_TIMEOUT_SQL: + self.session_settings.append((query, args)) + return 0 self.statements.append((query, args)) if self._execute_raw is not None: return self._execute_raw() @@ -752,7 +770,7 @@ async def test_update_tag_db_with_valid_tags(): """ Test that _update_tag_db correctly processes valid tags and adds them to the spend update queue. """ - from litellm.proxy._types import Litellm_EntityType, SpendUpdateQueueItem + from litellm.proxy._types import Litellm_EntityType writer = DBSpendUpdateWriter() mock_prisma = MagicMock() @@ -1048,7 +1066,7 @@ async def test_commit_spend_updates_to_db_writes_team_member_spend_in_one_roster ), ) - lock_call, spend_call = mock_transaction.execute_raw.await_args_list + _lock_timeout_call, lock_call, spend_call = mock_transaction.execute_raw.await_args_list lock_statement, locked_team_id = lock_call.args assert lock_statement is _TEAM_ADVISORY_LOCK_SQL assert locked_team_id == team_id @@ -1092,7 +1110,7 @@ async def test_commit_spend_updates_to_db_orders_team_member_rows_by_team_then_u ), ) - *lock_calls, spend_call = mock_transaction.execute_raw.await_args_list + _lock_timeout_call, *lock_calls, spend_call = mock_transaction.execute_raw.await_args_list _statement, members = spend_call.args assert [lock_call.args for lock_call in lock_calls] == [ (_TEAM_ADVISORY_LOCK_SQL, "eng"), @@ -1730,7 +1748,7 @@ async def test_endpoint_field_is_correctly_mapped_from_call_type(): for key, transaction in update_dict.items(): # Verify endpoint is included in the key - assert key == f"test-user_2024-01-01_test-key_gpt-4_openai_/chat/completions" + assert key == "test-user_2024-01-01_test-key_gpt-4_openai_/chat/completions" # Verify endpoint is set in the transaction assert transaction["endpoint"] == "/chat/completions" @@ -2912,17 +2930,22 @@ async def test_daily_transaction_carries_compression_saved_tokens(): @pytest.mark.asyncio -@pytest.mark.parametrize("estimate, recorded_savings, expected", [ - pytest.param(None, None, -0.005, id="plain-classifier-cost"), - pytest.param({"version": 1, "status": "unknown"}, None, 0.0, id="unknown"), - pytest.param({"version": 2, "status": "unknown"}, None, 0.0, id="unknown-v2"), - pytest.param({"version": 1, "status": "unknown"}, -0.003, 0.0, id="unknown-stale-value"), - pytest.param({"version": 0, "status": "estimated"}, -0.003, 0.0, id="unsupported-version"), - pytest.param({"version": 1, "status": "estimated"}, -0.003, -0.003, id="estimated"), - pytest.param(None, -0.003, -0.003, id="legacy"), -]) +@pytest.mark.parametrize( + "estimate, recorded_savings, expected", + [ + pytest.param(None, None, -0.005, id="plain-classifier-cost"), + pytest.param({"version": 1, "status": "unknown"}, None, 0.0, id="unknown"), + pytest.param({"version": 2, "status": "unknown"}, None, 0.0, id="unknown-v2"), + pytest.param({"version": 1, "status": "unknown"}, -0.003, 0.0, id="unknown-stale-value"), + pytest.param({"version": 0, "status": "estimated"}, -0.003, 0.0, id="unsupported-version"), + pytest.param({"version": 1, "status": "estimated"}, -0.003, -0.003, id="estimated"), + pytest.param(None, -0.003, -0.003, id="legacy"), + ], +) async def test_daily_transaction_compression_saved_tokens_zero_when_absent( - estimate: dict[str, object] | None, recorded_savings: float | None, expected: float, + estimate: dict[str, object] | None, + recorded_savings: float | None, + expected: float, ) -> None: """Requests without any compression metadata produce a zero count.""" writer = DBSpendUpdateWriter() @@ -2941,12 +2964,14 @@ async def test_daily_transaction_compression_saved_tokens_zero_when_absent( "prompt_tokens": 100, "completion_tokens": 10, "spend": 0.01, - "metadata": json.dumps({ - "usage_object": {"prompt_tokens": 100, "completion_tokens": 10}, - "routing_decision": {"savings_baseline_model": "anthropic/claude-sonnet-5", "classifier_cost": 0.005}, - "autorouter_savings": recorded_savings, - "autorouter_savings_estimate": estimate, - }), + "metadata": json.dumps( + { + "usage_object": {"prompt_tokens": 100, "completion_tokens": 10}, + "routing_decision": {"savings_baseline_model": "anthropic/claude-sonnet-5", "classifier_cost": 0.005}, + "autorouter_savings": recorded_savings, + "autorouter_savings_estimate": estimate, + } + ), } transaction = await writer._common_add_spend_log_transaction_to_daily_transaction( @@ -3417,6 +3442,9 @@ async def test_failed_per_entity_increment_from_redis_restores_only_what_may_sti def batch_(self): return _BatchContext() + async def execute_raw(self, query: str, *args: object) -> int: + return 0 + async def __aenter__(self): return self @@ -3439,9 +3467,7 @@ async def test_failed_per_entity_increment_from_redis_restores_only_what_may_sti ) mock_redis_update_buffer.restore_transactions_to_redis.assert_awaited_once() - restored = mock_redis_update_buffer.restore_transactions_to_redis.call_args.kwargs[ - "db_spend_update_transactions" - ] + restored = mock_redis_update_buffer.restore_transactions_to_redis.call_args.kwargs["db_spend_update_transactions"] assert restored["user_list_transactions"] is None assert restored["team_list_transactions"] == {"team-1": 1.5} assert restored["key_list_transactions"] == ({"key-1": 1.5} if safe_to_resend else None) @@ -3985,6 +4011,77 @@ async def test_commit_spend_updates_retries_deadlock_on_every_entity_path(monkey proxy_logging.failure_handler.assert_not_called() +_SPEND_UPDATE_PATHS: Final = [ + ("key_list_transactions", "sk-abc"), + ("user_list_transactions", "user-1"), + ("team_list_transactions", "team-1"), + ("team_member_list_transactions", "team_id::team-1::user_id::user-1"), + ("org_list_transactions", "org-1"), + ("org_member_list_transactions", "organization_id::org-1::user_id::user-1"), + ("tag_list_transactions", "tag-1"), + ("agent_list_transactions", "agent-1"), +] + + +@pytest.mark.parametrize("transactions_key, sample_key", _SPEND_UPDATE_PATHS) +@pytest.mark.asyncio +async def test_commit_spend_updates_retries_a_rollup_lock_timeout_then_commits( + monkeypatch, transactions_key, sample_key +): + """A spend UPDATE cancelled under ``lock_timeout`` (55P03) never took its row lock, so the + transaction rolled back with nothing applied. Every entity path retries it like a deadlock + and lands the increment exactly once, instead of raising and dropping the drained batch.""" + slept = [] + monkeypatch.setattr( + "litellm.proxy.db.db_spend_update_writer.asyncio.sleep", + AsyncMock(side_effect=lambda s: slept.append(s)), + ) + + mock_batcher = MagicMock() + mock_prisma_client = MagicMock() + mock_prisma_client.db.tx = MagicMock(side_effect=[_failing_tx(_lock_timeout_error()), _good_tx(mock_batcher)]) + + proxy_logging = MagicMock() + proxy_logging.failure_handler = AsyncMock() + proxy_logging.call_details = {} + + await DBSpendUpdateWriter()._commit_spend_updates_to_db( + prisma_client=mock_prisma_client, + n_retry_times=3, + proxy_logging_obj=proxy_logging, + db_spend_update_transactions=_empty_spend_transactions(**{transactions_key: {sample_key: 0.5}}), + ) + + assert { + "transactions_opened": mock_prisma_client.db.tx.call_count, + "backoff_sleeps": len(slept), + "failure_handler_calls": proxy_logging.failure_handler.await_count, + } == {"transactions_opened": 2, "backoff_sleeps": 1, "failure_handler_calls": 0} + + +@pytest.mark.asyncio +async def test_commit_spend_updates_raises_after_exhausting_lock_timeout_retries(monkeypatch): + """A row that stays locked past every retry still surfaces as a failure rather than + looping forever or being swallowed; the budget is the same one deadlocks get.""" + monkeypatch.setattr("litellm.proxy.db.db_spend_update_writer.asyncio.sleep", AsyncMock(return_value=None)) + + mock_prisma_client = MagicMock() + mock_prisma_client.db.tx = MagicMock(side_effect=lambda *a, **k: _failing_tx(_lock_timeout_error())) + + proxy_logging = MagicMock() + proxy_logging.failure_handler = AsyncMock() + + with pytest.raises(PrismaDataError, match="lock timeout"): + await DBSpendUpdateWriter()._commit_spend_updates_to_db( + prisma_client=mock_prisma_client, + n_retry_times=2, + proxy_logging_obj=proxy_logging, + db_spend_update_transactions=_empty_spend_transactions(key_list_transactions={"sk-abc": 0.5}), + ) + + assert mock_prisma_client.db.tx.call_count == 3 + + @pytest.mark.asyncio @pytest.mark.parametrize( "call_type, expects_flush", @@ -4881,3 +4978,92 @@ async def test_shutdown_drain_that_lands_before_the_interrupted_tag_commit_resol assert redis_buffer.restored == [drained], "a tag batch whose COMMIT came back failed must be restored to Redis" (upsert,) = _daily_upserts(final_db, "LiteLLM_DailyTagSpend") assert _row_values(upsert, "api_requests") == [1] + + +def _lock_timeout_error() -> PrismaDataError: + return PrismaDataError( + data={ + "user_facing_error": { + "is_panic": False, + "message": "Error querying the database: canceling statement due to lock timeout", + "meta": {"code": "55P03", "message": "canceling statement due to lock timeout"}, + } + } + ) + + +@pytest.mark.asyncio +async def test_daily_spend_upsert_runs_under_the_rollup_lock_timeout(): + """The bulk daily upsert opens its transaction by bounding how long it may wait on + rows another pod holds; the setting is transaction-local and bound as a parameter.""" + prisma_client = _RecordingPrisma() + daily_spend_transactions = { + "key": { + "user_id": "user-1", + "date": "2024-01-01", + "api_key": "test-api-key", + "model": "gpt-4", + "custom_llm_provider": "openai", + "prompt_tokens": 10, + "completion_tokens": 20, + "spend": 0.1, + "api_requests": 1, + "successful_requests": 1, + "failed_requests": 0, + } + } + + await DBSpendUpdateWriter._update_daily_spend( + n_retry_times=0, + prisma_client=prisma_client, + proxy_logging_obj=MagicMock(), + daily_spend_transactions=daily_spend_transactions, + entity_type="user", + entity_id_field="user_id", + ) + + assert prisma_client.db.session_settings == [(ROLLUP_LOCK_TIMEOUT_SQL, (f"{SPEND_ROLLUP_LOCK_TIMEOUT_MS}ms",))] + assert len(prisma_client.db.statements) == 1 + assert daily_spend_transactions == {} + + +@pytest.mark.asyncio +async def test_daily_spend_rows_survive_a_lock_timeout_for_the_next_flush(): + """A statement cancelled under ``lock_timeout`` (55P03) never applied, so its rows + are not data Postgres refused: they stay queued for the next flush instead of being + dropped the way a constraint violation is.""" + + def cancelled_waiting_for_the_row() -> int: + raise _lock_timeout_error() + + prisma_client = _RecordingPrisma(execute_raw=cancelled_waiting_for_the_row) + proxy_logging = MagicMock() + proxy_logging.failure_handler = AsyncMock() + daily_spend_transactions = { + "key": { + "user_id": "user-1", + "date": "2024-01-01", + "api_key": "test-api-key", + "model": "gpt-4", + "custom_llm_provider": "openai", + "prompt_tokens": 10, + "completion_tokens": 20, + "spend": 0.1, + "api_requests": 1, + "successful_requests": 1, + "failed_requests": 0, + } + } + + with pytest.raises(PrismaDataError, match="lock timeout"): + await DBSpendUpdateWriter._update_daily_spend( + n_retry_times=0, + prisma_client=prisma_client, + proxy_logging_obj=proxy_logging, + daily_spend_transactions=daily_spend_transactions, + entity_type="user", + entity_id_field="user_id", + ) + + assert len(prisma_client.db.statements) == 1 + assert list(daily_spend_transactions) == ["key"] diff --git a/tests/unit/proxy/db/test_exception_handler.py b/tests/unit/proxy/db/test_exception_handler.py index af647b2a3d9..75346ca08a6 100644 --- a/tests/unit/proxy/db/test_exception_handler.py +++ b/tests/unit/proxy/db/test_exception_handler.py @@ -1,12 +1,10 @@ import asyncio -import json import sys from typing import Final from unittest.mock import MagicMock, patch import httpx import pytest -from fastapi import HTTPException, Request from prisma import errors as prisma_errors from prisma.engine.errors import BinaryNotFoundError, EngineConnectionError, EngineRequestError from prisma.errors import ( @@ -22,9 +20,7 @@ from prisma.errors import ( UniqueViolationError, ) - import litellm -from litellm._logging import verbose_proxy_logger from litellm.proxy._types import ProxyErrorTypes, ProxyException from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler @@ -55,22 +51,12 @@ def test_is_database_infrastructure_error_prisma_connection_errors(prisma_error) PrismaError(), PrismaError("validation failed on query"), DataError(data={"user_facing_error": {"meta": {"table": "test_table"}}}), - UniqueViolationError( - data={"user_facing_error": {"meta": {"table": "test_table"}}} - ), - ForeignKeyViolationError( - data={"user_facing_error": {"meta": {"table": "test_table"}}} - ), - MissingRequiredValueError( - data={"user_facing_error": {"meta": {"table": "test_table"}}} - ), + UniqueViolationError(data={"user_facing_error": {"meta": {"table": "test_table"}}}), + ForeignKeyViolationError(data={"user_facing_error": {"meta": {"table": "test_table"}}}), + MissingRequiredValueError(data={"user_facing_error": {"meta": {"table": "test_table"}}}), RawQueryError(data={"user_facing_error": {"meta": {"table": "test_table"}}}), - TableNotFoundError( - data={"user_facing_error": {"meta": {"table": "test_table"}}} - ), - RecordNotFoundError( - data={"user_facing_error": {"meta": {"table": "test_table"}}} - ), + TableNotFoundError(data={"user_facing_error": {"meta": {"table": "test_table"}}}), + RecordNotFoundError(data={"user_facing_error": {"meta": {"table": "test_table"}}}), ], ) def test_is_database_transport_error_non_connection_prisma_errors(prisma_error): @@ -82,12 +68,7 @@ def test_is_database_connection_generic_errors(): """ Test non-Prisma error cases for database connection checking """ - assert ( - PrismaDBExceptionHandler.is_database_connection_error( - Exception("Regular error") - ) - == False - ) + assert PrismaDBExceptionHandler.is_database_connection_error(Exception("Regular error")) == False # Test with ProxyException (DB connection) db_proxy_exception = ProxyException( @@ -95,17 +76,11 @@ def test_is_database_connection_generic_errors(): type=ProxyErrorTypes.no_db_connection, param="test-param", ) - assert ( - PrismaDBExceptionHandler.is_database_connection_error(db_proxy_exception) - == True - ) + assert PrismaDBExceptionHandler.is_database_connection_error(db_proxy_exception) == True # Test with non-DB error regular_exception = Exception("Regular error") - assert ( - PrismaDBExceptionHandler.is_database_connection_error(regular_exception) - == False - ) + assert PrismaDBExceptionHandler.is_database_connection_error(regular_exception) == False @pytest.mark.parametrize( @@ -143,12 +118,7 @@ def test_is_database_service_unavailable_error_prisma_p1001_masquerades_as_datae } } ) - assert ( - PrismaDBExceptionHandler.is_database_service_unavailable_error( - p1001_as_dataerror - ) - is True - ) + assert PrismaDBExceptionHandler.is_database_service_unavailable_error(p1001_as_dataerror) is True def test_is_prisma_data_error_only_true_for_dataerror(): @@ -196,12 +166,7 @@ def test_is_database_service_unavailable_error_cached_plan_escapes_as_503(): } } ) - assert ( - PrismaDBExceptionHandler.is_database_service_unavailable_error( - cached_plan_error - ) - is True - ) + assert PrismaDBExceptionHandler.is_database_service_unavailable_error(cached_plan_error) is True def test_is_database_service_unavailable_error_prisma_engine_malformed_payload(): @@ -229,10 +194,7 @@ def test_is_database_service_unavailable_error_prisma_engine_malformed_payload() prisma_engine_utils.handle_response_errors(None, malformed_payload) assert "no attribute 'get'" in str(exc_info.value) - assert ( - PrismaDBExceptionHandler.is_database_service_unavailable_error(exc_info.value) - is True - ) + assert PrismaDBExceptionHandler.is_database_service_unavailable_error(exc_info.value) is True def test_is_prisma_engine_internal_error_excludes_application_attributeerror(): @@ -247,23 +209,15 @@ def test_is_prisma_engine_internal_error_excludes_application_attributeerror(): with pytest.raises(AttributeError) as exc_info: application_bug() - assert ( - PrismaDBExceptionHandler.is_prisma_engine_internal_error(exc_info.value) - is False - ) - assert ( - PrismaDBExceptionHandler.is_database_service_unavailable_error(exc_info.value) - is False - ) + assert PrismaDBExceptionHandler.is_prisma_engine_internal_error(exc_info.value) is False + assert PrismaDBExceptionHandler.is_database_service_unavailable_error(exc_info.value) is False def test_is_prisma_engine_internal_error_excludes_data_layer_prisma_error(): """A data-layer ``PrismaError`` (the DB IS reachable and rejected the data) must stay 401. These are always raised from prisma internals, so the check excludes any ``PrismaError`` by type before inspecting the traceback.""" - data_layer_error = UniqueViolationError( - data={"user_facing_error": {"meta": {"table": "t"}}} - ) + data_layer_error = UniqueViolationError(data={"user_facing_error": {"meta": {"table": "t"}}}) with pytest.raises(UniqueViolationError) as exc_info: raise data_layer_error e = exc_info.value @@ -284,9 +238,7 @@ def test_is_database_service_unavailable_error_excludes_non_infra(error): """Data-layer errors (the DB IS reachable and answered) and generic non-DB errors must NOT be classified as service-unavailable, otherwise a genuine 401 would be masked as a transient 503.""" - assert ( - PrismaDBExceptionHandler.is_database_service_unavailable_error(error) is False - ) + assert PrismaDBExceptionHandler.is_database_service_unavailable_error(error) is False def _wrapped_like_get_user_object(original): @@ -393,22 +345,14 @@ def test_is_database_service_unavailable_error_asyncpg(monkeypatch): monkeypatch.setitem(sys.modules, "asyncpg.exceptions", fake_exceptions) assert ( - PrismaDBExceptionHandler.is_database_service_unavailable_error( - PostgresConnectionError("connection reset") - ) + PrismaDBExceptionHandler.is_database_service_unavailable_error(PostgresConnectionError("connection reset")) is True ) assert ( - PrismaDBExceptionHandler.is_database_service_unavailable_error( - InterfaceError("connection was closed") - ) - is True + PrismaDBExceptionHandler.is_database_service_unavailable_error(InterfaceError("connection was closed")) is True ) assert ( - PrismaDBExceptionHandler.is_database_service_unavailable_error( - UniqueViolationError("duplicate key") - ) - is False + PrismaDBExceptionHandler.is_database_service_unavailable_error(UniqueViolationError("duplicate key")) is False ) @@ -675,7 +619,10 @@ def test_is_deadlock_error_excludes_non_deadlocks(error): "22021", ), (RawQueryError(data={"user_facing_error": {"error_code": "P2010", "meta": {"message": "m"}}}), None), - (RawQueryError(data={"user_facing_error": {"error_code": "P2010", "meta": {"code": 42, "message": "m"}}}), None), + ( + RawQueryError(data={"user_facing_error": {"error_code": "P2010", "meta": {"code": 42, "message": "m"}}}), + None, + ), (prisma_errors.DataError(data={"user_facing_error": {"meta": None}}), None), ( prisma_errors.DataError( @@ -695,7 +642,9 @@ def test_is_deadlock_error_excludes_non_deadlocks(error): (httpx.ReadTimeout("no reply"), None), ], ) -def test_postgres_sqlstate_reads_the_code_prisma_attached_to_the_failed_statement(error: Exception, sqlstate: str | None): +def test_postgres_sqlstate_reads_the_code_prisma_attached_to_the_failed_statement( + error: Exception, sqlstate: str | None +): """Only a prisma data error carrying Postgres's own error code yields a SQLSTATE, whether in ``meta`` or, for a batched statement, only in the message; a codeless or malformed payload, an engine-level error, and a transport error yield None.""" @@ -905,7 +854,7 @@ def _pool_timeout_error() -> DataError: RawQueryError( data={ "user_facing_error": { - "message": 'Raw query failed. Code: `53300`. Message: `db error: FATAL: sorry, too many clients already`', + "message": "Raw query failed. Code: `53300`. Message: `db error: FATAL: sorry, too many clients already`", "meta": {"code": "53300", "message": "FATAL: sorry, too many clients already"}, "error_code": "P2010", } @@ -945,3 +894,60 @@ def test_capacity_error_is_seen_through_a_wrapping_exception() -> None: wrapped: Final = RuntimeError("spend flush failed") wrapped.__cause__ = _capacity_error("sorry, too many clients already") assert PrismaDBExceptionHandler.is_database_service_unavailable_error_in_chain(wrapped) is True + + +def _lock_timeout_error( + message: str = "canceling statement due to lock timeout", code: str | None = "55P03" +) -> DataError: + """The shape prisma raises for a statement Postgres cancelled under ``lock_timeout``.""" + user_facing: Final[dict[str, object]] = {"is_panic": False, "message": f"Error querying the database: {message}"} + if code is not None: + user_facing["meta"] = {"code": code, "message": message} + return DataError(data={"user_facing_error": user_facing}) + + +@pytest.mark.parametrize( + "error", + [ + _lock_timeout_error(), + _lock_timeout_error(message="Abbruch der Anweisung wegen Zeitüberschreitung beim Warten auf eine Sperre"), + _lock_timeout_error(code=None), + DataError( + data={ + "user_facing_error": { + "message": 'Error occurred during query execution: ConnectorError(ConnectorError { user_facing_error: None, kind: QueryError(PostgresError { code: "55P03", message: "canceling statement due to lock timeout", severity: "ERROR" }) })' + } + } + ), + ], + ids=["meta_55P03", "meta_55P03_localised_message", "message_only", "batched_statement"], +) +def test_is_lock_timeout_error_recognises_a_statement_cancelled_under_lock_timeout(error: DataError) -> None: + """SQLSTATE 55P03 means the statement never took its lock, so it never applied + and its rows are safe to re-send. It is a wait budget, not a full server, so + it stays out of the connection-capacity classification.""" + assert PrismaDBExceptionHandler.is_lock_timeout_error(error) is True + assert PrismaDBExceptionHandler.is_database_capacity_error(error) is False + + +@pytest.mark.parametrize( + "error", + [ + _lock_timeout_error(message="canceling statement due to statement timeout", code="57014"), + _lock_timeout_error(message="deadlock detected", code="40P01"), + _capacity_error("sorry, too many clients already"), + _pool_timeout_error(), + httpx.ConnectError("connection refused"), + RuntimeError("canceling statement due to lock timeout"), + ], + ids=[ + "statement_timeout_57014", + "deadlock_40P01", + "capacity_53300", + "pool_timeout_P2024", + "transport", + "not_prisma", + ], +) +def test_is_lock_timeout_error_excludes_other_failures(error: Exception) -> None: + assert PrismaDBExceptionHandler.is_lock_timeout_error(error) is False diff --git a/tests/unit/proxy/db/test_model_access_group_spend.py b/tests/unit/proxy/db/test_model_access_group_spend.py index d2d079bb0e4..0ee86e783ea 100644 --- a/tests/unit/proxy/db/test_model_access_group_spend.py +++ b/tests/unit/proxy/db/test_model_access_group_spend.py @@ -64,6 +64,9 @@ class _FakeTransaction: def batch_(self) -> _FakeBatchManager: return _FakeBatchManager(self._batcher) + async def execute_raw(self, query: str, *args: object) -> int: + return 0 + async def __aenter__(self) -> "_FakeTransaction": return self diff --git a/tests/unit/proxy/db/test_model_usage_rollup.py b/tests/unit/proxy/db/test_model_usage_rollup.py index b9b806f4c54..0b7427d71f2 100644 --- a/tests/unit/proxy/db/test_model_usage_rollup.py +++ b/tests/unit/proxy/db/test_model_usage_rollup.py @@ -6,6 +6,7 @@ from unittest.mock import MagicMock import httpx import pytest +from litellm.proxy.db import rollup_lock_timeout from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter from litellm.proxy.db.model_usage_rollup import ( ModelUsageKey, @@ -14,11 +15,13 @@ from litellm.proxy.db.model_usage_rollup import ( flush_model_usage_transactions, model_usage_task_type, ) +from litellm.proxy.db.rollup_lock_timeout import ROLLUP_LOCK_TIMEOUT_SQL class _FakeBatcher: def __init__(self) -> None: self.litellm_dailymodelusage = MagicMock() + self.execute_raw = MagicMock() async def __aenter__(self) -> "_FakeBatcher": return self @@ -201,3 +204,26 @@ async def test_request_time_path_queues_usage_instead_of_writing_to_the_db() -> assert [transaction.key.model for transaction in prisma.model_usage_transactions] == ["openai/gpt-5.4-mini"] prisma.db.litellm_dailymodelusage.upsert.assert_not_called() + + +@pytest.mark.asyncio +async def test_rollup_transaction_sets_the_lock_timeout_before_its_first_upsert(monkeypatch) -> None: + """The DailyModelUsage rollup shares the DailyToolSpend write shape, so it runs + under the same per-transaction lock budget, issued before any upsert queues.""" + monkeypatch.setattr(rollup_lock_timeout, "SPEND_ROLLUP_LOCK_TIMEOUT_MS", 250) + batcher = _FakeBatcher() + prisma = _prisma(MagicMock(return_value=batcher)) + order: list[str] = [] + batcher.execute_raw.side_effect = lambda *_: order.append("lock_timeout") + batcher.litellm_dailymodelusage.upsert.side_effect = lambda **_: order.append("upsert") + + await flush_model_usage_transactions( + prisma_client=prisma, + transactions=[ + build_model_usage_transaction(_payload()), + build_model_usage_transaction(_payload(model="other")), + ], + ) + + assert batcher.execute_raw.call_args_list == [((ROLLUP_LOCK_TIMEOUT_SQL, "250ms"),)] + assert order == ["lock_timeout", "upsert", "upsert"] diff --git a/tests/unit/proxy/db/test_prisma_query_span.py b/tests/unit/proxy/db/test_prisma_query_span.py index 674679f8a18..abd95dec517 100644 --- a/tests/unit/proxy/db/test_prisma_query_span.py +++ b/tests/unit/proxy/db/test_prisma_query_span.py @@ -106,6 +106,7 @@ def test_a_payload_the_parser_does_not_know_stays_the_legacy_function_named_span ), ('INSERT INTO "LiteLLM_DailyUserSpend" (id) VALUES ($1)', ("insert", "LiteLLM_DailyUserSpend")), ("SET LOCAL lock_timeout = 1000", ("set", "lock_timeout")), + ("SELECT set_config('lock_timeout', $1::text, true)", ("set", "lock_timeout")), ("SELECT COUNT(*) FROM pg_stat_activity", ("select", "pg_catalog")), ("SELECT 1", ("ping", None)), ("SELECT current_setting('transaction_read_only') AS transaction_read_only", ("select", "pg_catalog")), diff --git a/tests/unit/proxy/db/test_spend_log_tool_index.py b/tests/unit/proxy/db/test_spend_log_tool_index.py index 610faebe17c..eaa94c0438f 100644 --- a/tests/unit/proxy/db/test_spend_log_tool_index.py +++ b/tests/unit/proxy/db/test_spend_log_tool_index.py @@ -4,22 +4,23 @@ only) and the flush that writes LiteLLM_SpendLogToolIndex in bounded statements plus the LiteLLM_DailyToolSpend rollup in one transaction. """ -from types import SimpleNamespace from collections.abc import Awaitable, Callable +from types import SimpleNamespace from typing import Any from unittest.mock import AsyncMock, MagicMock import httpx import pytest -from litellm.proxy.db import spend_log_tool_index +from litellm.proxy.db import rollup_lock_timeout, spend_log_tool_index +from litellm.proxy.db.log_db_metrics import record_db_io +from litellm.proxy.db.rollup_lock_timeout import ROLLUP_LOCK_TIMEOUT_SQL from litellm.proxy.db.spend_log_tool_index import ( ToolUsageTransaction, build_tool_usage_transaction, flush_tool_usage_transactions, response_tool_call_names, ) -from litellm.proxy.db.log_db_metrics import record_db_io from tests.unit.proxy.db.fake_prisma_engine import engine_call @@ -32,6 +33,7 @@ class _FakeBatcher: def __init__(self) -> None: self.litellm_spendlogtoolindex = MagicMock() self.litellm_dailytoolspend = MagicMock() + self.execute_raw = MagicMock() async def __aenter__(self) -> "_FakeBatcher": return self @@ -84,7 +86,9 @@ class TestBuildToolUsageTransaction: mcp_namespaced_tool_name=None, spend=0.5, total_tokens=100, - completion_response=SimpleNamespace(choices=[SimpleNamespace(message=SimpleNamespace(tool_calls=None))]), + completion_response=SimpleNamespace( + choices=[SimpleNamespace(message=SimpleNamespace(tool_calls=None))] + ), ) is None ) @@ -395,3 +399,25 @@ async def test_a_tool_usage_flush_renders_one_postgres_span_per_table_written( "postgres.insert LiteLLM_SpendLogToolIndex", "postgres.upsert LiteLLM_DailyToolSpend", ) + + +@pytest.mark.asyncio +async def test_rollup_transaction_sets_the_lock_timeout_before_its_first_upsert(monkeypatch) -> None: + """Every DailyToolSpend rollup transaction opens by bounding how long its upserts + may wait on a row another pod holds, so a waiter costs one cancelled statement + rather than a pooled connection pinned until the holder commits. The budget is the + SPEND_ROLLUP_LOCK_TIMEOUT_MS knob, bound as a parameter, not baked into the SQL.""" + monkeypatch.setattr(rollup_lock_timeout, "SPEND_ROLLUP_LOCK_TIMEOUT_MS", 250) + prisma, batcher = _prisma_with_batcher() + order: list[str] = [] + batcher.execute_raw.side_effect = lambda *_: order.append("lock_timeout") + batcher.litellm_dailytoolspend.upsert.side_effect = lambda **_: order.append("upsert") + + await flush_tool_usage_transactions( + prisma_client=prisma, + transactions=[_transaction("r1", tool_names=("tool_a", "tool_b"))], + ) + + assert batcher.execute_raw.call_args_list == [((ROLLUP_LOCK_TIMEOUT_SQL, "250ms"),)] + assert order == ["lock_timeout", "upsert", "upsert"] + assert "set_config('lock_timeout'" in ROLLUP_LOCK_TIMEOUT_SQL and ", true)" in ROLLUP_LOCK_TIMEOUT_SQL diff --git a/tests/unit/proxy/db/test_update_daily_tag_spend.py b/tests/unit/proxy/db/test_update_daily_tag_spend.py index 530c0d16767..dda41ab4543 100644 --- a/tests/unit/proxy/db/test_update_daily_tag_spend.py +++ b/tests/unit/proxy/db/test_update_daily_tag_spend.py @@ -1,12 +1,12 @@ -from typing import Dict from unittest.mock import AsyncMock, MagicMock, patch +import httpx import pytest -from litellm.proxy.utils import update_daily_tag_spend from litellm.proxy._types import DailyTagSpendTransaction -import httpx from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter +from litellm.proxy.db.rollup_lock_timeout import ROLLUP_LOCK_TIMEOUT_SQL +from litellm.proxy.utils import update_daily_tag_spend @pytest.mark.asyncio @@ -18,9 +18,7 @@ async def test_update_daily_tag_spend_delegates_to_tag_commit_writer(): proxy_logging_obj.db_spend_update_writer = MagicMock() proxy_logging_obj.db_spend_update_writer.redis_update_buffer = redis_update_buffer proxy_logging_obj.db_spend_update_writer._commit_daily_tag_spend_to_db = AsyncMock() - proxy_logging_obj.db_spend_update_writer._commit_daily_tag_spend_to_db_with_redis = ( - AsyncMock() - ) + proxy_logging_obj.db_spend_update_writer._commit_daily_tag_spend_to_db_with_redis = AsyncMock() await update_daily_tag_spend( prisma_client, @@ -43,12 +41,8 @@ async def test_update_daily_tag_spend_logs_error_and_does_not_raise(): redis_update_buffer._should_commit_spend_updates_to_redis.return_value = False proxy_logging_obj.db_spend_update_writer = MagicMock() proxy_logging_obj.db_spend_update_writer.redis_update_buffer = redis_update_buffer - proxy_logging_obj.db_spend_update_writer._commit_daily_tag_spend_to_db = AsyncMock( - side_effect=ValueError("boom") - ) - proxy_logging_obj.db_spend_update_writer._commit_daily_tag_spend_to_db_with_redis = ( - AsyncMock() - ) + proxy_logging_obj.db_spend_update_writer._commit_daily_tag_spend_to_db = AsyncMock(side_effect=ValueError("boom")) + proxy_logging_obj.db_spend_update_writer._commit_daily_tag_spend_to_db_with_redis = AsyncMock() with patch("litellm.proxy.utils.verbose_proxy_logger.error") as error_logger: await update_daily_tag_spend( @@ -69,9 +63,7 @@ async def test_update_daily_tag_spend_uses_redis_writer_when_enabled(): proxy_logging_obj.db_spend_update_writer = MagicMock() proxy_logging_obj.db_spend_update_writer._commit_daily_tag_spend_to_db = AsyncMock() proxy_logging_obj.db_spend_update_writer.redis_update_buffer = redis_update_buffer - proxy_logging_obj.db_spend_update_writer._commit_daily_tag_spend_to_db_with_redis = ( - AsyncMock() - ) + proxy_logging_obj.db_spend_update_writer._commit_daily_tag_spend_to_db_with_redis = AsyncMock() await update_daily_tag_spend( prisma_client, @@ -92,17 +84,20 @@ async def test_daily_tag_spend_retries_then_succeeds(): proxy_logging_obj = MagicMock() # Fail the upsert 3 times with retryable DB errors, then succeed. - prisma_client.db.execute_raw = AsyncMock( - side_effect=[ - httpx.ConnectError("x"), - httpx.ConnectError("x"), - httpx.ConnectError("x"), - 1, - ] - ) + upsert_outcomes = iter([httpx.ConnectError("x"), httpx.ConnectError("x"), httpx.ConnectError("x"), 1]) + + async def execute_raw(query: str, *args: object) -> int: + if query == ROLLUP_LOCK_TIMEOUT_SQL: + return 0 + outcome = next(upsert_outcomes) + if isinstance(outcome, Exception): + raise outcome + return outcome + + prisma_client.db.execute_raw = AsyncMock(side_effect=execute_raw) prisma_client.db.tx.return_value.__aenter__.return_value.execute_raw = prisma_client.db.execute_raw - daily_spend_transactions: Dict[str, DailyTagSpendTransaction] = { + daily_spend_transactions: dict[str, DailyTagSpendTransaction] = { "k": { "tag": "prod-tag", "date": "2026-04-03", @@ -135,7 +130,8 @@ async def test_daily_tag_spend_retries_then_succeeds(): daily_spend_transactions=daily_spend_transactions, ) - assert prisma_client.db.execute_raw.await_count == 4 + upsert_attempts = [c for c in prisma_client.db.execute_raw.await_args_list if c.args[0] != ROLLUP_LOCK_TIMEOUT_SQL] + assert len(upsert_attempts) == 4 assert sleep_mock.await_count == 3 # The batch is one statement, so the successful attempt is a single call carrying # the row rather than one call per key. diff --git a/tests/unit/proxy/lens/test_dataset_endpoints.py b/tests/unit/proxy/lens/test_dataset_endpoints.py new file mode 100644 index 00000000000..53d60d98d20 --- /dev/null +++ b/tests/unit/proxy/lens/test_dataset_endpoints.py @@ -0,0 +1,408 @@ +import json +from collections.abc import Mapping +from datetime import datetime +from typing import Final + +import pytest +from fastapi import HTTPException + +from litellm.constants import LENS_DATASET_MAX_CASE_CHARS, LENS_DATASET_MAX_CASES +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.lens.dataset_endpoints import ( + ProxyDatasetReader, + build_dataset_cases, + create_dataset, + eval_cases, + export_dataset, + list_datasets, + read_dataset, + save_revision, +) +from litellm.proxy.lens.dataset_repository import StoredSummary +from litellm.proxy.lens.datasets import case_id +from litellm.proxy.lens.models import ( + BuildRequest, + CaseSource, + Dataset, + DatasetCase, + DatasetCreate, + DatasetMessage, + DatasetSummary, + Evidence, + Lens, + RevisionSave, + Scope, + TextSource, +) +from litellm.proxy.lens.repository import LensRepository, Row +from litellm.rust_bridge.trace.errors import TraceChanged +from litellm.rust_bridge.trace.generated.types import SpanDetail, Trace, TraceScope +from litellm.rust_bridge.trace.storage import ClickHouseStorage +from litellm.tracing import TraceReceiver +from tests.unit.proxy.lens.test_datasets import detail, span, stored_finding, trace +from tests.unit.proxy.lens.test_state import lens + +ADMIN: Final = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin", team_id="alpha") + + +class MemoryStore: + def __init__(self) -> None: + self.rows: Final[dict[tuple[str, int], Dataset]] = {} # mutable-ok: stands in for the table + + async def summaries(self) -> tuple[StoredSummary, ...]: + latest: Final = {i: d for (i, _), d in sorted(self.rows.items(), key=lambda item: item[0][1])} + return tuple(StoredSummary(team_id=d.team_id, summary=summary(d)) for d in latest.values()) + + async def get(self, dataset_id: str, revision: int | None = None) -> Dataset | None: + revisions: Final = sorted(r for i, r in self.rows if i == dataset_id) + wanted: Final = revision if revision is not None else (revisions[-1] if revisions else None) + return self.rows.get((dataset_id, wanted)) if wanted is not None else None + + async def insert(self, dataset: Dataset, saved_at: datetime) -> bool: + key: Final = (dataset.id, dataset.revision) + if key in self.rows: + return False + self.rows[key] = dataset + return True + + +def case(text: str, included: bool = True) -> DatasetCase: + message: Final = (DatasetMessage(role="user", content=text),) + return DatasetCase(id=case_id(message, "", ()), messages=message, included=included, source=CaseSource()) + + +@pytest.mark.asyncio +async def test_saving_inserts_a_new_revision_and_leaves_the_older_one_unchanged() -> None: + store: Final = MemoryStore() + created: Final = await create_dataset_named(store) + first: Final = await save_revision(created.id, RevisionSave(base_revision=0, cases=(case("a"),)), ADMIN, store) + second: Final = await save_revision( + created.id, RevisionSave(base_revision=1, cases=(case("a"), case("b"))), ADMIN, store + ) + + assert (first.revision, second.revision) == (1, 2) + assert await read_dataset(created.id, ADMIN, store, revision=1) == first + assert (await read_dataset(created.id, ADMIN, store, revision=0)).cases == () + assert await read_dataset(created.id, ADMIN, store) == second + + +@pytest.mark.asyncio +async def test_saving_on_a_stale_revision_is_a_conflict_and_writes_nothing() -> None: + store: Final = MemoryStore() + created: Final = await create_dataset_named(store) + await save_revision(created.id, RevisionSave(base_revision=0, cases=(case("a"),)), ADMIN, store) + + with pytest.raises(HTTPException) as error: + await save_revision(created.id, RevisionSave(base_revision=0, cases=(case("b"),)), ADMIN, store) + assert error.value.status_code == 409 + assert sorted(store.rows) == [(created.id, 0), (created.id, 1)] + + +@pytest.mark.asyncio +async def test_saving_dedupes_cases_and_restores_content_hash_ids_but_keeps_edits() -> None: + store: Final = MemoryStore() + created: Final = await create_dataset_named(store) + edited: Final = case("a").model_copy(update={"id": "forged", "expected": "say hi"}) + saved: Final = await save_revision( + created.id, RevisionSave(base_revision=0, cases=(edited, case("a"))), ADMIN, store + ) + + assert len(saved.cases) == 1 + assert saved.cases[0].id == case("a").id + assert saved.cases[0].expected == "say hi" + + +@pytest.mark.asyncio +async def test_eval_cases_return_only_included_cases_of_the_requested_revision() -> None: + store: Final = MemoryStore() + created: Final = await create_dataset_named(store) + await save_revision( + created.id, RevisionSave(base_revision=0, cases=(case("a"), case("b", included=False))), ADMIN, store + ) + await save_revision(created.id, RevisionSave(base_revision=1, cases=(case("c"),)), ADMIN, store) + + cases: Final = await eval_cases(created.id, 1, ADMIN, store) + assert (cases.dataset_id, cases.revision) == (created.id, 1) + assert tuple(c.messages[0].content for c in cases.cases) == ("a",) + + +@pytest.mark.asyncio +async def test_read_only_admin_can_read_but_cannot_save_a_revision() -> None: + store: Final = MemoryStore() + created: Final = await create_dataset_named(store) + viewer: Final = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY) + + assert (await read_dataset(created.id, viewer, store)).id == created.id + with pytest.raises(HTTPException) as error: + await save_revision(created.id, RevisionSave(base_revision=0, cases=(case("a"),)), viewer, store) + assert error.value.status_code == 403 + + +async def create_dataset_named(store: MemoryStore) -> Dataset: + return await create_dataset(DatasetCreate(name="Refunds", agent_name="support"), ADMIN, store) + + +def summary(dataset: Dataset) -> DatasetSummary: + return DatasetSummary( + id=dataset.id, + name=dataset.name, + agent_name=dataset.agent_name, + revision=dataset.revision, + case_count=len(dataset.cases), + updated_at=dataset.created_at, + ) + + +class RejectingStore: + def __init__(self, stored: Dataset | None = None) -> None: + self.stored: Final = stored + + async def summaries(self) -> tuple[StoredSummary, ...]: + return () + + async def get(self, dataset_id: str, revision: int | None = None) -> Dataset | None: + return self.stored + + async def insert(self, dataset: Dataset, saved_at: datetime) -> bool: + return False + + +class TraceStorage(ClickHouseStorage): + def __init__( + self, + pages: Mapping[str | None, Trace], + spans: Mapping[str, SpanDetail] | None = None, + error: Exception | None = None, + ) -> None: + self.pages: Final = pages + self.spans: Final = spans or {} + self.error: Final = error + + async def get_trace( + self, + trace_id: str, + scope: TraceScope, + trace_ref: str = "", + cursor: str | None = None, + page_size: int | None = None, + ) -> Trace | None: + if self.error: + raise self.error + return self.pages.get(cursor) + + async def get_span(self, trace_id: str, span_id: str, scope: TraceScope, trace_ref: str = "") -> SpanDetail | None: + if self.error: + raise self.error + return self.spans.get(span_id) + + +class LensTable: + def __init__(self, stored: Lens) -> None: + self.stored: Final = stored + + async def query_raw(self, query: str, *args: object) -> tuple[Row, ...]: + return (Row(data=self.stored.model_dump(mode="json")),) if args == (self.stored.id,) else () + + async def execute_raw(self, query: str, *args: object) -> int: + return 0 + + +def page(next_cursor: str | None, *span_ids: str) -> Trace: + return {**trace(*(span(s, "llm", i) for i, s in enumerate(span_ids))), "next_cursor": next_cursor} + + +def lenses(stored: Lens | None = None) -> LensRepository: + return LensRepository(LensTable(stored or lens())) + + +def reader( + storage: TraceStorage | None = None, stored: Lens | None = None, scope: Scope | None = None +) -> ProxyDatasetReader: + return ProxyDatasetReader( + TraceReceiver(storage) if storage else None, + lenses(stored), + scope or Scope(all_teams=True), + ) + + +@pytest.mark.asyncio +async def test_reading_a_trace_follows_every_cursor_page_and_joins_the_spans() -> None: + storage: Final = TraceStorage({None: page("c1", "a", "b"), "c1": page("c2", "c"), "c2": page(None, "d")}) + read: Final = await reader(storage).trace("t1", "ref") + + assert read is not None + assert tuple(s["span_id"] for s in read["spans"]) == ("a", "b", "c", "d") + assert read.get("next_cursor") is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize("pages", ({}, {None: page("c1", "a")}), ids=("missing-trace", "page-disappears")) +async def test_reading_a_trace_is_none_when_the_trace_or_a_later_page_is_gone( + pages: Mapping[str | None, Trace], +) -> None: + assert await reader(TraceStorage(pages)).trace("t1", "ref") is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("error", "status"), ((TraceChanged("moved"), 409), (ValueError("bad cursor"), 400), (RuntimeError("down"), 503)) +) +async def test_trace_and_span_read_failures_become_the_matching_http_error(error: Exception, status: int) -> None: + failing: Final = reader(TraceStorage({}, error=error)) + + with pytest.raises(HTTPException) as traced: + await failing.trace("t1", "ref") + with pytest.raises(HTTPException) as spanned: + await failing.span("t1", "s1", "ref") + assert (traced.value.status_code, spanned.value.status_code) == (status, status) + + +@pytest.mark.asyncio +async def test_reading_traces_without_tracing_enabled_is_not_implemented() -> None: + disabled: Final = reader() + + with pytest.raises(HTTPException) as traced: + await disabled.trace("t1", "ref") + with pytest.raises(HTTPException) as spanned: + await disabled.span("t1", "s1", "ref") + assert (traced.value.status_code, spanned.value.status_code) == (501, 501) + + +@pytest.mark.asyncio +async def test_reading_a_span_returns_its_detail() -> None: + stored: Final = detail("s1", "refund?") + assert await reader(TraceStorage({}, {"s1": stored})).span("t1", "s1", "") == stored + + +def lens_with_findings() -> Lens: + evidence: Final = Evidence(execution_id="run", span_id="s1", quote="q") + return lens().model_copy( + update={ + "findings": (stored_finding("f1", evidence), stored_finding("f2", evidence), stored_finding("f3", evidence)) + } + ) + + +@pytest.mark.asyncio +async def test_findings_returns_only_the_requested_ids() -> None: + found: Final = await reader(stored=lens_with_findings()).findings("lens", ("f3", "f1", "unknown")) + assert tuple(f.id for f in found) == ("f1", "f3") + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("lens_id", "scope"), + (("missing", Scope(all_teams=True)), ("lens", Scope(team_id="beta"))), + ids=("unknown-lens", "lens-outside-scope"), +) +async def test_findings_of_an_unknown_or_inaccessible_lens_are_not_found(lens_id: str, scope: Scope) -> None: + with pytest.raises(HTTPException) as error: + await reader(stored=lens_with_findings(), scope=scope).findings(lens_id, ("f1",)) + assert error.value.status_code == 404 + + +@pytest.mark.asyncio +async def test_list_returns_a_summary_of_the_latest_revision_of_each_dataset() -> None: + store: Final = MemoryStore() + first: Final = await create_dataset_named(store) + second: Final = await create_dataset_named(store) + await save_revision(first.id, RevisionSave(base_revision=0, cases=(case("a"), case("b"))), ADMIN, store) + + listed: Final = await list_datasets(ADMIN, store) + assert sorted((s.id, s.revision, s.case_count) for s in listed) == sorted(((first.id, 1, 2), (second.id, 0, 0))) + + +@pytest.mark.asyncio +async def test_listing_requires_proxy_admin_access() -> None: + with pytest.raises(HTTPException) as error: + await list_datasets(UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER), MemoryStore()) + assert error.value.status_code == 403 + + +def text_source(*texts: str) -> TextSource: + return TextSource(text="\n".join(json.dumps({"messages": [{"role": "user", "content": t}]}) for t in texts)) + + +@pytest.mark.asyncio +async def test_building_into_a_dataset_skips_cases_it_already_holds() -> None: + store: Final = MemoryStore() + created: Final = await create_dataset_named(store) + await save_revision(created.id, RevisionSave(base_revision=0, cases=(case("a"),)), ADMIN, store) + request: Final = BuildRequest(sources=(text_source("a", "b"),), dataset_id=created.id) + + built: Final = await build_dataset_cases(request, ADMIN, store, lenses(), None) + fresh: Final = await build_dataset_cases( + request.model_copy(update={"dataset_id": ""}), ADMIN, store, lenses(), None + ) + + assert tuple(c.messages[0].content for c in built.cases) == ("b",) + assert tuple(s.reason for s in built.skipped) == ("duplicate",) + assert tuple(c.messages[0].content for c in fresh.cases) == ("a", "b") + + +@pytest.mark.asyncio +async def test_building_into_an_unknown_dataset_is_not_found() -> None: + request: Final = BuildRequest(sources=(text_source("a"),), dataset_id="missing") + with pytest.raises(HTTPException) as error: + await build_dataset_cases(request, ADMIN, MemoryStore(), lenses(), None) + assert error.value.status_code == 404 + + +def exported_texts(body: bytes) -> tuple[str, ...]: + return tuple(DatasetCase.model_validate_json(line).messages[0].content for line in body.splitlines()) + + +@pytest.mark.asyncio +async def test_export_downloads_included_cases_of_the_requested_revision_as_ndjson() -> None: + store: Final = MemoryStore() + created: Final = await create_dataset_named(store) + await save_revision( + created.id, RevisionSave(base_revision=0, cases=(case("a"), case("b", included=False))), ADMIN, store + ) + await save_revision(created.id, RevisionSave(base_revision=1, cases=(case("c"),)), ADMIN, store) + + exported: Final = await export_dataset(created.id, ADMIN, store, revision=1) + latest: Final = await export_dataset(created.id, ADMIN, store) + + assert exported.media_type == "application/x-ndjson" + assert exported.headers["content-disposition"] == f'attachment; filename="dataset-{created.id}-r1.jsonl"' + assert exported_texts(bytes(exported.body)) == ("a",) + assert latest.headers["content-disposition"] == f'attachment; filename="dataset-{created.id}-r2.jsonl"' + assert exported_texts(bytes(latest.body)) == ("c",) + + +@pytest.mark.asyncio +async def test_creating_a_dataset_whose_insert_is_rejected_is_a_conflict() -> None: + with pytest.raises(HTTPException) as error: + await create_dataset(DatasetCreate(name="Refunds"), ADMIN, RejectingStore()) + assert error.value.status_code == 409 + + +@pytest.mark.asyncio +async def test_saving_loses_to_a_concurrent_save_of_the_same_revision() -> None: + created: Final = await create_dataset_named(MemoryStore()) + + with pytest.raises(HTTPException) as error: + await save_revision( + created.id, RevisionSave(base_revision=0, cases=(case("a"),)), ADMIN, RejectingStore(created) + ) + assert error.value.status_code == 409 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "cases", + ( + tuple(case(str(i)) for i in range(LENS_DATASET_MAX_CASES + 1)), + (case("x" * (LENS_DATASET_MAX_CASE_CHARS + 1)),), + ), + ids=("too-many-cases", "case-too-large"), +) +async def test_saving_an_oversized_revision_is_rejected_and_writes_nothing(cases: tuple[DatasetCase, ...]) -> None: + store: Final = MemoryStore() + created: Final = await create_dataset_named(store) + + with pytest.raises(HTTPException) as error: + await save_revision(created.id, RevisionSave(base_revision=0, cases=cases), ADMIN, store) + assert error.value.status_code == 422 + assert sorted(store.rows) == [(created.id, 0)] diff --git a/tests/unit/proxy/lens/test_datasets.py b/tests/unit/proxy/lens/test_datasets.py new file mode 100644 index 00000000000..eda9b26927e --- /dev/null +++ b/tests/unit/proxy/lens/test_datasets.py @@ -0,0 +1,465 @@ +import json +from collections.abc import Mapping +from datetime import datetime, timezone +from typing import Final + +import pytest + +from litellm.constants import LENS_DATASET_MAX_CASE_CHARS, LENS_DATASET_MAX_CASES +from litellm.proxy.lens.datasets import build_cases, case_id, export_jsonl, revision_problem +from litellm.proxy.lens.models import ( + BuildRequest, + BuildResult, + CaseSource, + DatasetCase, + DatasetMessage, + DatasetToolCall, + Evidence, + Finding, + FindingSource, + SkippedCase, + TextSource, + TraceSource, +) +from litellm.proxy.lens.sources import execution_id +from litellm.rust_bridge.trace.generated.types import ( + Span, + SpanDetail, + SpanType, + Trace, + TraceSummary, + UIContent, + UIField, + UIFields, + UIMessage, + UIMessages, + UIText, +) + +NOW: Final = datetime(2026, 1, 15, tzinfo=timezone.utc) + + +def messages(*items: UIMessage) -> UIMessages: + return UIMessages(kind="messages", messages=items) + + +def detail( + span_id: str, + question: str, + answer: str = "Done", + attributes: Mapping[str, str] | None = None, + input_ui: UIContent | None = None, +) -> SpanDetail: + return SpanDetail( + span_id=span_id, + input_ui=input_ui + or messages(UIMessage(role="system", content="Be terse"), UIMessage(role="user", content=question)), + output_ui=messages( + UIMessage( + role="assistant", + content=answer, + tool_calls=({"name": "search", "arguments": json.dumps({"q": question})},), + ) + ), + input=question, + output=answer, + attributes=attributes or {}, + ) + + +def span(span_id: str, kind: SpanType, offset: float) -> Span: + return Span( + span_id=span_id, + parent_span_id=None, + name=span_id, + type=kind, + agent="support", + framework="", + start_offset_ms=offset, + duration_ms=1, + status="ok", + error=None, + error_truncated=False, + input_preview="", + model=None, + input_tokens=0, + output_tokens=0, + litellm_request_id=None, + spend=None, + ) + + +def trace(*spans: Span) -> Trace: + summary: Final = TraceSummary( + trace_id="t1", + name="run", + service="svc", + input_preview="", + start_time="", + duration_ms=1, + status="ok", + span_count=len(spans), + agent_count=1, + agent_invocations=1, + llm_calls=len(spans), + tool_calls=0, + error_count=0, + input_tokens=0, + output_tokens=0, + models=(), + spend=None, + ) + return Trace(summary=summary, agents=(), spans=spans) + + +def stored_finding(finding_id: str, *evidence: Evidence) -> Finding: + return Finding( + id=finding_id, + title="Repeated failed searches", + description="The agent repeats the same failed search", + check_id="retries", + evidence=evidence, + first_seen=NOW, + last_seen=NOW, + revision=1, + ) + + +class FakeReader: + def __init__( + self, + spans: Mapping[tuple[str, str], SpanDetail], + traces: Mapping[str, Trace] | None = None, + findings: tuple[Finding, ...] = (), + ) -> None: + self.spans: Final = spans + self.traces: Final = traces or {} + self.stored: Final = findings + + async def trace(self, trace_id: str, trace_ref: str) -> Trace | None: + return self.traces.get(trace_id) + + async def span(self, trace_id: str, span_id: str, trace_ref: str) -> SpanDetail | None: + return self.spans.get((trace_id, span_id)) + + async def findings(self, lens_id: str, ids: tuple[str, ...]) -> tuple[Finding, ...]: + return tuple(f for f in self.stored if f.id in ids) + + +async def build(reader: FakeReader, *sources: TraceSource | FindingSource | TextSource) -> BuildResult: + return await build_cases(BuildRequest(sources=sources), reader, ()) + + +@pytest.mark.asyncio +async def test_a_span_becomes_a_case_with_its_conversation_reply_and_tool_calls() -> None: + reader: Final = FakeReader({("t1", "s1"): detail("s1", "refund?", "Refunded", {"agent.version": "v7"})}) + result: Final = await build(reader, TraceSource(trace_id="t1", trace_ref="ref", span_id="s1")) + + case: Final = result.cases[0] + assert case.messages == ( + DatasetMessage(role="system", content="Be terse"), + DatasetMessage(role="user", content="refund?"), + ) + assert case.reply == "Refunded" + assert case.tool_calls == (DatasetToolCall(name="search", arguments=json.dumps({"q": "refund?"})),) + assert case.source == CaseSource(trace_id="t1", trace_ref="ref", span_id="s1") + assert case.agent_version == "v7" + assert case.id == case_id(case.messages, case.reply, case.tool_calls) + + +def history(tool: str) -> UIMessages: + return messages( + UIMessage(role="user", content="refund A1"), + UIMessage(role="assistant", content="", tool_calls=({"name": tool, "arguments": '{"id":"A1"}'},)), + UIMessage(role="tool", content="found"), + ) + + +@pytest.mark.asyncio +async def test_tool_calls_in_the_conversation_history_are_kept_and_change_the_case_id() -> None: + reader: Final = FakeReader( + { + ("t1", "s1"): detail("s1", "refund A1", input_ui=history("lookup_order")), + ("t1", "s2"): detail("s2", "refund A1", input_ui=history("cancel_order")), + } + ) + result: Final = await build( + reader, TraceSource(trace_id="t1", span_id="s1"), TraceSource(trace_id="t1", span_id="s2") + ) + + assert result.cases[0].messages[1].tool_calls == (DatasetToolCall(name="lookup_order", arguments='{"id":"A1"}'),) + assert result.cases[1].messages[1].tool_calls == (DatasetToolCall(name="cancel_order", arguments='{"id":"A1"}'),) + assert result.skipped == () + + +@pytest.mark.asyncio +async def test_agent_version_is_empty_when_the_span_does_not_report_one() -> None: + reader: Final = FakeReader({("t1", "s1"): detail("s1", "refund?")}) + result: Final = await build(reader, TraceSource(trace_id="t1", span_id="s1")) + assert result.cases[0].agent_version == "" + + +@pytest.mark.asyncio +async def test_whole_trace_uses_the_last_llm_span_that_has_a_conversation() -> None: + reader: Final = FakeReader( + { + ("t1", "early"): detail("early", "first"), + ("t1", "late"): detail("late", "second"), + ("t1", "tool"): detail("tool", "not an llm"), + ("t1", "text"): detail("text", "raw", input_ui=UIText(kind="text", text="raw")), + }, + traces={ + "t1": trace( + span("early", "llm", 1), span("late", "llm", 5), span("tool", "tool", 9), span("text", "llm", 7) + ) + }, + ) + result: Final = await build(reader, TraceSource(trace_id="t1")) + assert tuple(c.source.span_id for c in result.cases) == ("late",) + assert result.cases[0].messages[-1].content == "second" + + +@pytest.mark.asyncio +async def test_finding_yields_one_case_per_distinct_evidence_span_located_by_its_execution() -> None: + first: Final = execution_id("traces", "alpha", "t1", "ref1") + second: Final = execution_id("traces", "alpha", "t2") + reader: Final = FakeReader( + {("t1", "s1"): detail("s1", "one"), ("t2", "s2"): detail("s2", "two")}, + findings=( + stored_finding("f1", Evidence(execution_id=first, span_id="s1", quote="a")), + stored_finding( + "f2", + Evidence(execution_id=first, span_id="s1", quote="b"), + Evidence(execution_id=second, span_id="s2", quote="c"), + ), + ), + ) + result: Final = await build(reader, FindingSource(lens_id="lens", finding_ids=("f1", "f2"))) + + assert tuple(c.source for c in result.cases) == ( + CaseSource(trace_id="t1", trace_ref="ref1", span_id="s1", finding_id="f1", lens_id="lens"), + CaseSource(trace_id="t2", trace_ref="", span_id="s2", finding_id="f2", lens_id="lens"), + ) + assert result.skipped == () + + +@pytest.mark.asyncio +async def test_text_jsonl_lines_become_cases_and_plain_text_becomes_one_user_message() -> None: + lines: Final = "\n".join( + ( + json.dumps({"messages": [{"role": "user", "content": "hi"}], "reply": "hello", "expected": "greet"}), + "", + json.dumps({"messages": [{"role": "user", "content": "bye"}]}), + ) + ) + jsonl: Final = await build(FakeReader({}), TextSource(text=lines)) + plain: Final = await build(FakeReader({}), TextSource(text="Cancel my order\nplease")) + + assert tuple((c.messages[0].content, c.reply, c.expected) for c in jsonl.cases) == ( + ("hi", "hello", "greet"), + ("bye", "", ""), + ) + assert plain.cases[0].messages == (DatasetMessage(role="user", content="Cancel my order\nplease"),) + + +@pytest.mark.asyncio +async def test_building_again_from_the_same_sources_adds_nothing() -> None: + reader: Final = FakeReader({("t1", "s1"): detail("s1", "one"), ("t1", "s2"): detail("s2", "two")}) + request: Final = BuildRequest( + sources=(TraceSource(trace_id="t1", span_id="s1"), TraceSource(trace_id="t1", span_id="s2")) + ) + first: Final = await build_cases(request, reader, ()) + second: Final = await build_cases(request, reader, first.cases) + + assert len(first.cases) == 2 + assert second.cases == () + assert tuple(s.reason for s in second.skipped) == ("duplicate", "duplicate") + + +@pytest.mark.asyncio +async def test_each_skip_reason_is_reported_against_its_source() -> None: + huge: Final = "x" * (LENS_DATASET_MAX_CASE_CHARS + 1) + reader: Final = FakeReader( + {("t1", "s1"): detail("s1", "same"), ("t1", "big"): detail("big", huge), ("t1", "s3"): detail("s3", "new")} + ) + full: Final = tuple( + DatasetCase(id=str(i), messages=(DatasetMessage(role="user", content=str(i)),), source=CaseSource()) + for i in range(LENS_DATASET_MAX_CASES - 1) + ) + result: Final = await build_cases( + BuildRequest( + sources=( + TraceSource(trace_id="t1", span_id="missing"), + TraceSource(trace_id="t1", span_id="big"), + TraceSource(trace_id="t1", span_id="s1"), + TraceSource(trace_id="t1", span_id="s1"), + TraceSource(trace_id="t1", span_id="s3"), + ) + ), + reader, + full, + ) + + assert tuple(c.source.span_id for c in result.cases) == ("s1",) + assert tuple((s.source.span_id, s.reason) for s in result.skipped) == ( + ("missing", "no_content"), + ("big", "too_large"), + ("s1", "over_limit"), + ("s3", "over_limit"), + ) + + +class CountingReader(FakeReader): + def __init__(self, spans: Mapping[tuple[str, str], SpanDetail]) -> None: + super().__init__(spans) + self.reads: list[str] = [] # mutable-ok: records which spans the build actually fetched + + async def span(self, trace_id: str, span_id: str, trace_ref: str) -> SpanDetail | None: + self.reads.append(span_id) + return await super().span(trace_id, span_id, trace_ref) + + +@pytest.mark.asyncio +async def test_sources_past_the_case_limit_are_not_read() -> None: + reader: Final = CountingReader({("t1", "s1"): detail("s1", "a"), ("t1", "s2"): detail("s2", "b")}) + full: Final = tuple( + DatasetCase(id=str(i), messages=(DatasetMessage(role="user", content=str(i)),), source=CaseSource()) + for i in range(LENS_DATASET_MAX_CASES - 1) + ) + result: Final = await build_cases( + BuildRequest(sources=(TraceSource(trace_id="t1", span_id="s1"), TraceSource(trace_id="t1", span_id="s2"))), + reader, + full, + ) + + assert reader.reads == ["s1"] + assert result.skipped == (SkippedCase(source=CaseSource(trace_id="t1", span_id="s2"), reason="over_limit"),) + + +@pytest.mark.asyncio +async def test_a_malformed_jsonl_line_is_skipped_without_losing_the_valid_lines() -> None: + good: Final = json.dumps({"messages": [{"role": "user", "content": "refund?"}], "reply": "No"}) + result: Final = await build(FakeReader({}), TextSource(text=f'{good}\n{{not json\n{{"reply": "no messages"}}')) + + assert tuple(c.reply for c in result.cases) == ("No",) + assert tuple(s.reason for s in result.skipped) == ("invalid", "invalid") + + +@pytest.mark.asyncio +async def test_an_exported_case_rebuilds_with_its_source_and_agent_version() -> None: + original: Final = DatasetCase( + id="", + messages=(DatasetMessage(role="user", content="refund?"),), + reply="No", + expected="Decline politely", + source=CaseSource(trace_id="t1", span_id="s1"), + agent_version="v7", + ) + result: Final = await build(FakeReader({}), TextSource(text=export_jsonl((original,)))) + + rebuilt: Final = result.cases[0] + assert (rebuilt.source, rebuilt.agent_version, rebuilt.expected) == (original.source, "v7", "Decline politely") + + +@pytest.mark.parametrize( + "case", + ( + DatasetCase( + id="", + messages=(DatasetMessage(role="user", content="q"),), + expected="x" * LENS_DATASET_MAX_CASE_CHARS, + source=CaseSource(), + ), + DatasetCase( + id="", + messages=(DatasetMessage(role="user", content="q", name="x" * LENS_DATASET_MAX_CASE_CHARS),), + source=CaseSource(), + ), + ), +) +def test_expected_and_message_names_count_toward_the_case_size_limit(case: DatasetCase) -> None: + assert revision_problem((case,)) is not None + + +def raw_span(span_id: str, input_ui: UIContent, output_ui: UIContent, raw_input: str, raw_output: str) -> SpanDetail: + return SpanDetail( + span_id=span_id, input_ui=input_ui, output_ui=output_ui, input=raw_input, output=raw_output, attributes={} + ) + + +@pytest.mark.asyncio +async def test_spans_without_chat_messages_fall_back_to_their_text_or_raw_input_and_output() -> None: + fields: Final = UIFields(kind="fields", fields=(UIField(key="q", value="v"),)) + reader: Final = FakeReader( + { + ("t1", "text"): raw_span( + "text", UIText(kind="text", text="shown"), UIText(kind="text", text="answer"), "asked", "raw answer" + ), + ("t1", "fields"): raw_span("fields", fields, fields, '{"q":"v"}', '{"ok":true}'), + } + ) + result: Final = await build( + reader, TraceSource(trace_id="t1", span_id="text"), TraceSource(trace_id="t1", span_id="fields") + ) + + assert tuple((c.messages, c.reply) for c in result.cases) == ( + ((DatasetMessage(role="user", content="asked"),), "answer"), + ((DatasetMessage(role="user", content='{"q":"v"}'),), '{"ok":true}'), + ) + + +@pytest.mark.asyncio +async def test_a_span_with_blank_input_and_output_is_skipped_as_no_content() -> None: + blank: Final = UIText(kind="text", text=" ") + reader: Final = FakeReader({("t1", "s1"): raw_span("s1", blank, blank, " ", "")}) + result: Final = await build(reader, TraceSource(trace_id="t1", span_id="s1")) + + assert result.cases == () + assert result.skipped == (SkippedCase(source=CaseSource(trace_id="t1", span_id="s1"), reason="no_content"),) + + +@pytest.mark.asyncio +async def test_a_whole_trace_without_a_conversation_or_that_is_missing_is_skipped_as_no_content() -> None: + reader: Final = FakeReader( + {("t1", "text"): detail("text", "raw", input_ui=UIText(kind="text", text="raw"))}, + traces={"t1": trace(span("text", "llm", 1), span("gone", "llm", 2))}, + ) + result: Final = await build(reader, TraceSource(trace_id="t1"), TraceSource(trace_id="missing")) + + assert result.cases == () + assert result.skipped == ( + SkippedCase(source=CaseSource(trace_id="t1"), reason="no_content"), + SkippedCase(source=CaseSource(trace_id="missing"), reason="no_content"), + ) + + +@pytest.mark.asyncio +async def test_finding_evidence_with_an_undecodable_execution_or_a_missing_span_is_skipped() -> None: + located: Final = execution_id("traces", "alpha", "t1", "ref1") + reader: Final = FakeReader( + {}, + findings=( + stored_finding( + "f1", + Evidence(execution_id="not-an-execution", span_id="s1", quote="a"), + Evidence(execution_id=located, span_id="gone", quote="b"), + ), + ), + ) + result: Final = await build(reader, FindingSource(lens_id="lens", finding_ids=("f1",))) + + assert result.cases == () + assert result.skipped == ( + SkippedCase(source=CaseSource(span_id="s1", finding_id="f1", lens_id="lens"), reason="no_content"), + SkippedCase( + source=CaseSource(trace_id="t1", trace_ref="ref1", span_id="gone", finding_id="f1", lens_id="lens"), + reason="no_content", + ), + ) + + +def test_export_writes_only_included_cases_one_per_line() -> None: + kept: Final = DatasetCase(id="a", messages=(DatasetMessage(role="user", content="keep"),), source=CaseSource()) + dropped: Final = kept.model_copy(update={"id": "b", "included": False}) + lines: Final = export_jsonl((kept, dropped)).splitlines() + assert tuple(DatasetCase.model_validate_json(line) for line in lines) == (kept,) diff --git a/tests/unit/proxy/management_endpoints/test_auto_router_endpoints.py b/tests/unit/proxy/management_endpoints/test_auto_router_endpoints.py index 0aff5852c96..c0af21671ba 100644 --- a/tests/unit/proxy/management_endpoints/test_auto_router_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_auto_router_endpoints.py @@ -1222,15 +1222,14 @@ class TestAutoRouterBenchmarks: assert [group.router_name for group in response.groups] == ["tagged"] def test_the_listed_kinds_match_the_router_types_traffic_can_record(self): - """The one reason semantic is excluded, pinned against both declarations: a kind the - rollup can record must be listable, and a kind it cannot must not be.""" from typing import get_args, get_type_hints from litellm.router_utils.auto_router_model_naming import StrategyRouterKind from litellm.types.utils import StandardLoggingRoutingDecision - recorded = set(get_args(get_type_hints(StandardLoggingRoutingDecision)["router_type"])) - assert set(get_args(StrategyRouterKind)) - {"semantic"} == recorded + readonly_router_type: Final = get_type_hints(StandardLoggingRoutingDecision, include_extras=True)["router_type"] + recorded: Final = set(get_args(get_args(readonly_router_type)[0])) + assert set(get_args(StrategyRouterKind)) == recorded # --------------------------------------------------------------------------- diff --git a/tests/unit/proxy/management_endpoints/test_model_insights_endpoints.py b/tests/unit/proxy/management_endpoints/test_model_insights_endpoints.py index 7c8d1346944..40b0cbe11c5 100644 --- a/tests/unit/proxy/management_endpoints/test_model_insights_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_model_insights_endpoints.py @@ -225,6 +225,9 @@ class _InMemoryBatcher: def __init__(self, table: _InMemoryUsageTable) -> None: self.litellm_dailymodelusage = table + def execute_raw(self, query: str, *args: object) -> None: + return None + async def __aenter__(self) -> "_InMemoryBatcher": return self diff --git a/tests/unit/proxy/pass_through_endpoints/test_passthrough_endpoint_router.py b/tests/unit/proxy/pass_through_endpoints/test_passthrough_endpoint_router.py index 7a272a49853..87bc47f783f 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_passthrough_endpoint_router.py +++ b/tests/unit/proxy/pass_through_endpoints/test_passthrough_endpoint_router.py @@ -1,4 +1,5 @@ import pytest +from fastapi import HTTPException import litellm from litellm.litellm_core_utils.credential_accessor import CredentialAccessor @@ -45,10 +46,7 @@ def test_credential_loaded_after_deployment_registration_still_resolves(): CredentialAccessor.upsert_credentials([_credential("cred_openai", "sk-loaded-after-boot")]) - assert ( - passthrough_router.get_credentials(custom_llm_provider="openai", region_name=None) - == "sk-loaded-after-boot" - ) + assert passthrough_router.get_credentials(custom_llm_provider="openai", region_name=None) == "sk-loaded-after-boot" def test_credential_rotation_is_reflected_without_deployment_update(): @@ -58,17 +56,11 @@ def test_credential_rotation_is_reflected_without_deployment_update(): ) passthrough_router = _passthrough_router(llm_router) - assert ( - passthrough_router.get_credentials(custom_llm_provider="openai", region_name=None) - == "sk-before-rotation" - ) + assert passthrough_router.get_credentials(custom_llm_provider="openai", region_name=None) == "sk-before-rotation" CredentialAccessor.upsert_credentials([_credential("cred_openai", "sk-after-rotation")]) - assert ( - passthrough_router.get_credentials(custom_llm_provider="openai", region_name=None) - == "sk-after-rotation" - ) + assert passthrough_router.get_credentials(custom_llm_provider="openai", region_name=None) == "sk-after-rotation" def test_deleted_deployment_stops_serving_its_key(monkeypatch): @@ -80,9 +72,7 @@ def test_deleted_deployment_stops_serving_its_key(monkeypatch): llm_router.set_model_list([]) monkeypatch.setenv("OPENAI_API_KEY", "sk-from-env") - assert ( - passthrough_router.get_credentials(custom_llm_provider="openai", region_name=None) == "sk-from-env" - ) + assert passthrough_router.get_credentials(custom_llm_provider="openai", region_name=None) == "sk-from-env" def test_inline_api_key_resolves_without_credential_name(): @@ -91,10 +81,7 @@ def test_inline_api_key_resolves_without_credential_name(): ) passthrough_router = _passthrough_router(llm_router) - assert ( - passthrough_router.get_credentials(custom_llm_provider="anthropic", region_name=None) - == "sk-ant-inline" - ) + assert passthrough_router.get_credentials(custom_llm_provider="anthropic", region_name=None) == "sk-ant-inline" def test_missing_credential_and_no_inline_key_falls_back_to_env(monkeypatch): @@ -104,9 +91,7 @@ def test_missing_credential_and_no_inline_key_falls_back_to_env(monkeypatch): passthrough_router = _passthrough_router(llm_router) monkeypatch.setenv("OPENAI_API_KEY", "sk-from-env") - assert ( - passthrough_router.get_credentials(custom_llm_provider="openai", region_name=None) == "sk-from-env" - ) + assert passthrough_router.get_credentials(custom_llm_provider="openai", region_name=None) == "sk-from-env" def test_deployment_for_other_provider_does_not_match(): @@ -147,9 +132,7 @@ def test_first_matching_deployment_wins(): def test_assemblyai_region_matching(): llm_router = litellm.Router( model_list=[ - _flagged_deployment( - "assemblyai/best", api_key="sk-eu", api_base="https://api.eu.assemblyai.com" - ), + _flagged_deployment("assemblyai/best", api_key="sk-eu", api_base="https://api.eu.assemblyai.com"), _flagged_deployment("assemblyai/best", api_key="sk-us", api_base="https://api.assemblyai.com"), ] ) @@ -179,9 +162,7 @@ def test_env_fallback_when_no_router(monkeypatch): passthrough_router = _passthrough_router(None) monkeypatch.setenv("OPENAI_API_KEY", "sk-from-env") - assert ( - passthrough_router.get_credentials(custom_llm_provider="openai", region_name=None) == "sk-from-env" - ) + assert passthrough_router.get_credentials(custom_llm_provider="openai", region_name=None) == "sk-from-env" def test_returns_none_when_no_router_and_no_env(): @@ -216,9 +197,7 @@ def test_vertex_deployment_resolves_via_named_credential(): ) llm_router = litellm.Router( model_list=[ - _vertex_deployment( - "gemini-live", "vertex_ai/gemini-live-2.5-flash", litellm_credential_name="cred_gcp" - ) + _vertex_deployment("gemini-live", "vertex_ai/gemini-live-2.5-flash", litellm_credential_name="cred_gcp") ] ) passthrough_router = _passthrough_router(llm_router) @@ -276,9 +255,7 @@ def test_vertex_model_hint_prefers_matching_deployment(): passthrough_router = _passthrough_router(_two_vertex_deployments_router()) by_alias = passthrough_router.get_vertex_credentials_from_router_deployments(model="gemini-live") - by_upstream_id = passthrough_router.get_vertex_credentials_from_router_deployments( - model="gemini-live-2.5-flash" - ) + by_upstream_id = passthrough_router.get_vertex_credentials_from_router_deployments(model="gemini-live-2.5-flash") assert by_alias is not None and by_alias.vertex_project == "proj-live" assert by_upstream_id is not None and by_upstream_id.vertex_project == "proj-live" @@ -388,12 +365,100 @@ def test_vertex_deployment_with_deleted_credential_is_skipped(monkeypatch): ) llm_router = litellm.Router( model_list=[ - _vertex_deployment( - "gemini-live", "vertex_ai/gemini-live-2.5-flash", litellm_credential_name="cred_gone" - ) + _vertex_deployment("gemini-live", "vertex_ai/gemini-live-2.5-flash", litellm_credential_name="cred_gone") ] ) passthrough_router = _passthrough_router(llm_router) monkeypatch.setattr(litellm, "credential_list", []) assert passthrough_router.get_vertex_credentials_from_router_deployments(model=None) is None + + +def _token_file(tmp_path, monkeypatch, value: str): + monkeypatch.setenv("LITELLM_OIDC_ALLOWED_CREDENTIAL_DIRS", str(tmp_path)) + token = tmp_path / "token" + token.write_text(value) + return token + + +def test_env_oidc_file_reference_is_read_on_every_call(tmp_path, monkeypatch): + token = _token_file(tmp_path, monkeypatch, "token-one") + monkeypatch.setenv("OPENAI_API_KEY", f"oidc/file/{token}") + passthrough_router = _passthrough_router(None) + + assert passthrough_router.get_credentials(custom_llm_provider="openai", region_name=None) == "token-one" + + token.write_text("token-two") + + assert passthrough_router.get_credentials(custom_llm_provider="openai", region_name=None) == "token-two" + + +def test_deployment_oidc_file_reference_is_resolved(tmp_path, monkeypatch): + token = _token_file(tmp_path, monkeypatch, "deployment-token") + llm_router = litellm.Router(model_list=[_flagged_deployment("openai/gpt-4o", api_key=f"oidc/file/{token}")]) + + assert ( + _passthrough_router(llm_router).get_credentials(custom_llm_provider="openai", region_name=None) + == "deployment-token" + ) + + +def test_oidc_file_reference_outside_allowed_dirs_is_refused(tmp_path, monkeypatch): + allowed = tmp_path / "allowed" + allowed.mkdir() + monkeypatch.setenv("LITELLM_OIDC_ALLOWED_CREDENTIAL_DIRS", str(allowed)) + outside = tmp_path / "token" + outside.write_text("not-allowed") + monkeypatch.setenv("OPENAI_API_KEY", f"oidc/file/{outside}") + + with pytest.raises(HTTPException) as excinfo: + _passthrough_router(None).get_credentials(custom_llm_provider="openai", region_name=None) + + assert excinfo.value.status_code == 401 + assert f"oidc/file/{outside}" in str(excinfo.value.detail) + assert "outside the allowed credential directories" in str(excinfo.value.detail) + + +def test_missing_oidc_token_file_is_a_clear_auth_error(tmp_path, monkeypatch): + monkeypatch.setenv("LITELLM_OIDC_ALLOWED_CREDENTIAL_DIRS", str(tmp_path)) + monkeypatch.setenv("OPENAI_API_KEY", f"oidc/file/{tmp_path / 'missing-token'}") + + with pytest.raises(HTTPException) as excinfo: + _passthrough_router(None).get_credentials(custom_llm_provider="openai", region_name=None) + + assert excinfo.value.status_code == 401 + assert f"oidc/file/{tmp_path / 'missing-token'}" in str(excinfo.value.detail) + + +def test_oidc_env_reference_is_resolved(monkeypatch): + monkeypatch.setenv("UPSTREAM_TOKEN", "env-token") + monkeypatch.setenv("OPENAI_API_KEY", "oidc/env/UPSTREAM_TOKEN") + + assert _passthrough_router(None).get_credentials(custom_llm_provider="openai", region_name=None) == "env-token" + + +def test_oidc_env_path_reference_is_not_resolved(tmp_path, monkeypatch): + outside = tmp_path / "token" + outside.write_text("must-not-be-read") + monkeypatch.setenv("UPSTREAM_TOKEN_FILE", str(outside)) + monkeypatch.setenv("OPENAI_API_KEY", "oidc/env_path/UPSTREAM_TOKEN_FILE") + + assert ( + _passthrough_router(None).get_credentials(custom_llm_provider="openai", region_name=None) + == "oidc/env_path/UPSTREAM_TOKEN_FILE" + ) + + +def test_network_backed_oidc_reference_is_not_fetched_inline(monkeypatch): + def fail(*args, **kwargs): + raise AssertionError("network-backed oidc reference was resolved inside the request path") + + monkeypatch.setattr( + "litellm.proxy.pass_through_endpoints.passthrough_endpoint_router.get_secret_str", + lambda name, *a, **k: "oidc/google/https://example.com" if name == "OPENAI_API_KEY" else fail(), + ) + + assert ( + _passthrough_router(None).get_credentials(custom_llm_provider="openai", region_name=None) + == "oidc/google/https://example.com" + ) diff --git a/tests/unit/proxy/utils/prisma_and_spend/test_spend_functions.py b/tests/unit/proxy/utils/prisma_and_spend/test_spend_functions.py index c2921dd877e..7c4f4e582be 100644 --- a/tests/unit/proxy/utils/prisma_and_spend/test_spend_functions.py +++ b/tests/unit/proxy/utils/prisma_and_spend/test_spend_functions.py @@ -15,7 +15,7 @@ import json from collections.abc import Callable from contextlib import suppress from datetime import datetime, timezone -from typing import Any, Dict, Final, List +from typing import Any, Final from unittest.mock import AsyncMock, MagicMock import pytest @@ -111,15 +111,11 @@ async def test_update_daily_tag_spend_redis_path_when_buffered( writer = MagicMock() proxy_logging.db_spend_update_writer = writer writer.redis_update_buffer = MagicMock() - writer.redis_update_buffer._should_commit_spend_updates_to_redis = MagicMock( - return_value=True - ) + writer.redis_update_buffer._should_commit_spend_updates_to_redis = MagicMock(return_value=True) writer._commit_daily_tag_spend_to_db_with_redis = AsyncMock() writer._commit_daily_tag_spend_to_db = AsyncMock() - await update_daily_tag_spend( - prisma_client=mock_prisma_client, proxy_logging_obj=proxy_logging - ) + await update_daily_tag_spend(prisma_client=mock_prisma_client, proxy_logging_obj=proxy_logging) redis_kwargs = writer._commit_daily_tag_spend_to_db_with_redis.await_args.kwargs pinned = { "redis_calls": writer._commit_daily_tag_spend_to_db_with_redis.await_count, @@ -130,9 +126,7 @@ async def test_update_daily_tag_spend_redis_path_when_buffered( assert pinned == { "redis_calls": 1, "direct_calls": 0, - "redis_kwargs_keys": sorted( - ["prisma_client", "n_retry_times", "proxy_logging_obj"] - ), + "redis_kwargs_keys": sorted(["prisma_client", "n_retry_times", "proxy_logging_obj"]), "redis_n_retries": 3, } @@ -145,15 +139,11 @@ async def test_update_daily_tag_spend_direct_path_when_no_redis( writer = MagicMock() proxy_logging.db_spend_update_writer = writer writer.redis_update_buffer = MagicMock() - writer.redis_update_buffer._should_commit_spend_updates_to_redis = MagicMock( - return_value=False - ) + writer.redis_update_buffer._should_commit_spend_updates_to_redis = MagicMock(return_value=False) writer._commit_daily_tag_spend_to_db_with_redis = AsyncMock() writer._commit_daily_tag_spend_to_db = AsyncMock() - await update_daily_tag_spend( - prisma_client=mock_prisma_client, proxy_logging_obj=proxy_logging - ) + await update_daily_tag_spend(prisma_client=mock_prisma_client, proxy_logging_obj=proxy_logging) assert writer._commit_daily_tag_spend_to_db.await_count == 1 assert writer._commit_daily_tag_spend_to_db_with_redis.await_count == 0 @@ -175,9 +165,7 @@ async def test_update_daily_tag_spend_logs_and_swallows_errors( proxy_logging.db_spend_update_writer._commit_daily_tag_spend_to_db = AsyncMock( side_effect=RuntimeError("commit boom") ) - await update_daily_tag_spend( - prisma_client=mock_prisma_client, proxy_logging_obj=proxy_logging - ) + await update_daily_tag_spend(prisma_client=mock_prisma_client, proxy_logging_obj=proxy_logging) @pytest.mark.asyncio @@ -265,15 +253,11 @@ async def test_update_spend_logs_job_processes_and_clears_queue( mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock() # Stub auxiliary imports so the test focuses on the spend-logs write path. - import litellm.proxy.guardrails.usage_tracking as guard_mod import litellm.proxy.db.spend_log_tool_index as tool_mod + import litellm.proxy.guardrails.usage_tracking as guard_mod - monkeypatch.setattr( - guard_mod, "process_spend_logs_guardrail_usage", AsyncMock(), raising=False - ) - monkeypatch.setattr( - tool_mod, "flush_tool_usage_transactions", AsyncMock(), raising=False - ) + monkeypatch.setattr(guard_mod, "process_spend_logs_guardrail_usage", AsyncMock(), raising=False) + monkeypatch.setattr(tool_mod, "flush_tool_usage_transactions", AsyncMock(), raising=False) await update_spend_logs_job( prisma_client=mock_prisma_client, @@ -283,12 +267,10 @@ async def test_update_spend_logs_job_processes_and_clears_queue( pinned = { "create_many_calls": mock_prisma_client.db.litellm_spendlogs.create_many.await_count, "queue_after": mock_prisma_client.spend_log_transactions, - "first_data_request_id": mock_prisma_client.db.litellm_spendlogs.create_many.await_args.kwargs[ - "data" - ][0]["request_id"], - "skip_duplicates_set": mock_prisma_client.db.litellm_spendlogs.create_many.await_args.kwargs[ - "skip_duplicates" + "first_data_request_id": mock_prisma_client.db.litellm_spendlogs.create_many.await_args.kwargs["data"][0][ + "request_id" ], + "skip_duplicates_set": mock_prisma_client.db.litellm_spendlogs.create_many.await_args.kwargs["skip_duplicates"], } assert pinned == { "create_many_calls": 1, @@ -315,9 +297,7 @@ async def test_update_spend_logs_job_requeues_popped_rows_when_write_cancelled( mock_prisma_client.spend_log_transactions.append(row_arriving_mid_flush) raise asyncio.CancelledError() - mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock( - side_effect=_cancel_mid_write - ) + mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock(side_effect=_cancel_mid_write) with pytest.raises(asyncio.CancelledError): await update_spend_logs_job( @@ -326,9 +306,7 @@ async def test_update_spend_logs_job_requeues_popped_rows_when_write_cancelled( proxy_logging_obj=proxy_logging, ) - assert [ - row["request_id"] for row in mock_prisma_client.spend_log_transactions - ] == ["r1", "r2", "r3"] + assert [row["request_id"] for row in mock_prisma_client.spend_log_transactions] == ["r1", "r2", "r3"] @pytest.mark.asyncio @@ -369,12 +347,8 @@ async def test_drain_spend_logs_queue_flushes_rows_queued_while_draining( import litellm.proxy.db.spend_log_tool_index as tool_mod import litellm.proxy.guardrails.usage_tracking as guard_mod - monkeypatch.setattr( - guard_mod, "process_spend_logs_guardrail_usage", AsyncMock(), raising=False - ) - monkeypatch.setattr( - tool_mod, "process_spend_logs_tool_usage", AsyncMock(), raising=False - ) + monkeypatch.setattr(guard_mod, "process_spend_logs_guardrail_usage", AsyncMock(), raising=False) + monkeypatch.setattr(tool_mod, "process_spend_logs_tool_usage", AsyncMock(), raising=False) proxy_logging = MagicMock() proxy_logging.failure_handler = AsyncMock() @@ -385,9 +359,7 @@ async def test_drain_spend_logs_queue_flushes_rows_queued_while_draining( async def _write(*args: Any, **kwargs: Any) -> None: written.extend(row["request_id"] for row in kwargs["data"]) if len(written) == 1: - mock_prisma_client.spend_log_transactions.append( - make_spend_log_row(request_id="r2") - ) + mock_prisma_client.spend_log_transactions.append(make_spend_log_row(request_id="r2")) mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock(side_effect=_write) @@ -408,12 +380,8 @@ async def test_drain_spend_logs_queue_stops_monitor_and_keeps_its_popped_rows( import litellm.proxy.db.spend_log_tool_index as tool_mod import litellm.proxy.guardrails.usage_tracking as guard_mod - monkeypatch.setattr( - guard_mod, "process_spend_logs_guardrail_usage", AsyncMock(), raising=False - ) - monkeypatch.setattr( - tool_mod, "process_spend_logs_tool_usage", AsyncMock(), raising=False - ) + monkeypatch.setattr(guard_mod, "process_spend_logs_guardrail_usage", AsyncMock(), raising=False) + monkeypatch.setattr(tool_mod, "process_spend_logs_tool_usage", AsyncMock(), raising=False) proxy_logging = MagicMock() proxy_logging.failure_handler = AsyncMock() @@ -460,12 +428,8 @@ async def test_drain_spend_logs_queue_gives_up_after_max_passes( import litellm.proxy.db.spend_log_tool_index as tool_mod import litellm.proxy.guardrails.usage_tracking as guard_mod - monkeypatch.setattr( - guard_mod, "process_spend_logs_guardrail_usage", AsyncMock(), raising=False - ) - monkeypatch.setattr( - tool_mod, "process_spend_logs_tool_usage", AsyncMock(), raising=False - ) + monkeypatch.setattr(guard_mod, "process_spend_logs_guardrail_usage", AsyncMock(), raising=False) + monkeypatch.setattr(tool_mod, "process_spend_logs_tool_usage", AsyncMock(), raising=False) proxy_logging = MagicMock() proxy_logging.failure_handler = AsyncMock() @@ -474,9 +438,7 @@ async def test_drain_spend_logs_queue_gives_up_after_max_passes( async def _write_and_refill(*args: Any, **kwargs: Any) -> None: mock_prisma_client.spend_log_transactions.append(make_spend_log_row()) - mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock( - side_effect=_write_and_refill - ) + mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock(side_effect=_write_and_refill) await drain_spend_logs_queue( prisma_client=mock_prisma_client, @@ -484,10 +446,7 @@ async def test_drain_spend_logs_queue_gives_up_after_max_passes( proxy_logging_obj=proxy_logging, ) - assert ( - mock_prisma_client.db.litellm_spendlogs.create_many.await_count - == MAX_SPEND_LOG_DRAIN_ITERATIONS - ) + assert mock_prisma_client.db.litellm_spendlogs.create_many.await_count == MAX_SPEND_LOG_DRAIN_ITERATIONS @pytest.mark.asyncio @@ -496,8 +455,8 @@ async def test_monitor_spend_logs_queue_invokes_job_when_queue_nonempty( make_spend_log_row: Any, monkeypatch: pytest.MonkeyPatch, ) -> None: - import litellm.proxy.utils as utils_mod import litellm.constants as constants_mod + import litellm.proxy.utils as utils_mod monkeypatch.setattr(constants_mod, "SPEND_LOG_QUEUE_POLL_INTERVAL", 0.0, raising=False) monkeypatch.setattr(constants_mod, "SPEND_LOG_QUEUE_SIZE_THRESHOLD", 1, raising=False) @@ -530,8 +489,8 @@ async def test_monitor_spend_logs_queue_swallows_errors_and_backs_off( """An exception inside the loop is logged with backoff and the loop continues running rather than crashing the monitor task. """ - import litellm.proxy.utils as utils_mod import litellm.constants as constants_mod + import litellm.proxy.utils as utils_mod monkeypatch.setattr(constants_mod, "SPEND_LOG_QUEUE_POLL_INTERVAL", 0.0, raising=False) @@ -721,8 +680,7 @@ def test_raise_failed_update_spend_exception_emits_failure_handler() -> None: else None ), "non_blocking_in_traceback": ( - "Non-Blocking" - in proxy_logging.failure_handler.call_args.kwargs["traceback_str"] + "Non-Blocking" in proxy_logging.failure_handler.call_args.kwargs["traceback_str"] if proxy_logging.failure_handler.call_args else False ), @@ -1086,3 +1044,152 @@ async def test_tool_usage_flush_still_drops_ambiguous_failures( ) assert mock_prisma_client.tool_usage_transactions == [] + + +def _postgres_lock_timeout() -> Exception: + """prisma's shape for Postgres SQLSTATE 55P03: the rollup upsert waited past + ``lock_timeout`` for a row another pod held and was cancelled before it took the lock.""" + from prisma.errors import DataError + + return DataError( + data={ + "user_facing_error": { + "is_panic": False, + "message": "Error querying the database: canceling statement due to lock timeout", + "meta": {"code": "55P03", "message": "canceling statement due to lock timeout"}, + } + } + ) + + +@pytest.mark.asyncio +async def test_tool_usage_flush_requeues_when_the_rollup_row_lock_times_out( + mock_prisma_client: MagicMock, +) -> None: + """A rollup transaction cancelled by ``lock_timeout`` never applied: the waiter + gives its connection back instead of pinning it behind the holder, and its + increments go back to the queue head for the next flush rather than being dropped. + Unlike a full server, one contended row says nothing about the other writers, so the + job carries on: the model usage rollup still opens its own transaction and the + auto-router drain still runs.""" + mock_prisma_client.db.litellm_spendlogtoolindex.create_many = AsyncMock() + rollup_batch = MagicMock() + rollup_batch.__aenter__ = AsyncMock(return_value=MagicMock()) + rollup_batch.__aexit__ = AsyncMock(side_effect=_postgres_lock_timeout()) + mock_prisma_client.db.batch_ = MagicMock(return_value=rollup_batch) + mock_prisma_client.spend_log_transactions = [] + first, second = _tool_usage_transaction("r1"), _tool_usage_transaction("r2") + mock_prisma_client.tool_usage_transactions = [first, second] + mock_prisma_client.model_usage_transactions = [_model_usage_transaction("gpt-4o")] + mock_prisma_client.autorouter_turn_transactions = [MagicMock(name="autorouter_turn")] + + await update_spend_logs_job( + prisma_client=mock_prisma_client, + db_writer_client=None, + proxy_logging_obj=MagicMock(), + ) + + assert { + "rollup_transactions": mock_prisma_client.db.batch_.call_count, + "queue_after": mock_prisma_client.tool_usage_transactions, + "autorouter_queue_after": mock_prisma_client.autorouter_turn_transactions, + } == {"rollup_transactions": 2, "queue_after": [first, second], "autorouter_queue_after": []} + + +def _model_usage_transaction(model: str) -> object: + from litellm.proxy.db.model_usage_rollup import ModelUsageKey, ModelUsageTransaction + + return ModelUsageTransaction( + key=ModelUsageKey( + date="2026-10-03", model_group="gpt", model=model, custom_llm_provider="openai", task_type="chat" + ), + spend=0.001, + prompt_tokens=1, + completion_tokens=1, + successful=True, + ) + + +def _model_usage_batch_failing_with(error: Exception, mock_prisma_client: MagicMock) -> MagicMock: + """Point the model usage rollup's ``batch_()`` at a transaction that fails on commit, with + no tool usage queued so the tool rollup never opens one.""" + mock_prisma_client.db.litellm_spendlogtoolindex.create_many = AsyncMock() + mock_prisma_client.tool_usage_transactions = [] + mock_prisma_client.spend_log_transactions = [] + rollup_batch = MagicMock() + rollup_batch.__aenter__ = AsyncMock(return_value=MagicMock()) + rollup_batch.__aexit__ = AsyncMock(side_effect=error) + mock_prisma_client.db.batch_ = MagicMock(return_value=rollup_batch) + return rollup_batch + + +@pytest.mark.asyncio +async def test_model_usage_flush_requeues_and_carries_on_when_its_row_lock_times_out( + mock_prisma_client: MagicMock, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The DailyModelUsage rollup shares its rows across every pod exactly like the tool rollup. + A batch cancelled under ``lock_timeout`` never applied, so its transactions go back to the + queue head for the next flush instead of being dropped, and because only that rollup's row + was contended the job still drains the auto-router queue behind it.""" + monkeypatch.setattr("litellm.proxy.db.model_usage_rollup.asyncio.sleep", AsyncMock(return_value=None)) + _model_usage_batch_failing_with(_postgres_lock_timeout(), mock_prisma_client) + first, second = _model_usage_transaction("gpt-4o"), _model_usage_transaction("gpt-4o-mini") + mock_prisma_client.model_usage_transactions = [first, second] + mock_prisma_client.autorouter_turn_transactions = [MagicMock(name="autorouter_turn")] + + await update_spend_logs_job( + prisma_client=mock_prisma_client, + db_writer_client=None, + proxy_logging_obj=MagicMock(), + ) + + assert { + "queue_after": mock_prisma_client.model_usage_transactions, + "autorouter_queue_after": mock_prisma_client.autorouter_turn_transactions, + } == {"queue_after": [first, second], "autorouter_queue_after": []} + + +@pytest.mark.asyncio +async def test_model_usage_flush_requeues_and_stops_the_job_when_postgres_is_out_of_connections( + mock_prisma_client: MagicMock, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A batch refused for want of a connection (53300) was never sent either, so it is kept, but + a full server will refuse the writers behind it too: the job stops there like the tool + rollup does, so the drain loop backs off instead of re-hitting the server.""" + monkeypatch.setattr("litellm.proxy.db.model_usage_rollup.asyncio.sleep", AsyncMock(return_value=None)) + _model_usage_batch_failing_with(_postgres_out_of_connections(), mock_prisma_client) + first, second = _model_usage_transaction("gpt-4o"), _model_usage_transaction("gpt-4o-mini") + mock_prisma_client.model_usage_transactions = [first, second] + mock_prisma_client.autorouter_turn_transactions = [MagicMock(name="autorouter_turn")] + + with pytest.raises(Exception, match="too many clients already"): + await update_spend_logs_job( + prisma_client=mock_prisma_client, + db_writer_client=None, + proxy_logging_obj=MagicMock(), + ) + + assert { + "queue_after": mock_prisma_client.model_usage_transactions, + "autorouter_queue_after": len(mock_prisma_client.autorouter_turn_transactions), + } == {"queue_after": [first, second], "autorouter_queue_after": 1} + + +@pytest.mark.asyncio +async def test_model_usage_flush_still_drops_a_batch_that_may_have_applied( + mock_prisma_client: MagicMock, +) -> None: + """Any other failure is ambiguous about whether the increments landed, so replaying it could + double-count: the batch is logged and dropped, and the job carries on.""" + _model_usage_batch_failing_with(RuntimeError("engine lost mid-commit"), mock_prisma_client) + mock_prisma_client.model_usage_transactions = [_model_usage_transaction("gpt-4o")] + + await update_spend_logs_job( + prisma_client=mock_prisma_client, + db_writer_client=None, + proxy_logging_obj=MagicMock(), + ) + + assert mock_prisma_client.model_usage_transactions == [] diff --git a/tests/unit/router_strategy/test_auto_router.py b/tests/unit/router_strategy/test_auto_router.py index 01bf0c2a5ad..6cdee407ced 100644 --- a/tests/unit/router_strategy/test_auto_router.py +++ b/tests/unit/router_strategy/test_auto_router.py @@ -6,6 +6,7 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest +from litellm.exceptions import Timeout as LiteLLMTimeout from litellm.router_strategy.auto_router.auto_router import AutoRouter pytestmark_skip_beta = pytest.mark.skip( @@ -342,6 +343,8 @@ class FixedRouteLayer: self.route_choice = route_choice async def acall(self, vector: Any) -> Any: + if isinstance(self.route_choice, BaseException): + raise self.route_choice return self.route_choice @@ -372,11 +375,15 @@ class StubEmbeddingRouter: class FailingEmbeddingRouter(StubEmbeddingRouter): """Router whose query embedding fails, as it does when the prompt exceeds the encoder's window.""" - async def aembedding(self, input: List[str], model: str, **kwargs: Any) -> Any: - raise ValueError( + def __init__(self, error: Exception | None = None) -> None: + super().__init__() + self.error = error or ValueError( "litellm.InternalServerError: input is too large to process. increase the physical batch size" ) + async def aembedding(self, input: List[str], model: str, **kwargs: Any) -> Any: + raise self.error + def _auto_router(routelayer: Any, litellm_router_instance: Any = None, **kwargs: Any) -> AutoRouter: auto_router: Final = AutoRouter( @@ -395,24 +402,35 @@ class TestAutoRouterAlwaysResolvesARoutableModel: """The hook returns the alias's default model instead of failing or leaking the alias downstream.""" @pytest.mark.asyncio - async def test_should_fall_back_to_default_model_when_the_embedding_call_fails(self): + @pytest.mark.parametrize("request_model", ["my-auto-router", "public-router-alias"]) + async def test_should_fall_back_to_default_model_when_the_embedding_call_fails(self, request_model: str) -> None: auto_router: Final = _auto_router(FixedRouteLayer(None), litellm_router_instance=FailingEmbeddingRouter()) result: Final = await auto_router.async_pre_routing_hook( - model="my-auto-router", + model=request_model, request_kwargs={}, messages=[{"role": "user", "content": "a" * 100_000}], ) assert result is not None assert result.model == "fallback-model" + assert result.routing_decision == { + "router_model_name": "my-auto-router", + "router_type": "semantic", + "routed_model": "fallback-model", + "cause": "semantic_error", + "classifier_model": auto_router.embedding_model, + "classifier_failure_reason": "classifier_error", + "classifier_error_type": "ValueError", + } @pytest.mark.asyncio - async def test_should_fall_back_to_default_model_when_no_route_matches(self): + @pytest.mark.parametrize("request_model", ["my-auto-router", "public-router-alias"]) + async def test_should_fall_back_to_default_model_when_no_route_matches(self, request_model: str) -> None: auto_router: Final = _auto_router(FixedRouteLayer(None)) result: Final = await auto_router.async_pre_routing_hook( - model="my-auto-router", + model=request_model, request_kwargs={}, messages=[{"role": "user", "content": "nothing like any route"}], ) @@ -420,6 +438,13 @@ class TestAutoRouterAlwaysResolvesARoutableModel: assert result is not None # Leaving "my-auto-router" here fails downstream with "Unmapped LLM provider". assert result.model == "fallback-model" + assert result.routing_decision == { + "router_model_name": "my-auto-router", + "router_type": "semantic", + "routed_model": "fallback-model", + "cause": "semantic_no_match", + "classifier_model": auto_router.embedding_model, + } @pytest.mark.asyncio async def test_should_fall_back_to_default_model_when_the_route_layer_returns_an_empty_list(self): @@ -452,14 +477,15 @@ class TestAutoRouterAlwaysResolvesARoutableModel: assert result.model == "code-model" @pytest.mark.asyncio - async def test_should_still_route_to_the_matched_route_when_one_matches(self): + @pytest.mark.parametrize("request_model", ["my-auto-router", "public-router-alias"]) + async def test_should_still_route_to_the_matched_route_when_one_matches(self, request_model: str) -> None: from semantic_router.schema import RouteChoice router: Final = StubEmbeddingRouter() auto_router: Final = _auto_router(FixedRouteLayer(RouteChoice(name="code-model")), litellm_router_instance=router) result: Final = await auto_router.async_pre_routing_hook( - model="my-auto-router", + model=request_model, request_kwargs={}, messages=[{"role": "user", "content": "fix this stack trace"}], ) @@ -467,6 +493,43 @@ class TestAutoRouterAlwaysResolvesARoutableModel: assert result is not None assert result.model == "code-model" assert router.seen_text == "fix this stack trace" + assert result.routing_decision == { + "router_model_name": "my-auto-router", + "router_type": "semantic", + "routed_model": "code-model", + "cause": "semantic_match", + "classifier_model": auto_router.embedding_model, + } + + @pytest.mark.asyncio + @pytest.mark.parametrize("error", ( + TimeoutError("private request content"), + asyncio.TimeoutError("private request content"), + LiteLLMTimeout(message="private request content", model="embedding", llm_provider="openai"), + )) + @pytest.mark.parametrize("wrapped", [False, True]) + async def test_should_record_timeout_without_exception_message(self, error: Exception, wrapped: bool) -> None: + auto_router: Final = _auto_router( + FixedRouteLayer(None if wrapped else error), + litellm_router_instance=FailingEmbeddingRouter(error) if wrapped else StubEmbeddingRouter(), + ) + result: Final = await auto_router.async_pre_routing_hook( + model="my-auto-router", request_kwargs={}, messages=[{"role": "user", "content": "route this"}] + ) + assert result is not None and result.model == "fallback-model" + assert result.routing_decision is not None + assert result.routing_decision["cause"] == "semantic_error" + assert result.routing_decision["classifier_failure_reason"] == "timeout" + assert result.routing_decision["classifier_error_type"] == type(error).__name__ + assert "private request content" not in str(result.routing_decision) + + @pytest.mark.asyncio + async def test_should_propagate_cancellation_during_matching(self) -> None: + auto_router: Final = _auto_router(FixedRouteLayer(asyncio.CancelledError())) + with pytest.raises(asyncio.CancelledError): + await auto_router.async_pre_routing_hook( + model="my-auto-router", request_kwargs={}, messages=[{"role": "user", "content": "route this"}] + ) class TestAutoRouterEmbeddingInputCap: diff --git a/tests/unit/router_strategy/test_complexity_router.py b/tests/unit/router_strategy/test_complexity_router.py index 67e0649958b..d943e57fa51 100644 --- a/tests/unit/router_strategy/test_complexity_router.py +++ b/tests/unit/router_strategy/test_complexity_router.py @@ -20,11 +20,12 @@ from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest import respx -from pydantic import ValidationError +from pydantic import TypeAdapter, ValidationError import litellm from litellm import Router from litellm.integrations.custom_logger import CustomLogger +from litellm.types.utils import StandardLoggingRoutingDecision from litellm.router_utils.auto_router_model_naming import ( CUSTOMIZATION_CAPABILITY, GATED_AUTO_ROUTER_CAPABILITIES, @@ -484,6 +485,10 @@ class TestComplexityRouterInit: assert first.cause != "jev_classifier" assert second.cause != "jev_classifier" assert client.calls == 1 + assert first.classifier_failure_reason == "timeout" + assert first.classifier_error_type == "TimeoutError" + assert second.classifier_failure_reason == "circuit_open" + assert second.classifier_error_type is None assert _CLASSIFIER_CIRCUIT_OPEN_SIGNAL in second.signals @pytest.mark.asyncio @@ -518,6 +523,8 @@ class TestComplexityRouterInit: outcome = await router.aclassify("Explain this") assert outcome.cause != "jev_classifier" + assert outcome.classifier_failure_reason == "classifier_error" + assert outcome.classifier_error_type == ("RuntimeError" if isinstance(response, RuntimeError) else "ValueError") class TestTokenScoring: @@ -3342,6 +3349,8 @@ class TestCapabilityClassifier: assert outcome.tier == ComplexityTier.REASONING assert outcome.cause == "capability_classifier_fallback" assert outcome.signals == ("capability-classifier-fallback",) + assert outcome.classifier_failure_reason == "invalid_response" + assert outcome.classifier_error_type == "ValidationError" @pytest.mark.asyncio async def test_classifier_call_failure_fails_closed_to_capable_model(self, mock_router_instance): @@ -3353,6 +3362,8 @@ class TestCapabilityClassifier: ) assert response.model == "capable-model" assert response.routing_decision["cause"] == "capability_classifier_fallback" + assert response.routing_decision["classifier_failure_reason"] == "timeout" + assert response.routing_decision["classifier_error_type"] == "TimeoutError" assert "classifier_p_solve" not in response.routing_decision assert "classifier_threshold" not in response.routing_decision @@ -4226,6 +4237,9 @@ class TestLLMClassifier: assert first.cause == "heuristic_scorer" assert second.cause == "heuristic_scorer" assert "classifier-circuit-open" in second.signals + assert first.classifier_failure_reason == "timeout" + assert second.classifier_failure_reason == "circuit_open" + assert second.classifier_error_type is None mock_router_instance.acompletion.assert_awaited_once() def test_classifier_circuit_allows_one_probe_and_closes_on_success(self): @@ -8189,18 +8203,24 @@ class TestClassifierPlugin: router = _plugin_router(mock_router_instance, _FixedTierClassifier(None)) outcome = await router.aclassify("what is 2+2?") assert outcome.cause == "heuristic_scorer" + assert outcome.classifier_failure_reason == "declined" + assert outcome.classifier_error_type is None @pytest.mark.asyncio async def test_plugin_error_falls_back_to_heuristic(self, mock_router_instance): router = _plugin_router(mock_router_instance, _RaisingClassifier()) outcome = await router.aclassify("what is 2+2?") assert outcome.cause == "heuristic_scorer" + assert outcome.classifier_failure_reason == "classifier_error" + assert outcome.classifier_error_type == "RuntimeError" @pytest.mark.asyncio async def test_plugin_timeout_falls_back_to_heuristic(self, mock_router_instance): router = _plugin_router(mock_router_instance, _SlowClassifier(), classifier_plugin_timeout_ms=20) outcome = await router.aclassify("what is 2+2?") assert outcome.cause == "heuristic_scorer" + assert outcome.classifier_failure_reason == "timeout" + assert outcome.classifier_error_type == "TimeoutError" @pytest.mark.asyncio async def test_plugin_non_string_verdict_falls_back_to_heuristic(self, mock_router_instance): @@ -8208,12 +8228,16 @@ class TestClassifierPlugin: router = _plugin_router(mock_router_instance, _FixedTierClassifier(42)) outcome = await router.aclassify("what is 2+2?") assert outcome.cause == "heuristic_scorer" + assert outcome.classifier_failure_reason == "invalid_response" + assert outcome.classifier_error_type is None @pytest.mark.asyncio async def test_plugin_unknown_tier_falls_back_to_heuristic(self, mock_router_instance): router = _plugin_router(mock_router_instance, _FixedTierClassifier("galactic")) outcome = await router.aclassify("what is 2+2?") assert outcome.cause == "heuristic_scorer" + assert outcome.classifier_failure_reason == "invalid_response" + assert outcome.classifier_error_type is None @pytest.mark.asyncio async def test_plugin_tier_without_pool_falls_back(self, mock_router_instance): @@ -8229,6 +8253,8 @@ class TestClassifierPlugin: ) outcome = await router.aclassify("what is 2+2?") assert outcome.cause == "heuristic_scorer" + assert outcome.classifier_failure_reason == "invalid_response" + assert outcome.classifier_error_type is None @pytest.mark.asyncio async def test_plugin_failure_with_default_model_fallback(self, mock_router_instance): @@ -9564,6 +9590,101 @@ class TestRedactedLoggingDropsPromptText: assert decision["cause"] == "literal_keyword_match" +@pytest.mark.asyncio +@pytest.mark.parametrize( + "overrides, expected_cause", + [ + ({"classifier_type": "llm"}, "heuristic_scorer"), + ({"classifier_type": "llm", "classifier_fallback": "default_model"}, "default_model_fallback"), + ( + { + "classifier_type": "capability", + "capability_classifier_config": { + "efficient_tier": "SIMPLE", "capable_tier": "REASONING", + "base_threshold": 0.5, "threshold_step": 0.1, + }, + }, + "capability_classifier_fallback", + ), + ( + { + "classifier_type": "llm_v2", + "llm_v2_config": { + "efficient_profile": "Routine tasks", "capable_profile": "Complex tasks", + "harness": "Read files and run checks", "max_quality_gap": 0.05, + }, + }, + "llm_v2_fallback", + ), + ], +) +async def test_classifier_failure_survives_real_router_decision_and_redaction( + overrides: Mapping[str, object], expected_cause: str, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router: Final = Router(model_list=[ + { + "model_name": "auto", + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_config": { + "tiers": {"SIMPLE": "cheap", "REASONING": "capable"}, + "default_model": "cheap", + "classifier_llm_config": {"model": "classifier", "timeout_ms": 2000}, + **overrides, + }, + }, + }, + { + "model_name": "classifier", + "litellm_params": { + "model": "openai/test-classifier", "api_key": "test-key", + "api_base": "https://classifier.test/v1", + }, + }, + {"model_name": "cheap", "litellm_params": {"model": "openai/test-cheap", "api_key": "test-key"}}, + {"model_name": "capable", "litellm_params": {"model": "openai/test-capable", "api_key": "test-key"}}, + ]) + with respx.mock: + upstream: Final = respx.post("https://classifier.test/v1/chat/completions").respond( + 200, + json={ + "id": "classifier-result", "object": "chat.completion", "model": "test-classifier", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "PRIVATE_INVALID_REPLY"}, + "finish_reason": "stop"}], + }, + ) + metadata: Final[dict[str, object]] = {"headers": {"x-litellm-enable-message-redaction": True}} + request: Final[dict[str, object]] = {"metadata": metadata, "turn_off_message_logging": True} + response: Final = await router.async_pre_routing_hook( + model="auto", request_kwargs=request, messages=[{"role": "user", "content": "hi"}] + ) + assert upstream.called + assert response is not None and response.routing_decision is not None + decision: Final = TypeAdapter(StandardLoggingRoutingDecision).validate_python(metadata["routing_decision"]) + assert decision["cause"] == expected_cause + assert decision["classifier_failure_reason"] == "invalid_response" + assert decision["classifier_error_type"] == "ValidationError" + assert "PRIVATE_INVALID_REPLY" not in json.dumps(decision) + assert "signals" not in decision + if overrides["classifier_type"] == "llm": + upstream.respond(200, json={ + "id": "classifier-recovered", "object": "chat.completion", "model": "test-classifier", + "choices": [{"index": 0, "message": {"role": "assistant", "content": '{"tier":"SIMPLE"}'}, + "finish_reason": "stop"}], + }) + recovered: Final = await router.async_pre_routing_hook( + model="auto", request_kwargs=request, messages=[{"role": "user", "content": "hi"}] + ) + assert recovered is not None and recovered.routing_decision is not None + assert recovered.routing_decision["cause"] == "llm_classifier" + recovered_decision: Final = TypeAdapter(StandardLoggingRoutingDecision).validate_python( + metadata["routing_decision"] + ) + assert "classifier_failure_reason" not in recovered_decision + assert "classifier_error_type" not in recovered_decision + + def test_every_routing_decision_field_is_classified(): """Redaction is derived from a declaration, not a list at the call site, so every field has to be classified as quoting the prompt or aggregating it. A field added @@ -14148,6 +14269,8 @@ async def test_v2_chain_judge_failure_uses_selected_fallback( assert outcome.cause == ("heuristic_v2" if fallback == "heuristic" else "default_model_fallback") assert outcome.tier == (ComplexityTier.COMPLEX if fallback == "heuristic" else ComplexityTier.MEDIUM) assert (outcome.heuristic_v2_forecast is not None) == (fallback == "heuristic") + assert outcome.classifier_failure_reason == "classifier_error" + assert outcome.classifier_error_type == "RuntimeError" assert dependency.aresponses.await_count == int(encrypted) assert dependency.acompletion.await_count == int(not encrypted) diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 290b4e62852..03b53f36e6f 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -12,7 +12,7 @@ import warnings from collections.abc import AsyncIterable, AsyncIterator, Awaitable, Callable, Mapping from datetime import datetime, timedelta, timezone from types import SimpleNamespace -from typing import Final, Literal +from typing import TYPE_CHECKING, Final, Literal from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -31,6 +31,7 @@ from litellm.exceptions import GuardrailRaisedException, MidStreamFallbackError, from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.custom_logger import CustomLogger +from litellm.integrations.otel.plumbing.context import set_request_root_span from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.llms.anthropic.pass_through.messages.agentic_streaming_iterator import ( SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES, @@ -68,8 +69,14 @@ from litellm.types.router import ( ModelInfo, PreRoutingHookResponse, RetryPolicy, + RoutingContext, ) +if TYPE_CHECKING: + from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + + from litellm.integrations.otel.logger import OpenTelemetryV2 + def test_update_kwargs_does_not_mutate_defaults_and_merges_metadata(): # initialize a real Router (env‑vars can be empty) @@ -13213,15 +13220,22 @@ class TestTeamPublicNameReachesPreRoutingStrategies: @pytest.mark.asyncio async def test_team_public_name_shadows_a_global_model_for_that_team_only(self): - router = self._router( - {self.INTERNAL_NAME: self._RewriteStrategy()}, - extra_deployments=({"model_name": self.PUBLIC_NAME, "litellm_params": {"model": "openai/gpt-4o"}},), + router: Final = litellm.Router( + model_list=[ + self._team_marker(self.INTERNAL_NAME), + {"model_name": "gemini-flash", "litellm_params": {"model": "gemini/gemini-3.6-flash"}}, + {"model_name": self.PUBLIC_NAME, "litellm_params": {"model": "openai/gpt-4o"}}, + ], ) async def routed(request_kwargs: dict) -> str | None: response = await router.async_pre_routing_hook( model=self.PUBLIC_NAME, request_kwargs=request_kwargs, messages=self._messages() ) + if response is not None: + assert response.routing_decision is not None + assert response.routing_decision["router_model_name"] == self.INTERNAL_NAME + assert response.routing_decision["router_config_id"] == router.model_list[0]["model_info"]["id"] return response.model if response else None async def selected(request_kwargs: dict) -> str: @@ -20267,3 +20281,270 @@ async def test_non_chat_surfaces_mark_their_deployment_pick(monkeypatch: pytest. router.completion(model="gpt-4o", messages=[{"role": "user", "content": "hi"}]) assert events == [_pick("embed", "initial", 1), _pick("gpt-4o", "initial", 1)] + + +class TestAutoRouterTraceProvenance: + @pytest.fixture + def _tracing(self, monkeypatch: pytest.MonkeyPatch) -> "tuple[OpenTelemetryV2, InMemorySpanExporter]": + from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + from litellm.integrations.otel import OpenTelemetryV2Config + from litellm.integrations.otel.logger import OpenTelemetryV2 + from litellm.integrations.otel.plumbing import providers + from litellm.proxy import proxy_server + + config: Final = OpenTelemetryV2Config(exporter="in_memory") + exporter: Final = InMemorySpanExporter() + logger: Final = OpenTelemetryV2(config=config, tracer_provider=providers.build_tracer_provider(config, exporter=exporter)) + monkeypatch.setattr(proxy_server, "open_telemetry_logger", logger) + return logger, exporter + + @staticmethod + def _marker(tag: str, target: str = "answer") -> dict[str, object]: + return { + "model_name": "traced-router", + "model_info": {"id": f"definition-{tag}", "updated_at": "2026-10-01T12:00:00+00:00"}, + "litellm_params": { + "model": "auto_router/complexity_router", "tags": [tag], + "complexity_router_config": { + "tiers": dict.fromkeys(("SIMPLE", "MEDIUM", "COMPLEX", "REASONING"), target), + }, + }, + } + + def test_definition_fingerprint_is_canonical_and_tracks_config_not_credentials(self): + original: Final = Deployment.model_validate(self._marker("blue")) + reordered: Final = Deployment.model_validate({ + **self._marker("blue"), + "litellm_params": { + "api_key": "credential-only-change", "tags": ["blue"], "model": "auto_router/complexity_router", + "complexity_router_config": {"tiers": dict.fromkeys(("REASONING", "COMPLEX", "MEDIUM", "SIMPLE"), "answer")}, + }, + }) + fingerprint: Final = Router._routing_definition_fingerprint(original) + assert fingerprint is not None + assert fingerprint == Router._routing_definition_fingerprint(reordered) + assert fingerprint != Router._routing_definition_fingerprint(Deployment.model_validate(self._marker("blue", "other"))) + + @pytest.mark.asyncio + async def test_direct_classifier_and_plugin_prelude_never_inherit_parent_decisions( + self, _tracing: "tuple[OpenTelemetryV2, InMemorySpanExporter]", monkeypatch: pytest.MonkeyPatch + ) -> None: + logger, exporter = _tracing + completed: Final = asyncio.Event() + + class ObservedClassifier: + prelude: Mapping[str, object] = {} + + async def run(self, context: RoutingContext) -> RoutingContext: + self.prelude = dict(context.metadata) + return context + + async def classify(self, context: RoutingContext) -> str: + await litellm.acompletion( + model="openai/gpt-5.6-luna", api_key="test", mock_response="SIMPLE", + messages=[{"role": "user", "content": "choose a tier"}], + metadata={**context.metadata, "internal_call_origin": "autorouter_classifier"}, + ) + return "SIMPLE" + + class CallbackComplete(CustomLogger): + async def async_log_success_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime + ) -> None: + completed.set() + + monkeypatch.setattr(litellm, "_async_success_callback", [logger, CallbackComplete()]) + classifier: Final = ObservedClassifier() + router: Final = Router(model_list=[ + {**self._marker("blue"), "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_config": { + "classifier_type": "custom", "classifier_plugin": classifier, + "tiers": {"SIMPLE": "answer"}, "session_affinity": False, "deployment_affinity": False, + }, + }}, + {"model_name": "answer", "litellm_params": {"model": "openai/gpt-5.6-luna", "api_key": "test"}}, + ], plugins=[classifier]) + stale: Final = {"routing_decision": {"router_model_name": "previous"}, "_autorouter_baseline_route": "old"} + request: Final = {"metadata": {**stale}, "litellm_metadata": {**stale, "user_api_key_team_id": "team"}} + with logger.tracer.start_as_current_span("request") as root: + set_request_root_span(root) + await router.async_get_available_deployment( + model="traced-router", request_kwargs=request, messages=[{"role": "user", "content": "hello"}] + ) + await completed.wait() + classifier_span: Final = next( + span for span in exporter.get_finished_spans() + if (span.attributes or {}).get("litellm.request.purpose") == "autorouter_classifier" + ) + assert not any(key.startswith("litellm.routing.") for key in classifier_span.attributes) + assert "routing_decision" not in classifier.prelude + assert "_autorouter_baseline_route" not in classifier.prelude + assert classifier.prelude["user_api_key_team_id"] == "team" + assert "routing_decision" not in request["metadata"] + assert "_autorouter_baseline_route" not in request["metadata"] + assert request["litellm_metadata"]["routing_decision"]["cause"] == "classifier_plugin" + + @pytest.mark.asyncio + @pytest.mark.parametrize("build_fails", [False, True]) + async def test_semantic_internal_failure_keeps_phase_identity_without_parent_retry_attribution( + self, _tracing: "tuple[OpenTelemetryV2, InMemorySpanExporter]", monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, build_fails: bool, + ) -> None: + logger, exporter = _tracing + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + query_statuses: Final = iter((503, 200)) + + def embedding_response(request: httpx.Request) -> httpx.Response: + status: Final = 400 if build_fails else ( + next(query_statuses) if json.loads(request.content)["input"] == ["query-retry"] else 200 + ) + return httpx.Response(status, headers={"x-should-retry": "false"}, json={ + "object": "list", "model": "text-embedding-3-small", + "data": [{"object": "embedding", "index": 0, "embedding": [1.0, 0.0]}], + "usage": {"prompt_tokens": 1, "total_tokens": 1}, + }) + + upstream: Final = respx_mock.post("https://embedding.example/v1/embeddings").mock(side_effect=embedding_response) + router: Final = Router(model_list=[ + {"model_name": "semantic", "model_info": {"id": "semantic-definition"}, "litellm_params": { + "model": "auto_router/semantic-router", "auto_router_default_model": "answer", + "auto_router_embedding_model": "embed", + "auto_router_config": json.dumps({"routes": [{"name": "answer", "utterances": ["hello"]}]}), + }}, + *({"model_name": "embed", "model_info": {"id": f"embed-{index}"}, "litellm_params": { + "model": "openai/text-embedding-3-small", "api_base": "https://embedding.example/v1", + "api_key": "test", "max_retries": 0, + }} for index in range(2)), + {"model_name": "answer", "litellm_params": {"model": "openai/gpt-5.6-luna", "api_key": "test"}}, + ], num_retries=1, retry_after=0) + stale: Final = {"routing_decision": {"router_model_name": "previous"}, "_autorouter_baseline_route": "old"} + request: Final = {"metadata": {**stale}, "litellm_metadata": {**stale}} + with logger.tracer.start_as_current_span("request") as root: + set_request_root_span(root) + call: Final = router.async_get_available_deployment( + model="semantic", request_kwargs=request, messages=[{"role": "user", "content": "query-retry"}] + ) + if build_fails: + with pytest.raises(ValueError, match="API call failed"): + await call + else: + await call + spans: Final = {span.name: span for span in exporter.get_finished_spans()} + assert spans["route semantic"].attributes["litellm.routing.router_config_id"] == "semantic-definition" + assert not any(event.name == "litellm.routing.retry" for event in spans["request"].events) + if build_fails: + assert all( + "routing_decision" not in bucket and "_autorouter_baseline_route" not in bucket + for bucket in (request["metadata"], request["litellm_metadata"]) + ) + else: + assert sum(json.loads(call.request.content)["input"] == ["query-retry"] for call in upstream.calls) == 2 + assert request["litellm_metadata"]["routing_decision"]["cause"] == "semantic_match" + assert sum(span.name == "route embed" for span in exporter.get_finished_spans()) == 2 + assert any(event.name == "litellm.request.deployment_selected" for event in spans["request"].events) + + @pytest.mark.asyncio + async def test_same_alias_and_tags_across_strategy_families_keep_selected_definition( + self, _tracing: "tuple[OpenTelemetryV2, InMemorySpanExporter]" + ) -> None: + logger, exporter = _tracing + router: Final = Router(model_list=[ + self._marker("blue"), + { + "model_name": "traced-router", + "model_info": {"id": "definition-semantic"}, + "litellm_params": { + "model": "auto_router/semantic-router", + "tags": ["blue"], + "auto_router_config": json.dumps({"routes": [{"name": "answer", "utterances": ["hello"]}]}), + "auto_router_default_model": "answer", + "auto_router_embedding_model": "embed", + }, + }, + ]) + with logger.start_phase_span("route traced-router"): + response: Final = await router.async_pre_routing_hook( + model="traced-router", request_kwargs={"metadata": {"tags": ["blue"]}} + ) + assert response is None + span: Final = next(span for span in exporter.get_finished_spans() if span.name == "route traced-router") + assert span.attributes is not None + assert span.attributes["litellm.routing.router_config_id"] == router.model_list[1]["model_info"]["id"] + assert span.attributes["litellm.routing.router_type"] == "semantic" + + @pytest.mark.asyncio + async def test_semantic_team_alias_keeps_registered_identity_after_routing_and_retry( + self, + _tracing: "tuple[OpenTelemetryV2, InMemorySpanExporter]", + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + ) -> None: + logger, exporter = _tracing + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post("https://embedding.example/v1/embeddings").respond(200, json={ + "object": "list", + "model": "text-embedding-3-small", + "data": [{"object": "embedding", "index": 0, "embedding": [1.0, 0.0]}], + "usage": {"prompt_tokens": 1, "total_tokens": 1}, + }) + router: Final = Router(model_list=[ + { + "model_name": "internal-semantic-router", + "model_info": {"id": "semantic-definition", "team_id": "trace-team", "team_public_model_name": "public-router"}, + "litellm_params": { + "model": "auto_router/semantic-router", + "auto_router_config": json.dumps({"routes": [{"name": "answer", "utterances": ["hello"]}]}), + "auto_router_default_model": "answer", + "auto_router_embedding_model": "embed", + }, + }, + {"model_name": "embed", "litellm_params": {"model": "openai/text-embedding-3-small", "api_base": "https://embedding.example/v1", "api_key": "test"}}, + {"model_name": "answer", "litellm_params": {"model": "openai/gpt-5.6-luna", "api_key": "test"}}, + ]) + request: Final = {"metadata": {"user_api_key_team_id": "trace-team"}} + with logger.tracer.start_as_current_span("request"): + await router.async_get_available_deployment( + model="public-router", request_kwargs=request, messages=[{"role": "user", "content": "hello"}] + ) + router.log_retry({**request, "model": "answer"}, ValueError("retry")) + decision: Final = request["metadata"]["routing_decision"] + assert decision["router_model_name"] == router.model_list[0]["model_name"] + assert decision["router_config_id"] == router.model_list[0]["model_info"]["id"] + assert decision["cause"] == "semantic_match" + spans: Final = {span.name: span for span in exporter.get_finished_spans()} + assert spans["route public-router"].attributes["litellm.routing.router_model_name"] == decision["router_model_name"] + assert spans["route public-router"].attributes["litellm.routing.router_config_id"] == decision["router_config_id"] + retry: Final = next(event for event in spans["request"].events if event.name == "litellm.routing.retry") + assert retry.attributes["litellm.routing.router_model_name"] == decision["router_model_name"] + + @pytest.mark.asyncio + @pytest.mark.parametrize("passthrough", [False, True]) + async def test_attempt_spans_keep_selected_config_and_plain_fallback_clears_it( + self, _tracing: "tuple[OpenTelemetryV2, InMemorySpanExporter]", passthrough: bool + ) -> None: + logger, exporter = _tracing + router: Final = Router(model_list=[ + self._marker("blue"), self._marker("green"), + {"model_name": "answer", "litellm_params": {"model": "openai/gpt-5.6-luna", "api_key": "test", "use_in_pass_through": True, "tags": ["blue", "green"]}}, + ], enable_tag_filtering=True) + request: Final = {"metadata": {"tags": ["green"]}, "litellm_metadata": {"tags": ["green"]}} + pick: Final = router.async_get_available_deployment_for_pass_through if passthrough else router.async_get_available_deployment + with logger.tracer.start_as_current_span("request"): + await pick(model="traced-router", request_kwargs=request, messages=[{"role": "user", "content": "hello"}]) + decision: Final = request["litellm_metadata"]["routing_decision"] + assert decision["router_config_id"] == "definition-green" + assert decision["router_config_updated_at"] == "2026-10-01T12:00:00+00:00" + assert decision["router_config_fingerprint"] == router.complexity_routers["traced-router"][1].definition_fingerprint + assert "routing_decision" not in request["metadata"] + router.log_retry({**request, "model": "answer"}, ValueError("private-error-text")) + await pick(model="answer", request_kwargs=request, messages=[{"role": "user", "content": "hello"}]) + assert "routing_decision" not in request["litellm_metadata"] + spans: Final = {span.name: span for span in exporter.get_finished_spans()} + assert spans["route traced-router"].attributes["litellm.routing.router_config_id"] == "definition-green" + assert spans["route traced-router"].attributes["litellm.routing.routed_model"] == "answer" + assert not any(key.startswith("litellm.routing.") for key in spans["route answer"].attributes) + retry: Final = next(event for event in spans["request"].events if event.name == "litellm.routing.retry") + assert retry.attributes["error.type"] == "ValueError" + assert retry.attributes["litellm.retry.count"] == 1 + assert "private-error-text" not in str(retry.attributes) diff --git a/ui/litellm-dashboard/src/components/lens/LensWorkspace.integration.test.tsx b/ui/litellm-dashboard/src/components/lens/LensWorkspace.integration.test.tsx index f42f2bd7e8c..c251b36d190 100644 --- a/ui/litellm-dashboard/src/components/lens/LensWorkspace.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/lens/LensWorkspace.integration.test.tsx @@ -85,8 +85,8 @@ describe("Lens interactive demo", () => { expect(lastUrl(onUrlUpdate).get("trace")).toBe(openRun.get("trace")); await user.click(within(drawer).getByRole("tab", { name: "Attributes" })); await expectUrl(onUrlUpdate, (url) => expect(url.get("span_tab")).toBe("attributes")); - await user.click(within(drawer).getByRole("tab", { name: "Conversation" })); - await expectUrl(onUrlUpdate, (url) => expect(url.get("view")).toBe("conversation")); + await user.click(within(drawer).getByRole("tab", { name: "Thread" })); + await expectUrl(onUrlUpdate, (url) => expect(url.get("view")).toBe("thread")); expect(network).not.toHaveBeenCalled(); await user.click(screen.getByRole("switch", { name: "Demo data" })); expect(await screen.findByRole("region", { name: "Get started with Lens" })).toBeVisible(); diff --git a/ui/litellm-dashboard/src/components/lens/LensWorkspace.tsx b/ui/litellm-dashboard/src/components/lens/LensWorkspace.tsx index 2c796ebd6fe..1a483f55e90 100644 --- a/ui/litellm-dashboard/src/components/lens/LensWorkspace.tsx +++ b/ui/litellm-dashboard/src/components/lens/LensWorkspace.tsx @@ -11,6 +11,7 @@ import { Tabs, TabsContent } from "@/components/ui/tabs"; import { LensServicesProvider, useLensAccessToken, useLensApi, useLiveLensServices } from "./data/LensServices"; import { isProxyAdminRole, isProxyAdminTierRole } from "@/utils/roles"; import { InvestigationsView } from "./investigations/InvestigationsView"; +import { DatasetsView } from "./datasets/DatasetsView"; import { LensSettings } from "./settings/LensSettings"; import { createLensDemo } from "./data/demo/createLensDemo"; import { lensQueries } from "./data/queries"; @@ -213,6 +214,9 @@ function LensContent({ userRole, readOnly }: Omit

)} + + + )} {workers && list && ( @@ -237,6 +241,12 @@ function LensContent({ userRole, readOnly }: Omit ); } +function DatasetsPanel({ canView, isAdmin, readOnly }: { canView: boolean; isAdmin: boolean; readOnly: boolean }) { + if (!canView) + return

Datasets require proxy administrator access.

; + return ; +} + function needsSetup( state: LensReadiness, location: { @@ -249,7 +259,7 @@ function needsSetup( issueKey: string | null; }, ) { - if (location.tab === "settings") return false; + if (location.tab === "settings" || location.tab === "datasets") return false; if (location.requested) return true; const selected = location.tab === "traces" ? location.trace : location.lensId || location.dialog || location.issueKey; if (!state.missingTraces || selected) return false; diff --git a/ui/litellm-dashboard/src/components/lens/data/LensServices.tsx b/ui/litellm-dashboard/src/components/lens/data/LensServices.tsx index a4804212012..2895976de01 100644 --- a/ui/litellm-dashboard/src/components/lens/data/LensServices.tsx +++ b/ui/litellm-dashboard/src/components/lens/data/LensServices.tsx @@ -26,6 +26,8 @@ function useLensServices(): LensServices { export const useLensApi = (): LensApi => useLensServices().lens; +export const useOptionalLensApi = (): LensApi | null => useContext(LensServicesContext)?.lens ?? null; + export const useLensAccessToken = (): string => useLensServices().accessToken; export function useLiveLensServices(accessToken: string): LensServices { diff --git a/ui/litellm-dashboard/src/components/lens/data/demo/createLensDemo.ts b/ui/litellm-dashboard/src/components/lens/data/demo/createLensDemo.ts index 36c1d9238b2..422601c44c2 100644 --- a/ui/litellm-dashboard/src/components/lens/data/demo/createLensDemo.ts +++ b/ui/litellm-dashboard/src/components/lens/data/demo/createLensDemo.ts @@ -2,6 +2,7 @@ import { ApiError } from "@/lib/http/client"; import type { TracesApi } from "@/components/lens/traces/api"; import type { LensServices } from "../LensServices"; import type { LensApi } from "../service"; +import { demoDatasetsApi } from "./demoDatasets"; import { createLensDemoData, type LensDemoData } from "./fixtures"; const notInDemo = (): Promise => @@ -14,6 +15,7 @@ function demoLensApi(data: LensDemoData): LensApi { const jobs = (lensId: string) => data.lenses.find((lens) => lens.id === lensId)?.jobs; return { scope: "demo", + datasets: demoDatasetsApi(data), lenses: async () => ({ lenses: data.lenses, workers: [], tracing_enabled: true }), activity: async () => ({ traces: true, requests: false }), runs: (lensId, offset) => found(jobs(lensId)?.slice(offset)), diff --git a/ui/litellm-dashboard/src/components/lens/data/demo/demoDatasets.ts b/ui/litellm-dashboard/src/components/lens/data/demo/demoDatasets.ts new file mode 100644 index 00000000000..d3bd54e3d38 --- /dev/null +++ b/ui/litellm-dashboard/src/components/lens/data/demo/demoDatasets.ts @@ -0,0 +1,206 @@ +import { ApiError } from "@/lib/http/client"; +import type { SpanDetail } from "@/components/lens/traces/types"; + +import { evidenceTarget } from "../../model/findings"; +import type { DatasetsApi } from "../../datasets/client"; +import type { BuildSource, CaseSource, Dataset, DatasetCase, DatasetMessage, SkippedCase } from "../../datasets/types"; +import type { LensDemoData } from "./fixtures"; + +const MAX_CASES = 200; +const NO_SOURCE: CaseSource = { trace_id: "", trace_ref: "", span_id: "", finding_id: "", lens_id: "" }; +const ROLES: ReadonlySet = new Set(["system", "user", "assistant", "tool"]); + +type Candidate = DatasetCase | SkippedCase; + +const isRole = (role: unknown): role is DatasetMessage["role"] => typeof role === "string" && ROLES.has(role); + +function contentHash(text: string): string { + const hash = [...text].reduce((acc, char) => (Math.imul(acc, 31) + char.charCodeAt(0)) | 0, 7); + return (hash >>> 0).toString(16).padStart(8, "0"); +} + +function parsedMessages(raw: string): DatasetMessage[] { + try { + const parsed: unknown = JSON.parse(raw); + if (!Array.isArray(parsed)) return []; + return parsed.flatMap((item: { role?: unknown; content?: unknown }) => + isRole(item?.role) && typeof item.content === "string" + ? [{ role: item.role, content: item.content, name: "", tool_calls: [] }] + : [], + ); + } catch { + return []; + } +} + +function makeCase(messages: DatasetMessage[], reply: string, source: CaseSource): Candidate { + if (messages.length === 0 && !reply) return { source, reason: "no_content" }; + const id = contentHash(JSON.stringify({ messages, reply })); + return { id, messages, reply, tool_calls: [], expected: "", included: true, source, agent_version: "" }; +} + +function caseFromSpan(detail: SpanDetail, source: CaseSource): Candidate { + const messages = parsedMessages(detail.input); + const replies = parsedMessages(detail.output).filter((message) => message.role === "assistant"); + const reply = replies.at(-1)?.content ?? detail.output; + return makeCase( + messages.length ? messages : [{ role: "user", content: detail.input, name: "", tool_calls: [] }], + reply, + source, + ); +} + +function textCases(text: string): Candidate[] { + const lines = text + .split("\n") + .map((line) => line.trim()) + .filter(Boolean); + const parsed = lines.map((line) => { + try { + const value: { messages?: unknown; reply?: unknown } = JSON.parse(line); + return Array.isArray(value.messages) ? value : null; + } catch { + return null; + } + }); + if (parsed.length === 0 || parsed.some((value) => value === null)) + return [makeCase([{ role: "user", content: text, name: "", tool_calls: [] }], "", NO_SOURCE)]; + return parsed.map((value) => + makeCase( + parsedMessages(JSON.stringify(value?.messages)), + typeof value?.reply === "string" ? value.reply : "", + NO_SOURCE, + ), + ); +} + +const isCase = (candidate: Candidate): candidate is DatasetCase => "id" in candidate; + +export function demoDatasetsApi(data: LensDemoData, now: () => Date = () => new Date()): DatasetsApi { + const revisions = new Map(); + const missing = () => Promise.reject(new ApiError("Dataset not found", 404, { detail: "Dataset not found" })); + const latest = (id: string) => revisions.get(id)?.at(-1); + const run = (traceId: string) => data.runs.find(({ trace }) => trace.summary.trace_id === traceId); + const spanCase = (traceId: string, spanId: string, source: CaseSource): Candidate => { + const found = run(traceId); + const detail = spanId + ? found?.details.find((span) => span.span_id === spanId) + : found?.details.findLast((span) => found.trace.spans.find((s) => s.span_id === span.span_id)?.type === "llm"); + return detail ? caseFromSpan(detail, source) : { source, reason: "no_content" }; + }; + const sourceCases = (source: BuildSource): Candidate[] => { + if (source.kind === "text") return textCases(source.text); + if (source.kind === "finding") { + const findings = data.lenses + .flatMap((lens) => lens.jobs.flatMap((job) => job.findings ?? [])) + .filter((finding) => source.finding_ids.includes(finding.id)); + const spans = new Map( + findings.flatMap((finding) => + finding.evidence.map((evidence) => { + const traceId = evidenceTarget(evidence.execution_id)?.id ?? ""; + const origin = { ...NO_SOURCE, trace_id: traceId, span_id: evidence.span_id, finding_id: finding.id }; + return [`${traceId}/${evidence.span_id}`, { ...origin, lens_id: source.lens_id }] as const; + }), + ), + ); + return [...spans.values()].map((origin) => spanCase(origin.trace_id, origin.span_id, origin)); + } + const origin = { + ...NO_SOURCE, + trace_id: source.trace_id, + trace_ref: source.trace_ref ?? "", + span_id: source.span_id ?? "", + }; + return [spanCase(source.trace_id, source.span_id ?? "", origin)]; + }; + return { + list: async () => + [...revisions.values()].map((history) => { + const dataset = history.at(-1)!; + return { + id: dataset.id, + name: dataset.name, + agent_name: dataset.agent_name, + revision: dataset.revision, + case_count: dataset.cases.length, + updated_at: dataset.created_at, + }; + }), + get: (id, revision) => { + const history = revisions.get(id); + const found = revision === undefined ? history?.at(-1) : history?.find((item) => item.revision === revision); + return found ? Promise.resolve(found) : missing(); + }, + create: async ({ name, agent_name }) => { + const dataset: Dataset = { + id: `demo-dataset-${revisions.size + 1}`, + name, + agent_name: agent_name ?? "", + team_id: "", + created_at: now().toISOString(), + revision: 0, + created_by: "demo", + cases: [], + }; + revisions.set(dataset.id, [dataset]); + return dataset; + }, + build: async ({ sources, dataset_id }) => { + const existing = (dataset_id && latest(dataset_id)?.cases) || []; + const seen = new Set(existing.map((item) => item.id)); + const candidates = sources.flatMap(sourceCases); + const cases: DatasetCase[] = []; + const skipped: SkippedCase[] = []; + for (const candidate of candidates) { + if (!isCase(candidate)) skipped.push(candidate); + else if (seen.has(candidate.id)) skipped.push({ source: candidate.source, reason: "duplicate" }); + else if (existing.length + cases.length >= MAX_CASES) + skipped.push({ source: candidate.source, reason: "over_limit" }); + else { + seen.add(candidate.id); + cases.push(candidate); + } + } + return { cases, skipped }; + }, + saveRevision: async (id, { base_revision, cases }) => { + const current = latest(id); + if (!current) return missing(); + if (current.revision !== base_revision) + throw new ApiError("Dataset changed, reload", 409, { detail: "Dataset changed, reload" }); + const saved: Dataset = { + ...current, + revision: current.revision + 1, + created_at: now().toISOString(), + cases: cases.map((item) => ({ + ...item, + reply: item.reply ?? "", + tool_calls: item.tool_calls ?? [], + expected: item.expected ?? "", + included: item.included ?? true, + agent_version: item.agent_version ?? "", + messages: item.messages.map((message) => ({ + ...message, + name: message.name ?? "", + tool_calls: message.tool_calls ?? [], + })), + source: { ...NO_SOURCE, ...item.source }, + })), + }; + revisions.get(id)!.push(saved); + return saved; + }, + exportJsonl: async (id, revision) => { + const history = revisions.get(id); + const found = revision === undefined ? history?.at(-1) : history?.find((item) => item.revision === revision); + if (!found) return missing(); + const lines = found.cases.filter((item) => item.included).map((item) => `${JSON.stringify(item)}\n`); + return new Blob(lines, { type: "application/x-ndjson" }); + }, + evalCases: async (id, revision) => { + const found = revisions.get(id)?.find((item) => item.revision === revision); + if (!found) return missing(); + return { dataset_id: id, revision, cases: found.cases.filter((item) => item.included) }; + }, + }; +} diff --git a/ui/litellm-dashboard/src/components/lens/data/service.ts b/ui/litellm-dashboard/src/components/lens/data/service.ts index f035d882259..9bd7919308b 100644 --- a/ui/litellm-dashboard/src/components/lens/data/service.ts +++ b/ui/litellm-dashboard/src/components/lens/data/service.ts @@ -3,6 +3,7 @@ import type { ApiClient } from "@/lib/http/client"; import { getAuthHeaderName } from "@/lib/http/runtime"; import type { Client } from "openapi-fetch"; import type { components, paths } from "@/lib/http/schema"; +import { liveDatasetsApi, type DatasetsApi } from "../datasets/client"; import type { ActivitySelection, AnalysisModelInfo, @@ -45,6 +46,7 @@ export interface AnalysisKeyRequest { export interface LensApi { /** Partitions query caches between backends (one token, or the demo). */ readonly scope: string; + readonly datasets: DatasetsApi; lenses(): Promise; activity(): Promise; runs(lensId: string, offset: number): Promise; @@ -87,6 +89,7 @@ export function liveLensApi(client: LensClient, apiClient: ApiClient, accessToke const worker = (worker_id: string) => ({ headers, params: { path: { worker_id } } }); return { scope: accessToken, + datasets: liveDatasetsApi(client, apiClient, accessToken), lenses: () => required(client.GET("/lens", { headers })), activity: () => required(client.GET("/lens/activity/available", { headers })), runs: (lensId, offset) => diff --git a/ui/litellm-dashboard/src/components/lens/datasets/AddToDatasetDialog.integration.test.tsx b/ui/litellm-dashboard/src/components/lens/datasets/AddToDatasetDialog.integration.test.tsx new file mode 100644 index 00000000000..c1a06150988 --- /dev/null +++ b/ui/litellm-dashboard/src/components/lens/datasets/AddToDatasetDialog.integration.test.tsx @@ -0,0 +1,138 @@ +import { fireEvent, screen, waitFor, within } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { beforeEach, expect, it, vi } from "vitest"; + +import { renderWithLens, stubGateway } from "@/../tests/lens-test-utils"; +import { testQueryClient } from "@/../tests/test-utils"; + +import { AddToDatasetDialog } from "./AddToDatasetDialog"; +import type { Dataset, DatasetCase } from "./types"; + +const source = { trace_id: "trace-1", trace_ref: "", span_id: "", finding_id: "", lens_id: "" }; +const makeCase = (id: string, question: string, reply: string): DatasetCase => ({ + id, + messages: [{ role: "user", content: question, name: "", tool_calls: [] }], + reply, + tool_calls: [], + expected: "", + included: true, + source, + agent_version: "", +}); + +const kept = makeCase("kept", "Where is order 1042?", "It ships tomorrow."); +const junk = makeCase("junk", "asdf", "I did not understand."); +const existing = makeCase("existing", "Can I return headphones?", "Yes, within 30 days."); +const dataset: Dataset = { + id: "ds-1", + name: "support cases", + agent_name: "support_agent", + team_id: "", + created_at: "2026-10-01T00:00:00Z", + revision: 3, + created_by: "admin", + cases: [existing], +}; +const sources = [{ kind: "trace" as const, trace_id: "trace-1", trace_ref: "", span_id: "" }]; + +let proxy = stubGateway(); + +beforeEach(() => { + testQueryClient.clear(); + proxy = stubGateway(); + proxy.get.mockImplementation(async (path) => { + if (path === "/lens/datasets") + return [ + { id: "ds-1", name: "support cases", agent_name: "support_agent", revision: 3, case_count: 1, updated_at: "" }, + ]; + if (path === "/lens/datasets/ds-1") return dataset; + throw new Error(`unexpected GET ${path}`); + }); + proxy.post.mockImplementation(async (path) => { + if (path === "/lens/datasets/build") + return { cases: [kept, junk], skipped: [{ source: { ...source, trace_id: "trace-2" }, reason: "duplicate" }] }; + if (path === "/lens/datasets/ds-1/revisions") return { ...dataset, revision: 4 }; + throw new Error(`unexpected POST ${path}`); + }); +}); + +it("saves the dataset's existing cases plus only the ticked new ones, with their expected text", async () => { + const user = userEvent.setup(); + const onClose = vi.fn(); + renderWithLens(); + + const list = await screen.findByRole("list", { name: "Cases to add" }); + expect(within(list).getAllByRole("listitem")[0]).toHaveTextContent("Where is order 1042?It ships tomorrow."); + expect(within(screen.getByRole("list", { name: "Skipped" })).getByText("Already in the dataset")).toBeVisible(); + expect(proxy.post.mock.calls.find(([path]) => path === "/lens/datasets/build")?.[1].body).toEqual({ + sources, + dataset_id: "ds-1", + }); + + await user.click(screen.getByRole("checkbox", { name: "Include Case 2" })); + expect(screen.getByText("1 of 2 selected")).toBeVisible(); + fireEvent.change(screen.getByRole("textbox", { name: "Expected for Case 1" }), { + target: { value: "Gives the ship date" }, + }); + await user.click(screen.getByRole("button", { name: "Save 1 case" })); + + await waitFor(() => expect(onClose).toHaveBeenCalled()); + const saved = proxy.post.mock.calls.find(([path]) => path === "/lens/datasets/ds-1/revisions")?.[1].body; + expect(saved).toEqual({ + base_revision: 3, + cases: [existing, { ...kept, expected: "Gives the ship date" }], + }); +}); + +it("tells the user to reload when someone else saved the dataset first", async () => { + const user = userEvent.setup(); + const onClose = vi.fn(); + vi.stubGlobal( + "fetch", + vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { + const url = new URL(input instanceof Request ? input.url : String(input), "http://localhost"); + const method = input instanceof Request ? input.method : init?.method ?? "GET"; + if (method === "POST" && url.pathname === "/lens/datasets/ds-1/revisions") + return Response.json({ detail: "Dataset changed, reload" }, { status: 409 }); + if (method === "POST") return Response.json({ cases: [kept], skipped: [] }); + if (url.pathname === "/lens/datasets/ds-1") return Response.json(dataset); + return Response.json([ + { id: "ds-1", name: "support cases", agent_name: "support_agent", revision: 3, case_count: 1, updated_at: "" }, + ]); + }), + ); + renderWithLens(); + + await user.click(await screen.findByRole("button", { name: "Save 1 case" })); + + expect(await screen.findByRole("alert")).toHaveTextContent("Reload to add to the latest version"); + expect(screen.getByRole("button", { name: "Save 1 case" })).toBeDisabled(); + expect(onClose).not.toHaveBeenCalled(); +}); + +it("creates a new dataset by name and saves the cases into its first revision", async () => { + const user = userEvent.setup(); + const created: Dataset = { ...dataset, id: "ds-new", name: "refund cases", revision: 0, cases: [] }; + proxy.get.mockImplementation(async (path) => { + if (path === "/lens/datasets") return []; + throw new Error(`unexpected GET ${path}`); + }); + proxy.post.mockImplementation(async (path) => { + if (path === "/lens/datasets/build") return { cases: [kept], skipped: [] }; + if (path === "/lens/datasets") return created; + if (path === "/lens/datasets/ds-new/revisions") return { ...created, revision: 1 }; + throw new Error(`unexpected POST ${path}`); + }); + renderWithLens(); + + fireEvent.change(await screen.findByRole("textbox", { name: "Name" }), { target: { value: "refund cases" } }); + await user.click(await screen.findByRole("button", { name: "Save 1 case" })); + + await waitFor(() => + expect(proxy.post.mock.calls.map(([path, request]) => [path, request.body])).toEqual([ + ["/lens/datasets/build", { sources, dataset_id: "" }], + ["/lens/datasets", { name: "refund cases", agent_name: "support_agent" }], + ["/lens/datasets/ds-new/revisions", { base_revision: 0, cases: [kept] }], + ]), + ); +}); diff --git a/ui/litellm-dashboard/src/components/lens/datasets/AddToDatasetDialog.tsx b/ui/litellm-dashboard/src/components/lens/datasets/AddToDatasetDialog.tsx new file mode 100644 index 00000000000..e36a8a09e97 --- /dev/null +++ b/ui/litellm-dashboard/src/components/lens/datasets/AddToDatasetDialog.tsx @@ -0,0 +1,424 @@ +"use client"; + +import { useQueryClient } from "@tanstack/react-query"; +import { Check, DatabaseZap, Loader2, RotateCw, TriangleAlert } from "lucide-react"; +import { useId, useState } from "react"; + +import { Button } from "@/components/ui/button"; +import { Checkbox } from "@/components/ui/checkbox"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { Input } from "@/components/ui/input"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import { Textarea } from "@/components/ui/textarea"; +import { extractProxyErrorMessage } from "@/lib/http/client"; +import { cn } from "@/lib/cva.config"; +import { toast } from "@/lib/toast"; + +import { useTracesLive } from "../traces/api"; +import { useOptionalLensApi } from "../data/LensServices"; +import { useOptionalOnboarding } from "../onboarding/OnboardingContext"; +import { useDatasetRoute, useLensRoute } from "../route"; +import { + datasetKeys, + isRevisionConflict, + useBuildCases, + useCreateDataset, + useDataset, + useDatasets, + useSaveRevision, +} from "./api"; +import { + casePrompt, + caseReplySummary, + EMPTY_DRAFT, + revisionCases, + setExpected, + SKIP_REASON_TEXT, + toggleCase, + type Draft, +} from "./draft"; +import type { BuildSource, Dataset, DatasetCase, DatasetMessage, DatasetSummary, SkippedCase } from "./types"; + +const NEW_DATASET = "__new__"; + +const cases = (count: number): string => `${count} ${count === 1 ? "case" : "cases"}`; + +/** Admins can save cases; the sample session keeps its datasets in memory so anyone can try it. */ +export function useCanAddToDataset(): boolean { + const api = useOptionalLensApi(); + const onboarding = useOptionalOnboarding(); + const live = useTracesLive(); + const canWrite = !!onboarding && onboarding.canInvestigate && !onboarding.readOnly; + return !!api && (!live || canWrite); +} + +export interface AddToDatasetDialogProps { + readonly sources: readonly BuildSource[]; + readonly agentName?: string; + readonly onClose: () => void; +} + +function useOpenSavedDataset() { + const { setTab } = useLensRoute(); + const { openDataset } = useDatasetRoute(); + return (id: string) => { + setTab("datasets"); + openDataset(id); + }; +} + +/** Defaults to the agent's dataset when one exists, otherwise to a new one once the list has loaded. */ +function useDatasetTarget(agentName: string) { + const datasets = useDatasets(); + const [picked, setPicked] = useState(null); + const forAgent = datasets.data?.find((item) => agentName && item.agent_name === agentName)?.id; + const target = picked ?? forAgent ?? (datasets.isPending ? null : NEW_DATASET); + const existingId = target === NEW_DATASET ? null : target; + return { datasets: datasets.data ?? [], target, existingId, setPicked }; +} + +export function AddToDatasetDialog({ sources, agentName = "", onClose }: AddToDatasetDialogProps) { + const { datasets, target, existingId, setPicked } = useDatasetTarget(agentName); + const dataset = useDataset(existingId); + const build = useBuildCases({ sources: [...sources], dataset_id: existingId ?? "" }, target !== null); + const [draft, setDraft] = useState(EMPTY_DRAFT); + const [name, setName] = useState(agentName ? `${agentName} cases` : ""); + const [conflict, setConflict] = useState(false); + const create = useCreateDataset(); + const save = useSaveRevision(); + const queryClient = useQueryClient(); + const openSaved = useOpenSavedDataset(); + + const built = build.data?.cases ?? []; + const chosen = built.filter((item) => !draft.excluded.has(item.id)).length; + const busy = create.isPending || save.isPending; + const targetReady = existingId ? dataset.isSuccess : name.trim().length > 0; + const hasCases = build.isSuccess && chosen > 0; + const idle = !busy && !conflict; + const canSave = hasCases && targetReady && idle; + + const pickTarget = (next: string) => { + setPicked(next); + setConflict(false); + }; + + const reload = async () => { + setConflict(false); + await queryClient.invalidateQueries({ queryKey: datasetKeys.all() }); + }; + + const baseDataset = async (): Promise => { + if (existingId) return dataset.data; + return create.mutateAsync({ name: name.trim(), agent_name: agentName }).catch((error: unknown) => { + toast.fromError(error); + return undefined; + }); + }; + + const submit = async () => { + const base = await baseDataset(); + if (!base) return; + try { + const saved = await save.mutateAsync({ + datasetId: base.id, + body: { base_revision: base.revision, cases: revisionCases(base.cases, built, draft) }, + }); + toast.success(`Added ${cases(chosen)} to ${saved.name}`, { + description: ( + + ), + }); + onClose(); + } catch (error) { + if (isRevisionConflict(error)) setConflict(true); + else toast.fromError(error); + } + }; + + return ( + !open && !busy && onClose()}> + + + Add to dataset + Saves a copy of each conversation, so it stays after the trace expires. + + + setDraft((current) => toggleCase(current, id))} + onExpected={(id, text) => setDraft((current) => setExpected(current, id, text))} + /> + + {conflict ? void reload()} /> : } +
+ + +
+ + + + ); +} + +interface TargetFieldsProps { + readonly datasets: readonly DatasetSummary[]; + readonly target: string | null; + readonly name: string; + readonly onTarget: (target: string) => void; + readonly onName: (name: string) => void; +} + +function TargetFields({ datasets, target, name, onTarget, onName }: TargetFieldsProps) { + const nameId = useId(); + const items = [ + { value: NEW_DATASET, label: "New dataset" }, + ...datasets.map((item) => ({ value: item.id, label: item.name })), + ]; + return ( +
+ + {target === NEW_DATASET && ( + + )} +
+ ); +} + +function ConflictNotice({ onReload }: { onReload: () => void }) { + return ( +

+

+ ); +} + +interface CasePreviewProps { + readonly build: ReturnType; + readonly draft: Draft; + readonly chosen: number; + readonly onToggle: (id: string) => void; + readonly onExpected: (id: string, text: string) => void; +} + +function CasePreview({ build, draft, chosen, onToggle, onExpected }: CasePreviewProps) { + if (build.isPending) + return ( +

+

+ ); + if (build.isError) + return ( +

+

+ ); + const { cases, skipped } = build.data; + return ( +
+
+

Cases

+

+ {chosen} of {cases.length} selected +

+
+ {cases.length === 0 ? ( +

Nothing new to add.

+ ) : ( +
    + {cases.map((item, index) => ( + onToggle(item.id)} + onExpected={(text) => onExpected(item.id, text)} + /> + ))} +
+ )} + {skipped.length > 0 && } +
+ ); +} + +function CaseTile({ + item, + index, + selected, + expected, + onToggle, + onExpected, +}: { + item: DatasetCase; + index: number; + selected: boolean; + expected: string; + onToggle: () => void; + onExpected: (text: string) => void; +}) { + const label = `Case ${index + 1}`; + return ( +
  • + {selected && ( +